2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
30 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
31 * All rights reserved.
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
43 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
46 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
47 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
48 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
49 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
50 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
51 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * @(#)kern_event.c 1.0 (3/31/2000)
58 #include <machine/atomic.h>
60 #include <sys/param.h>
61 #include <sys/systm.h>
62 #include <sys/filedesc.h>
63 #include <sys/kernel.h>
64 #include <sys/proc_internal.h>
65 #include <sys/kauth.h>
66 #include <sys/malloc.h>
67 #include <sys/unistd.h>
68 #include <sys/file_internal.h>
69 #include <sys/fcntl.h>
70 #include <sys/select.h>
71 #include <sys/queue.h>
72 #include <sys/event.h>
73 #include <sys/eventvar.h>
74 #include <sys/protosw.h>
75 #include <sys/socket.h>
76 #include <sys/socketvar.h>
78 #include <sys/syscall.h> // SYS_* constants
79 #include <sys/sysctl.h>
81 #include <sys/sysproto.h>
83 #include <sys/vnode_internal.h>
85 #include <sys/proc_info.h>
86 #include <sys/codesign.h>
87 #include <sys/pthread_shims.h>
88 #include <sys/kdebug.h>
90 #include <pexpert/pexpert.h>
92 #include <kern/locks.h>
93 #include <kern/clock.h>
94 #include <kern/cpu_data.h>
95 #include <kern/policy_internal.h>
96 #include <kern/thread_call.h>
97 #include <kern/sched_prim.h>
98 #include <kern/waitq.h>
99 #include <kern/zalloc.h>
100 #include <kern/kalloc.h>
101 #include <kern/assert.h>
102 #include <kern/ast.h>
103 #include <kern/thread.h>
104 #include <kern/kcdata.h>
106 #include <pthread/priority_private.h>
107 #include <pthread/workqueue_syscalls.h>
108 #include <pthread/workqueue_internal.h>
109 #include <libkern/libkern.h>
111 #include "net/net_str_id.h"
113 #include <mach/task.h>
114 #include <libkern/section_keywords.h>
116 #if CONFIG_MEMORYSTATUS
117 #include <sys/kern_memorystatus.h>
120 extern mach_port_name_t
ipc_entry_name_mask(mach_port_name_t name
); /* osfmk/ipc/ipc_entry.h */
122 #define KEV_EVTID(code) BSDDBG_CODE(DBG_BSD_KEVENT, (code))
124 MALLOC_DEFINE(M_KQUEUE
, "kqueue", "memory for kqueue system");
126 #define KQ_EVENT NO_EVENT64
128 static int kqueue_select(struct fileproc
*fp
, int which
, void *wq_link_id
,
130 static int kqueue_close(struct fileglob
*fg
, vfs_context_t ctx
);
131 static int kqueue_kqfilter(struct fileproc
*fp
, struct knote
*kn
,
132 struct kevent_qos_s
*kev
);
133 static int kqueue_drain(struct fileproc
*fp
, vfs_context_t ctx
);
135 static const struct fileops kqueueops
= {
136 .fo_type
= DTYPE_KQUEUE
,
137 .fo_read
= fo_no_read
,
138 .fo_write
= fo_no_write
,
139 .fo_ioctl
= fo_no_ioctl
,
140 .fo_select
= kqueue_select
,
141 .fo_close
= kqueue_close
,
142 .fo_drain
= kqueue_drain
,
143 .fo_kqfilter
= kqueue_kqfilter
,
146 static inline int kevent_modern_copyout(struct kevent_qos_s
*, user_addr_t
*);
147 static int kevent_register_wait_prepare(struct knote
*kn
, struct kevent_qos_s
*kev
, int result
);
148 static void kevent_register_wait_block(struct turnstile
*ts
, thread_t handoff_thread
,
149 thread_continue_t cont
, struct _kevent_register
*cont_args
) __dead2
;
150 static void kevent_register_wait_return(struct _kevent_register
*cont_args
) __dead2
;
151 static void kevent_register_wait_cleanup(struct knote
*kn
);
153 static struct kqtailq
*kqueue_get_suppressed_queue(kqueue_t kq
, struct knote
*kn
);
154 static void kqueue_threadreq_initiate(struct kqueue
*kq
, workq_threadreq_t
, kq_index_t qos
, int flags
);
156 static void kqworkq_unbind(proc_t p
, workq_threadreq_t
);
157 static thread_qos_t
kqworkq_unbind_locked(struct kqworkq
*kqwq
, workq_threadreq_t
, thread_t thread
);
158 static workq_threadreq_t
kqworkq_get_request(struct kqworkq
*kqwq
, kq_index_t qos_index
);
160 static void kqworkloop_unbind(struct kqworkloop
*kwql
);
162 enum kqwl_unbind_locked_mode
{
163 KQWL_OVERRIDE_DROP_IMMEDIATELY
,
164 KQWL_OVERRIDE_DROP_DELAYED
,
166 static void kqworkloop_unbind_locked(struct kqworkloop
*kwql
, thread_t thread
,
167 enum kqwl_unbind_locked_mode how
);
168 static void kqworkloop_unbind_delayed_override_drop(thread_t thread
);
169 static kq_index_t
kqworkloop_override(struct kqworkloop
*kqwl
);
170 static void kqworkloop_set_overcommit(struct kqworkloop
*kqwl
);
174 * The wakeup qos is the qos of QUEUED knotes.
176 * This QoS is accounted for with the events override in the
177 * kqr_override_index field. It is raised each time a new knote is queued at
178 * a given QoS. The kqwl_wakeup_indexes field is a superset of the non empty
179 * knote buckets and is recomputed after each event delivery.
181 KQWL_UTQ_UPDATE_WAKEUP_QOS
,
182 KQWL_UTQ_UPDATE_STAYACTIVE_QOS
,
183 KQWL_UTQ_RECOMPUTE_WAKEUP_QOS
,
184 KQWL_UTQ_UNBINDING
, /* attempt to rebind */
187 * The wakeup override is for suppressed knotes that have fired again at
188 * a higher QoS than the one for which they are suppressed already.
189 * This override is cleared when the knote suppressed list becomes empty.
191 KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE
,
192 KQWL_UTQ_RESET_WAKEUP_OVERRIDE
,
194 * The QoS is the maximum QoS of an event enqueued on this workloop in
195 * userland. It is copied from the only EVFILT_WORKLOOP knote with
196 * a NOTE_WL_THREAD_REQUEST bit set allowed on this workloop. If there is no
197 * such knote, this QoS is 0.
199 KQWL_UTQ_SET_QOS_INDEX
,
200 KQWL_UTQ_REDRIVE_EVENTS
,
202 static void kqworkloop_update_threads_qos(struct kqworkloop
*kqwl
, int op
, kq_index_t qos
);
203 static int kqworkloop_end_processing(struct kqworkloop
*kqwl
, int flags
, int kevent_flags
);
205 static struct knote
*knote_alloc(void);
206 static void knote_free(struct knote
*kn
);
207 static int kq_add_knote(struct kqueue
*kq
, struct knote
*kn
,
208 struct knote_lock_ctx
*knlc
, struct proc
*p
);
209 static struct knote
*kq_find_knote_and_kq_lock(struct kqueue
*kq
,
210 struct kevent_qos_s
*kev
, bool is_fd
, struct proc
*p
);
212 static void knote_activate(kqueue_t kqu
, struct knote
*kn
, int result
);
213 static void knote_dequeue(kqueue_t kqu
, struct knote
*kn
);
215 static void knote_apply_touch(kqueue_t kqu
, struct knote
*kn
,
216 struct kevent_qos_s
*kev
, int result
);
217 static void knote_suppress(kqueue_t kqu
, struct knote
*kn
);
218 static void knote_unsuppress(kqueue_t kqu
, struct knote
*kn
);
219 static void knote_drop(kqueue_t kqu
, struct knote
*kn
, struct knote_lock_ctx
*knlc
);
221 // both these functions may dequeue the knote and it is up to the caller
222 // to enqueue the knote back
223 static void knote_adjust_qos(struct kqueue
*kq
, struct knote
*kn
, int result
);
224 static void knote_reset_priority(kqueue_t kqu
, struct knote
*kn
, pthread_priority_t pp
);
226 static zone_t knote_zone
;
227 static zone_t kqfile_zone
;
228 static zone_t kqworkq_zone
;
229 static zone_t kqworkloop_zone
;
230 #if DEVELOPMENT || DEBUG
231 #define KEVENT_PANIC_ON_WORKLOOP_OWNERSHIP_LEAK (1U << 0)
232 #define KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS (1U << 1)
233 #define KEVENT_PANIC_BOOT_ARG_INITIALIZED (1U << 31)
235 #define KEVENT_PANIC_DEFAULT_VALUE (0)
237 kevent_debug_flags(void)
239 static uint32_t flags
= KEVENT_PANIC_DEFAULT_VALUE
;
241 if ((flags
& KEVENT_PANIC_BOOT_ARG_INITIALIZED
) == 0) {
243 if (!PE_parse_boot_argn("kevent_debug", &value
, sizeof(value
))) {
244 value
= KEVENT_PANIC_DEFAULT_VALUE
;
246 value
|= KEVENT_PANIC_BOOT_ARG_INITIALIZED
;
247 os_atomic_store(&flags
, value
, relaxed
);
253 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
255 static int filt_no_attach(struct knote
*kn
, struct kevent_qos_s
*kev
);
256 static void filt_no_detach(struct knote
*kn
);
257 static int filt_bad_event(struct knote
*kn
, long hint
);
258 static int filt_bad_touch(struct knote
*kn
, struct kevent_qos_s
*kev
);
259 static int filt_bad_process(struct knote
*kn
, struct kevent_qos_s
*kev
);
261 SECURITY_READ_ONLY_EARLY(static struct filterops
) bad_filtops
= {
262 .f_attach
= filt_no_attach
,
263 .f_detach
= filt_no_detach
,
264 .f_event
= filt_bad_event
,
265 .f_touch
= filt_bad_touch
,
266 .f_process
= filt_bad_process
,
269 #if CONFIG_MEMORYSTATUS
270 extern const struct filterops memorystatus_filtops
;
271 #endif /* CONFIG_MEMORYSTATUS */
272 extern const struct filterops fs_filtops
;
273 extern const struct filterops sig_filtops
;
274 extern const struct filterops machport_filtops
;
275 extern const struct filterops pipe_nfiltops
;
276 extern const struct filterops pipe_rfiltops
;
277 extern const struct filterops pipe_wfiltops
;
278 extern const struct filterops ptsd_kqops
;
279 extern const struct filterops ptmx_kqops
;
280 extern const struct filterops soread_filtops
;
281 extern const struct filterops sowrite_filtops
;
282 extern const struct filterops sock_filtops
;
283 extern const struct filterops soexcept_filtops
;
284 extern const struct filterops spec_filtops
;
285 extern const struct filterops bpfread_filtops
;
286 extern const struct filterops necp_fd_rfiltops
;
287 extern const struct filterops fsevent_filtops
;
288 extern const struct filterops vnode_filtops
;
289 extern const struct filterops tty_filtops
;
291 const static struct filterops file_filtops
;
292 const static struct filterops kqread_filtops
;
293 const static struct filterops proc_filtops
;
294 const static struct filterops timer_filtops
;
295 const static struct filterops user_filtops
;
296 const static struct filterops workloop_filtops
;
300 * Rules for adding new filters to the system:
302 * - Add a new "EVFILT_" option value to bsd/sys/event.h (typically a negative value)
303 * in the exported section of the header
304 * - Update the EVFILT_SYSCOUNT value to reflect the new addition
305 * - Add a filterops to the sysfilt_ops array. Public filters should be added at the end
306 * of the Public Filters section in the array.
308 * - Add a new "EVFILT_" value to bsd/sys/event.h (typically a positive value)
309 * in the XNU_KERNEL_PRIVATE section of the header
310 * - Update the EVFILTID_MAX value to reflect the new addition
311 * - Add a filterops to the sysfilt_ops. Private filters should be added at the end of
312 * the Private filters section of the array.
314 static_assert(EVFILTID_MAX
< UINT8_MAX
, "kn_filtid expects this to be true");
315 static const struct filterops
* const sysfilt_ops
[EVFILTID_MAX
] = {
317 [~EVFILT_READ
] = &file_filtops
,
318 [~EVFILT_WRITE
] = &file_filtops
,
319 [~EVFILT_AIO
] = &bad_filtops
,
320 [~EVFILT_VNODE
] = &file_filtops
,
321 [~EVFILT_PROC
] = &proc_filtops
,
322 [~EVFILT_SIGNAL
] = &sig_filtops
,
323 [~EVFILT_TIMER
] = &timer_filtops
,
324 [~EVFILT_MACHPORT
] = &machport_filtops
,
325 [~EVFILT_FS
] = &fs_filtops
,
326 [~EVFILT_USER
] = &user_filtops
,
327 [~EVFILT_UNUSED_11
] = &bad_filtops
,
328 [~EVFILT_VM
] = &bad_filtops
,
329 [~EVFILT_SOCK
] = &file_filtops
,
330 #if CONFIG_MEMORYSTATUS
331 [~EVFILT_MEMORYSTATUS
] = &memorystatus_filtops
,
333 [~EVFILT_MEMORYSTATUS
] = &bad_filtops
,
335 [~EVFILT_EXCEPT
] = &file_filtops
,
336 [~EVFILT_WORKLOOP
] = &workloop_filtops
,
338 /* Private filters */
339 [EVFILTID_KQREAD
] = &kqread_filtops
,
340 [EVFILTID_PIPE_N
] = &pipe_nfiltops
,
341 [EVFILTID_PIPE_R
] = &pipe_rfiltops
,
342 [EVFILTID_PIPE_W
] = &pipe_wfiltops
,
343 [EVFILTID_PTSD
] = &ptsd_kqops
,
344 [EVFILTID_SOREAD
] = &soread_filtops
,
345 [EVFILTID_SOWRITE
] = &sowrite_filtops
,
346 [EVFILTID_SCK
] = &sock_filtops
,
347 [EVFILTID_SOEXCEPT
] = &soexcept_filtops
,
348 [EVFILTID_SPEC
] = &spec_filtops
,
349 [EVFILTID_BPFREAD
] = &bpfread_filtops
,
350 [EVFILTID_NECP_FD
] = &necp_fd_rfiltops
,
351 [EVFILTID_FSEVENT
] = &fsevent_filtops
,
352 [EVFILTID_VN
] = &vnode_filtops
,
353 [EVFILTID_TTY
] = &tty_filtops
,
354 [EVFILTID_PTMX
] = &ptmx_kqops
,
356 /* fake filter for detached knotes, keep last */
357 [EVFILTID_DETACHED
] = &bad_filtops
,
360 /* waitq prepost callback */
361 void waitq_set__CALLING_PREPOST_HOOK__(waitq_set_prepost_hook_t
*kq_hook
);
364 kqr_thread_bound(workq_threadreq_t kqr
)
366 return kqr
->tr_state
== WORKQ_TR_STATE_BOUND
;
370 kqr_thread_requested_pending(workq_threadreq_t kqr
)
372 workq_tr_state_t tr_state
= kqr
->tr_state
;
373 return tr_state
> WORKQ_TR_STATE_IDLE
&& tr_state
< WORKQ_TR_STATE_BOUND
;
377 kqr_thread_requested(workq_threadreq_t kqr
)
379 return kqr
->tr_state
!= WORKQ_TR_STATE_IDLE
;
382 static inline thread_t
383 kqr_thread_fast(workq_threadreq_t kqr
)
385 assert(kqr_thread_bound(kqr
));
386 return kqr
->tr_thread
;
389 static inline thread_t
390 kqr_thread(workq_threadreq_t kqr
)
392 return kqr_thread_bound(kqr
) ? kqr
->tr_thread
: THREAD_NULL
;
395 static inline struct kqworkloop
*
396 kqr_kqworkloop(workq_threadreq_t kqr
)
398 if (kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
) {
399 return __container_of(kqr
, struct kqworkloop
, kqwl_request
);
404 static inline kqueue_t
405 kqr_kqueue(proc_t p
, workq_threadreq_t kqr
)
408 if (kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
) {
409 kqu
.kqwl
= kqr_kqworkloop(kqr
);
411 kqu
.kqwq
= p
->p_fd
->fd_wqkqueue
;
412 assert(kqr
>= kqu
.kqwq
->kqwq_request
&&
413 kqr
< kqu
.kqwq
->kqwq_request
+ KQWQ_NBUCKETS
);
419 * kqueue/note lock implementations
421 * The kqueue lock guards the kq state, the state of its queues,
422 * and the kqueue-aware status and locks of individual knotes.
424 * The kqueue workq lock is used to protect state guarding the
425 * interaction of the kqueue with the workq. This state cannot
426 * be guarded by the kq lock - as it needs to be taken when we
427 * already have the waitq set lock held (during the waitq hook
428 * callback). It might be better to use the waitq lock itself
429 * for this, but the IRQ requirements make that difficult).
431 * Knote flags, filter flags, and associated data are protected
432 * by the underlying object lock - and are only ever looked at
433 * by calling the filter to get a [consistent] snapshot of that
436 static lck_grp_attr_t
*kq_lck_grp_attr
;
437 static lck_grp_t
*kq_lck_grp
;
438 static lck_attr_t
*kq_lck_attr
;
443 lck_spin_lock(&kqu
.kq
->kq_lock
);
447 kqlock_held(__assert_only kqueue_t kqu
)
449 LCK_SPIN_ASSERT(&kqu
.kq
->kq_lock
, LCK_ASSERT_OWNED
);
453 kqunlock(kqueue_t kqu
)
455 lck_spin_unlock(&kqu
.kq
->kq_lock
);
459 knhash_lock(struct filedesc
*fdp
)
461 lck_mtx_lock(&fdp
->fd_knhashlock
);
465 knhash_unlock(struct filedesc
*fdp
)
467 lck_mtx_unlock(&fdp
->fd_knhashlock
);
470 /* wait event for knote locks */
471 static inline event_t
472 knote_lock_wev(struct knote
*kn
)
474 return (event_t
)(&kn
->kn_hook
);
477 /* wait event for kevent_register_wait_* */
478 static inline event64_t
479 knote_filt_wev64(struct knote
*kn
)
481 /* kdp_workloop_sync_wait_find_owner knows about this */
482 return CAST_EVENT64_T(kn
);
485 /* wait event for knote_post/knote_drop */
486 static inline event64_t
487 knote_post_wev64(struct knote
*kn
)
489 return CAST_EVENT64_T(&kn
->kn_kevent
);
493 * @function knote_has_qos
496 * Whether the knote has a regular QoS.
499 * kn_qos_override is:
501 * - THREAD_QOS_LAST for special buckets (stayactive, manager)
503 * Other values mean the knote participates to QoS propagation.
506 knote_has_qos(struct knote
*kn
)
508 return kn
->kn_qos_override
> 0 && kn
->kn_qos_override
< THREAD_QOS_LAST
;
511 #pragma mark knote locks
514 * Enum used by the knote_lock_* functions.
516 * KNOTE_KQ_LOCK_ALWAYS
517 * The function will always return with the kq lock held.
519 * KNOTE_KQ_LOCK_ON_SUCCESS
520 * The function will return with the kq lock held if it was successful
521 * (knote_lock() is the only function that can fail).
523 * KNOTE_KQ_LOCK_ON_FAILURE
524 * The function will return with the kq lock held if it was unsuccessful
525 * (knote_lock() is the only function that can fail).
528 * The function returns with the kq unlocked.
531 KNOTE_KQ_LOCK_ALWAYS
,
532 KNOTE_KQ_LOCK_ON_SUCCESS
,
533 KNOTE_KQ_LOCK_ON_FAILURE
,
537 static struct knote_lock_ctx
*
538 knote_lock_ctx_find(kqueue_t kqu
, struct knote
*kn
)
540 struct knote_lock_ctx
*ctx
;
541 LIST_FOREACH(ctx
, &kqu
.kq
->kq_knlocks
, knlc_link
) {
542 if (ctx
->knlc_knote
== kn
) {
546 panic("knote lock context not found: %p", kn
);
550 /* slowpath of knote_lock() */
551 __attribute__((noinline
))
552 static bool __result_use_check
553 knote_lock_slow(kqueue_t kqu
, struct knote
*kn
,
554 struct knote_lock_ctx
*knlc
, int kqlocking
)
556 struct knote_lock_ctx
*owner_lc
;
557 struct uthread
*uth
= current_uthread();
562 owner_lc
= knote_lock_ctx_find(kqu
, kn
);
563 #if DEBUG || DEVELOPMENT
564 knlc
->knlc_state
= KNOTE_LOCK_CTX_WAITING
;
566 owner_lc
->knlc_waiters
++;
569 * Make our lock context visible to knote_unlock()
571 uth
->uu_knlock
= knlc
;
573 wr
= lck_spin_sleep_with_inheritor(&kqu
.kq
->kq_lock
, LCK_SLEEP_UNLOCK
,
574 knote_lock_wev(kn
), owner_lc
->knlc_thread
,
575 THREAD_UNINT
| THREAD_WAIT_NOREPORT
, TIMEOUT_WAIT_FOREVER
);
577 if (wr
== THREAD_RESTART
) {
579 * We haven't been woken up by knote_unlock() but knote_unlock_cancel.
580 * We need to cleanup the state since no one did.
582 uth
->uu_knlock
= NULL
;
583 #if DEBUG || DEVELOPMENT
584 assert(knlc
->knlc_state
== KNOTE_LOCK_CTX_WAITING
);
585 knlc
->knlc_state
= KNOTE_LOCK_CTX_UNLOCKED
;
588 if (kqlocking
== KNOTE_KQ_LOCK_ALWAYS
||
589 kqlocking
== KNOTE_KQ_LOCK_ON_FAILURE
) {
594 if (kqlocking
== KNOTE_KQ_LOCK_ALWAYS
||
595 kqlocking
== KNOTE_KQ_LOCK_ON_SUCCESS
) {
597 #if DEBUG || DEVELOPMENT
599 * This state is set under the lock so we can't
600 * really assert this unless we hold the lock.
602 assert(knlc
->knlc_state
== KNOTE_LOCK_CTX_LOCKED
);
610 * Attempts to take the "knote" lock.
612 * Called with the kqueue lock held.
614 * Returns true if the knote lock is acquired, false if it has been dropped
616 static bool __result_use_check
617 knote_lock(kqueue_t kqu
, struct knote
*kn
, struct knote_lock_ctx
*knlc
,
618 enum kqlocking kqlocking
)
622 #if DEBUG || DEVELOPMENT
623 assert(knlc
->knlc_state
== KNOTE_LOCK_CTX_UNLOCKED
);
625 knlc
->knlc_knote
= kn
;
626 knlc
->knlc_thread
= current_thread();
627 knlc
->knlc_waiters
= 0;
629 if (__improbable(kn
->kn_status
& KN_LOCKED
)) {
630 return knote_lock_slow(kqu
, kn
, knlc
, kqlocking
);
634 * When the knote will be dropped, the knote lock is taken before
635 * KN_DROPPING is set, and then the knote will be removed from any
636 * hash table that references it before the lock is canceled.
638 assert((kn
->kn_status
& KN_DROPPING
) == 0);
639 LIST_INSERT_HEAD(&kqu
.kq
->kq_knlocks
, knlc
, knlc_link
);
640 kn
->kn_status
|= KN_LOCKED
;
641 #if DEBUG || DEVELOPMENT
642 knlc
->knlc_state
= KNOTE_LOCK_CTX_LOCKED
;
645 if (kqlocking
== KNOTE_KQ_UNLOCK
||
646 kqlocking
== KNOTE_KQ_LOCK_ON_FAILURE
) {
653 * Unlocks a knote successfully locked with knote_lock().
655 * Called with the kqueue lock held.
657 * Returns with the kqueue lock held according to KNOTE_KQ_* mode.
660 knote_unlock(kqueue_t kqu
, struct knote
*kn
,
661 struct knote_lock_ctx
*knlc
, enum kqlocking kqlocking
)
665 assert(knlc
->knlc_knote
== kn
);
666 assert(kn
->kn_status
& KN_LOCKED
);
667 #if DEBUG || DEVELOPMENT
668 assert(knlc
->knlc_state
== KNOTE_LOCK_CTX_LOCKED
);
671 LIST_REMOVE(knlc
, knlc_link
);
673 if (knlc
->knlc_waiters
) {
674 thread_t thread
= THREAD_NULL
;
676 wakeup_one_with_inheritor(knote_lock_wev(kn
), THREAD_AWAKENED
,
677 LCK_WAKE_DEFAULT
, &thread
);
680 * knote_lock_slow() publishes the lock context of waiters
681 * in uthread::uu_knlock.
683 * Reach out and make this context the new owner.
685 struct uthread
*ut
= get_bsdthread_info(thread
);
686 struct knote_lock_ctx
*next_owner_lc
= ut
->uu_knlock
;
688 assert(next_owner_lc
->knlc_knote
== kn
);
689 next_owner_lc
->knlc_waiters
= knlc
->knlc_waiters
- 1;
690 LIST_INSERT_HEAD(&kqu
.kq
->kq_knlocks
, next_owner_lc
, knlc_link
);
691 #if DEBUG || DEVELOPMENT
692 next_owner_lc
->knlc_state
= KNOTE_LOCK_CTX_LOCKED
;
694 ut
->uu_knlock
= NULL
;
695 thread_deallocate_safe(thread
);
697 kn
->kn_status
&= ~KN_LOCKED
;
700 if ((kn
->kn_status
& KN_MERGE_QOS
) && !(kn
->kn_status
& KN_POSTING
)) {
702 * No f_event() in flight anymore, we can leave QoS "Merge" mode
704 * See knote_adjust_qos()
706 kn
->kn_status
&= ~KN_MERGE_QOS
;
708 if (kqlocking
== KNOTE_KQ_UNLOCK
) {
711 #if DEBUG || DEVELOPMENT
712 knlc
->knlc_state
= KNOTE_LOCK_CTX_UNLOCKED
;
717 * Aborts all waiters for a knote lock, and unlock the knote.
719 * Called with the kqueue lock held.
721 * Returns with the kqueue unlocked.
724 knote_unlock_cancel(struct kqueue
*kq
, struct knote
*kn
,
725 struct knote_lock_ctx
*knlc
)
729 assert(knlc
->knlc_knote
== kn
);
730 assert(kn
->kn_status
& KN_LOCKED
);
731 assert(kn
->kn_status
& KN_DROPPING
);
733 LIST_REMOVE(knlc
, knlc_link
);
734 kn
->kn_status
&= ~KN_LOCKED
;
737 if (knlc
->knlc_waiters
) {
738 wakeup_all_with_inheritor(knote_lock_wev(kn
), THREAD_RESTART
);
740 #if DEBUG || DEVELOPMENT
741 knlc
->knlc_state
= KNOTE_LOCK_CTX_UNLOCKED
;
746 * Call the f_event hook of a given filter.
748 * Takes a use count to protect against concurrent drops.
751 knote_post(struct knote
*kn
, long hint
)
753 struct kqueue
*kq
= knote_get_kq(kn
);
754 int dropping
, result
;
758 if (__improbable(kn
->kn_status
& (KN_DROPPING
| KN_VANISHED
))) {
762 if (__improbable(kn
->kn_status
& KN_POSTING
)) {
763 panic("KNOTE() called concurrently on knote %p", kn
);
766 kn
->kn_status
|= KN_POSTING
;
769 result
= filter_call(knote_fops(kn
), f_event(kn
, hint
));
772 dropping
= (kn
->kn_status
& KN_DROPPING
);
774 if (!dropping
&& (result
& FILTER_ACTIVE
)) {
775 knote_activate(kq
, kn
, result
);
778 if ((kn
->kn_status
& KN_LOCKED
) == 0) {
780 * There's no other f_* call in flight, we can leave QoS "Merge" mode.
782 * See knote_adjust_qos()
784 kn
->kn_status
&= ~(KN_POSTING
| KN_MERGE_QOS
);
786 kn
->kn_status
&= ~KN_POSTING
;
789 if (__improbable(dropping
)) {
790 waitq_wakeup64_all((struct waitq
*)&kq
->kq_wqs
, knote_post_wev64(kn
),
791 THREAD_AWAKENED
, WAITQ_ALL_PRIORITIES
);
798 * Called by knote_drop() to wait for the last f_event() caller to be done.
800 * - kq locked at entry
801 * - kq unlocked at exit
804 knote_wait_for_post(struct kqueue
*kq
, struct knote
*kn
)
806 wait_result_t wr
= THREAD_NOT_WAITING
;
810 assert(kn
->kn_status
& KN_DROPPING
);
812 if (kn
->kn_status
& KN_POSTING
) {
813 wr
= waitq_assert_wait64((struct waitq
*)&kq
->kq_wqs
,
814 knote_post_wev64(kn
), THREAD_UNINT
| THREAD_WAIT_NOREPORT
,
815 TIMEOUT_WAIT_FOREVER
);
818 if (wr
== THREAD_WAITING
) {
819 thread_block(THREAD_CONTINUE_NULL
);
823 #pragma mark knote helpers for filters
827 knote_set_error(struct knote
*kn
, int error
)
829 kn
->kn_flags
|= EV_ERROR
;
830 kn
->kn_sdata
= error
;
835 knote_low_watermark(const struct knote
*kn
)
837 return (kn
->kn_sfflags
& NOTE_LOWAT
) ? kn
->kn_sdata
: 1;
841 * @function knote_fill_kevent_with_sdata
844 * Fills in a kevent from the current content of a knote.
847 * This is meant to be called from filter's f_event hooks.
848 * The kevent data is filled with kn->kn_sdata.
850 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
852 * Using knote_fill_kevent is typically preferred.
856 knote_fill_kevent_with_sdata(struct knote
*kn
, struct kevent_qos_s
*kev
)
858 #define knote_assert_aliases(name1, offs1, name2) \
859 static_assert(offsetof(struct kevent_qos_s, name1) + offs1 == \
860 offsetof(struct kevent_internal_s, name2), \
861 "kevent_qos_s::" #name1 " and kevent_internal_s::" #name2 "need to alias")
863 * All the code makes assumptions on these aliasing,
864 * so make sure we fail the build if we ever ever ever break them.
866 knote_assert_aliases(ident
, 0, kei_ident
);
867 #ifdef __LITTLE_ENDIAN__
868 knote_assert_aliases(filter
, 0, kei_filter
); // non trivial overlap
869 knote_assert_aliases(filter
, 1, kei_filtid
); // non trivial overlap
871 knote_assert_aliases(filter
, 0, kei_filtid
); // non trivial overlap
872 knote_assert_aliases(filter
, 1, kei_filter
); // non trivial overlap
874 knote_assert_aliases(flags
, 0, kei_flags
);
875 knote_assert_aliases(qos
, 0, kei_qos
);
876 knote_assert_aliases(udata
, 0, kei_udata
);
877 knote_assert_aliases(fflags
, 0, kei_fflags
);
878 knote_assert_aliases(xflags
, 0, kei_sfflags
); // non trivial overlap
879 knote_assert_aliases(data
, 0, kei_sdata
); // non trivial overlap
880 knote_assert_aliases(ext
, 0, kei_ext
);
881 #undef knote_assert_aliases
884 * Fix the differences between kevent_qos_s and kevent_internal_s:
885 * - xflags is where kn_sfflags lives, we need to zero it
886 * - fixup the high bits of `filter` where kn_filtid lives
888 *kev
= *(struct kevent_qos_s
*)&kn
->kn_kevent
;
890 kev
->filter
|= 0xff00;
891 if (kn
->kn_flags
& EV_CLEAR
) {
897 * @function knote_fill_kevent
900 * Fills in a kevent from the current content of a knote.
903 * This is meant to be called from filter's f_event hooks.
904 * The kevent data is filled with the passed in data.
906 * kn->kn_fflags is cleared if kn->kn_flags has EV_CLEAR set.
910 knote_fill_kevent(struct knote
*kn
, struct kevent_qos_s
*kev
, int64_t data
)
912 knote_fill_kevent_with_sdata(kn
, kev
);
913 kev
->filter
= kn
->kn_filter
;
918 #pragma mark file_filtops
921 filt_fileattach(struct knote
*kn
, struct kevent_qos_s
*kev
)
923 return fo_kqfilter(kn
->kn_fp
, kn
, kev
);
926 SECURITY_READ_ONLY_EARLY(static struct filterops
) file_filtops
= {
928 .f_attach
= filt_fileattach
,
931 #pragma mark kqread_filtops
933 #define f_flag f_fglob->fg_flag
934 #define f_ops f_fglob->fg_ops
935 #define f_data f_fglob->fg_data
936 #define f_lflags f_fglob->fg_lflags
939 filt_kqdetach(struct knote
*kn
)
941 struct kqfile
*kqf
= (struct kqfile
*)kn
->kn_fp
->f_data
;
942 struct kqueue
*kq
= &kqf
->kqf_kqueue
;
945 KNOTE_DETACH(&kqf
->kqf_sel
.si_note
, kn
);
950 filt_kqueue(struct knote
*kn
, __unused
long hint
)
952 struct kqueue
*kq
= (struct kqueue
*)kn
->kn_fp
->f_data
;
954 return kq
->kq_count
> 0;
958 filt_kqtouch(struct knote
*kn
, struct kevent_qos_s
*kev
)
961 struct kqueue
*kq
= (struct kqueue
*)kn
->kn_fp
->f_data
;
965 res
= (kq
->kq_count
> 0);
972 filt_kqprocess(struct knote
*kn
, struct kevent_qos_s
*kev
)
974 struct kqueue
*kq
= (struct kqueue
*)kn
->kn_fp
->f_data
;
979 knote_fill_kevent(kn
, kev
, kq
->kq_count
);
987 SECURITY_READ_ONLY_EARLY(static struct filterops
) kqread_filtops
= {
989 .f_detach
= filt_kqdetach
,
990 .f_event
= filt_kqueue
,
991 .f_touch
= filt_kqtouch
,
992 .f_process
= filt_kqprocess
,
995 #pragma mark proc_filtops
998 filt_procattach(struct knote
*kn
, __unused
struct kevent_qos_s
*kev
)
1002 assert(PID_MAX
< NOTE_PDATAMASK
);
1004 if ((kn
->kn_sfflags
& (NOTE_TRACK
| NOTE_TRACKERR
| NOTE_CHILD
)) != 0) {
1005 knote_set_error(kn
, ENOTSUP
);
1009 p
= proc_find(kn
->kn_id
);
1011 knote_set_error(kn
, ESRCH
);
1015 const uint32_t NoteExitStatusBits
= NOTE_EXIT
| NOTE_EXITSTATUS
;
1017 if ((kn
->kn_sfflags
& NoteExitStatusBits
) == NoteExitStatusBits
) {
1019 pid_t selfpid
= proc_selfpid();
1021 if (p
->p_ppid
== selfpid
) {
1022 break; /* parent => ok */
1024 if ((p
->p_lflag
& P_LTRACED
) != 0 &&
1025 (p
->p_oppid
== selfpid
)) {
1026 break; /* parent-in-waiting => ok */
1029 knote_set_error(kn
, EACCES
);
1035 kn
->kn_flags
|= EV_CLEAR
; /* automatically set */
1036 kn
->kn_sdata
= 0; /* incoming data is ignored */
1040 KNOTE_ATTACH(&p
->p_klist
, kn
);
1042 proc_klist_unlock();
1047 * only captures edge-triggered events after this point
1048 * so it can't already be fired.
1055 * The knote may be attached to a different process, which may exit,
1056 * leaving nothing for the knote to be attached to. In that case,
1057 * the pointer to the process will have already been nulled out.
1060 filt_procdetach(struct knote
*kn
)
1067 if (p
!= PROC_NULL
) {
1068 kn
->kn_proc
= PROC_NULL
;
1069 KNOTE_DETACH(&p
->p_klist
, kn
);
1072 proc_klist_unlock();
1076 filt_procevent(struct knote
*kn
, long hint
)
1080 /* ALWAYS CALLED WITH proc_klist_lock */
1083 * Note: a lot of bits in hint may be obtained from the knote
1084 * To free some of those bits, see <rdar://problem/12592988> Freeing up
1085 * bits in hint for filt_procevent
1087 * mask off extra data
1089 event
= (u_int
)hint
& NOTE_PCTRLMASK
;
1092 * termination lifecycle events can happen while a debugger
1093 * has reparented a process, in which case notifications
1094 * should be quashed except to the tracing parent. When
1095 * the debugger reaps the child (either via wait4(2) or
1096 * process exit), the child will be reparented to the original
1097 * parent and these knotes re-fired.
1099 if (event
& NOTE_EXIT
) {
1100 if ((kn
->kn_proc
->p_oppid
!= 0)
1101 && (knote_get_kq(kn
)->kq_p
->p_pid
!= kn
->kn_proc
->p_ppid
)) {
1103 * This knote is not for the current ptrace(2) parent, ignore.
1110 * if the user is interested in this event, record it.
1112 if (kn
->kn_sfflags
& event
) {
1113 kn
->kn_fflags
|= event
;
1116 #pragma clang diagnostic push
1117 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
1118 if ((event
== NOTE_REAP
) || ((event
== NOTE_EXIT
) && !(kn
->kn_sfflags
& NOTE_REAP
))) {
1119 kn
->kn_flags
|= (EV_EOF
| EV_ONESHOT
);
1121 #pragma clang diagnostic pop
1125 * The kernel has a wrapper in place that returns the same data
1126 * as is collected here, in kn_hook32. Any changes to how
1127 * NOTE_EXITSTATUS and NOTE_EXIT_DETAIL are collected
1128 * should also be reflected in the proc_pidnoteexit() wrapper.
1130 if (event
== NOTE_EXIT
) {
1132 if ((kn
->kn_sfflags
& NOTE_EXITSTATUS
) != 0) {
1133 kn
->kn_fflags
|= NOTE_EXITSTATUS
;
1134 kn
->kn_hook32
|= (hint
& NOTE_PDATAMASK
);
1136 if ((kn
->kn_sfflags
& NOTE_EXIT_DETAIL
) != 0) {
1137 kn
->kn_fflags
|= NOTE_EXIT_DETAIL
;
1138 if ((kn
->kn_proc
->p_lflag
&
1139 P_LTERM_DECRYPTFAIL
) != 0) {
1140 kn
->kn_hook32
|= NOTE_EXIT_DECRYPTFAIL
;
1142 if ((kn
->kn_proc
->p_lflag
&
1143 P_LTERM_JETSAM
) != 0) {
1144 kn
->kn_hook32
|= NOTE_EXIT_MEMORY
;
1145 switch (kn
->kn_proc
->p_lflag
& P_JETSAM_MASK
) {
1146 case P_JETSAM_VMPAGESHORTAGE
:
1147 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_VMPAGESHORTAGE
;
1149 case P_JETSAM_VMTHRASHING
:
1150 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_VMTHRASHING
;
1152 case P_JETSAM_FCTHRASHING
:
1153 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_FCTHRASHING
;
1155 case P_JETSAM_VNODE
:
1156 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_VNODE
;
1158 case P_JETSAM_HIWAT
:
1159 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_HIWAT
;
1162 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_PID
;
1164 case P_JETSAM_IDLEEXIT
:
1165 kn
->kn_hook32
|= NOTE_EXIT_MEMORY_IDLE
;
1169 if ((kn
->kn_proc
->p_csflags
&
1171 kn
->kn_hook32
|= NOTE_EXIT_CSERROR
;
1176 /* if we have any matching state, activate the knote */
1177 return kn
->kn_fflags
!= 0;
1181 filt_proctouch(struct knote
*kn
, struct kevent_qos_s
*kev
)
1187 /* accept new filter flags and mask off output events no long interesting */
1188 kn
->kn_sfflags
= kev
->fflags
;
1190 /* restrict the current results to the (smaller?) set of new interest */
1192 * For compatibility with previous implementations, we leave kn_fflags
1193 * as they were before.
1195 //kn->kn_fflags &= kn->kn_sfflags;
1197 res
= (kn
->kn_fflags
!= 0);
1199 proc_klist_unlock();
1205 filt_procprocess(struct knote
*kn
, struct kevent_qos_s
*kev
)
1210 if (kn
->kn_fflags
) {
1211 knote_fill_kevent(kn
, kev
, kn
->kn_hook32
);
1215 proc_klist_unlock();
1219 SECURITY_READ_ONLY_EARLY(static struct filterops
) proc_filtops
= {
1220 .f_attach
= filt_procattach
,
1221 .f_detach
= filt_procdetach
,
1222 .f_event
= filt_procevent
,
1223 .f_touch
= filt_proctouch
,
1224 .f_process
= filt_procprocess
,
1227 #pragma mark timer_filtops
1229 struct filt_timer_params
{
1230 uint64_t deadline
; /* deadline in abs/cont time
1231 * (or 0 if NOTE_ABSOLUTE and deadline is in past) */
1232 uint64_t leeway
; /* leeway in abstime, or 0 if none */
1233 uint64_t interval
; /* interval in abstime or 0 if non-repeating timer */
1237 * Values stored in the knote at rest (using Mach absolute time units)
1239 * kn->kn_thcall where the thread_call object is stored
1240 * kn->kn_ext[0] next deadline or 0 if immediate expiration
1241 * kn->kn_ext[1] leeway value
1242 * kn->kn_sdata interval timer: the interval
1243 * absolute/deadline timer: 0
1244 * kn->kn_hook32 timer state
1247 * The timer has either never been scheduled or been cancelled.
1248 * It is safe to schedule a new one in this state.
1251 * The timer has been scheduled
1254 * The timer has fired and an event needs to be delivered.
1255 * When in this state, the callout may still be running.
1258 * The timer has fired at registration time, and the callout was never
1261 #define TIMER_IDLE 0x0
1262 #define TIMER_ARMED 0x1
1263 #define TIMER_FIRED 0x2
1264 #define TIMER_IMMEDIATE 0x3
1267 filt_timer_set_params(struct knote
*kn
, struct filt_timer_params
*params
)
1269 kn
->kn_ext
[0] = params
->deadline
;
1270 kn
->kn_ext
[1] = params
->leeway
;
1271 kn
->kn_sdata
= params
->interval
;
1275 * filt_timervalidate - process data from user
1277 * Sets up the deadline, interval, and leeway from the provided user data
1280 * kn_sdata timer deadline or interval time
1281 * kn_sfflags style of timer, unit of measurement
1284 * struct filter_timer_params to apply to the filter with
1285 * filt_timer_set_params when changes are ready to be commited.
1288 * EINVAL Invalid user data parameters
1289 * ERANGE Various overflows with the parameters
1291 * Called with timer filter lock held.
1294 filt_timervalidate(const struct kevent_qos_s
*kev
,
1295 struct filt_timer_params
*params
)
1298 * There are 5 knobs that need to be chosen for a timer registration:
1300 * A) Units of time (what is the time duration of the specified number)
1301 * Absolute and interval take:
1302 * NOTE_SECONDS, NOTE_USECONDS, NOTE_NSECONDS, NOTE_MACHTIME
1303 * Defaults to milliseconds if not specified
1305 * B) Clock epoch (what is the zero point of the specified number)
1306 * For interval, there is none
1307 * For absolute, defaults to the gettimeofday/calendar epoch
1308 * With NOTE_MACHTIME, uses mach_absolute_time()
1309 * With NOTE_MACHTIME and NOTE_MACH_CONTINUOUS_TIME, uses mach_continuous_time()
1311 * C) The knote's behavior on delivery
1312 * Interval timer causes the knote to arm for the next interval unless one-shot is set
1313 * Absolute is a forced one-shot timer which deletes on delivery
1314 * TODO: Add a way for absolute to be not forced one-shot
1316 * D) Whether the time duration is relative to now or absolute
1317 * Interval fires at now + duration when it is set up
1318 * Absolute fires at now + difference between now walltime and passed in walltime
1319 * With NOTE_MACHTIME it fires at an absolute MAT or MCT.
1321 * E) Whether the timer continues to tick across sleep
1322 * By default all three do not.
1323 * For interval and absolute, NOTE_MACH_CONTINUOUS_TIME causes them to tick across sleep
1324 * With NOTE_ABSOLUTE | NOTE_MACHTIME | NOTE_MACH_CONTINUOUS_TIME:
1325 * expires when mach_continuous_time() is > the passed in value.
1328 uint64_t multiplier
;
1330 boolean_t use_abstime
= FALSE
;
1332 switch (kev
->fflags
& (NOTE_SECONDS
| NOTE_USECONDS
| NOTE_NSECONDS
| NOTE_MACHTIME
)) {
1334 multiplier
= NSEC_PER_SEC
;
1337 multiplier
= NSEC_PER_USEC
;
1346 case 0: /* milliseconds (default) */
1347 multiplier
= NSEC_PER_SEC
/ 1000;
1353 /* transform the leeway in kn_ext[1] to same time scale */
1354 if (kev
->fflags
& NOTE_LEEWAY
) {
1355 uint64_t leeway_abs
;
1358 leeway_abs
= (uint64_t)kev
->ext
[1];
1361 if (os_mul_overflow((uint64_t)kev
->ext
[1], multiplier
, &leeway_ns
)) {
1365 nanoseconds_to_absolutetime(leeway_ns
, &leeway_abs
);
1368 params
->leeway
= leeway_abs
;
1373 if (kev
->fflags
& NOTE_ABSOLUTE
) {
1374 uint64_t deadline_abs
;
1377 deadline_abs
= (uint64_t)kev
->data
;
1379 uint64_t calendar_deadline_ns
;
1381 if (os_mul_overflow((uint64_t)kev
->data
, multiplier
, &calendar_deadline_ns
)) {
1385 /* calendar_deadline_ns is in nanoseconds since the epoch */
1387 clock_sec_t seconds
;
1388 clock_nsec_t nanoseconds
;
1391 * Note that the conversion through wall-time is only done once.
1393 * If the relationship between MAT and gettimeofday changes,
1394 * the underlying timer does not update.
1396 * TODO: build a wall-time denominated timer_call queue
1397 * and a flag to request DTRTing with wall-time timers
1399 clock_get_calendar_nanotime(&seconds
, &nanoseconds
);
1401 uint64_t calendar_now_ns
= (uint64_t)seconds
* NSEC_PER_SEC
+ nanoseconds
;
1403 /* if deadline is in the future */
1404 if (calendar_now_ns
< calendar_deadline_ns
) {
1405 uint64_t interval_ns
= calendar_deadline_ns
- calendar_now_ns
;
1406 uint64_t interval_abs
;
1408 nanoseconds_to_absolutetime(interval_ns
, &interval_abs
);
1411 * Note that the NOTE_MACH_CONTINUOUS_TIME flag here only
1412 * causes the timer to keep ticking across sleep, but
1413 * it does not change the calendar timebase.
1416 if (kev
->fflags
& NOTE_MACH_CONTINUOUS_TIME
) {
1417 clock_continuoustime_interval_to_deadline(interval_abs
,
1420 clock_absolutetime_interval_to_deadline(interval_abs
,
1424 deadline_abs
= 0; /* cause immediate expiration */
1428 params
->deadline
= deadline_abs
;
1429 params
->interval
= 0; /* NOTE_ABSOLUTE is non-repeating */
1430 } else if (kev
->data
< 0) {
1432 * Negative interval timers fire immediately, once.
1434 * Ideally a negative interval would be an error, but certain clients
1435 * pass negative values on accident, and expect an event back.
1437 * In the old implementation the timer would repeat with no delay
1438 * N times until mach_absolute_time() + (N * interval) underflowed,
1439 * then it would wait ~forever by accidentally arming a timer for the far future.
1441 * We now skip the power-wasting hot spin phase and go straight to the idle phase.
1444 params
->deadline
= 0; /* expire immediately */
1445 params
->interval
= 0; /* non-repeating */
1447 uint64_t interval_abs
= 0;
1450 interval_abs
= (uint64_t)kev
->data
;
1452 uint64_t interval_ns
;
1453 if (os_mul_overflow((uint64_t)kev
->data
, multiplier
, &interval_ns
)) {
1457 nanoseconds_to_absolutetime(interval_ns
, &interval_abs
);
1460 uint64_t deadline
= 0;
1462 if (kev
->fflags
& NOTE_MACH_CONTINUOUS_TIME
) {
1463 clock_continuoustime_interval_to_deadline(interval_abs
, &deadline
);
1465 clock_absolutetime_interval_to_deadline(interval_abs
, &deadline
);
1468 params
->deadline
= deadline
;
1469 params
->interval
= interval_abs
;
1476 * filt_timerexpire - the timer callout routine
1479 filt_timerexpire(void *knx
, __unused
void *spare
)
1481 struct knote
*kn
= knx
;
1484 if (os_atomic_cmpxchgv(&kn
->kn_hook32
, TIMER_ARMED
, TIMER_FIRED
,
1486 // our f_event always would say FILTER_ACTIVE,
1487 // so be leaner and just do it.
1488 struct kqueue
*kq
= knote_get_kq(kn
);
1490 knote_activate(kq
, kn
, FILTER_ACTIVE
);
1494 * From TIMER_ARMED, the only allowed transition are:
1495 * - to TIMER_FIRED through the timer callout just above
1496 * - to TIMER_IDLE due to filt_timercancel() which will wait for the
1497 * timer callout (and any possible invocation of filt_timerexpire) to
1498 * have finished before the state is changed again.
1500 assert(v
== TIMER_IDLE
);
1505 filt_timercancel(struct knote
*kn
)
1507 if (os_atomic_xchg(&kn
->kn_hook32
, TIMER_IDLE
, relaxed
) == TIMER_ARMED
) {
1508 /* cancel the thread call and wait for any filt_timerexpire in flight */
1509 thread_call_cancel_wait(kn
->kn_thcall
);
1514 * Does this deadline needs a timer armed for it, or has it expired?
1517 filt_timer_is_ready(struct knote
*kn
)
1519 uint64_t now
, deadline
= kn
->kn_ext
[0];
1521 if (deadline
== 0) {
1525 if (kn
->kn_sfflags
& NOTE_MACH_CONTINUOUS_TIME
) {
1526 now
= mach_continuous_time();
1528 now
= mach_absolute_time();
1530 return deadline
<= now
;
1536 * It is the responsibility of the caller to make sure the timer call
1537 * has completed or been cancelled properly prior to arming it.
1540 filt_timerarm(struct knote
*kn
)
1542 uint64_t deadline
= kn
->kn_ext
[0];
1543 uint64_t leeway
= kn
->kn_ext
[1];
1545 int filter_flags
= kn
->kn_sfflags
;
1546 unsigned int timer_flags
= 0;
1548 assert(os_atomic_load(&kn
->kn_hook32
, relaxed
) == TIMER_IDLE
);
1550 if (filter_flags
& NOTE_CRITICAL
) {
1551 timer_flags
|= THREAD_CALL_DELAY_USER_CRITICAL
;
1552 } else if (filter_flags
& NOTE_BACKGROUND
) {
1553 timer_flags
|= THREAD_CALL_DELAY_USER_BACKGROUND
;
1555 timer_flags
|= THREAD_CALL_DELAY_USER_NORMAL
;
1558 if (filter_flags
& NOTE_LEEWAY
) {
1559 timer_flags
|= THREAD_CALL_DELAY_LEEWAY
;
1562 if (filter_flags
& NOTE_MACH_CONTINUOUS_TIME
) {
1563 timer_flags
|= THREAD_CALL_CONTINUOUS
;
1566 os_atomic_store(&kn
->kn_hook32
, TIMER_ARMED
, relaxed
);
1567 thread_call_enter_delayed_with_leeway(kn
->kn_thcall
, NULL
,
1568 deadline
, leeway
, timer_flags
);
1572 * Allocate a thread call for the knote's lifetime, and kick off the timer.
1575 filt_timerattach(struct knote
*kn
, struct kevent_qos_s
*kev
)
1577 thread_call_t callout
;
1578 struct filt_timer_params params
;
1581 if ((error
= filt_timervalidate(kev
, ¶ms
)) != 0) {
1582 knote_set_error(kn
, error
);
1586 callout
= thread_call_allocate_with_options(filt_timerexpire
,
1587 (thread_call_param_t
)kn
, THREAD_CALL_PRIORITY_HIGH
,
1588 THREAD_CALL_OPTIONS_ONCE
);
1590 if (NULL
== callout
) {
1591 knote_set_error(kn
, ENOMEM
);
1595 filt_timer_set_params(kn
, ¶ms
);
1596 kn
->kn_thcall
= callout
;
1597 kn
->kn_flags
|= EV_CLEAR
;
1598 os_atomic_store(&kn
->kn_hook32
, TIMER_IDLE
, relaxed
);
1600 /* NOTE_ABSOLUTE implies EV_ONESHOT */
1601 if (kn
->kn_sfflags
& NOTE_ABSOLUTE
) {
1602 kn
->kn_flags
|= EV_ONESHOT
;
1605 if (filt_timer_is_ready(kn
)) {
1606 os_atomic_store(&kn
->kn_hook32
, TIMER_IMMEDIATE
, relaxed
);
1607 return FILTER_ACTIVE
;
1615 * Shut down the timer if it's running, and free the callout.
1618 filt_timerdetach(struct knote
*kn
)
1620 __assert_only boolean_t freed
;
1623 * Unconditionally cancel to make sure there can't be any filt_timerexpire()
1626 thread_call_cancel_wait(kn
->kn_thcall
);
1627 freed
= thread_call_free(kn
->kn_thcall
);
1632 * filt_timertouch - update timer knote with new user input
1634 * Cancel and restart the timer based on new user data. When
1635 * the user picks up a knote, clear the count of how many timer
1636 * pops have gone off (in kn_data).
1639 filt_timertouch(struct knote
*kn
, struct kevent_qos_s
*kev
)
1641 struct filt_timer_params params
;
1642 uint32_t changed_flags
= (kn
->kn_sfflags
^ kev
->fflags
);
1645 if (changed_flags
& NOTE_ABSOLUTE
) {
1646 kev
->flags
|= EV_ERROR
;
1651 if ((error
= filt_timervalidate(kev
, ¶ms
)) != 0) {
1652 kev
->flags
|= EV_ERROR
;
1657 /* capture the new values used to compute deadline */
1658 filt_timercancel(kn
);
1659 filt_timer_set_params(kn
, ¶ms
);
1660 kn
->kn_sfflags
= kev
->fflags
;
1662 if (filt_timer_is_ready(kn
)) {
1663 os_atomic_store(&kn
->kn_hook32
, TIMER_IMMEDIATE
, relaxed
);
1664 return FILTER_ACTIVE
| FILTER_UPDATE_REQ_QOS
;
1667 return FILTER_UPDATE_REQ_QOS
;
1672 * filt_timerprocess - query state of knote and snapshot event data
1674 * Determine if the timer has fired in the past, snapshot the state
1675 * of the kevent for returning to user-space, and clear pending event
1676 * counters for the next time.
1679 filt_timerprocess(struct knote
*kn
, struct kevent_qos_s
*kev
)
1682 * filt_timerprocess is serialized with any filter routine except for
1683 * filt_timerexpire which atomically does a TIMER_ARMED -> TIMER_FIRED
1684 * transition, and on success, activates the knote.
1686 * Hence, we don't need atomic modifications of the state, only to peek at
1687 * whether we see any of the "FIRED" state, and if we do, it is safe to
1688 * do simple state machine transitions.
1690 switch (os_atomic_load(&kn
->kn_hook32
, relaxed
)) {
1694 * This can happen if a touch resets a timer that had fired
1695 * without being processed
1700 os_atomic_store(&kn
->kn_hook32
, TIMER_IDLE
, relaxed
);
1703 * Copy out the interesting kevent state,
1704 * but don't leak out the raw time calculations.
1706 * TODO: potential enhancements - tell the user about:
1707 * - deadline to which this timer thought it was expiring
1708 * - return kn_sfflags in the fflags field so the client can know
1709 * under what flags the timer fired
1711 knote_fill_kevent(kn
, kev
, 1);
1713 /* kev->ext[1] = 0; JMM - shouldn't we hide this too? */
1715 if (kn
->kn_sdata
!= 0) {
1717 * This is a 'repeating' timer, so we have to emit
1718 * how many intervals expired between the arm
1721 * A very strange style of interface, because
1722 * this could easily be done in the client...
1727 if (kn
->kn_sfflags
& NOTE_MACH_CONTINUOUS_TIME
) {
1728 now
= mach_continuous_time();
1730 now
= mach_absolute_time();
1733 uint64_t first_deadline
= kn
->kn_ext
[0];
1734 uint64_t interval_abs
= kn
->kn_sdata
;
1735 uint64_t orig_arm_time
= first_deadline
- interval_abs
;
1737 assert(now
> orig_arm_time
);
1738 assert(now
> first_deadline
);
1740 uint64_t elapsed
= now
- orig_arm_time
;
1742 uint64_t num_fired
= elapsed
/ interval_abs
;
1745 * To reach this code, we must have seen the timer pop
1746 * and be in repeating mode, so therefore it must have been
1747 * more than 'interval' time since the attach or last
1750 assert(num_fired
> 0);
1752 /* report how many intervals have elapsed to the user */
1753 kev
->data
= (int64_t)num_fired
;
1755 /* We only need to re-arm the timer if it's not about to be destroyed */
1756 if ((kn
->kn_flags
& EV_ONESHOT
) == 0) {
1757 /* fire at the end of the next interval */
1758 uint64_t new_deadline
= first_deadline
+ num_fired
* interval_abs
;
1760 assert(new_deadline
> now
);
1762 kn
->kn_ext
[0] = new_deadline
;
1765 * This can't shortcut setting up the thread call, because
1766 * knote_process deactivates EV_CLEAR knotes unconditionnally.
1772 return FILTER_ACTIVE
;
1775 SECURITY_READ_ONLY_EARLY(static struct filterops
) timer_filtops
= {
1776 .f_extended_codes
= true,
1777 .f_attach
= filt_timerattach
,
1778 .f_detach
= filt_timerdetach
,
1779 .f_event
= filt_bad_event
,
1780 .f_touch
= filt_timertouch
,
1781 .f_process
= filt_timerprocess
,
1784 #pragma mark user_filtops
1787 filt_userattach(struct knote
*kn
, __unused
struct kevent_qos_s
*kev
)
1789 if (kn
->kn_sfflags
& NOTE_TRIGGER
) {
1790 kn
->kn_hook32
= FILTER_ACTIVE
;
1794 return kn
->kn_hook32
;
1798 filt_usertouch(struct knote
*kn
, struct kevent_qos_s
*kev
)
1803 ffctrl
= kev
->fflags
& NOTE_FFCTRLMASK
;
1804 fflags
= kev
->fflags
& NOTE_FFLAGSMASK
;
1809 kn
->kn_sfflags
&= fflags
;
1812 kn
->kn_sfflags
|= fflags
;
1815 kn
->kn_sfflags
= fflags
;
1818 kn
->kn_sdata
= kev
->data
;
1820 if (kev
->fflags
& NOTE_TRIGGER
) {
1821 kn
->kn_hook32
= FILTER_ACTIVE
;
1823 return (int)kn
->kn_hook32
;
1827 filt_userprocess(struct knote
*kn
, struct kevent_qos_s
*kev
)
1829 int result
= (int)kn
->kn_hook32
;
1832 /* EVFILT_USER returns the data that was passed in */
1833 knote_fill_kevent_with_sdata(kn
, kev
);
1834 kev
->fflags
= kn
->kn_sfflags
;
1835 if (kn
->kn_flags
& EV_CLEAR
) {
1836 /* knote_fill_kevent cleared kn_fflags */
1844 SECURITY_READ_ONLY_EARLY(static struct filterops
) user_filtops
= {
1845 .f_extended_codes
= true,
1846 .f_attach
= filt_userattach
,
1847 .f_detach
= filt_no_detach
,
1848 .f_event
= filt_bad_event
,
1849 .f_touch
= filt_usertouch
,
1850 .f_process
= filt_userprocess
,
1853 #pragma mark workloop_filtops
1855 #define EPREEMPTDISABLED (-1)
1858 filt_wllock(struct kqworkloop
*kqwl
)
1860 lck_spin_lock(&kqwl
->kqwl_statelock
);
1864 filt_wlunlock(struct kqworkloop
*kqwl
)
1866 lck_spin_unlock(&kqwl
->kqwl_statelock
);
1870 * Returns true when the interlock for the turnstile is the workqueue lock
1872 * When this is the case, all turnstiles operations are delegated
1873 * to the workqueue subsystem.
1875 * This is required because kqueue_threadreq_bind_prepost only holds the
1876 * workqueue lock but needs to move the inheritor from the workloop turnstile
1877 * away from the creator thread, so that this now fulfilled request cannot be
1878 * picked anymore by other threads.
1881 filt_wlturnstile_interlock_is_workq(struct kqworkloop
*kqwl
)
1883 return kqr_thread_requested_pending(&kqwl
->kqwl_request
);
1887 filt_wlupdate_inheritor(struct kqworkloop
*kqwl
, struct turnstile
*ts
,
1888 turnstile_update_flags_t flags
)
1890 turnstile_inheritor_t inheritor
= TURNSTILE_INHERITOR_NULL
;
1891 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
1894 * binding to the workq should always happen through
1895 * workq_kern_threadreq_update_inheritor()
1897 assert(!filt_wlturnstile_interlock_is_workq(kqwl
));
1899 if ((inheritor
= kqwl
->kqwl_owner
)) {
1900 flags
|= TURNSTILE_INHERITOR_THREAD
;
1901 } else if ((inheritor
= kqr_thread(kqr
))) {
1902 flags
|= TURNSTILE_INHERITOR_THREAD
;
1905 turnstile_update_inheritor(ts
, inheritor
, flags
);
1908 #define EVFILT_WORKLOOP_EFAULT_RETRY_COUNT 100
1909 #define FILT_WLATTACH 0
1910 #define FILT_WLTOUCH 1
1911 #define FILT_WLDROP 2
1915 filt_wlupdate(struct kqworkloop
*kqwl
, struct knote
*kn
,
1916 struct kevent_qos_s
*kev
, kq_index_t qos_index
, int op
)
1918 user_addr_t uaddr
= CAST_USER_ADDR_T(kev
->ext
[EV_EXTIDX_WL_ADDR
]);
1919 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
1920 thread_t cur_owner
, new_owner
, extra_thread_ref
= THREAD_NULL
;
1921 kq_index_t cur_override
= THREAD_QOS_UNSPECIFIED
;
1922 int efault_retry
= EVFILT_WORKLOOP_EFAULT_RETRY_COUNT
;
1923 int action
= KQWL_UTQ_NONE
, error
= 0;
1924 bool wl_inheritor_updated
= false, needs_wake
= false;
1925 uint64_t kdata
= kev
->ext
[EV_EXTIDX_WL_VALUE
];
1926 uint64_t mask
= kev
->ext
[EV_EXTIDX_WL_MASK
];
1928 struct turnstile
*ts
= TURNSTILE_NULL
;
1933 new_owner
= cur_owner
= kqwl
->kqwl_owner
;
1938 * If asked, load the uint64 value at the user provided address and compare
1939 * it against the passed in mask and expected value.
1941 * If NOTE_WL_DISCOVER_OWNER is specified, translate the loaded name as
1942 * a thread reference.
1944 * If NOTE_WL_END_OWNERSHIP is specified and the currently known owner is
1945 * the current thread, then end ownership.
1947 * Lastly decide whether we need to perform a QoS update.
1951 * Until <rdar://problem/24999882> exists,
1952 * disabling preemption copyin forces any
1953 * vm_fault we encounter to fail.
1955 error
= copyin_atomic64(uaddr
, &udata
);
1958 * If we get EFAULT, drop locks, and retry.
1959 * If we still get an error report it,
1960 * else assume the memory has been faulted
1961 * and attempt to copyin under lock again.
1967 if (efault_retry
-- > 0) {
1968 filt_wlunlock(kqwl
);
1969 error
= copyin_atomic64(uaddr
, &udata
);
1980 /* Update state as copied in. */
1981 kev
->ext
[EV_EXTIDX_WL_VALUE
] = udata
;
1983 if ((udata
& mask
) != (kdata
& mask
)) {
1985 } else if (kev
->fflags
& NOTE_WL_DISCOVER_OWNER
) {
1987 * Decipher the owner port name, and translate accordingly.
1988 * The low 2 bits were borrowed for other flags, so mask them off.
1990 * Then attempt translation to a thread reference or fail.
1992 mach_port_name_t name
= (mach_port_name_t
)udata
& ~0x3;
1993 if (name
!= MACH_PORT_NULL
) {
1994 name
= ipc_entry_name_mask(name
);
1995 extra_thread_ref
= port_name_to_thread(name
,
1996 PORT_TO_THREAD_IN_CURRENT_TASK
);
1997 if (extra_thread_ref
== THREAD_NULL
) {
2001 new_owner
= extra_thread_ref
;
2006 if ((kev
->fflags
& NOTE_WL_END_OWNERSHIP
) && new_owner
== current_thread()) {
2007 new_owner
= THREAD_NULL
;
2011 if ((kev
->fflags
& NOTE_WL_THREAD_REQUEST
) && (kev
->flags
& EV_DELETE
)) {
2012 action
= KQWL_UTQ_SET_QOS_INDEX
;
2013 } else if (qos_index
&& kqr
->tr_kq_qos_index
!= qos_index
) {
2014 action
= KQWL_UTQ_SET_QOS_INDEX
;
2017 if (op
== FILT_WLTOUCH
) {
2019 * Save off any additional fflags/data we just accepted
2020 * But only keep the last round of "update" bits we acted on which helps
2023 kn
->kn_sfflags
&= ~NOTE_WL_UPDATES_MASK
;
2024 kn
->kn_sfflags
|= kev
->fflags
;
2025 if (kev
->fflags
& NOTE_WL_SYNC_WAKE
) {
2026 needs_wake
= (kn
->kn_thread
!= THREAD_NULL
);
2028 } else if (op
== FILT_WLDROP
) {
2029 if ((kn
->kn_sfflags
& (NOTE_WL_SYNC_WAIT
| NOTE_WL_SYNC_WAKE
)) ==
2030 NOTE_WL_SYNC_WAIT
) {
2032 * When deleting a SYNC_WAIT knote that hasn't been woken up
2033 * explicitly, issue a wake up.
2035 kn
->kn_sfflags
|= NOTE_WL_SYNC_WAKE
;
2036 needs_wake
= (kn
->kn_thread
!= THREAD_NULL
);
2044 * Commit ownership and QoS changes if any, possibly wake up waiters
2047 if (cur_owner
== new_owner
&& action
== KQWL_UTQ_NONE
&& !needs_wake
) {
2053 /* If already tracked as servicer, don't track as owner */
2054 if (new_owner
== kqr_thread(kqr
)) {
2055 new_owner
= THREAD_NULL
;
2058 if (cur_owner
!= new_owner
) {
2059 kqwl
->kqwl_owner
= new_owner
;
2060 if (new_owner
== extra_thread_ref
) {
2061 /* we just transfered this ref to kqwl_owner */
2062 extra_thread_ref
= THREAD_NULL
;
2064 cur_override
= kqworkloop_override(kqwl
);
2067 /* override it before we drop the old */
2068 if (cur_override
!= THREAD_QOS_UNSPECIFIED
) {
2069 thread_add_kevent_override(new_owner
, cur_override
);
2071 if (kqr_thread_requested_pending(kqr
)) {
2072 if (action
== KQWL_UTQ_NONE
) {
2073 action
= KQWL_UTQ_REDRIVE_EVENTS
;
2077 if (!kqr_thread_requested(kqr
) && kqr
->tr_kq_wakeup
) {
2078 if (action
== KQWL_UTQ_NONE
) {
2079 action
= KQWL_UTQ_REDRIVE_EVENTS
;
2085 if (action
!= KQWL_UTQ_NONE
) {
2086 kqworkloop_update_threads_qos(kqwl
, action
, qos_index
);
2089 ts
= kqwl
->kqwl_turnstile
;
2090 if (cur_owner
!= new_owner
&& ts
) {
2091 if (action
== KQWL_UTQ_REDRIVE_EVENTS
) {
2093 * Note that when action is KQWL_UTQ_REDRIVE_EVENTS,
2094 * the code went through workq_kern_threadreq_initiate()
2095 * and the workqueue has set the inheritor already
2097 assert(filt_wlturnstile_interlock_is_workq(kqwl
));
2098 } else if (filt_wlturnstile_interlock_is_workq(kqwl
)) {
2099 workq_kern_threadreq_lock(kqwl
->kqwl_p
);
2100 workq_kern_threadreq_update_inheritor(kqwl
->kqwl_p
, kqr
, new_owner
,
2101 ts
, TURNSTILE_IMMEDIATE_UPDATE
);
2102 workq_kern_threadreq_unlock(kqwl
->kqwl_p
);
2103 if (!filt_wlturnstile_interlock_is_workq(kqwl
)) {
2105 * If the workq is no longer the interlock, then
2106 * workq_kern_threadreq_update_inheritor() has finished a bind
2107 * and we need to fallback to the regular path.
2109 filt_wlupdate_inheritor(kqwl
, ts
, TURNSTILE_IMMEDIATE_UPDATE
);
2111 wl_inheritor_updated
= true;
2113 filt_wlupdate_inheritor(kqwl
, ts
, TURNSTILE_IMMEDIATE_UPDATE
);
2114 wl_inheritor_updated
= true;
2118 * We need a turnstile reference because we are dropping the interlock
2119 * and the caller has not called turnstile_prepare.
2121 if (wl_inheritor_updated
) {
2122 turnstile_reference(ts
);
2126 if (needs_wake
&& ts
) {
2127 waitq_wakeup64_thread(&ts
->ts_waitq
, knote_filt_wev64(kn
),
2128 kn
->kn_thread
, THREAD_AWAKENED
);
2129 if (op
== FILT_WLATTACH
|| op
== FILT_WLTOUCH
) {
2130 disable_preemption();
2131 error
= EPREEMPTDISABLED
;
2141 * Unlock and cleanup various lingering references and things.
2143 filt_wlunlock(kqwl
);
2145 #if CONFIG_WORKLOOP_DEBUG
2146 KQWL_HISTORY_WRITE_ENTRY(kqwl
, {
2147 .updater
= current_thread(),
2148 .servicer
= kqr_thread(kqr
), /* Note: racy */
2149 .old_owner
= cur_owner
,
2150 .new_owner
= new_owner
,
2152 .kev_ident
= kev
->ident
,
2153 .error
= (int16_t)error
,
2154 .kev_flags
= kev
->flags
,
2155 .kev_fflags
= kev
->fflags
,
2161 #endif // CONFIG_WORKLOOP_DEBUG
2163 if (wl_inheritor_updated
) {
2164 turnstile_update_inheritor_complete(ts
, TURNSTILE_INTERLOCK_NOT_HELD
);
2165 turnstile_deallocate_safe(ts
);
2168 if (cur_owner
&& new_owner
!= cur_owner
) {
2169 if (cur_override
!= THREAD_QOS_UNSPECIFIED
) {
2170 thread_drop_kevent_override(cur_owner
);
2172 thread_deallocate_safe(cur_owner
);
2174 if (extra_thread_ref
) {
2175 thread_deallocate_safe(extra_thread_ref
);
2181 * Remembers the last updated that came in from userspace for debugging reasons.
2182 * - fflags is mirrored from the userspace kevent
2183 * - ext[i, i != VALUE] is mirrored from the userspace kevent
2184 * - ext[VALUE] is set to what the kernel loaded atomically
2185 * - data is set to the error if any
2188 filt_wlremember_last_update(struct knote
*kn
, struct kevent_qos_s
*kev
,
2191 kn
->kn_fflags
= kev
->fflags
;
2192 kn
->kn_sdata
= error
;
2193 memcpy(kn
->kn_ext
, kev
->ext
, sizeof(kev
->ext
));
2197 filt_wlupdate_sync_ipc(struct kqworkloop
*kqwl
, struct knote
*kn
,
2198 struct kevent_qos_s
*kev
, int op
)
2200 uint64_t uaddr
= kev
->ext
[EV_EXTIDX_WL_ADDR
];
2201 uint64_t kdata
= kev
->ext
[EV_EXTIDX_WL_VALUE
];
2202 uint64_t mask
= kev
->ext
[EV_EXTIDX_WL_MASK
];
2204 int efault_retry
= EVFILT_WORKLOOP_EFAULT_RETRY_COUNT
;
2207 if (op
== FILT_WLATTACH
) {
2208 (void)kqueue_alloc_turnstile(&kqwl
->kqwl_kqueue
);
2209 } else if (uaddr
== 0) {
2218 * Do the debounce thing, the lock serializing the state is the knote lock.
2222 * Until <rdar://problem/24999882> exists,
2223 * disabling preemption copyin forces any
2224 * vm_fault we encounter to fail.
2226 error
= copyin_atomic64(uaddr
, &udata
);
2229 * If we get EFAULT, drop locks, and retry.
2230 * If we still get an error report it,
2231 * else assume the memory has been faulted
2232 * and attempt to copyin under lock again.
2238 if (efault_retry
-- > 0) {
2239 filt_wlunlock(kqwl
);
2240 error
= copyin_atomic64(uaddr
, &udata
);
2251 kev
->ext
[EV_EXTIDX_WL_VALUE
] = udata
;
2252 kn
->kn_ext
[EV_EXTIDX_WL_VALUE
] = udata
;
2254 if ((udata
& mask
) != (kdata
& mask
)) {
2260 if (op
== FILT_WLATTACH
) {
2261 error
= filt_wlattach_sync_ipc(kn
);
2263 disable_preemption();
2264 error
= EPREEMPTDISABLED
;
2269 filt_wlunlock(kqwl
);
2274 filt_wlattach(struct knote
*kn
, struct kevent_qos_s
*kev
)
2276 struct kqueue
*kq
= knote_get_kq(kn
);
2277 struct kqworkloop
*kqwl
= (struct kqworkloop
*)kq
;
2278 int error
= 0, result
= 0;
2279 kq_index_t qos_index
= 0;
2281 if (__improbable((kq
->kq_state
& KQ_WORKLOOP
) == 0)) {
2286 uint32_t command
= (kn
->kn_sfflags
& NOTE_WL_COMMANDS_MASK
);
2288 case NOTE_WL_THREAD_REQUEST
:
2289 if (kn
->kn_id
!= kqwl
->kqwl_dynamicid
) {
2293 qos_index
= _pthread_priority_thread_qos(kn
->kn_qos
);
2294 if (qos_index
== THREAD_QOS_UNSPECIFIED
) {
2298 if (kqwl
->kqwl_request
.tr_kq_qos_index
) {
2300 * There already is a thread request, and well, you're only allowed
2301 * one per workloop, so fail the attach.
2307 case NOTE_WL_SYNC_WAIT
:
2308 case NOTE_WL_SYNC_WAKE
:
2309 if (kn
->kn_id
== kqwl
->kqwl_dynamicid
) {
2313 if ((kn
->kn_flags
& EV_DISABLE
) == 0) {
2317 if (kn
->kn_sfflags
& NOTE_WL_END_OWNERSHIP
) {
2323 case NOTE_WL_SYNC_IPC
:
2324 if ((kn
->kn_flags
& EV_DISABLE
) == 0) {
2328 if (kn
->kn_sfflags
& (NOTE_WL_UPDATE_QOS
| NOTE_WL_DISCOVER_OWNER
)) {
2338 if (command
== NOTE_WL_SYNC_IPC
) {
2339 error
= filt_wlupdate_sync_ipc(kqwl
, kn
, kev
, FILT_WLATTACH
);
2341 error
= filt_wlupdate(kqwl
, kn
, kev
, qos_index
, FILT_WLATTACH
);
2344 if (error
== EPREEMPTDISABLED
) {
2346 result
= FILTER_THREADREQ_NODEFEER
;
2350 /* If userland wants ESTALE to be hidden, fail the attach anyway */
2351 if (error
== ESTALE
&& (kn
->kn_sfflags
& NOTE_WL_IGNORE_ESTALE
)) {
2354 knote_set_error(kn
, error
);
2357 if (command
== NOTE_WL_SYNC_WAIT
) {
2358 return kevent_register_wait_prepare(kn
, kev
, result
);
2360 /* Just attaching the thread request successfully will fire it */
2361 if (command
== NOTE_WL_THREAD_REQUEST
) {
2363 * Thread Request knotes need an explicit touch to be active again,
2364 * so delivering an event needs to also consume it.
2366 kn
->kn_flags
|= EV_CLEAR
;
2367 return result
| FILTER_ACTIVE
;
2373 filt_wlwait_continue(void *parameter
, wait_result_t wr
)
2375 struct _kevent_register
*cont_args
= parameter
;
2376 struct kqworkloop
*kqwl
= cont_args
->kqwl
;
2379 if (filt_wlturnstile_interlock_is_workq(kqwl
)) {
2380 workq_kern_threadreq_lock(kqwl
->kqwl_p
);
2381 turnstile_complete((uintptr_t)kqwl
, &kqwl
->kqwl_turnstile
, NULL
, TURNSTILE_WORKLOOPS
);
2382 workq_kern_threadreq_unlock(kqwl
->kqwl_p
);
2384 turnstile_complete((uintptr_t)kqwl
, &kqwl
->kqwl_turnstile
, NULL
, TURNSTILE_WORKLOOPS
);
2388 turnstile_cleanup();
2390 if (wr
== THREAD_INTERRUPTED
) {
2391 cont_args
->kev
.flags
|= EV_ERROR
;
2392 cont_args
->kev
.data
= EINTR
;
2393 } else if (wr
!= THREAD_AWAKENED
) {
2394 panic("Unexpected wait result: %d", wr
);
2397 kevent_register_wait_return(cont_args
);
2401 * Called with the workloop mutex held, most of the time never returns as it
2402 * calls filt_wlwait_continue through a continuation.
2405 filt_wlpost_register_wait(struct uthread
*uth
, struct knote
*kn
,
2406 struct _kevent_register
*cont_args
)
2408 struct kqworkloop
*kqwl
= cont_args
->kqwl
;
2409 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
2410 struct turnstile
*ts
;
2411 bool workq_locked
= false;
2415 if (filt_wlturnstile_interlock_is_workq(kqwl
)) {
2416 workq_kern_threadreq_lock(kqwl
->kqwl_p
);
2417 workq_locked
= true;
2420 ts
= turnstile_prepare((uintptr_t)kqwl
, &kqwl
->kqwl_turnstile
,
2421 TURNSTILE_NULL
, TURNSTILE_WORKLOOPS
);
2424 workq_kern_threadreq_update_inheritor(kqwl
->kqwl_p
,
2425 &kqwl
->kqwl_request
, kqwl
->kqwl_owner
, ts
,
2426 TURNSTILE_DELAYED_UPDATE
);
2427 if (!filt_wlturnstile_interlock_is_workq(kqwl
)) {
2429 * if the interlock is no longer the workqueue lock,
2430 * then we don't need to hold it anymore.
2432 workq_kern_threadreq_unlock(kqwl
->kqwl_p
);
2433 workq_locked
= false;
2436 if (!workq_locked
) {
2438 * If the interlock is the workloop's, then it's our responsibility to
2439 * call update_inheritor, so just do it.
2441 filt_wlupdate_inheritor(kqwl
, ts
, TURNSTILE_DELAYED_UPDATE
);
2444 thread_set_pending_block_hint(uth
->uu_thread
, kThreadWaitWorkloopSyncWait
);
2445 waitq_assert_wait64(&ts
->ts_waitq
, knote_filt_wev64(kn
),
2446 THREAD_ABORTSAFE
, TIMEOUT_WAIT_FOREVER
);
2449 workq_kern_threadreq_unlock(kqwl
->kqwl_p
);
2452 thread_t thread
= kqwl
->kqwl_owner
?: kqr_thread(kqr
);
2454 thread_reference(thread
);
2457 kevent_register_wait_block(ts
, thread
, filt_wlwait_continue
, cont_args
);
2460 /* called in stackshot context to report the thread responsible for blocking this thread */
2462 kdp_workloop_sync_wait_find_owner(__assert_only thread_t thread
,
2463 event64_t event
, thread_waitinfo_t
*waitinfo
)
2465 struct knote
*kn
= (struct knote
*)event
;
2466 assert(kdp_is_in_zone(kn
, "knote zone"));
2468 assert(kn
->kn_thread
== thread
);
2470 struct kqueue
*kq
= knote_get_kq(kn
);
2471 assert(kdp_is_in_zone(kq
, "kqueue workloop zone"));
2472 assert(kq
->kq_state
& KQ_WORKLOOP
);
2474 struct kqworkloop
*kqwl
= (struct kqworkloop
*)kq
;
2475 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
2477 thread_t kqwl_owner
= kqwl
->kqwl_owner
;
2479 if (kqwl_owner
!= THREAD_NULL
) {
2480 assert(kdp_is_in_zone(kqwl_owner
, "threads"));
2482 waitinfo
->owner
= thread_tid(kqwl
->kqwl_owner
);
2483 } else if (kqr_thread_requested_pending(kqr
)) {
2484 waitinfo
->owner
= STACKSHOT_WAITOWNER_THREQUESTED
;
2485 } else if (kqr
->tr_state
>= WORKQ_TR_STATE_BINDING
) {
2486 assert(kdp_is_in_zone(kqr
->tr_thread
, "threads"));
2487 waitinfo
->owner
= thread_tid(kqr
->tr_thread
);
2489 waitinfo
->owner
= 0;
2492 waitinfo
->context
= kqwl
->kqwl_dynamicid
;
2496 filt_wldetach(struct knote
*kn
)
2498 if (kn
->kn_sfflags
& NOTE_WL_SYNC_IPC
) {
2499 filt_wldetach_sync_ipc(kn
);
2500 } else if (kn
->kn_thread
) {
2501 kevent_register_wait_cleanup(kn
);
2506 filt_wlvalidate_kev_flags(struct knote
*kn
, struct kevent_qos_s
*kev
,
2507 thread_qos_t
*qos_index
)
2509 uint32_t new_commands
= kev
->fflags
& NOTE_WL_COMMANDS_MASK
;
2510 uint32_t sav_commands
= kn
->kn_sfflags
& NOTE_WL_COMMANDS_MASK
;
2512 if ((kev
->fflags
& NOTE_WL_DISCOVER_OWNER
) && (kev
->flags
& EV_DELETE
)) {
2515 if (kev
->fflags
& NOTE_WL_UPDATE_QOS
) {
2516 if (kev
->flags
& EV_DELETE
) {
2519 if (sav_commands
!= NOTE_WL_THREAD_REQUEST
) {
2522 if (!(*qos_index
= _pthread_priority_thread_qos(kev
->qos
))) {
2527 switch (new_commands
) {
2528 case NOTE_WL_THREAD_REQUEST
:
2529 /* thread requests can only update themselves */
2530 if (sav_commands
!= NOTE_WL_THREAD_REQUEST
) {
2535 case NOTE_WL_SYNC_WAIT
:
2536 if (kev
->fflags
& NOTE_WL_END_OWNERSHIP
) {
2541 case NOTE_WL_SYNC_WAKE
:
2543 if (!(sav_commands
& (NOTE_WL_SYNC_WAIT
| NOTE_WL_SYNC_WAKE
))) {
2546 if ((kev
->flags
& (EV_ENABLE
| EV_DELETE
)) == EV_ENABLE
) {
2551 case NOTE_WL_SYNC_IPC
:
2552 if (sav_commands
!= NOTE_WL_SYNC_IPC
) {
2555 if ((kev
->flags
& (EV_ENABLE
| EV_DELETE
)) == EV_ENABLE
) {
2567 filt_wltouch(struct knote
*kn
, struct kevent_qos_s
*kev
)
2569 struct kqworkloop
*kqwl
= (struct kqworkloop
*)knote_get_kq(kn
);
2570 thread_qos_t qos_index
= THREAD_QOS_UNSPECIFIED
;
2573 int error
= filt_wlvalidate_kev_flags(kn
, kev
, &qos_index
);
2578 uint32_t command
= kev
->fflags
& NOTE_WL_COMMANDS_MASK
;
2579 if (command
== NOTE_WL_SYNC_IPC
) {
2580 error
= filt_wlupdate_sync_ipc(kqwl
, kn
, kev
, FILT_WLTOUCH
);
2582 error
= filt_wlupdate(kqwl
, kn
, kev
, qos_index
, FILT_WLTOUCH
);
2583 filt_wlremember_last_update(kn
, kev
, error
);
2585 if (error
== EPREEMPTDISABLED
) {
2587 result
= FILTER_THREADREQ_NODEFEER
;
2592 if (error
== ESTALE
&& (kev
->fflags
& NOTE_WL_IGNORE_ESTALE
)) {
2593 /* If userland wants ESTALE to be hidden, do not activate */
2596 kev
->flags
|= EV_ERROR
;
2600 if (command
== NOTE_WL_SYNC_WAIT
&& !(kn
->kn_sfflags
& NOTE_WL_SYNC_WAKE
)) {
2601 return kevent_register_wait_prepare(kn
, kev
, result
);
2603 /* Just touching the thread request successfully will fire it */
2604 if (command
== NOTE_WL_THREAD_REQUEST
) {
2605 if (kev
->fflags
& NOTE_WL_UPDATE_QOS
) {
2606 result
|= FILTER_UPDATE_REQ_QOS
;
2608 result
|= FILTER_ACTIVE
;
2614 filt_wlallow_drop(struct knote
*kn
, struct kevent_qos_s
*kev
)
2616 struct kqworkloop
*kqwl
= (struct kqworkloop
*)knote_get_kq(kn
);
2618 int error
= filt_wlvalidate_kev_flags(kn
, kev
, NULL
);
2623 uint32_t command
= (kev
->fflags
& NOTE_WL_COMMANDS_MASK
);
2624 if (command
== NOTE_WL_SYNC_IPC
) {
2625 error
= filt_wlupdate_sync_ipc(kqwl
, kn
, kev
, FILT_WLDROP
);
2627 error
= filt_wlupdate(kqwl
, kn
, kev
, 0, FILT_WLDROP
);
2628 filt_wlremember_last_update(kn
, kev
, error
);
2630 assert(error
!= EPREEMPTDISABLED
);
2634 if (error
== ESTALE
&& (kev
->fflags
& NOTE_WL_IGNORE_ESTALE
)) {
2637 kev
->flags
|= EV_ERROR
;
2645 filt_wlprocess(struct knote
*kn
, struct kevent_qos_s
*kev
)
2647 struct kqworkloop
*kqwl
= (struct kqworkloop
*)knote_get_kq(kn
);
2650 assert(kn
->kn_sfflags
& NOTE_WL_THREAD_REQUEST
);
2654 if (kqwl
->kqwl_owner
) {
2656 * <rdar://problem/33584321> userspace sometimes due to events being
2657 * delivered but not triggering a drain session can cause a process
2658 * of the thread request knote.
2660 * When that happens, the automatic deactivation due to process
2661 * would swallow the event, so we have to activate the knote again.
2663 knote_activate(kqwl
, kn
, FILTER_ACTIVE
);
2665 #if DEBUG || DEVELOPMENT
2666 if (kevent_debug_flags() & KEVENT_PANIC_ON_NON_ENQUEUED_PROCESS
) {
2668 * see src/queue_internal.h in libdispatch
2670 #define DISPATCH_QUEUE_ENQUEUED 0x1ull
2671 user_addr_t addr
= CAST_USER_ADDR_T(kn
->kn_ext
[EV_EXTIDX_WL_ADDR
]);
2672 task_t t
= current_task();
2674 if (addr
&& task_is_active(t
) && !task_is_halting(t
) &&
2675 copyin_atomic64(addr
, &val
) == 0 &&
2676 val
&& (val
& DISPATCH_QUEUE_ENQUEUED
) == 0 &&
2677 (val
>> 48) != 0xdead && (val
>> 48) != 0 && (val
>> 48) != 0xffff) {
2678 panic("kevent: workloop %#016llx is not enqueued "
2679 "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)",
2680 kn
->kn_udata
, kn
, val
, kn
->kn_ext
[EV_EXTIDX_WL_VALUE
]);
2684 knote_fill_kevent(kn
, kev
, 0);
2685 kev
->fflags
= kn
->kn_sfflags
;
2686 rc
|= FILTER_ACTIVE
;
2691 if (rc
& FILTER_ACTIVE
) {
2692 workq_thread_set_max_qos(kqwl
->kqwl_p
, &kqwl
->kqwl_request
);
2697 SECURITY_READ_ONLY_EARLY(static struct filterops
) workloop_filtops
= {
2698 .f_extended_codes
= true,
2699 .f_attach
= filt_wlattach
,
2700 .f_detach
= filt_wldetach
,
2701 .f_event
= filt_bad_event
,
2702 .f_touch
= filt_wltouch
,
2703 .f_process
= filt_wlprocess
,
2704 .f_allow_drop
= filt_wlallow_drop
,
2705 .f_post_register_wait
= filt_wlpost_register_wait
,
2708 #pragma mark - kqueues allocation and deallocation
2711 * @enum kqworkloop_dealloc_flags_t
2714 * Flags that alter kqworkloop_dealloc() behavior.
2716 * @const KQWL_DEALLOC_NONE
2717 * Convenient name for "no flags".
2719 * @const KQWL_DEALLOC_SKIP_HASH_REMOVE
2720 * Do not remove the workloop fromt he hash table.
2721 * This is used for process tear-down codepaths as the workloops have been
2722 * removed by the caller already.
2724 OS_OPTIONS(kqworkloop_dealloc_flags
, unsigned,
2725 KQWL_DEALLOC_NONE
= 0x0000,
2726 KQWL_DEALLOC_SKIP_HASH_REMOVE
= 0x0001,
2730 kqworkloop_dealloc(struct kqworkloop
*, kqworkloop_dealloc_flags_t
, uint32_t);
2732 OS_NOINLINE OS_COLD OS_NORETURN
2734 kqworkloop_retain_panic(struct kqworkloop
*kqwl
, uint32_t previous
)
2736 if (previous
== 0) {
2737 panic("kq(%p) resurrection", kqwl
);
2739 panic("kq(%p) retain overflow", kqwl
);
2743 OS_NOINLINE OS_COLD OS_NORETURN
2745 kqworkloop_release_panic(struct kqworkloop
*kqwl
)
2747 panic("kq(%p) over-release", kqwl
);
2752 kqworkloop_try_retain(struct kqworkloop
*kqwl
)
2754 uint32_t old_ref
, new_ref
;
2755 os_atomic_rmw_loop(&kqwl
->kqwl_retains
, old_ref
, new_ref
, relaxed
, {
2756 if (__improbable(old_ref
== 0)) {
2757 os_atomic_rmw_loop_give_up(return false);
2759 if (__improbable(old_ref
>= KQ_WORKLOOP_RETAINS_MAX
)) {
2760 kqworkloop_retain_panic(kqwl
, old_ref
);
2762 new_ref
= old_ref
+ 1;
2769 kqworkloop_retain(struct kqworkloop
*kqwl
)
2771 uint32_t previous
= os_atomic_inc_orig(&kqwl
->kqwl_retains
, relaxed
);
2772 if (__improbable(previous
== 0 || previous
>= KQ_WORKLOOP_RETAINS_MAX
)) {
2773 kqworkloop_retain_panic(kqwl
, previous
);
2779 kqueue_retain(kqueue_t kqu
)
2781 if (kqu
.kq
->kq_state
& KQ_DYNAMIC
) {
2782 kqworkloop_retain(kqu
.kqwl
);
2788 kqworkloop_release_live(struct kqworkloop
*kqwl
)
2790 uint32_t refs
= os_atomic_dec_orig(&kqwl
->kqwl_retains
, relaxed
);
2791 if (__improbable(refs
<= 1)) {
2792 kqworkloop_release_panic(kqwl
);
2798 kqueue_release_live(kqueue_t kqu
)
2800 if (kqu
.kq
->kq_state
& KQ_DYNAMIC
) {
2801 kqworkloop_release_live(kqu
.kqwl
);
2807 kqworkloop_release(struct kqworkloop
*kqwl
)
2809 uint32_t refs
= os_atomic_dec_orig(&kqwl
->kqwl_retains
, relaxed
);
2811 if (__improbable(refs
<= 1)) {
2812 kqworkloop_dealloc(kqwl
, KQWL_DEALLOC_NONE
, refs
- 1);
2818 kqueue_release(kqueue_t kqu
)
2820 if (kqu
.kq
->kq_state
& KQ_DYNAMIC
) {
2821 kqworkloop_release(kqu
.kqwl
);
2826 * @function kqueue_destroy
2829 * Common part to all kqueue dealloc functions.
2833 kqueue_destroy(kqueue_t kqu
, zone_t zone
)
2836 * waitq_set_deinit() remove the KQ's waitq set from
2837 * any select sets to which it may belong.
2839 * The order of these deinits matter: before waitq_set_deinit() returns,
2840 * waitq_set__CALLING_PREPOST_HOOK__ may be called and it will take the
2843 waitq_set_deinit(&kqu
.kq
->kq_wqs
);
2844 lck_spin_destroy(&kqu
.kq
->kq_lock
, kq_lck_grp
);
2846 zfree(zone
, kqu
.kq
);
2850 * @function kqueue_init
2853 * Common part to all kqueue alloc functions.
2856 kqueue_init(kqueue_t kqu
, waitq_set_prepost_hook_t
*hook
, int policy
)
2858 waitq_set_init(&kqu
.kq
->kq_wqs
, policy
, NULL
, hook
);
2859 lck_spin_init(&kqu
.kq
->kq_lock
, kq_lck_grp
, kq_lck_attr
);
2863 #pragma mark kqfile allocation and deallocation
2866 * @function kqueue_dealloc
2869 * Detach all knotes from a kqfile and free it.
2872 * We walk each list looking for knotes referencing this
2873 * this kqueue. If we find one, we try to drop it. But
2874 * if we fail to get a drop reference, that will wait
2875 * until it is dropped. So, we can just restart again
2876 * safe in the assumption that the list will eventually
2877 * not contain any more references to this kqueue (either
2878 * we dropped them all, or someone else did).
2880 * Assumes no new events are being added to the kqueue.
2881 * Nothing locked on entry or exit.
2884 kqueue_dealloc(struct kqueue
*kq
)
2886 KNOTE_LOCK_CTX(knlc
);
2887 struct proc
*p
= kq
->kq_p
;
2888 struct filedesc
*fdp
= p
->p_fd
;
2891 assert(kq
&& (kq
->kq_state
& (KQ_WORKLOOP
| KQ_WORKQ
)) == 0);
2894 for (int i
= 0; i
< fdp
->fd_knlistsize
; i
++) {
2895 kn
= SLIST_FIRST(&fdp
->fd_knlist
[i
]);
2896 while (kn
!= NULL
) {
2897 if (kq
== knote_get_kq(kn
)) {
2900 if (knote_lock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ON_SUCCESS
)) {
2901 knote_drop(kq
, kn
, &knlc
);
2904 /* start over at beginning of list */
2905 kn
= SLIST_FIRST(&fdp
->fd_knlist
[i
]);
2908 kn
= SLIST_NEXT(kn
, kn_link
);
2915 if (fdp
->fd_knhashmask
!= 0) {
2916 for (int i
= 0; i
< (int)fdp
->fd_knhashmask
+ 1; i
++) {
2917 kn
= SLIST_FIRST(&fdp
->fd_knhash
[i
]);
2918 while (kn
!= NULL
) {
2919 if (kq
== knote_get_kq(kn
)) {
2922 if (knote_lock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ON_SUCCESS
)) {
2923 knote_drop(kq
, kn
, &knlc
);
2926 /* start over at beginning of list */
2927 kn
= SLIST_FIRST(&fdp
->fd_knhash
[i
]);
2930 kn
= SLIST_NEXT(kn
, kn_link
);
2936 kqueue_destroy(kq
, kqfile_zone
);
2940 * @function kqueue_alloc
2943 * Allocate a kqfile.
2946 kqueue_alloc(struct proc
*p
)
2950 kqf
= (struct kqfile
*)zalloc(kqfile_zone
);
2951 if (__improbable(kqf
== NULL
)) {
2954 bzero(kqf
, sizeof(struct kqfile
));
2957 * kqfiles are created with kqueue() so we need to wait for
2958 * the first kevent syscall to know which bit among
2959 * KQ_KEV_{32,64,QOS} will be set in kqf_state
2962 TAILQ_INIT_AFTER_BZERO(&kqf
->kqf_queue
);
2963 TAILQ_INIT_AFTER_BZERO(&kqf
->kqf_suppressed
);
2965 return kqueue_init(kqf
, NULL
, SYNC_POLICY_FIFO
| SYNC_POLICY_PREPOST
).kq
;
2969 * @function kqueue_internal
2972 * Core implementation for kqueue and guarded_kqueue_np()
2975 kqueue_internal(struct proc
*p
, fp_allocfn_t fp_zalloc
, void *cra
, int32_t *retval
)
2978 struct fileproc
*fp
;
2981 error
= falloc_withalloc(p
, &fp
, &fd
, vfs_context_current(), fp_zalloc
, cra
);
2986 kq
= kqueue_alloc(p
);
2992 fp
->f_flag
= FREAD
| FWRITE
;
2993 fp
->f_ops
= &kqueueops
;
2995 fp
->f_lflags
|= FG_CONFINED
;
2998 *fdflags(p
, fd
) |= UF_EXCLOSE
| UF_FORKCLOSE
;
2999 procfdtbl_releasefd(p
, fd
, NULL
);
3000 fp_drop(p
, fd
, fp
, 1);
3011 * The kqueue syscall.
3014 kqueue(struct proc
*p
, __unused
struct kqueue_args
*uap
, int32_t *retval
)
3016 return kqueue_internal(p
, fileproc_alloc_init
, NULL
, retval
);
3019 #pragma mark kqworkq allocation and deallocation
3022 * @function kqworkq_dealloc
3025 * Deallocates a workqueue kqueue.
3028 * This only happens at process death, or for races with concurrent
3029 * kevent_get_kqwq calls, hence we don't have to care about knotes referencing
3030 * this kqueue, either there are none, or someone else took care of them.
3033 kqworkq_dealloc(struct kqworkq
*kqwq
)
3035 kqueue_destroy(kqwq
, kqworkq_zone
);
3039 * @function kqworkq_alloc
3042 * Allocates a workqueue kqueue.
3045 * This is the slow path of kevent_get_kqwq.
3046 * This takes care of making sure procs have a single workq kqueue.
3049 static struct kqworkq
*
3050 kqworkq_alloc(struct proc
*p
, unsigned int flags
)
3052 struct kqworkq
*kqwq
, *tmp
;
3054 kqwq
= (struct kqworkq
*)zalloc(kqworkq_zone
);
3055 if (__improbable(kqwq
== NULL
)) {
3058 bzero(kqwq
, sizeof(struct kqworkq
));
3060 assert((flags
& KEVENT_FLAG_LEGACY32
) == 0);
3061 if (flags
& KEVENT_FLAG_LEGACY64
) {
3062 kqwq
->kqwq_state
= KQ_WORKQ
| KQ_KEV64
;
3064 kqwq
->kqwq_state
= KQ_WORKQ
| KQ_KEV_QOS
;
3068 for (int i
= 0; i
< KQWQ_NBUCKETS
; i
++) {
3069 TAILQ_INIT_AFTER_BZERO(&kqwq
->kqwq_queue
[i
]);
3070 TAILQ_INIT_AFTER_BZERO(&kqwq
->kqwq_suppressed
[i
]);
3072 for (int i
= 0; i
< KQWQ_NBUCKETS
; i
++) {
3074 * Because of how the bucketized system works, we mix overcommit
3075 * sources with not overcommit: each time we move a knote from
3076 * one bucket to the next due to overrides, we'd had to track
3077 * overcommitness, and it's really not worth it in the workloop
3078 * enabled world that track this faithfully.
3080 * Incidentally, this behaves like the original manager-based
3081 * kqwq where event delivery always happened (hence is
3084 kqwq
->kqwq_request
[i
].tr_state
= WORKQ_TR_STATE_IDLE
;
3085 kqwq
->kqwq_request
[i
].tr_flags
= WORKQ_TR_FLAG_KEVENT
;
3086 if (i
!= KQWQ_QOS_MANAGER
) {
3087 kqwq
->kqwq_request
[i
].tr_flags
|= WORKQ_TR_FLAG_OVERCOMMIT
;
3089 kqwq
->kqwq_request
[i
].tr_kq_qos_index
= i
;
3092 kqueue_init(kqwq
, &kqwq
->kqwq_waitq_hook
, SYNC_POLICY_FIFO
);
3094 if (!os_atomic_cmpxchgv(&p
->p_fd
->fd_wqkqueue
, NULL
, kqwq
, &tmp
, release
)) {
3095 kqworkq_dealloc(kqwq
);
3102 #pragma mark kqworkloop allocation and deallocation
3104 #define KQ_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
3105 #define CONFIG_KQ_HASHSIZE CONFIG_KN_HASHSIZE
3109 kqhash_lock(struct filedesc
*fdp
)
3111 lck_mtx_lock_spin_always(&fdp
->fd_kqhashlock
);
3116 kqhash_unlock(struct filedesc
*fdp
)
3118 lck_mtx_unlock(&fdp
->fd_kqhashlock
);
3123 kqworkloop_hash_insert_locked(struct filedesc
*fdp
, kqueue_id_t id
,
3124 struct kqworkloop
*kqwl
)
3126 struct kqwllist
*list
= &fdp
->fd_kqhash
[KQ_HASH(id
, fdp
->fd_kqhashmask
)];
3127 LIST_INSERT_HEAD(list
, kqwl
, kqwl_hashlink
);
3131 static inline struct kqworkloop
*
3132 kqworkloop_hash_lookup_locked(struct filedesc
*fdp
, kqueue_id_t id
)
3134 struct kqwllist
*list
= &fdp
->fd_kqhash
[KQ_HASH(id
, fdp
->fd_kqhashmask
)];
3135 struct kqworkloop
*kqwl
;
3137 LIST_FOREACH(kqwl
, list
, kqwl_hashlink
) {
3138 if (kqwl
->kqwl_dynamicid
== id
) {
3145 static struct kqworkloop
*
3146 kqworkloop_hash_lookup_and_retain(struct filedesc
*fdp
, kqueue_id_t kq_id
)
3148 struct kqworkloop
*kqwl
= NULL
;
3151 if (__probable(fdp
->fd_kqhash
)) {
3152 kqwl
= kqworkloop_hash_lookup_locked(fdp
, kq_id
);
3153 if (kqwl
&& !kqworkloop_try_retain(kqwl
)) {
3163 kqworkloop_hash_init(struct filedesc
*fdp
)
3165 struct kqwllist
*alloc_hash
;
3169 alloc_hash
= hashinit(CONFIG_KQ_HASHSIZE
, M_KQUEUE
, &alloc_mask
);
3172 /* See if we won the race */
3173 if (__probable(fdp
->fd_kqhashmask
== 0)) {
3174 fdp
->fd_kqhash
= alloc_hash
;
3175 fdp
->fd_kqhashmask
= alloc_mask
;
3178 FREE(alloc_hash
, M_KQUEUE
);
3184 * @function kqworkloop_dealloc
3187 * Deallocates a workloop kqueue.
3190 * Knotes hold references on the workloop, so we can't really reach this
3191 * function unless all of these are already gone.
3193 * Nothing locked on entry or exit.
3196 * Unless KQWL_DEALLOC_SKIP_HASH_REMOVE is set, the workloop is removed
3197 * from its hash table.
3199 * @param current_ref
3200 * This function is also called to undo a kqworkloop_alloc in case of
3201 * allocation races, expected_ref is the current refcount that is expected
3202 * on the workloop object, usually 0, and 1 when a dealloc race is resolved.
3205 kqworkloop_dealloc(struct kqworkloop
*kqwl
, kqworkloop_dealloc_flags_t flags
,
3206 uint32_t current_ref
)
3210 if (__improbable(current_ref
> 1)) {
3211 kqworkloop_release_panic(kqwl
);
3213 assert(kqwl
->kqwl_retains
== current_ref
);
3215 /* pair with kqunlock() and other kq locks */
3216 os_atomic_thread_fence(acquire
);
3218 cur_owner
= kqwl
->kqwl_owner
;
3220 if (kqworkloop_override(kqwl
) != THREAD_QOS_UNSPECIFIED
) {
3221 thread_drop_kevent_override(cur_owner
);
3223 thread_deallocate(cur_owner
);
3224 kqwl
->kqwl_owner
= THREAD_NULL
;
3227 if (kqwl
->kqwl_state
& KQ_HAS_TURNSTILE
) {
3228 struct turnstile
*ts
;
3229 turnstile_complete((uintptr_t)kqwl
, &kqwl
->kqwl_turnstile
,
3230 &ts
, TURNSTILE_WORKLOOPS
);
3231 turnstile_cleanup();
3232 turnstile_deallocate(ts
);
3235 if ((flags
& KQWL_DEALLOC_SKIP_HASH_REMOVE
) == 0) {
3236 struct filedesc
*fdp
= kqwl
->kqwl_p
->p_fd
;
3239 LIST_REMOVE(kqwl
, kqwl_hashlink
);
3243 assert(TAILQ_EMPTY(&kqwl
->kqwl_suppressed
));
3244 assert(kqwl
->kqwl_owner
== THREAD_NULL
);
3245 assert(kqwl
->kqwl_turnstile
== TURNSTILE_NULL
);
3247 lck_spin_destroy(&kqwl
->kqwl_statelock
, kq_lck_grp
);
3248 kqueue_destroy(kqwl
, kqworkloop_zone
);
3252 * @function kqworkloop_alloc
3255 * Allocates a workloop kqueue.
3258 kqworkloop_init(struct kqworkloop
*kqwl
, proc_t p
,
3259 kqueue_id_t id
, workq_threadreq_param_t
*trp
)
3261 bzero(kqwl
, sizeof(struct kqworkloop
));
3263 kqwl
->kqwl_state
= KQ_WORKLOOP
| KQ_DYNAMIC
| KQ_KEV_QOS
;
3264 kqwl
->kqwl_retains
= 1; /* donate a retain to creator */
3265 kqwl
->kqwl_dynamicid
= id
;
3268 kqwl
->kqwl_params
= trp
->trp_value
;
3271 workq_tr_flags_t tr_flags
= WORKQ_TR_FLAG_WORKLOOP
;
3273 if (trp
->trp_flags
& TRP_PRIORITY
) {
3274 tr_flags
|= WORKQ_TR_FLAG_WL_OUTSIDE_QOS
;
3276 if (trp
->trp_flags
) {
3277 tr_flags
|= WORKQ_TR_FLAG_WL_PARAMS
;
3280 kqwl
->kqwl_request
.tr_state
= WORKQ_TR_STATE_IDLE
;
3281 kqwl
->kqwl_request
.tr_flags
= tr_flags
;
3283 for (int i
= 0; i
< KQWL_NBUCKETS
; i
++) {
3284 TAILQ_INIT_AFTER_BZERO(&kqwl
->kqwl_queue
[i
]);
3286 TAILQ_INIT_AFTER_BZERO(&kqwl
->kqwl_suppressed
);
3288 lck_spin_init(&kqwl
->kqwl_statelock
, kq_lck_grp
, kq_lck_attr
);
3290 kqueue_init(kqwl
, &kqwl
->kqwl_waitq_hook
, SYNC_POLICY_FIFO
);
3294 * @function kqworkloop_get_or_create
3297 * Wrapper around kqworkloop_alloc that handles the uniquing of workloops.
3301 * EINVAL: invalid parameters
3302 * EEXIST: KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST is set and a collision exists.
3303 * ENOENT: KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST is set and the entry wasn't found.
3304 * ENOMEM: allocation failed
3307 kqworkloop_get_or_create(struct proc
*p
, kqueue_id_t id
,
3308 workq_threadreq_param_t
*trp
, unsigned int flags
, struct kqworkloop
**kqwlp
)
3310 struct filedesc
*fdp
= p
->p_fd
;
3311 struct kqworkloop
*alloc_kqwl
= NULL
;
3312 struct kqworkloop
*kqwl
= NULL
;
3315 assert(!trp
|| (flags
& KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST
));
3317 if (id
== 0 || id
== (kqueue_id_t
)-1) {
3323 if (__improbable(fdp
->fd_kqhash
== NULL
)) {
3324 kqworkloop_hash_init(fdp
);
3327 kqwl
= kqworkloop_hash_lookup_locked(fdp
, id
);
3329 if (__improbable(flags
& KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST
)) {
3331 * If MUST_NOT_EXIST was passed, even if we would have failed
3332 * the try_retain, it could have gone the other way, and
3333 * userspace can't tell. Let'em fix their race.
3339 if (__probable(kqworkloop_try_retain(kqwl
))) {
3341 * This is a valid live workloop !
3349 if (__improbable(flags
& KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST
)) {
3355 * We didn't find what we were looking for.
3357 * If this is the second time we reach this point (alloc_kqwl != NULL),
3360 * If this is the first time we reach this point (alloc_kqwl == NULL),
3361 * then try to allocate one without blocking.
3363 if (__probable(alloc_kqwl
== NULL
)) {
3364 alloc_kqwl
= (struct kqworkloop
*)zalloc_noblock(kqworkloop_zone
);
3366 if (__probable(alloc_kqwl
)) {
3367 kqworkloop_init(alloc_kqwl
, p
, id
, trp
);
3368 kqworkloop_hash_insert_locked(fdp
, id
, alloc_kqwl
);
3370 *kqwlp
= alloc_kqwl
;
3375 * We have to block to allocate a workloop, drop the lock,
3376 * allocate one, but then we need to retry lookups as someone
3377 * else could race with us.
3381 alloc_kqwl
= (struct kqworkloop
*)zalloc(kqworkloop_zone
);
3382 if (__improbable(!alloc_kqwl
)) {
3389 if (__improbable(alloc_kqwl
)) {
3390 zfree(kqworkloop_zone
, alloc_kqwl
);
3396 #pragma mark - knotes
3399 filt_no_attach(struct knote
*kn
, __unused
struct kevent_qos_s
*kev
)
3401 knote_set_error(kn
, ENOTSUP
);
3406 filt_no_detach(__unused
struct knote
*kn
)
3411 filt_bad_event(struct knote
*kn
, long hint
)
3413 panic("%s[%d](%p, %ld)", __func__
, kn
->kn_filter
, kn
, hint
);
3417 filt_bad_touch(struct knote
*kn
, struct kevent_qos_s
*kev
)
3419 panic("%s[%d](%p, %p)", __func__
, kn
->kn_filter
, kn
, kev
);
3423 filt_bad_process(struct knote
*kn
, struct kevent_qos_s
*kev
)
3425 panic("%s[%d](%p, %p)", __func__
, kn
->kn_filter
, kn
, kev
);
3429 * knotes_dealloc - detach all knotes for the process and drop them
3431 * Called with proc_fdlock held.
3432 * Returns with it locked.
3433 * May drop it temporarily.
3434 * Process is in such a state that it will not try to allocate
3435 * any more knotes during this process (stopped for exit or exec).
3438 knotes_dealloc(proc_t p
)
3440 struct filedesc
*fdp
= p
->p_fd
;
3443 struct klist
*kn_hash
= NULL
;
3446 /* Close all the fd-indexed knotes up front */
3447 if (fdp
->fd_knlistsize
> 0) {
3448 for (i
= 0; i
< fdp
->fd_knlistsize
; i
++) {
3449 while ((kn
= SLIST_FIRST(&fdp
->fd_knlist
[i
])) != NULL
) {
3450 kq
= knote_get_kq(kn
);
3453 knote_drop(kq
, kn
, NULL
);
3457 /* free the table */
3458 FREE(fdp
->fd_knlist
, M_KQUEUE
);
3459 fdp
->fd_knlist
= NULL
;
3461 fdp
->fd_knlistsize
= 0;
3466 /* Clean out all the hashed knotes as well */
3467 if (fdp
->fd_knhashmask
!= 0) {
3468 for (i
= 0; i
<= (int)fdp
->fd_knhashmask
; i
++) {
3469 while ((kn
= SLIST_FIRST(&fdp
->fd_knhash
[i
])) != NULL
) {
3470 kq
= knote_get_kq(kn
);
3473 knote_drop(kq
, kn
, NULL
);
3477 kn_hash
= fdp
->fd_knhash
;
3478 fdp
->fd_knhashmask
= 0;
3479 fdp
->fd_knhash
= NULL
;
3484 /* free the kn_hash table */
3486 FREE(kn_hash
, M_KQUEUE
);
3493 * kqworkloops_dealloc - rebalance retains on kqworkloops created with
3494 * scheduling parameters
3496 * Called with proc_fdlock held.
3497 * Returns with it locked.
3498 * Process is in such a state that it will not try to allocate
3499 * any more knotes during this process (stopped for exit or exec).
3502 kqworkloops_dealloc(proc_t p
)
3504 struct filedesc
*fdp
= p
->p_fd
;
3505 struct kqworkloop
*kqwl
, *kqwln
;
3506 struct kqwllist tofree
;
3508 if (!(fdp
->fd_flags
& FD_WORKLOOP
)) {
3514 if (fdp
->fd_kqhashmask
== 0) {
3521 for (size_t i
= 0; i
<= fdp
->fd_kqhashmask
; i
++) {
3522 LIST_FOREACH_SAFE(kqwl
, &fdp
->fd_kqhash
[i
], kqwl_hashlink
, kqwln
) {
3524 * kqworkloops that have scheduling parameters have an
3525 * implicit retain from kqueue_workloop_ctl that needs
3526 * to be balanced on process exit.
3528 assert(kqwl
->kqwl_params
);
3529 LIST_REMOVE(kqwl
, kqwl_hashlink
);
3530 LIST_INSERT_HEAD(&tofree
, kqwl
, kqwl_hashlink
);
3536 LIST_FOREACH_SAFE(kqwl
, &tofree
, kqwl_hashlink
, kqwln
) {
3537 kqworkloop_dealloc(kqwl
, KQWL_DEALLOC_SKIP_HASH_REMOVE
, 1);
3542 kevent_register_validate_priority(struct kqueue
*kq
, struct knote
*kn
,
3543 struct kevent_qos_s
*kev
)
3545 /* We don't care about the priority of a disabled or deleted knote */
3546 if (kev
->flags
& (EV_DISABLE
| EV_DELETE
)) {
3550 if (kq
->kq_state
& KQ_WORKLOOP
) {
3552 * Workloops need valid priorities with a QOS (excluding manager) for
3553 * any enabled knote.
3555 * When it is pre-existing, just make sure it has a valid QoS as
3556 * kevent_register() will not use the incoming priority (filters who do
3557 * have the responsibility to validate it again, see filt_wltouch).
3559 * If the knote is being made, validate the incoming priority.
3561 if (!_pthread_priority_thread_qos(kn
? kn
->kn_qos
: kev
->qos
)) {
3570 * Prepare a filter for waiting after register.
3572 * The f_post_register_wait hook will be called later by kevent_register()
3573 * and should call kevent_register_wait_block()
3576 kevent_register_wait_prepare(struct knote
*kn
, struct kevent_qos_s
*kev
, int rc
)
3578 thread_t thread
= current_thread();
3580 assert(knote_fops(kn
)->f_extended_codes
);
3582 if (kn
->kn_thread
== NULL
) {
3583 thread_reference(thread
);
3584 kn
->kn_thread
= thread
;
3585 } else if (kn
->kn_thread
!= thread
) {
3587 * kn_thread may be set from a previous aborted wait
3588 * However, it has to be from the same thread.
3590 kev
->flags
|= EV_ERROR
;
3595 return FILTER_REGISTER_WAIT
| rc
;
3599 * Cleanup a kevent_register_wait_prepare() effect for threads that have been
3600 * aborted instead of properly woken up with thread_wakeup_thread().
3603 kevent_register_wait_cleanup(struct knote
*kn
)
3605 thread_t thread
= kn
->kn_thread
;
3606 kn
->kn_thread
= NULL
;
3607 thread_deallocate(thread
);
3611 * Must be called at the end of a f_post_register_wait call from a filter.
3614 kevent_register_wait_block(struct turnstile
*ts
, thread_t thread
,
3615 thread_continue_t cont
, struct _kevent_register
*cont_args
)
3617 turnstile_update_inheritor_complete(ts
, TURNSTILE_INTERLOCK_HELD
);
3618 kqunlock(cont_args
->kqwl
);
3619 cont_args
->handoff_thread
= thread
;
3620 thread_handoff_parameter(thread
, cont
, cont_args
);
3624 * Called by Filters using a f_post_register_wait to return from their wait.
3627 kevent_register_wait_return(struct _kevent_register
*cont_args
)
3629 struct kqworkloop
*kqwl
= cont_args
->kqwl
;
3630 struct kevent_qos_s
*kev
= &cont_args
->kev
;
3633 if (cont_args
->handoff_thread
) {
3634 thread_deallocate(cont_args
->handoff_thread
);
3637 if (kev
->flags
& (EV_ERROR
| EV_RECEIPT
)) {
3638 if ((kev
->flags
& EV_ERROR
) == 0) {
3639 kev
->flags
|= EV_ERROR
;
3642 error
= kevent_modern_copyout(kev
, &cont_args
->ueventlist
);
3644 cont_args
->eventout
++;
3648 kqworkloop_release(kqwl
);
3650 *(int32_t *)¤t_uthread()->uu_rval
= cont_args
->eventout
;
3652 unix_syscall_return(error
);
3656 * kevent_register - add a new event to a kqueue
3658 * Creates a mapping between the event source and
3659 * the kqueue via a knote data structure.
3661 * Because many/most the event sources are file
3662 * descriptor related, the knote is linked off
3663 * the filedescriptor table for quick access.
3665 * called with nothing locked
3666 * caller holds a reference on the kqueue
3670 kevent_register(struct kqueue
*kq
, struct kevent_qos_s
*kev
,
3671 struct knote
**kn_out
)
3673 struct proc
*p
= kq
->kq_p
;
3674 const struct filterops
*fops
;
3675 struct knote
*kn
= NULL
;
3676 int result
= 0, error
= 0;
3677 unsigned short kev_flags
= kev
->flags
;
3678 KNOTE_LOCK_CTX(knlc
);
3680 if (__probable(kev
->filter
< 0 && kev
->filter
+ EVFILT_SYSCOUNT
>= 0)) {
3681 fops
= sysfilt_ops
[~kev
->filter
]; /* to 0-base index */
3687 /* restrict EV_VANISHED to adding udata-specific dispatch kevents */
3688 if (__improbable((kev
->flags
& EV_VANISHED
) &&
3689 (kev
->flags
& (EV_ADD
| EV_DISPATCH2
)) != (EV_ADD
| EV_DISPATCH2
))) {
3694 /* Simplify the flags - delete and disable overrule */
3695 if (kev
->flags
& EV_DELETE
) {
3696 kev
->flags
&= ~EV_ADD
;
3698 if (kev
->flags
& EV_DISABLE
) {
3699 kev
->flags
&= ~EV_ENABLE
;
3702 if (kq
->kq_state
& KQ_WORKLOOP
) {
3703 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_REGISTER
),
3704 ((struct kqworkloop
*)kq
)->kqwl_dynamicid
,
3705 kev
->udata
, kev
->flags
, kev
->filter
);
3706 } else if (kq
->kq_state
& KQ_WORKQ
) {
3707 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_REGISTER
),
3708 0, kev
->udata
, kev
->flags
, kev
->filter
);
3710 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_REGISTER
),
3711 VM_KERNEL_UNSLIDE_OR_PERM(kq
),
3712 kev
->udata
, kev
->flags
, kev
->filter
);
3716 /* find the matching knote from the fd tables/hashes */
3717 kn
= kq_find_knote_and_kq_lock(kq
, kev
, fops
->f_isfd
, p
);
3718 error
= kevent_register_validate_priority(kq
, kn
, kev
);
3724 if (kn
== NULL
&& (kev
->flags
& EV_ADD
) == 0) {
3726 * No knote found, EV_ADD wasn't specified
3729 if ((kev_flags
& EV_ADD
) && (kev_flags
& EV_DELETE
) &&
3730 (kq
->kq_state
& KQ_WORKLOOP
)) {
3732 * For workloops, understand EV_ADD|EV_DELETE as a "soft" delete
3733 * that doesn't care about ENOENT, so just pretend the deletion
3740 } else if (kn
== NULL
) {
3742 * No knote found, need to attach a new one (attach)
3745 struct fileproc
*knote_fp
= NULL
;
3747 /* grab a file reference for the new knote */
3749 if ((error
= fp_lookup(p
, kev
->ident
, &knote_fp
, 0)) != 0) {
3757 if (knote_fp
!= NULL
) {
3758 fp_drop(p
, kev
->ident
, knote_fp
, 0);
3763 kn
->kn_fp
= knote_fp
;
3764 kn
->kn_is_fd
= fops
->f_isfd
;
3765 kn
->kn_kq_packed
= (intptr_t)(struct kqueue
*)kq
;
3768 /* was vanish support requested */
3769 if (kev
->flags
& EV_VANISHED
) {
3770 kev
->flags
&= ~EV_VANISHED
;
3771 kn
->kn_status
|= KN_REQVANISH
;
3774 /* snapshot matching/dispatching protcol flags into knote */
3775 if (kev
->flags
& EV_DISABLE
) {
3776 kn
->kn_status
|= KN_DISABLED
;
3780 * copy the kevent state into knote
3781 * protocol is that fflags and data
3782 * are saved off, and cleared before
3783 * calling the attach routine.
3785 * - kn->kn_sfflags aliases with kev->xflags
3786 * - kn->kn_sdata aliases with kev->data
3787 * - kn->kn_filter is the top 8 bits of kev->filter
3789 kn
->kn_kevent
= *(struct kevent_internal_s
*)kev
;
3790 kn
->kn_sfflags
= kev
->fflags
;
3791 kn
->kn_filtid
= (uint8_t)~kev
->filter
;
3793 knote_reset_priority(kq
, kn
, kev
->qos
);
3795 /* Add the knote for lookup thru the fd table */
3796 error
= kq_add_knote(kq
, kn
, &knlc
, p
);
3799 if (knote_fp
!= NULL
) {
3800 fp_drop(p
, kev
->ident
, knote_fp
, 0);
3803 if (error
== ERESTART
) {
3809 /* fp reference count now applies to knote */
3812 * we can't use filter_call() because f_attach can change the filter ops
3813 * for a filter that supports f_extended_codes, so we need to reload
3814 * knote_fops() and not use `fops`.
3816 result
= fops
->f_attach(kn
, kev
);
3817 if (result
&& !knote_fops(kn
)->f_extended_codes
) {
3818 result
= FILTER_ACTIVE
;
3823 if (result
& FILTER_THREADREQ_NODEFEER
) {
3824 enable_preemption();
3827 if (kn
->kn_flags
& EV_ERROR
) {
3829 * Failed to attach correctly, so drop.
3831 kn
->kn_filtid
= EVFILTID_DETACHED
;
3832 error
= kn
->kn_sdata
;
3833 knote_drop(kq
, kn
, &knlc
);
3839 * end "attaching" phase - now just attached
3841 * Mark the thread request overcommit, if appropos
3843 * If the attach routine indicated that an
3844 * event is already fired, activate the knote.
3846 if ((kn
->kn_qos
& _PTHREAD_PRIORITY_OVERCOMMIT_FLAG
) &&
3847 (kq
->kq_state
& KQ_WORKLOOP
)) {
3848 kqworkloop_set_overcommit((struct kqworkloop
*)kq
);
3850 } else if (!knote_lock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ON_SUCCESS
)) {
3852 * The knote was dropped while we were waiting for the lock,
3853 * we need to re-evaluate entirely
3857 } else if (kev
->flags
& EV_DELETE
) {
3859 * Deletion of a knote (drop)
3861 * If the filter wants to filter drop events, let it do so.
3863 * defer-delete: when trying to delete a disabled EV_DISPATCH2 knote,
3864 * we must wait for the knote to be re-enabled (unless it is being
3865 * re-enabled atomically here).
3868 if (knote_fops(kn
)->f_allow_drop
) {
3872 drop
= knote_fops(kn
)->f_allow_drop(kn
, kev
);
3880 if ((kev
->flags
& EV_ENABLE
) == 0 &&
3881 (kn
->kn_flags
& EV_DISPATCH2
) == EV_DISPATCH2
&&
3882 (kn
->kn_status
& KN_DISABLED
) != 0) {
3883 kn
->kn_status
|= KN_DEFERDELETE
;
3884 error
= EINPROGRESS
;
3888 knote_drop(kq
, kn
, &knlc
);
3892 * Regular update of a knote (touch)
3894 * Call touch routine to notify filter of changes in filter values
3895 * (and to re-determine if any events are fired).
3897 * If the knote is in defer-delete, avoid calling the filter touch
3898 * routine (it has delivered its last event already).
3900 * If the touch routine had no failure,
3901 * apply the requested side effects to the knote.
3904 if (kn
->kn_status
& (KN_DEFERDELETE
| KN_VANISHED
)) {
3905 if (kev
->flags
& EV_ENABLE
) {
3906 result
= FILTER_ACTIVE
;
3910 result
= filter_call(knote_fops(kn
), f_touch(kn
, kev
));
3912 if (result
& FILTER_THREADREQ_NODEFEER
) {
3913 enable_preemption();
3917 if (kev
->flags
& EV_ERROR
) {
3922 if ((kn
->kn_flags
& EV_UDATA_SPECIFIC
) == 0 &&
3923 kn
->kn_udata
!= kev
->udata
) {
3924 // this allows klist_copy_udata() not to take locks
3925 os_atomic_store_wide(&kn
->kn_udata
, kev
->udata
, relaxed
);
3927 if ((kev
->flags
& EV_DISABLE
) && !(kn
->kn_status
& KN_DISABLED
)) {
3928 kn
->kn_status
|= KN_DISABLED
;
3929 knote_dequeue(kq
, kn
);
3933 /* accept new kevent state */
3934 knote_apply_touch(kq
, kn
, kev
, result
);
3938 * When the filter asked for a post-register wait,
3939 * we leave the kqueue locked for kevent_register()
3940 * to call the filter's f_post_register_wait hook.
3942 if (result
& FILTER_REGISTER_WAIT
) {
3943 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ALWAYS
);
3946 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_UNLOCK
);
3950 /* output local errors through the kevent */
3952 kev
->flags
|= EV_ERROR
;
3959 * knote_process - process a triggered event
3961 * Validate that it is really still a triggered event
3962 * by calling the filter routines (if necessary). Hold
3963 * a use reference on the knote to avoid it being detached.
3965 * If it is still considered triggered, we will have taken
3966 * a copy of the state under the filter lock. We use that
3967 * snapshot to dispatch the knote for future processing (or
3968 * not, if this was a lost event).
3970 * Our caller assures us that nobody else can be processing
3971 * events from this knote during the whole operation. But
3972 * others can be touching or posting events to the knote
3973 * interspersed with our processing it.
3975 * caller holds a reference on the kqueue.
3976 * kqueue locked on entry and exit - but may be dropped
3979 knote_process(struct knote
*kn
, kevent_ctx_t kectx
,
3980 kevent_callback_t callback
)
3982 struct kevent_qos_s kev
;
3983 struct kqueue
*kq
= knote_get_kq(kn
);
3984 KNOTE_LOCK_CTX(knlc
);
3985 int result
= FILTER_ACTIVE
;
3990 * Must be active or stayactive
3991 * Must be queued and not disabled/suppressed or dropping
3993 assert(kn
->kn_status
& KN_QUEUED
);
3994 assert(kn
->kn_status
& (KN_ACTIVE
| KN_STAYACTIVE
));
3995 assert(!(kn
->kn_status
& (KN_DISABLED
| KN_SUPPRESSED
| KN_DROPPING
)));
3997 if (kq
->kq_state
& KQ_WORKLOOP
) {
3998 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS
),
3999 ((struct kqworkloop
*)kq
)->kqwl_dynamicid
,
4000 kn
->kn_udata
, kn
->kn_status
| (kn
->kn_id
<< 32),
4002 } else if (kq
->kq_state
& KQ_WORKQ
) {
4003 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS
),
4004 0, kn
->kn_udata
, kn
->kn_status
| (kn
->kn_id
<< 32),
4007 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS
),
4008 VM_KERNEL_UNSLIDE_OR_PERM(kq
), kn
->kn_udata
,
4009 kn
->kn_status
| (kn
->kn_id
<< 32), kn
->kn_filtid
);
4012 if (!knote_lock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ALWAYS
)) {
4014 * When the knote is dropping or has dropped,
4015 * then there's nothing we want to process.
4021 * While waiting for the knote lock, we may have dropped the kq lock.
4022 * and a touch may have disabled and dequeued the knote.
4024 if (!(kn
->kn_status
& KN_QUEUED
)) {
4025 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ALWAYS
);
4030 * For deferred-drop or vanished events, we just create a fake
4031 * event to acknowledge end-of-life. Otherwise, we call the
4032 * filter's process routine to snapshot the kevent state under
4033 * the filter's locking protocol.
4035 * suppress knotes to avoid returning the same event multiple times in
4038 knote_suppress(kq
, kn
);
4040 if (kn
->kn_status
& (KN_DEFERDELETE
| KN_VANISHED
)) {
4041 int kev_flags
= EV_DISPATCH2
| EV_ONESHOT
;
4042 if (kn
->kn_status
& KN_DEFERDELETE
) {
4043 kev_flags
|= EV_DELETE
;
4045 kev_flags
|= EV_VANISHED
;
4048 /* create fake event */
4049 kev
= (struct kevent_qos_s
){
4050 .filter
= kn
->kn_filter
,
4053 .udata
= kn
->kn_udata
,
4057 kev
= (struct kevent_qos_s
) { };
4058 result
= filter_call(knote_fops(kn
), f_process(kn
, &kev
));
4063 * Determine how to dispatch the knote for future event handling.
4064 * not-fired: just return (do not callout, leave deactivated).
4065 * One-shot: If dispatch2, enter deferred-delete mode (unless this is
4066 * is the deferred delete event delivery itself). Otherwise,
4068 * Dispatch: don't clear state, just mark it disabled.
4069 * Cleared: just leave it deactivated.
4070 * Others: re-activate as there may be more events to handle.
4071 * This will not wake up more handlers right now, but
4072 * at the completion of handling events it may trigger
4073 * more handler threads (TODO: optimize based on more than
4074 * just this one event being detected by the filter).
4076 if ((result
& FILTER_ACTIVE
) == 0) {
4077 if ((kn
->kn_status
& (KN_ACTIVE
| KN_STAYACTIVE
)) == 0) {
4079 * Stay active knotes should not be unsuppressed or we'd create an
4082 * Some knotes (like EVFILT_WORKLOOP) can be reactivated from
4083 * within f_process() but that doesn't necessarily make them
4084 * ready to process, so we should leave them be.
4086 * For other knotes, since we will not return an event,
4087 * there's no point keeping the knote suppressed.
4089 knote_unsuppress(kq
, kn
);
4091 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ALWAYS
);
4095 if (result
& FILTER_ADJUST_EVENT_QOS_BIT
) {
4096 knote_adjust_qos(kq
, kn
, result
);
4098 kev
.qos
= _pthread_priority_combine(kn
->kn_qos
, kn
->kn_qos_override
);
4100 if (kev
.flags
& EV_ONESHOT
) {
4101 if ((kn
->kn_flags
& EV_DISPATCH2
) == EV_DISPATCH2
&&
4102 (kn
->kn_status
& KN_DEFERDELETE
) == 0) {
4103 /* defer dropping non-delete oneshot dispatch2 events */
4104 kn
->kn_status
|= KN_DEFERDELETE
| KN_DISABLED
;
4108 } else if (kn
->kn_flags
& EV_DISPATCH
) {
4109 /* disable all dispatch knotes */
4110 kn
->kn_status
|= KN_DISABLED
;
4111 } else if ((kn
->kn_flags
& EV_CLEAR
) == 0) {
4112 /* re-activate in case there are more events */
4113 knote_activate(kq
, kn
, FILTER_ACTIVE
);
4117 * callback to handle each event as we find it.
4118 * If we have to detach and drop the knote, do
4119 * it while we have the kq unlocked.
4122 knote_drop(kq
, kn
, &knlc
);
4124 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_UNLOCK
);
4127 if (kev
.flags
& EV_VANISHED
) {
4128 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KNOTE_VANISHED
),
4129 kev
.ident
, kn
->kn_udata
, kn
->kn_status
| (kn
->kn_id
<< 32),
4133 error
= (callback
)(&kev
, kectx
);
4139 * Returns -1 if the kqueue was unbound and processing should not happen
4141 #define KQWQAE_BEGIN_PROCESSING 1
4142 #define KQWQAE_END_PROCESSING 2
4143 #define KQWQAE_UNBIND 3
4145 kqworkq_acknowledge_events(struct kqworkq
*kqwq
, workq_threadreq_t kqr
,
4146 int kevent_flags
, int kqwqae_op
)
4148 thread_qos_t old_override
= THREAD_QOS_UNSPECIFIED
;
4149 thread_t thread
= kqr_thread_fast(kqr
);
4153 struct kqtailq
*suppressq
= &kqwq
->kqwq_suppressed
[kqr
->tr_kq_qos_index
];
4155 kqlock_held(&kqwq
->kqwq_kqueue
);
4157 if (!TAILQ_EMPTY(suppressq
)) {
4159 * Return suppressed knotes to their original state.
4160 * For workq kqueues, suppressed ones that are still
4161 * truly active (not just forced into the queue) will
4162 * set flags we check below to see if anything got
4165 while ((kn
= TAILQ_FIRST(suppressq
)) != NULL
) {
4166 assert(kn
->kn_status
& KN_SUPPRESSED
);
4167 knote_unsuppress(kqwq
, kn
);
4171 #if DEBUG || DEVELOPMENT
4172 thread_t self
= current_thread();
4173 struct uthread
*ut
= get_bsdthread_info(self
);
4175 assert(thread
== self
);
4176 assert(ut
->uu_kqr_bound
== kqr
);
4177 #endif // DEBUG || DEVELOPMENT
4179 if (kqwqae_op
== KQWQAE_UNBIND
) {
4181 } else if ((kevent_flags
& KEVENT_FLAG_PARKING
) == 0) {
4184 unbind
= !kqr
->tr_kq_wakeup
;
4187 old_override
= kqworkq_unbind_locked(kqwq
, kqr
, thread
);
4190 * request a new thread if we didn't process the whole queue or real events
4191 * have happened (not just putting stay-active events back).
4193 if (kqr
->tr_kq_wakeup
) {
4194 kqueue_threadreq_initiate(&kqwq
->kqwq_kqueue
, kqr
,
4195 kqr
->tr_kq_qos_index
, 0);
4201 * Reset wakeup bit to notice events firing while we are processing,
4202 * as we cannot rely on the bucket queue emptiness because of stay
4205 kqr
->tr_kq_wakeup
= false;
4209 thread_drop_kevent_override(thread
);
4216 * Return 0 to indicate that processing should proceed,
4217 * -1 if there is nothing to process.
4219 * Called with kqueue locked and returns the same way,
4220 * but may drop lock temporarily.
4223 kqworkq_begin_processing(struct kqworkq
*kqwq
, workq_threadreq_t kqr
,
4228 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN
) | DBG_FUNC_START
,
4229 0, kqr
->tr_kq_qos_index
);
4231 rc
= kqworkq_acknowledge_events(kqwq
, kqr
, kevent_flags
,
4232 KQWQAE_BEGIN_PROCESSING
);
4234 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_PROCESS_BEGIN
) | DBG_FUNC_END
,
4235 thread_tid(kqr_thread(kqr
)), kqr
->tr_kq_wakeup
);
4241 kqworkloop_acknowledge_events(struct kqworkloop
*kqwl
)
4243 kq_index_t qos
= THREAD_QOS_UNSPECIFIED
;
4244 struct knote
*kn
, *tmp
;
4248 TAILQ_FOREACH_SAFE(kn
, &kqwl
->kqwl_suppressed
, kn_tqe
, tmp
) {
4250 * If a knote that can adjust QoS is disabled because of the automatic
4251 * behavior of EV_DISPATCH, the knotes should stay suppressed so that
4252 * further overrides keep pushing.
4254 if (knote_fops(kn
)->f_adjusts_qos
&& (kn
->kn_status
& KN_DISABLED
) &&
4255 (kn
->kn_status
& (KN_STAYACTIVE
| KN_DROPPING
)) == 0 &&
4256 (kn
->kn_flags
& (EV_DISPATCH
| EV_DISABLE
)) == EV_DISPATCH
) {
4257 qos
= MAX(qos
, kn
->kn_qos_override
);
4260 knote_unsuppress(kqwl
, kn
);
4267 kqworkloop_begin_processing(struct kqworkloop
*kqwl
, unsigned int kevent_flags
)
4269 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
4270 struct kqueue
*kq
= &kqwl
->kqwl_kqueue
;
4271 thread_qos_t qos_override
;
4272 thread_t thread
= kqr_thread_fast(kqr
);
4273 int rc
= 0, op
= KQWL_UTQ_NONE
;
4277 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN
) | DBG_FUNC_START
,
4278 kqwl
->kqwl_dynamicid
, 0, 0);
4280 /* nobody else should still be processing */
4281 assert((kq
->kq_state
& KQ_PROCESSING
) == 0);
4283 kq
->kq_state
|= KQ_PROCESSING
;
4285 if (!TAILQ_EMPTY(&kqwl
->kqwl_suppressed
)) {
4286 op
= KQWL_UTQ_RESET_WAKEUP_OVERRIDE
;
4289 if (kevent_flags
& KEVENT_FLAG_PARKING
) {
4291 * When "parking" we want to process events and if no events are found
4294 * However, non overcommit threads sometimes park even when they have
4295 * more work so that the pool can narrow. For these, we need to unbind
4296 * early, so that calling kqworkloop_update_threads_qos() can ask the
4297 * workqueue subsystem whether the thread should park despite having
4300 if (kqr
->tr_flags
& WORKQ_TR_FLAG_OVERCOMMIT
) {
4301 op
= KQWL_UTQ_PARKING
;
4303 op
= KQWL_UTQ_UNBINDING
;
4306 if (op
== KQWL_UTQ_NONE
) {
4310 qos_override
= kqworkloop_acknowledge_events(kqwl
);
4312 if (op
== KQWL_UTQ_UNBINDING
) {
4313 kqworkloop_unbind_locked(kqwl
, thread
, KQWL_OVERRIDE_DROP_IMMEDIATELY
);
4314 kqworkloop_release_live(kqwl
);
4316 kqworkloop_update_threads_qos(kqwl
, op
, qos_override
);
4317 if (op
== KQWL_UTQ_PARKING
) {
4318 if (!TAILQ_EMPTY(&kqwl
->kqwl_queue
[KQWL_BUCKET_STAYACTIVE
])) {
4320 * We cannot trust tr_kq_wakeup when looking at stay active knotes.
4321 * We need to process once, and kqworkloop_end_processing will
4322 * handle the unbind.
4324 } else if (!kqr
->tr_kq_wakeup
|| kqwl
->kqwl_owner
) {
4325 kqworkloop_unbind_locked(kqwl
, thread
, KQWL_OVERRIDE_DROP_DELAYED
);
4326 kqworkloop_release_live(kqwl
);
4329 } else if (op
== KQWL_UTQ_UNBINDING
) {
4330 if (kqr_thread(kqr
) == thread
) {
4332 * The thread request fired again, passed the admission check and
4333 * got bound to the current thread again.
4342 * Reset wakeup bit to notice stay active events firing while we are
4343 * processing, as we cannot rely on the stayactive bucket emptiness.
4345 kqwl
->kqwl_wakeup_indexes
&= ~KQWL_STAYACTIVE_FIRED_BIT
;
4347 kq
->kq_state
&= ~KQ_PROCESSING
;
4351 kqworkloop_unbind_delayed_override_drop(thread
);
4355 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_BEGIN
) | DBG_FUNC_END
,
4356 kqwl
->kqwl_dynamicid
, 0, 0);
4362 * Return 0 to indicate that processing should proceed,
4363 * -1 if there is nothing to process.
4364 * EBADF if the kqueue is draining
4366 * Called with kqueue locked and returns the same way,
4367 * but may drop lock temporarily.
4371 kqfile_begin_processing(struct kqfile
*kq
)
4373 struct kqtailq
*suppressq
;
4377 assert((kq
->kqf_state
& (KQ_WORKQ
| KQ_WORKLOOP
)) == 0);
4378 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN
) | DBG_FUNC_START
,
4379 VM_KERNEL_UNSLIDE_OR_PERM(kq
), 0);
4381 /* wait to become the exclusive processing thread */
4383 if (kq
->kqf_state
& KQ_DRAIN
) {
4384 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN
) | DBG_FUNC_END
,
4385 VM_KERNEL_UNSLIDE_OR_PERM(kq
), 2);
4389 if ((kq
->kqf_state
& KQ_PROCESSING
) == 0) {
4393 /* if someone else is processing the queue, wait */
4394 kq
->kqf_state
|= KQ_PROCWAIT
;
4395 suppressq
= &kq
->kqf_suppressed
;
4396 waitq_assert_wait64((struct waitq
*)&kq
->kqf_wqs
,
4397 CAST_EVENT64_T(suppressq
), THREAD_UNINT
| THREAD_WAIT_NOREPORT
,
4398 TIMEOUT_WAIT_FOREVER
);
4401 thread_block(THREAD_CONTINUE_NULL
);
4405 /* Nobody else processing */
4407 /* clear pre-posts and KQ_WAKEUP now, in case we bail early */
4408 waitq_set_clear_preposts(&kq
->kqf_wqs
);
4409 kq
->kqf_state
&= ~KQ_WAKEUP
;
4411 /* anything left to process? */
4412 if (TAILQ_EMPTY(&kq
->kqf_queue
)) {
4413 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN
) | DBG_FUNC_END
,
4414 VM_KERNEL_UNSLIDE_OR_PERM(kq
), 1);
4418 /* convert to processing mode */
4419 kq
->kqf_state
|= KQ_PROCESSING
;
4421 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_BEGIN
) | DBG_FUNC_END
,
4422 VM_KERNEL_UNSLIDE_OR_PERM(kq
));
4428 * Try to end the processing, only called when a workq thread is attempting to
4429 * park (KEVENT_FLAG_PARKING is set).
4431 * When returning -1, the kqworkq is setup again so that it is ready to be
4435 kqworkq_end_processing(struct kqworkq
*kqwq
, workq_threadreq_t kqr
,
4438 if (!TAILQ_EMPTY(&kqwq
->kqwq_queue
[kqr
->tr_kq_qos_index
])) {
4439 /* remember we didn't process everything */
4440 kqr
->tr_kq_wakeup
= true;
4443 if (kevent_flags
& KEVENT_FLAG_PARKING
) {
4445 * if acknowledge events "succeeds" it means there are events,
4446 * which is a failure condition for end_processing.
4448 int rc
= kqworkq_acknowledge_events(kqwq
, kqr
, kevent_flags
,
4449 KQWQAE_END_PROCESSING
);
4459 * Try to end the processing, only called when a workq thread is attempting to
4460 * park (KEVENT_FLAG_PARKING is set).
4462 * When returning -1, the kqworkq is setup again so that it is ready to be
4463 * processed (as if kqworkloop_begin_processing had just been called).
4465 * If successful and KEVENT_FLAG_PARKING was set in the kevent_flags,
4466 * the kqworkloop is unbound from its servicer as a side effect.
4469 kqworkloop_end_processing(struct kqworkloop
*kqwl
, int flags
, int kevent_flags
)
4471 struct kqueue
*kq
= &kqwl
->kqwl_kqueue
;
4472 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
4473 thread_qos_t qos_override
;
4474 thread_t thread
= kqr_thread_fast(kqr
);
4479 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END
) | DBG_FUNC_START
,
4480 kqwl
->kqwl_dynamicid
, 0, 0);
4482 if (flags
& KQ_PROCESSING
) {
4483 assert(kq
->kq_state
& KQ_PROCESSING
);
4486 * If we still have queued stayactive knotes, remember we didn't finish
4487 * processing all of them. This should be extremely rare and would
4488 * require to have a lot of them registered and fired.
4490 if (!TAILQ_EMPTY(&kqwl
->kqwl_queue
[KQWL_BUCKET_STAYACTIVE
])) {
4491 kqworkloop_update_threads_qos(kqwl
, KQWL_UTQ_UPDATE_WAKEUP_QOS
,
4492 KQWL_BUCKET_STAYACTIVE
);
4496 * When KEVENT_FLAG_PARKING is set, we need to attempt an unbind while
4497 * still under the lock.
4499 * So we do everything kqworkloop_unbind() would do, but because we're
4500 * inside kqueue_process(), if the workloop actually received events
4501 * while our locks were dropped, we have the opportunity to fail the end
4502 * processing and loop again.
4504 * This avoids going through the process-wide workqueue lock hence
4507 if (kevent_flags
& KEVENT_FLAG_PARKING
) {
4508 qos_override
= kqworkloop_acknowledge_events(kqwl
);
4512 if (kevent_flags
& KEVENT_FLAG_PARKING
) {
4513 kqworkloop_update_threads_qos(kqwl
, KQWL_UTQ_PARKING
, qos_override
);
4514 if (kqr
->tr_kq_wakeup
&& !kqwl
->kqwl_owner
) {
4516 * Reset wakeup bit to notice stay active events firing while we are
4517 * processing, as we cannot rely on the stayactive bucket emptiness.
4519 kqwl
->kqwl_wakeup_indexes
&= ~KQWL_STAYACTIVE_FIRED_BIT
;
4522 kqworkloop_unbind_locked(kqwl
, thread
, KQWL_OVERRIDE_DROP_DELAYED
);
4523 kqworkloop_release_live(kqwl
);
4524 kq
->kq_state
&= ~flags
;
4527 kq
->kq_state
&= ~flags
;
4528 kq
->kq_state
|= KQ_R2K_ARMED
;
4529 kqworkloop_update_threads_qos(kqwl
, KQWL_UTQ_RECOMPUTE_WAKEUP_QOS
, 0);
4532 if ((kevent_flags
& KEVENT_FLAG_PARKING
) && rc
== 0) {
4533 kqworkloop_unbind_delayed_override_drop(thread
);
4536 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_PROCESS_END
) | DBG_FUNC_END
,
4537 kqwl
->kqwl_dynamicid
, 0, 0);
4543 * Called with kqueue lock held.
4546 * -1: has more events
4547 * EBADF: kqueue is in draining mode
4550 kqfile_end_processing(struct kqfile
*kq
)
4552 struct kqtailq
*suppressq
= &kq
->kqf_suppressed
;
4558 assert((kq
->kqf_state
& (KQ_WORKQ
| KQ_WORKLOOP
)) == 0);
4560 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQ_PROCESS_END
),
4561 VM_KERNEL_UNSLIDE_OR_PERM(kq
), 0);
4564 * Return suppressed knotes to their original state.
4566 while ((kn
= TAILQ_FIRST(suppressq
)) != NULL
) {
4567 assert(kn
->kn_status
& KN_SUPPRESSED
);
4568 knote_unsuppress(kq
, kn
);
4571 procwait
= (kq
->kqf_state
& KQ_PROCWAIT
);
4572 kq
->kqf_state
&= ~(KQ_PROCESSING
| KQ_PROCWAIT
);
4575 /* first wake up any thread already waiting to process */
4576 waitq_wakeup64_all((struct waitq
*)&kq
->kqf_wqs
,
4577 CAST_EVENT64_T(suppressq
), THREAD_AWAKENED
, WAITQ_ALL_PRIORITIES
);
4580 if (kq
->kqf_state
& KQ_DRAIN
) {
4583 return (kq
->kqf_state
& KQ_WAKEUP
) ? -1 : 0;
4587 kqueue_workloop_ctl_internal(proc_t p
, uintptr_t cmd
, uint64_t __unused options
,
4588 struct kqueue_workloop_params
*params
, int *retval
)
4591 struct kqworkloop
*kqwl
;
4592 struct filedesc
*fdp
= p
->p_fd
;
4593 workq_threadreq_param_t trp
= { };
4596 case KQ_WORKLOOP_CREATE
:
4597 if (!params
->kqwlp_flags
) {
4602 if ((params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_SCHED_PRI
) &&
4603 (params
->kqwlp_sched_pri
< 1 ||
4604 params
->kqwlp_sched_pri
> 63 /* MAXPRI_USER */)) {
4609 if ((params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_SCHED_POL
) &&
4610 invalid_policy(params
->kqwlp_sched_pol
)) {
4615 if ((params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_CPU_PERCENT
) &&
4616 (params
->kqwlp_cpu_percent
<= 0 ||
4617 params
->kqwlp_cpu_percent
> 100 ||
4618 params
->kqwlp_cpu_refillms
<= 0 ||
4619 params
->kqwlp_cpu_refillms
> 0x00ffffff)) {
4624 if (params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_SCHED_PRI
) {
4625 trp
.trp_flags
|= TRP_PRIORITY
;
4626 trp
.trp_pri
= params
->kqwlp_sched_pri
;
4628 if (params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_SCHED_POL
) {
4629 trp
.trp_flags
|= TRP_POLICY
;
4630 trp
.trp_pol
= params
->kqwlp_sched_pol
;
4632 if (params
->kqwlp_flags
& KQ_WORKLOOP_CREATE_CPU_PERCENT
) {
4633 trp
.trp_flags
|= TRP_CPUPERCENT
;
4634 trp
.trp_cpupercent
= (uint8_t)params
->kqwlp_cpu_percent
;
4635 trp
.trp_refillms
= params
->kqwlp_cpu_refillms
;
4638 error
= kqworkloop_get_or_create(p
, params
->kqwlp_id
, &trp
,
4639 KEVENT_FLAG_DYNAMIC_KQUEUE
| KEVENT_FLAG_WORKLOOP
|
4640 KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST
, &kqwl
);
4645 if (!(fdp
->fd_flags
& FD_WORKLOOP
)) {
4646 /* FD_WORKLOOP indicates we've ever created a workloop
4647 * via this syscall but its only ever added to a process, never
4651 fdp
->fd_flags
|= FD_WORKLOOP
;
4655 case KQ_WORKLOOP_DESTROY
:
4656 error
= kqworkloop_get_or_create(p
, params
->kqwlp_id
, NULL
,
4657 KEVENT_FLAG_DYNAMIC_KQUEUE
| KEVENT_FLAG_WORKLOOP
|
4658 KEVENT_FLAG_DYNAMIC_KQ_MUST_EXIST
, &kqwl
);
4663 trp
.trp_value
= kqwl
->kqwl_params
;
4664 if (trp
.trp_flags
&& !(trp
.trp_flags
& TRP_RELEASED
)) {
4665 trp
.trp_flags
|= TRP_RELEASED
;
4666 kqwl
->kqwl_params
= trp
.trp_value
;
4667 kqworkloop_release_live(kqwl
);
4672 kqworkloop_release(kqwl
);
4680 kqueue_workloop_ctl(proc_t p
, struct kqueue_workloop_ctl_args
*uap
, int *retval
)
4682 struct kqueue_workloop_params params
= {
4685 if (uap
->sz
< sizeof(params
.kqwlp_version
)) {
4689 size_t copyin_sz
= MIN(sizeof(params
), uap
->sz
);
4690 int rv
= copyin(uap
->addr
, ¶ms
, copyin_sz
);
4695 if (params
.kqwlp_version
!= (int)uap
->sz
) {
4699 return kqueue_workloop_ctl_internal(p
, uap
->cmd
, uap
->options
, ¶ms
,
4705 kqueue_select(struct fileproc
*fp
, int which
, void *wq_link_id
,
4706 __unused vfs_context_t ctx
)
4708 struct kqfile
*kq
= (struct kqfile
*)fp
->f_data
;
4709 struct kqtailq
*suppressq
= &kq
->kqf_suppressed
;
4710 struct kqtailq
*queue
= &kq
->kqf_queue
;
4714 if (which
!= FREAD
) {
4720 assert((kq
->kqf_state
& KQ_WORKQ
) == 0);
4723 * If this is the first pass, link the wait queue associated with the
4724 * the kqueue onto the wait queue set for the select(). Normally we
4725 * use selrecord() for this, but it uses the wait queue within the
4726 * selinfo structure and we need to use the main one for the kqueue to
4727 * catch events from KN_STAYQUEUED sources. So we do the linkage manually.
4728 * (The select() call will unlink them when it ends).
4730 if (wq_link_id
!= NULL
) {
4731 thread_t cur_act
= current_thread();
4732 struct uthread
* ut
= get_bsdthread_info(cur_act
);
4734 kq
->kqf_state
|= KQ_SEL
;
4735 waitq_link((struct waitq
*)&kq
->kqf_wqs
, ut
->uu_wqset
,
4736 WAITQ_SHOULD_LOCK
, (uint64_t *)wq_link_id
);
4738 /* always consume the reserved link object */
4739 waitq_link_release(*(uint64_t *)wq_link_id
);
4740 *(uint64_t *)wq_link_id
= 0;
4743 * selprocess() is expecting that we send it back the waitq
4744 * that was just added to the thread's waitq set. In order
4745 * to not change the selrecord() API (which is exported to
4746 * kexts), we pass this value back through the
4747 * void *wq_link_id pointer we were passed. We need to use
4748 * memcpy here because the pointer may not be properly aligned
4749 * on 32-bit systems.
4751 void *wqptr
= &kq
->kqf_wqs
;
4752 memcpy(wq_link_id
, (void *)&wqptr
, sizeof(void *));
4755 if (kqfile_begin_processing(kq
) == -1) {
4760 if (!TAILQ_EMPTY(queue
)) {
4762 * there is something queued - but it might be a
4763 * KN_STAYACTIVE knote, which may or may not have
4764 * any events pending. Otherwise, we have to walk
4765 * the list of knotes to see, and peek at the
4766 * (non-vanished) stay-active ones to be really sure.
4768 while ((kn
= (struct knote
*)TAILQ_FIRST(queue
)) != NULL
) {
4769 if (kn
->kn_status
& KN_ACTIVE
) {
4773 assert(kn
->kn_status
& KN_STAYACTIVE
);
4774 knote_suppress(kq
, kn
);
4778 * There were no regular events on the queue, so take
4779 * a deeper look at the stay-queued ones we suppressed.
4781 while ((kn
= (struct knote
*)TAILQ_FIRST(suppressq
)) != NULL
) {
4782 KNOTE_LOCK_CTX(knlc
);
4785 /* If didn't vanish while suppressed - peek at it */
4786 if ((kn
->kn_status
& KN_DROPPING
) || !knote_lock(kq
, kn
, &knlc
,
4787 KNOTE_KQ_LOCK_ON_FAILURE
)) {
4791 result
= filter_call(knote_fops(kn
), f_peek(kn
));
4794 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ALWAYS
);
4797 knote_unsuppress(kq
, kn
);
4799 /* has data or it has to report a vanish */
4800 if (result
& FILTER_ACTIVE
) {
4808 kqfile_end_processing(kq
);
4818 kqueue_close(struct fileglob
*fg
, __unused vfs_context_t ctx
)
4820 struct kqfile
*kqf
= (struct kqfile
*)fg
->fg_data
;
4822 assert((kqf
->kqf_state
& KQ_WORKQ
) == 0);
4823 kqueue_dealloc(&kqf
->kqf_kqueue
);
4829 * Max depth of the nested kq path that can be created.
4830 * Note that this has to be less than the size of kq_level
4831 * to avoid wrapping around and mislabeling the level.
4833 #define MAX_NESTED_KQ 1000
4837 * The callers has taken a use-count reference on this kqueue and will donate it
4838 * to the kqueue we are being added to. This keeps the kqueue from closing until
4839 * that relationship is torn down.
4842 kqueue_kqfilter(struct fileproc
*fp
, struct knote
*kn
,
4843 __unused
struct kevent_qos_s
*kev
)
4845 struct kqfile
*kqf
= (struct kqfile
*)fp
->f_data
;
4846 struct kqueue
*kq
= &kqf
->kqf_kqueue
;
4847 struct kqueue
*parentkq
= knote_get_kq(kn
);
4849 assert((kqf
->kqf_state
& KQ_WORKQ
) == 0);
4851 if (parentkq
== kq
|| kn
->kn_filter
!= EVFILT_READ
) {
4852 knote_set_error(kn
, EINVAL
);
4857 * We have to avoid creating a cycle when nesting kqueues
4858 * inside another. Rather than trying to walk the whole
4859 * potential DAG of nested kqueues, we just use a simple
4860 * ceiling protocol. When a kqueue is inserted into another,
4861 * we check that the (future) parent is not already nested
4862 * into another kqueue at a lower level than the potenial
4863 * child (because it could indicate a cycle). If that test
4864 * passes, we just mark the nesting levels accordingly.
4866 * Only up to MAX_NESTED_KQ can be nested.
4868 * Note: kqworkq and kqworkloop cannot be nested and have reused their
4869 * kq_level field, so ignore these as parent.
4874 if ((parentkq
->kq_state
& (KQ_WORKQ
| KQ_WORKLOOP
)) == 0) {
4875 if (parentkq
->kq_level
> 0 &&
4876 parentkq
->kq_level
< kq
->kq_level
) {
4878 knote_set_error(kn
, EINVAL
);
4882 /* set parent level appropriately */
4883 uint16_t plevel
= (parentkq
->kq_level
== 0)? 2: parentkq
->kq_level
;
4884 if (plevel
< kq
->kq_level
+ 1) {
4885 if (kq
->kq_level
+ 1 > MAX_NESTED_KQ
) {
4887 knote_set_error(kn
, EINVAL
);
4890 plevel
= kq
->kq_level
+ 1;
4893 parentkq
->kq_level
= plevel
;
4898 kn
->kn_filtid
= EVFILTID_KQREAD
;
4900 KNOTE_ATTACH(&kqf
->kqf_sel
.si_note
, kn
);
4901 /* indicate nesting in child, if needed */
4902 if (kq
->kq_level
== 0) {
4906 int count
= kq
->kq_count
;
4912 * kqueue_drain - called when kq is closed
4916 kqueue_drain(struct fileproc
*fp
, __unused vfs_context_t ctx
)
4918 struct kqfile
*kqf
= (struct kqfile
*)fp
->f_fglob
->fg_data
;
4920 assert((kqf
->kqf_state
& KQ_WORKQ
) == 0);
4923 kqf
->kqf_state
|= KQ_DRAIN
;
4925 /* wakeup sleeping threads */
4926 if ((kqf
->kqf_state
& (KQ_SLEEP
| KQ_SEL
)) != 0) {
4927 kqf
->kqf_state
&= ~(KQ_SLEEP
| KQ_SEL
);
4928 (void)waitq_wakeup64_all((struct waitq
*)&kqf
->kqf_wqs
,
4931 WAITQ_ALL_PRIORITIES
);
4934 /* wakeup threads waiting their turn to process */
4935 if (kqf
->kqf_state
& KQ_PROCWAIT
) {
4936 assert(kqf
->kqf_state
& KQ_PROCESSING
);
4938 kqf
->kqf_state
&= ~KQ_PROCWAIT
;
4939 (void)waitq_wakeup64_all((struct waitq
*)&kqf
->kqf_wqs
,
4940 CAST_EVENT64_T(&kqf
->kqf_suppressed
),
4941 THREAD_RESTART
, WAITQ_ALL_PRIORITIES
);
4950 kqueue_stat(struct kqueue
*kq
, void *ub
, int isstat64
, proc_t p
)
4952 assert((kq
->kq_state
& KQ_WORKQ
) == 0);
4955 if (isstat64
!= 0) {
4956 struct stat64
*sb64
= (struct stat64
*)ub
;
4958 bzero((void *)sb64
, sizeof(*sb64
));
4959 sb64
->st_size
= kq
->kq_count
;
4960 if (kq
->kq_state
& KQ_KEV_QOS
) {
4961 sb64
->st_blksize
= sizeof(struct kevent_qos_s
);
4962 } else if (kq
->kq_state
& KQ_KEV64
) {
4963 sb64
->st_blksize
= sizeof(struct kevent64_s
);
4964 } else if (IS_64BIT_PROCESS(p
)) {
4965 sb64
->st_blksize
= sizeof(struct user64_kevent
);
4967 sb64
->st_blksize
= sizeof(struct user32_kevent
);
4969 sb64
->st_mode
= S_IFIFO
;
4971 struct stat
*sb
= (struct stat
*)ub
;
4973 bzero((void *)sb
, sizeof(*sb
));
4974 sb
->st_size
= kq
->kq_count
;
4975 if (kq
->kq_state
& KQ_KEV_QOS
) {
4976 sb
->st_blksize
= sizeof(struct kevent_qos_s
);
4977 } else if (kq
->kq_state
& KQ_KEV64
) {
4978 sb
->st_blksize
= sizeof(struct kevent64_s
);
4979 } else if (IS_64BIT_PROCESS(p
)) {
4980 sb
->st_blksize
= sizeof(struct user64_kevent
);
4982 sb
->st_blksize
= sizeof(struct user32_kevent
);
4984 sb
->st_mode
= S_IFIFO
;
4991 kqueue_threadreq_can_use_ast(struct kqueue
*kq
)
4993 if (current_proc() == kq
->kq_p
) {
4995 * Setting an AST from a non BSD syscall is unsafe: mach_msg_trap() can
4996 * do combined send/receive and in the case of self-IPC, the AST may bet
4997 * set on a thread that will not return to userspace and needs the
4998 * thread the AST would create to unblock itself.
5000 * At this time, we really want to target:
5002 * - kevent variants that can cause thread creations, and dispatch
5003 * really only uses kevent_qos and kevent_id,
5005 * - workq_kernreturn (directly about thread creations)
5007 * - bsdthread_ctl which is used for qos changes and has direct impact
5008 * on the creator thread scheduling decisions.
5010 switch (current_uthread()->syscall_code
) {
5011 case SYS_kevent_qos
:
5013 case SYS_workq_kernreturn
:
5014 case SYS_bsdthread_ctl
:
5022 * Interact with the pthread kext to request a servicing there at a specific QoS
5025 * - Caller holds the workq request lock
5027 * - May be called with the kqueue's wait queue set locked,
5028 * so cannot do anything that could recurse on that.
5031 kqueue_threadreq_initiate(struct kqueue
*kq
, workq_threadreq_t kqr
,
5032 kq_index_t qos
, int flags
)
5034 assert(kqr
->tr_kq_wakeup
);
5035 assert(kqr_thread(kqr
) == THREAD_NULL
);
5036 assert(!kqr_thread_requested(kqr
));
5037 struct turnstile
*ts
= TURNSTILE_NULL
;
5039 if (workq_is_exiting(kq
->kq_p
)) {
5045 if (kq
->kq_state
& KQ_WORKLOOP
) {
5046 __assert_only
struct kqworkloop
*kqwl
= (struct kqworkloop
*)kq
;
5048 assert(kqwl
->kqwl_owner
== THREAD_NULL
);
5049 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THREQUEST
),
5050 kqwl
->kqwl_dynamicid
, 0, qos
, kqr
->tr_kq_wakeup
);
5051 ts
= kqwl
->kqwl_turnstile
;
5052 /* Add a thread request reference on the kqueue. */
5053 kqworkloop_retain(kqwl
);
5055 assert(kq
->kq_state
& KQ_WORKQ
);
5056 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_THREQUEST
),
5057 -1, 0, qos
, kqr
->tr_kq_wakeup
);
5061 * New-style thread request supported.
5062 * Provide the pthread kext a pointer to a workq_threadreq_s structure for
5063 * its use until a corresponding kqueue_threadreq_bind callback.
5065 if (kqueue_threadreq_can_use_ast(kq
)) {
5066 flags
|= WORKQ_THREADREQ_SET_AST_ON_FAILURE
;
5068 if (qos
== KQWQ_QOS_MANAGER
) {
5069 qos
= WORKQ_THREAD_QOS_MANAGER
;
5071 if (!workq_kern_threadreq_initiate(kq
->kq_p
, kqr
, ts
, qos
, flags
)) {
5073 * Process is shutting down or exec'ing.
5074 * All the kqueues are going to be cleaned up
5075 * soon. Forget we even asked for a thread -
5076 * and make sure we don't ask for more.
5078 kq
->kq_state
&= ~KQ_R2K_ARMED
;
5079 kqueue_release_live(kq
);
5084 * kqueue_threadreq_bind_prepost - prepost the bind to kevent
5086 * This is used when kqueue_threadreq_bind may cause a lock inversion.
5088 __attribute__((always_inline
))
5090 kqueue_threadreq_bind_prepost(struct proc
*p __unused
, workq_threadreq_t kqr
,
5093 ut
->uu_kqr_bound
= kqr
;
5094 kqr
->tr_thread
= ut
->uu_thread
;
5095 kqr
->tr_state
= WORKQ_TR_STATE_BINDING
;
5099 * kqueue_threadreq_bind_commit - commit a bind prepost
5101 * The workq code has to commit any binding prepost before the thread has
5102 * a chance to come back to userspace (and do kevent syscalls) or be aborted.
5105 kqueue_threadreq_bind_commit(struct proc
*p
, thread_t thread
)
5107 struct uthread
*ut
= get_bsdthread_info(thread
);
5108 workq_threadreq_t kqr
= ut
->uu_kqr_bound
;
5109 kqueue_t kqu
= kqr_kqueue(p
, kqr
);
5112 if (kqr
->tr_state
== WORKQ_TR_STATE_BINDING
) {
5113 kqueue_threadreq_bind(p
, kqr
, thread
, 0);
5119 kqueue_threadreq_modify(kqueue_t kqu
, workq_threadreq_t kqr
, kq_index_t qos
,
5120 workq_kern_threadreq_flags_t flags
)
5122 assert(kqr_thread_requested_pending(kqr
));
5126 if (kqueue_threadreq_can_use_ast(kqu
.kq
)) {
5127 flags
|= WORKQ_THREADREQ_SET_AST_ON_FAILURE
;
5129 workq_kern_threadreq_modify(kqu
.kq
->kq_p
, kqr
, qos
, flags
);
5133 * kqueue_threadreq_bind - bind thread to processing kqrequest
5135 * The provided thread will be responsible for delivering events
5136 * associated with the given kqrequest. Bind it and get ready for
5137 * the thread to eventually arrive.
5140 kqueue_threadreq_bind(struct proc
*p
, workq_threadreq_t kqr
, thread_t thread
,
5143 kqueue_t kqu
= kqr_kqueue(p
, kqr
);
5144 struct uthread
*ut
= get_bsdthread_info(thread
);
5148 assert(ut
->uu_kqueue_override
== 0);
5150 if (kqr
->tr_state
== WORKQ_TR_STATE_BINDING
) {
5151 assert(ut
->uu_kqr_bound
== kqr
);
5152 assert(kqr
->tr_thread
== thread
);
5154 assert(kqr_thread_requested_pending(kqr
));
5155 assert(kqr
->tr_thread
== THREAD_NULL
);
5156 assert(ut
->uu_kqr_bound
== NULL
);
5157 ut
->uu_kqr_bound
= kqr
;
5158 kqr
->tr_thread
= thread
;
5161 kqr
->tr_state
= WORKQ_TR_STATE_BOUND
;
5163 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
5164 struct turnstile
*ts
= kqu
.kqwl
->kqwl_turnstile
;
5166 if (__improbable(thread
== kqu
.kqwl
->kqwl_owner
)) {
5168 * <rdar://problem/38626999> shows that asserting here is not ok.
5170 * This is not supposed to happen for correct use of the interface,
5171 * but it is sadly possible for userspace (with the help of memory
5172 * corruption, such as over-release of a dispatch queue) to make
5173 * the creator thread the "owner" of a workloop.
5175 * Once that happens, and that creator thread picks up the same
5176 * workloop as a servicer, we trip this codepath. We need to fixup
5177 * the state to forget about this thread being the owner, as the
5178 * entire workloop state machine expects servicers to never be
5179 * owners and everything would basically go downhill from here.
5181 kqu
.kqwl
->kqwl_owner
= THREAD_NULL
;
5182 if (kqworkloop_override(kqu
.kqwl
)) {
5183 thread_drop_kevent_override(thread
);
5187 if (ts
&& (flags
& KQUEUE_THREADERQ_BIND_NO_INHERITOR_UPDATE
) == 0) {
5189 * Past this point, the interlock is the kq req lock again,
5190 * so we can fix the inheritor for good.
5192 filt_wlupdate_inheritor(kqu
.kqwl
, ts
, TURNSTILE_IMMEDIATE_UPDATE
);
5193 turnstile_update_inheritor_complete(ts
, TURNSTILE_INTERLOCK_HELD
);
5196 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_BIND
), kqu
.kqwl
->kqwl_dynamicid
,
5197 thread_tid(thread
), kqr
->tr_kq_qos_index
,
5198 (kqr
->tr_kq_override_index
<< 16) | kqr
->tr_kq_wakeup
);
5200 ut
->uu_kqueue_override
= kqr
->tr_kq_override_index
;
5201 if (kqr
->tr_kq_override_index
) {
5202 thread_add_servicer_override(thread
, kqr
->tr_kq_override_index
);
5205 assert(kqr
->tr_kq_override_index
== 0);
5207 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_BIND
), -1,
5208 thread_tid(thread
), kqr
->tr_kq_qos_index
,
5209 (kqr
->tr_kq_override_index
<< 16) | kqr
->tr_kq_wakeup
);
5214 * kqueue_threadreq_cancel - abort a pending thread request
5216 * Called when exiting/exec'ing. Forget our pending request.
5219 kqueue_threadreq_cancel(struct proc
*p
, workq_threadreq_t kqr
)
5221 kqueue_release(kqr_kqueue(p
, kqr
));
5224 workq_threadreq_param_t
5225 kqueue_threadreq_workloop_param(workq_threadreq_t kqr
)
5227 struct kqworkloop
*kqwl
;
5228 workq_threadreq_param_t trp
;
5230 assert(kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
);
5231 kqwl
= __container_of(kqr
, struct kqworkloop
, kqwl_request
);
5232 trp
.trp_value
= kqwl
->kqwl_params
;
5237 * kqueue_threadreq_unbind - unbind thread from processing kqueue
5239 * End processing the per-QoS bucket of events and allow other threads
5240 * to be requested for future servicing.
5242 * caller holds a reference on the kqueue.
5245 kqueue_threadreq_unbind(struct proc
*p
, workq_threadreq_t kqr
)
5247 if (kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
) {
5248 kqworkloop_unbind(kqr_kqworkloop(kqr
));
5250 kqworkq_unbind(p
, kqr
);
5255 * If we aren't already busy processing events [for this QoS],
5256 * request workq thread support as appropriate.
5258 * TBD - for now, we don't segregate out processing by QoS.
5260 * - May be called with the kqueue's wait queue set locked,
5261 * so cannot do anything that could recurse on that.
5264 kqworkq_wakeup(struct kqworkq
*kqwq
, kq_index_t qos_index
)
5266 workq_threadreq_t kqr
= kqworkq_get_request(kqwq
, qos_index
);
5268 /* convert to thread qos value */
5269 assert(qos_index
< KQWQ_NBUCKETS
);
5271 if (!kqr
->tr_kq_wakeup
) {
5272 kqr
->tr_kq_wakeup
= true;
5273 if (!kqr_thread_requested(kqr
)) {
5274 kqueue_threadreq_initiate(&kqwq
->kqwq_kqueue
, kqr
, qos_index
, 0);
5280 * This represent the asynchronous QoS a given workloop contributes,
5281 * hence is the max of the current active knotes (override index)
5282 * and the workloop max qos (userspace async qos).
5285 kqworkloop_override(struct kqworkloop
*kqwl
)
5287 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5288 return MAX(kqr
->tr_kq_qos_index
, kqr
->tr_kq_override_index
);
5292 kqworkloop_request_fire_r2k_notification(struct kqworkloop
*kqwl
)
5294 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5298 if (kqwl
->kqwl_state
& KQ_R2K_ARMED
) {
5299 kqwl
->kqwl_state
&= ~KQ_R2K_ARMED
;
5300 act_set_astkevent(kqr_thread_fast(kqr
), AST_KEVENT_RETURN_TO_KERNEL
);
5305 kqworkloop_update_threads_qos(struct kqworkloop
*kqwl
, int op
, kq_index_t qos
)
5307 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5308 struct kqueue
*kq
= &kqwl
->kqwl_kqueue
;
5309 kq_index_t old_override
= kqworkloop_override(kqwl
);
5315 case KQWL_UTQ_UPDATE_WAKEUP_QOS
:
5316 if (qos
== KQWL_BUCKET_STAYACTIVE
) {
5318 * the KQWL_BUCKET_STAYACTIVE is not a QoS bucket, we only remember
5319 * a high watermark (kqwl_stayactive_qos) of any stay active knote
5320 * that was ever registered with this workloop.
5322 * When waitq_set__CALLING_PREPOST_HOOK__() wakes up any stay active
5323 * knote, we use this high-watermark as a wakeup-index, and also set
5324 * the magic KQWL_BUCKET_STAYACTIVE bit to make sure we remember
5325 * there is at least one stay active knote fired until the next full
5326 * processing of this bucket.
5328 kqwl
->kqwl_wakeup_indexes
|= KQWL_STAYACTIVE_FIRED_BIT
;
5329 qos
= kqwl
->kqwl_stayactive_qos
;
5332 if (kqwl
->kqwl_wakeup_indexes
& (1 << qos
)) {
5333 assert(kqr
->tr_kq_wakeup
);
5337 kqwl
->kqwl_wakeup_indexes
|= (1 << qos
);
5338 kqr
->tr_kq_wakeup
= true;
5339 kqworkloop_request_fire_r2k_notification(kqwl
);
5342 case KQWL_UTQ_UPDATE_STAYACTIVE_QOS
:
5344 if (kqwl
->kqwl_stayactive_qos
< qos
) {
5345 kqwl
->kqwl_stayactive_qos
= qos
;
5346 if (kqwl
->kqwl_wakeup_indexes
& KQWL_STAYACTIVE_FIRED_BIT
) {
5347 assert(kqr
->tr_kq_wakeup
);
5348 kqwl
->kqwl_wakeup_indexes
|= (1 << qos
);
5354 case KQWL_UTQ_PARKING
:
5355 case KQWL_UTQ_UNBINDING
:
5356 kqr
->tr_kq_override_index
= qos
;
5358 case KQWL_UTQ_RECOMPUTE_WAKEUP_QOS
:
5359 if (op
== KQWL_UTQ_RECOMPUTE_WAKEUP_QOS
) {
5360 assert(qos
== THREAD_QOS_UNSPECIFIED
);
5362 i
= KQWL_BUCKET_STAYACTIVE
;
5363 if (TAILQ_EMPTY(&kqwl
->kqwl_suppressed
)) {
5364 kqr
->tr_kq_override_index
= THREAD_QOS_UNSPECIFIED
;
5366 if (!TAILQ_EMPTY(&kqwl
->kqwl_queue
[i
]) &&
5367 (kqwl
->kqwl_wakeup_indexes
& KQWL_STAYACTIVE_FIRED_BIT
)) {
5369 * If the KQWL_STAYACTIVE_FIRED_BIT is set, it means a stay active
5370 * knote may have fired, so we need to merge in kqwl_stayactive_qos.
5372 * Unlike other buckets, this one is never empty but could be idle.
5374 kqwl
->kqwl_wakeup_indexes
&= KQWL_STAYACTIVE_FIRED_BIT
;
5375 kqwl
->kqwl_wakeup_indexes
|= (1 << kqwl
->kqwl_stayactive_qos
);
5377 kqwl
->kqwl_wakeup_indexes
= 0;
5379 for (i
= THREAD_QOS_UNSPECIFIED
+ 1; i
< KQWL_BUCKET_STAYACTIVE
; i
++) {
5380 if (!TAILQ_EMPTY(&kqwl
->kqwl_queue
[i
])) {
5381 kqwl
->kqwl_wakeup_indexes
|= (1 << i
);
5384 if (kqwl
->kqwl_wakeup_indexes
) {
5385 kqr
->tr_kq_wakeup
= true;
5386 kqworkloop_request_fire_r2k_notification(kqwl
);
5388 kqr
->tr_kq_wakeup
= false;
5392 case KQWL_UTQ_RESET_WAKEUP_OVERRIDE
:
5393 kqr
->tr_kq_override_index
= qos
;
5396 case KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE
:
5399 * When modifying the wakeup QoS or the override QoS, we always need to
5400 * maintain our invariant that kqr_override_index is at least as large
5401 * as the highest QoS for which an event is fired.
5403 * However this override index can be larger when there is an overriden
5404 * suppressed knote pushing on the kqueue.
5406 if (kqwl
->kqwl_wakeup_indexes
> (1 << qos
)) {
5407 qos
= fls(kqwl
->kqwl_wakeup_indexes
) - 1; /* fls is 1-based */
5409 if (kqr
->tr_kq_override_index
< qos
) {
5410 kqr
->tr_kq_override_index
= qos
;
5414 case KQWL_UTQ_REDRIVE_EVENTS
:
5417 case KQWL_UTQ_SET_QOS_INDEX
:
5418 kqr
->tr_kq_qos_index
= qos
;
5422 panic("unknown kqwl thread qos update operation: %d", op
);
5425 thread_t kqwl_owner
= kqwl
->kqwl_owner
;
5426 thread_t servicer
= kqr_thread(kqr
);
5427 boolean_t qos_changed
= FALSE
;
5428 kq_index_t new_override
= kqworkloop_override(kqwl
);
5431 * Apply the diffs to the owner if applicable
5435 /* JMM - need new trace hooks for owner overrides */
5436 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST
),
5437 kqwl
->kqwl_dynamicid
, thread_tid(kqwl_owner
), kqr
->tr_kq_qos_index
,
5438 (kqr
->tr_kq_override_index
<< 16) | kqr
->tr_kq_wakeup
);
5440 if (new_override
== old_override
) {
5442 } else if (old_override
== THREAD_QOS_UNSPECIFIED
) {
5443 thread_add_kevent_override(kqwl_owner
, new_override
);
5444 } else if (new_override
== THREAD_QOS_UNSPECIFIED
) {
5445 thread_drop_kevent_override(kqwl_owner
);
5446 } else { /* old_override != new_override */
5447 thread_update_kevent_override(kqwl_owner
, new_override
);
5452 * apply the diffs to the servicer
5454 if (!kqr_thread_requested(kqr
)) {
5456 * No servicer, nor thread-request
5458 * Make a new thread request, unless there is an owner (or the workloop
5459 * is suspended in userland) or if there is no asynchronous work in the
5463 if (kqwl_owner
== NULL
&& kqr
->tr_kq_wakeup
) {
5464 int initiate_flags
= 0;
5465 if (op
== KQWL_UTQ_UNBINDING
) {
5466 initiate_flags
= WORKQ_THREADREQ_ATTEMPT_REBIND
;
5468 kqueue_threadreq_initiate(kq
, kqr
, new_override
, initiate_flags
);
5470 } else if (servicer
) {
5472 * Servicer in flight
5474 * Just apply the diff to the servicer
5476 struct uthread
*ut
= get_bsdthread_info(servicer
);
5477 if (ut
->uu_kqueue_override
!= new_override
) {
5478 if (ut
->uu_kqueue_override
== THREAD_QOS_UNSPECIFIED
) {
5479 thread_add_servicer_override(servicer
, new_override
);
5480 } else if (new_override
== THREAD_QOS_UNSPECIFIED
) {
5481 thread_drop_servicer_override(servicer
);
5482 } else { /* ut->uu_kqueue_override != new_override */
5483 thread_update_servicer_override(servicer
, new_override
);
5485 ut
->uu_kqueue_override
= new_override
;
5488 } else if (new_override
== THREAD_QOS_UNSPECIFIED
) {
5490 * No events to deliver anymore.
5492 * However canceling with turnstiles is challenging, so the fact that
5493 * the request isn't useful will be discovered by the servicer himself
5496 } else if (old_override
!= new_override
) {
5498 * Request is in flight
5500 * Apply the diff to the thread request
5502 kqueue_threadreq_modify(kq
, kqr
, new_override
, WORKQ_THREADREQ_NONE
);
5507 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_THADJUST
), kqwl
->kqwl_dynamicid
,
5508 thread_tid(servicer
), kqr
->tr_kq_qos_index
,
5509 (kqr
->tr_kq_override_index
<< 16) | kqr
->tr_kq_wakeup
);
5514 kqworkloop_wakeup(struct kqworkloop
*kqwl
, kq_index_t qos
)
5516 if ((kqwl
->kqwl_state
& KQ_PROCESSING
) &&
5517 kqr_thread(&kqwl
->kqwl_request
) == current_thread()) {
5519 * kqworkloop_end_processing() will perform the required QoS
5520 * computations when it unsets the processing mode.
5525 kqworkloop_update_threads_qos(kqwl
, KQWL_UTQ_UPDATE_WAKEUP_QOS
, qos
);
5528 static struct kqtailq
*
5529 kqueue_get_suppressed_queue(kqueue_t kq
, struct knote
*kn
)
5531 if (kq
.kq
->kq_state
& KQ_WORKLOOP
) {
5532 return &kq
.kqwl
->kqwl_suppressed
;
5533 } else if (kq
.kq
->kq_state
& KQ_WORKQ
) {
5534 return &kq
.kqwq
->kqwq_suppressed
[kn
->kn_qos_index
];
5536 return &kq
.kqf
->kqf_suppressed
;
5541 kqueue_alloc_turnstile(kqueue_t kqu
)
5543 struct kqworkloop
*kqwl
= kqu
.kqwl
;
5544 kq_state_t kq_state
;
5546 kq_state
= os_atomic_load(&kqu
.kq
->kq_state
, dependency
);
5547 if (kq_state
& KQ_HAS_TURNSTILE
) {
5548 /* force a dependency to pair with the atomic or with release below */
5549 return os_atomic_load_with_dependency_on(&kqwl
->kqwl_turnstile
,
5550 (uintptr_t)kq_state
);
5553 if (!(kq_state
& KQ_WORKLOOP
)) {
5554 return TURNSTILE_NULL
;
5557 struct turnstile
*ts
= turnstile_alloc(), *free_ts
= TURNSTILE_NULL
;
5558 bool workq_locked
= false;
5562 if (filt_wlturnstile_interlock_is_workq(kqwl
)) {
5563 workq_locked
= true;
5564 workq_kern_threadreq_lock(kqwl
->kqwl_p
);
5567 if (kqwl
->kqwl_state
& KQ_HAS_TURNSTILE
) {
5569 ts
= kqwl
->kqwl_turnstile
;
5571 ts
= turnstile_prepare((uintptr_t)kqwl
, &kqwl
->kqwl_turnstile
,
5572 ts
, TURNSTILE_WORKLOOPS
);
5574 /* release-barrier to pair with the unlocked load of kqwl_turnstile above */
5575 os_atomic_or(&kqwl
->kqwl_state
, KQ_HAS_TURNSTILE
, release
);
5577 if (filt_wlturnstile_interlock_is_workq(kqwl
)) {
5578 workq_kern_threadreq_update_inheritor(kqwl
->kqwl_p
,
5579 &kqwl
->kqwl_request
, kqwl
->kqwl_owner
,
5580 ts
, TURNSTILE_IMMEDIATE_UPDATE
);
5582 * The workq may no longer be the interlock after this.
5583 * In which case the inheritor wasn't updated.
5586 if (!filt_wlturnstile_interlock_is_workq(kqwl
)) {
5587 filt_wlupdate_inheritor(kqwl
, ts
, TURNSTILE_IMMEDIATE_UPDATE
);
5592 workq_kern_threadreq_unlock(kqwl
->kqwl_p
);
5598 turnstile_deallocate(free_ts
);
5600 turnstile_update_inheritor_complete(ts
, TURNSTILE_INTERLOCK_NOT_HELD
);
5605 __attribute__((always_inline
))
5607 kqueue_turnstile(kqueue_t kqu
)
5609 kq_state_t kq_state
= os_atomic_load(&kqu
.kq
->kq_state
, relaxed
);
5610 if (kq_state
& KQ_WORKLOOP
) {
5611 return os_atomic_load(&kqu
.kqwl
->kqwl_turnstile
, relaxed
);
5613 return TURNSTILE_NULL
;
5616 __attribute__((always_inline
))
5618 kqueue_threadreq_get_turnstile(workq_threadreq_t kqr
)
5620 struct kqworkloop
*kqwl
= kqr_kqworkloop(kqr
);
5622 return os_atomic_load(&kqwl
->kqwl_turnstile
, relaxed
);
5624 return TURNSTILE_NULL
;
5628 kqworkloop_set_overcommit(struct kqworkloop
*kqwl
)
5630 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5633 * This test is racy, but since we never remove this bit,
5634 * it allows us to avoid taking a lock.
5636 if (kqr
->tr_flags
& WORKQ_TR_FLAG_OVERCOMMIT
) {
5642 if (kqr_thread_requested_pending(kqr
)) {
5643 kqueue_threadreq_modify(kqwl
, kqr
, kqr
->tr_qos
,
5644 WORKQ_THREADREQ_MAKE_OVERCOMMIT
);
5646 kqr
->tr_flags
|= WORKQ_TR_FLAG_OVERCOMMIT
;
5651 kqworkq_update_override(struct kqworkq
*kqwq
, struct knote
*kn
,
5652 kq_index_t override_index
)
5654 workq_threadreq_t kqr
;
5655 kq_index_t old_override_index
;
5656 kq_index_t queue_index
= kn
->kn_qos_index
;
5658 if (override_index
<= queue_index
) {
5662 kqr
= kqworkq_get_request(kqwq
, queue_index
);
5666 old_override_index
= kqr
->tr_kq_override_index
;
5667 if (override_index
> MAX(kqr
->tr_kq_qos_index
, old_override_index
)) {
5668 thread_t servicer
= kqr_thread(kqr
);
5669 kqr
->tr_kq_override_index
= override_index
;
5671 /* apply the override to [incoming?] servicing thread */
5673 if (old_override_index
) {
5674 thread_update_kevent_override(servicer
, override_index
);
5676 thread_add_kevent_override(servicer
, override_index
);
5683 kqueue_update_override(kqueue_t kqu
, struct knote
*kn
, thread_qos_t qos
)
5685 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
5686 kqworkloop_update_threads_qos(kqu
.kqwl
, KQWL_UTQ_UPDATE_WAKEUP_OVERRIDE
,
5689 kqworkq_update_override(kqu
.kqwq
, kn
, qos
);
5694 kqworkloop_unbind_locked(struct kqworkloop
*kqwl
, thread_t thread
,
5695 enum kqwl_unbind_locked_mode how
)
5697 struct uthread
*ut
= get_bsdthread_info(thread
);
5698 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5700 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWL_UNBIND
), kqwl
->kqwl_dynamicid
,
5701 thread_tid(thread
), 0, 0);
5705 assert(ut
->uu_kqr_bound
== kqr
);
5706 ut
->uu_kqr_bound
= NULL
;
5707 if (how
== KQWL_OVERRIDE_DROP_IMMEDIATELY
&&
5708 ut
->uu_kqueue_override
!= THREAD_QOS_UNSPECIFIED
) {
5709 thread_drop_servicer_override(thread
);
5710 ut
->uu_kqueue_override
= THREAD_QOS_UNSPECIFIED
;
5713 if (kqwl
->kqwl_owner
== NULL
&& kqwl
->kqwl_turnstile
) {
5714 turnstile_update_inheritor(kqwl
->kqwl_turnstile
,
5715 TURNSTILE_INHERITOR_NULL
, TURNSTILE_IMMEDIATE_UPDATE
);
5716 turnstile_update_inheritor_complete(kqwl
->kqwl_turnstile
,
5717 TURNSTILE_INTERLOCK_HELD
);
5720 kqr
->tr_thread
= THREAD_NULL
;
5721 kqr
->tr_state
= WORKQ_TR_STATE_IDLE
;
5722 kqwl
->kqwl_state
&= ~KQ_R2K_ARMED
;
5726 kqworkloop_unbind_delayed_override_drop(thread_t thread
)
5728 struct uthread
*ut
= get_bsdthread_info(thread
);
5729 assert(ut
->uu_kqr_bound
== NULL
);
5730 if (ut
->uu_kqueue_override
!= THREAD_QOS_UNSPECIFIED
) {
5731 thread_drop_servicer_override(thread
);
5732 ut
->uu_kqueue_override
= THREAD_QOS_UNSPECIFIED
;
5737 * kqworkloop_unbind - Unbind the servicer thread of a workloop kqueue
5739 * It will acknowledge events, and possibly request a new thread if:
5740 * - there were active events left
5741 * - we pended waitq hook callouts during processing
5742 * - we pended wakeups while processing (or unsuppressing)
5744 * Called with kqueue lock held.
5747 kqworkloop_unbind(struct kqworkloop
*kqwl
)
5749 struct kqueue
*kq
= &kqwl
->kqwl_kqueue
;
5750 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
5751 thread_t thread
= kqr_thread_fast(kqr
);
5752 int op
= KQWL_UTQ_PARKING
;
5753 kq_index_t qos_override
= THREAD_QOS_UNSPECIFIED
;
5755 assert(thread
== current_thread());
5760 * Forcing the KQ_PROCESSING flag allows for QoS updates because of
5761 * unsuppressing knotes not to be applied until the eventual call to
5762 * kqworkloop_update_threads_qos() below.
5764 assert((kq
->kq_state
& KQ_PROCESSING
) == 0);
5765 if (!TAILQ_EMPTY(&kqwl
->kqwl_suppressed
)) {
5766 kq
->kq_state
|= KQ_PROCESSING
;
5767 qos_override
= kqworkloop_acknowledge_events(kqwl
);
5768 kq
->kq_state
&= ~KQ_PROCESSING
;
5771 kqworkloop_unbind_locked(kqwl
, thread
, KQWL_OVERRIDE_DROP_DELAYED
);
5772 kqworkloop_update_threads_qos(kqwl
, op
, qos_override
);
5777 * Drop the override on the current thread last, after the call to
5778 * kqworkloop_update_threads_qos above.
5780 kqworkloop_unbind_delayed_override_drop(thread
);
5782 /* If last reference, dealloc the workloop kq */
5783 kqworkloop_release(kqwl
);
5787 kqworkq_unbind_locked(struct kqworkq
*kqwq
,
5788 workq_threadreq_t kqr
, thread_t thread
)
5790 struct uthread
*ut
= get_bsdthread_info(thread
);
5791 kq_index_t old_override
= kqr
->tr_kq_override_index
;
5793 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KQWQ_UNBIND
), -1,
5794 thread_tid(kqr_thread(kqr
)), kqr
->tr_kq_qos_index
, 0);
5798 assert(ut
->uu_kqr_bound
== kqr
);
5799 ut
->uu_kqr_bound
= NULL
;
5800 kqr
->tr_thread
= THREAD_NULL
;
5801 kqr
->tr_state
= WORKQ_TR_STATE_IDLE
;
5802 kqr
->tr_kq_override_index
= THREAD_QOS_UNSPECIFIED
;
5803 kqwq
->kqwq_state
&= ~KQ_R2K_ARMED
;
5805 return old_override
;
5809 * kqworkq_unbind - unbind of a workq kqueue from a thread
5811 * We may have to request new threads.
5812 * This can happen there are no waiting processing threads and:
5813 * - there were active events we never got to (count > 0)
5814 * - we pended waitq hook callouts during processing
5815 * - we pended wakeups while processing (or unsuppressing)
5818 kqworkq_unbind(proc_t p
, workq_threadreq_t kqr
)
5820 struct kqworkq
*kqwq
= (struct kqworkq
*)p
->p_fd
->fd_wqkqueue
;
5821 __assert_only
int rc
;
5824 rc
= kqworkq_acknowledge_events(kqwq
, kqr
, 0, KQWQAE_UNBIND
);
5830 kqworkq_get_request(struct kqworkq
*kqwq
, kq_index_t qos_index
)
5832 assert(qos_index
< KQWQ_NBUCKETS
);
5833 return &kqwq
->kqwq_request
[qos_index
];
5837 knote_reset_priority(kqueue_t kqu
, struct knote
*kn
, pthread_priority_t pp
)
5839 kq_index_t qos
= _pthread_priority_thread_qos(pp
);
5841 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
5842 assert((pp
& _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
) == 0);
5843 pp
= _pthread_priority_normalize(pp
);
5844 } else if (kqu
.kq
->kq_state
& KQ_WORKQ
) {
5845 if (qos
== THREAD_QOS_UNSPECIFIED
) {
5846 /* On workqueues, outside of QoS means MANAGER */
5847 qos
= KQWQ_QOS_MANAGER
;
5848 pp
= _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG
;
5850 pp
= _pthread_priority_normalize(pp
);
5853 pp
= _pthread_unspecified_priority();
5854 qos
= THREAD_QOS_UNSPECIFIED
;
5859 if ((kn
->kn_status
& KN_MERGE_QOS
) == 0 || qos
> kn
->kn_qos_override
) {
5860 /* Never lower QoS when in "Merge" mode */
5861 kn
->kn_qos_override
= qos
;
5864 /* only adjust in-use qos index when not suppressed */
5865 if (kn
->kn_status
& KN_SUPPRESSED
) {
5866 kqueue_update_override(kqu
, kn
, qos
);
5867 } else if (kn
->kn_qos_index
!= qos
) {
5868 knote_dequeue(kqu
, kn
);
5869 kn
->kn_qos_index
= qos
;
5874 knote_adjust_qos(struct kqueue
*kq
, struct knote
*kn
, int result
)
5876 thread_qos_t qos_index
= (result
>> FILTER_ADJUST_EVENT_QOS_SHIFT
) & 7;
5880 assert(result
& FILTER_ADJUST_EVENT_QOS_BIT
);
5881 assert(qos_index
< THREAD_QOS_LAST
);
5884 * Early exit for knotes that should not change QoS
5886 if (__improbable(!knote_fops(kn
)->f_adjusts_qos
)) {
5887 panic("filter %d cannot change QoS", kn
->kn_filtid
);
5888 } else if (__improbable(!knote_has_qos(kn
))) {
5893 * knotes with the FALLBACK flag will only use their registration QoS if the
5894 * incoming event has no QoS, else, the registration QoS acts as a floor.
5896 thread_qos_t req_qos
= _pthread_priority_thread_qos_fast(kn
->kn_qos
);
5897 if (kn
->kn_qos
& _PTHREAD_PRIORITY_FALLBACK_FLAG
) {
5898 if (qos_index
== THREAD_QOS_UNSPECIFIED
) {
5899 qos_index
= req_qos
;
5902 if (qos_index
< req_qos
) {
5903 qos_index
= req_qos
;
5906 if ((kn
->kn_status
& KN_MERGE_QOS
) && (qos_index
< kn
->kn_qos_override
)) {
5907 /* Never lower QoS when in "Merge" mode */
5911 if ((kn
->kn_status
& KN_LOCKED
) && (kn
->kn_status
& KN_POSTING
)) {
5913 * When we're trying to update the QoS override and that both an
5914 * f_event() and other f_* calls are running concurrently, any of these
5915 * in flight calls may want to perform overrides that aren't properly
5916 * serialized with each other.
5918 * The first update that observes this racy situation enters a "Merge"
5919 * mode which causes subsequent override requests to saturate the
5920 * override instead of replacing its value.
5922 * This mode is left when knote_unlock() or knote_post()
5923 * observe that no other f_* routine is in flight.
5925 kn
->kn_status
|= KN_MERGE_QOS
;
5929 * Now apply the override if it changed.
5932 if (kn
->kn_qos_override
== qos_index
) {
5936 kn
->kn_qos_override
= qos_index
;
5938 if (kn
->kn_status
& KN_SUPPRESSED
) {
5940 * For suppressed events, the kn_qos_index field cannot be touched as it
5941 * allows us to know on which supress queue the knote is for a kqworkq.
5943 * Also, there's no natural push applied on the kqueues when this field
5944 * changes anyway. We hence need to apply manual overrides in this case,
5945 * which will be cleared when the events are later acknowledged.
5947 kqueue_update_override(kq
, kn
, qos_index
);
5948 } else if (kn
->kn_qos_index
!= qos_index
) {
5949 knote_dequeue(kq
, kn
);
5950 kn
->kn_qos_index
= qos_index
;
5955 * Called back from waitq code when no threads waiting and the hook was set.
5957 * Preemption is disabled - minimal work can be done in this context!!!
5960 waitq_set__CALLING_PREPOST_HOOK__(waitq_set_prepost_hook_t
*kq_hook
)
5964 kqu
.kq
= __container_of(kq_hook
, struct kqueue
, kq_waitq_hook
);
5965 assert(kqu
.kq
->kq_state
& (KQ_WORKQ
| KQ_WORKLOOP
));
5969 if (kqu
.kq
->kq_count
> 0) {
5970 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
5971 kqworkloop_wakeup(kqu
.kqwl
, KQWL_BUCKET_STAYACTIVE
);
5973 kqworkq_wakeup(kqu
.kqwq
, KQWQ_QOS_MANAGER
);
5981 klist_init(struct klist
*list
)
5988 * Query/Post each knote in the object's list
5990 * The object lock protects the list. It is assumed
5991 * that the filter/event routine for the object can
5992 * determine that the object is already locked (via
5993 * the hint) and not deadlock itself.
5995 * The object lock should also hold off pending
5996 * detach/drop operations.
5999 knote(struct klist
*list
, long hint
)
6003 SLIST_FOREACH(kn
, list
, kn_selnext
) {
6004 knote_post(kn
, hint
);
6009 * attach a knote to the specified list. Return true if this is the first entry.
6010 * The list is protected by whatever lock the object it is associated with uses.
6013 knote_attach(struct klist
*list
, struct knote
*kn
)
6015 int ret
= SLIST_EMPTY(list
);
6016 SLIST_INSERT_HEAD(list
, kn
, kn_selnext
);
6021 * detach a knote from the specified list. Return true if that was the last entry.
6022 * The list is protected by whatever lock the object it is associated with uses.
6025 knote_detach(struct klist
*list
, struct knote
*kn
)
6027 SLIST_REMOVE(list
, kn
, knote
, kn_selnext
);
6028 return SLIST_EMPTY(list
);
6032 * knote_vanish - Indicate that the source has vanished
6034 * If the knote has requested EV_VANISHED delivery,
6035 * arrange for that. Otherwise, deliver a NOTE_REVOKE
6036 * event for backward compatibility.
6038 * The knote is marked as having vanished, but is not
6039 * actually detached from the source in this instance.
6040 * The actual detach is deferred until the knote drop.
6042 * Our caller already has the object lock held. Calling
6043 * the detach routine would try to take that lock
6044 * recursively - which likely is not supported.
6047 knote_vanish(struct klist
*list
, bool make_active
)
6050 struct knote
*kn_next
;
6052 SLIST_FOREACH_SAFE(kn
, list
, kn_selnext
, kn_next
) {
6053 struct kqueue
*kq
= knote_get_kq(kn
);
6056 if (__probable(kn
->kn_status
& KN_REQVANISH
)) {
6058 * If EV_VANISH supported - prepare to deliver one
6060 kn
->kn_status
|= KN_VANISHED
;
6063 * Handle the legacy way to indicate that the port/portset was
6064 * deallocated or left the current Mach portspace (modern technique
6065 * is with an EV_VANISHED protocol).
6067 * Deliver an EV_EOF event for these changes (hopefully it will get
6068 * delivered before the port name recycles to the same generation
6069 * count and someone tries to re-register a kevent for it or the
6070 * events are udata-specific - avoiding a conflict).
6072 kn
->kn_flags
|= EV_EOF
| EV_ONESHOT
;
6075 knote_activate(kq
, kn
, FILTER_ACTIVE
);
6082 * Force a lazy allocation of the waitqset link
6083 * of the kq_wqs associated with the kn
6084 * if it wasn't already allocated.
6086 * This allows knote_link_waitq to never block
6087 * if reserved_link is not NULL.
6090 knote_link_waitqset_lazy_alloc(struct knote
*kn
)
6092 struct kqueue
*kq
= knote_get_kq(kn
);
6093 waitq_set_lazy_init_link(&kq
->kq_wqs
);
6097 * Check if a lazy allocation for the waitqset link
6098 * of the kq_wqs is needed.
6101 knote_link_waitqset_should_lazy_alloc(struct knote
*kn
)
6103 struct kqueue
*kq
= knote_get_kq(kn
);
6104 return waitq_set_should_lazy_init_link(&kq
->kq_wqs
);
6108 * For a given knote, link a provided wait queue directly with the kqueue.
6109 * Wakeups will happen via recursive wait queue support. But nothing will move
6110 * the knote to the active list at wakeup (nothing calls knote()). Instead,
6111 * we permanently enqueue them here.
6113 * kqueue and knote references are held by caller.
6114 * waitq locked by caller.
6116 * caller provides the wait queue link structure and insures that the kq->kq_wqs
6117 * is linked by previously calling knote_link_waitqset_lazy_alloc.
6120 knote_link_waitq(struct knote
*kn
, struct waitq
*wq
, uint64_t *reserved_link
)
6122 struct kqueue
*kq
= knote_get_kq(kn
);
6125 kr
= waitq_link(wq
, &kq
->kq_wqs
, WAITQ_ALREADY_LOCKED
, reserved_link
);
6126 if (kr
== KERN_SUCCESS
) {
6127 knote_markstayactive(kn
);
6135 * Unlink the provided wait queue from the kqueue associated with a knote.
6136 * Also remove it from the magic list of directly attached knotes.
6138 * Note that the unlink may have already happened from the other side, so
6139 * ignore any failures to unlink and just remove it from the kqueue list.
6141 * On success, caller is responsible for the link structure
6144 knote_unlink_waitq(struct knote
*kn
, struct waitq
*wq
)
6146 struct kqueue
*kq
= knote_get_kq(kn
);
6149 kr
= waitq_unlink(wq
, &kq
->kq_wqs
);
6150 knote_clearstayactive(kn
);
6151 return (kr
!= KERN_SUCCESS
) ? EINVAL
: 0;
6155 * remove all knotes referencing a specified fd
6157 * Entered with the proc_fd lock already held.
6158 * It returns the same way, but may drop it temporarily.
6161 knote_fdclose(struct proc
*p
, int fd
)
6165 KNOTE_LOCK_CTX(knlc
);
6168 list
= &p
->p_fd
->fd_knlist
[fd
];
6169 SLIST_FOREACH(kn
, list
, kn_link
) {
6170 struct kqueue
*kq
= knote_get_kq(kn
);
6174 if (kq
->kq_p
!= p
) {
6175 panic("%s: proc mismatch (kq->kq_p=%p != p=%p)",
6176 __func__
, kq
->kq_p
, p
);
6180 * If the knote supports EV_VANISHED delivery,
6181 * transition it to vanished mode (or skip over
6182 * it if already vanished).
6184 if (kn
->kn_status
& KN_VANISHED
) {
6190 if (!knote_lock(kq
, kn
, &knlc
, KNOTE_KQ_LOCK_ON_SUCCESS
)) {
6191 /* the knote was dropped by someone, nothing to do */
6192 } else if (kn
->kn_status
& KN_REQVANISH
) {
6193 kn
->kn_status
|= KN_VANISHED
;
6196 knote_fops(kn
)->f_detach(kn
);
6198 fp_drop(p
, kn
->kn_id
, kn
->kn_fp
, 0);
6200 kn
->kn_filtid
= EVFILTID_DETACHED
;
6203 knote_activate(kq
, kn
, FILTER_ACTIVE
);
6204 knote_unlock(kq
, kn
, &knlc
, KNOTE_KQ_UNLOCK
);
6206 knote_drop(kq
, kn
, &knlc
);
6215 * knote_fdfind - lookup a knote in the fd table for process
6217 * If the filter is file-based, lookup based on fd index.
6218 * Otherwise use a hash based on the ident.
6220 * Matching is based on kq, filter, and ident. Optionally,
6221 * it may also be based on the udata field in the kevent -
6222 * allowing multiple event registration for the file object
6225 * fd_knhashlock or fdlock held on entry (and exit)
6227 static struct knote
*
6228 knote_fdfind(struct kqueue
*kq
,
6229 const struct kevent_internal_s
*kev
,
6233 struct filedesc
*fdp
= p
->p_fd
;
6234 struct klist
*list
= NULL
;
6235 struct knote
*kn
= NULL
;
6238 * determine where to look for the knote
6241 /* fd-based knotes are linked off the fd table */
6242 if (kev
->kei_ident
< (u_int
)fdp
->fd_knlistsize
) {
6243 list
= &fdp
->fd_knlist
[kev
->kei_ident
];
6245 } else if (fdp
->fd_knhashmask
!= 0) {
6246 /* hash non-fd knotes here too */
6247 list
= &fdp
->fd_knhash
[KN_HASH((u_long
)kev
->kei_ident
, fdp
->fd_knhashmask
)];
6251 * scan the selected list looking for a match
6254 SLIST_FOREACH(kn
, list
, kn_link
) {
6255 if (kq
== knote_get_kq(kn
) &&
6256 kev
->kei_ident
== kn
->kn_id
&&
6257 kev
->kei_filter
== kn
->kn_filter
) {
6258 if (kev
->kei_flags
& EV_UDATA_SPECIFIC
) {
6259 if ((kn
->kn_flags
& EV_UDATA_SPECIFIC
) &&
6260 kev
->kei_udata
== kn
->kn_udata
) {
6261 break; /* matching udata-specific knote */
6263 } else if ((kn
->kn_flags
& EV_UDATA_SPECIFIC
) == 0) {
6264 break; /* matching non-udata-specific knote */
6273 * kq_add_knote- Add knote to the fd table for process
6274 * while checking for duplicates.
6276 * All file-based filters associate a list of knotes by file
6277 * descriptor index. All other filters hash the knote by ident.
6279 * May have to grow the table of knote lists to cover the
6280 * file descriptor index presented.
6282 * fd_knhashlock and fdlock unheld on entry (and exit).
6284 * Takes a rwlock boost if inserting the knote is successful.
6287 kq_add_knote(struct kqueue
*kq
, struct knote
*kn
, struct knote_lock_ctx
*knlc
,
6290 struct filedesc
*fdp
= p
->p_fd
;
6291 struct klist
*list
= NULL
;
6293 bool is_fd
= kn
->kn_is_fd
;
6301 if (knote_fdfind(kq
, &kn
->kn_kevent
, is_fd
, p
) != NULL
) {
6302 /* found an existing knote: we can't add this one */
6307 /* knote was not found: add it now */
6309 if (fdp
->fd_knhashmask
== 0) {
6312 list
= hashinit(CONFIG_KN_HASHSIZE
, M_KQUEUE
, &size
);
6318 fdp
->fd_knhash
= list
;
6319 fdp
->fd_knhashmask
= size
;
6322 list
= &fdp
->fd_knhash
[KN_HASH(kn
->kn_id
, fdp
->fd_knhashmask
)];
6323 SLIST_INSERT_HEAD(list
, kn
, kn_link
);
6327 /* knote is fd based */
6329 if ((u_int
)fdp
->fd_knlistsize
<= kn
->kn_id
) {
6332 if (kn
->kn_id
>= (uint64_t)p
->p_rlimit
[RLIMIT_NOFILE
].rlim_cur
6333 || kn
->kn_id
>= (uint64_t)maxfiles
) {
6337 /* have to grow the fd_knlist */
6338 size
= fdp
->fd_knlistsize
;
6339 while (size
<= kn
->kn_id
) {
6343 if (size
>= (UINT_MAX
/ sizeof(struct klist
*))) {
6348 MALLOC(list
, struct klist
*,
6349 size
* sizeof(struct klist
*), M_KQUEUE
, M_WAITOK
);
6355 bcopy((caddr_t
)fdp
->fd_knlist
, (caddr_t
)list
,
6356 fdp
->fd_knlistsize
* sizeof(struct klist
*));
6357 bzero((caddr_t
)list
+
6358 fdp
->fd_knlistsize
* sizeof(struct klist
*),
6359 (size
- fdp
->fd_knlistsize
) * sizeof(struct klist
*));
6360 FREE(fdp
->fd_knlist
, M_KQUEUE
);
6361 fdp
->fd_knlist
= list
;
6362 fdp
->fd_knlistsize
= size
;
6365 list
= &fdp
->fd_knlist
[kn
->kn_id
];
6366 SLIST_INSERT_HEAD(list
, kn
, kn_link
);
6374 assert((kn
->kn_status
& KN_LOCKED
) == 0);
6375 (void)knote_lock(kq
, kn
, knlc
, KNOTE_KQ_UNLOCK
);
6376 kqueue_retain(kq
); /* retain a kq ref */
6388 * kq_remove_knote - remove a knote from the fd table for process
6390 * If the filter is file-based, remove based on fd index.
6391 * Otherwise remove from the hash based on the ident.
6393 * fd_knhashlock and fdlock unheld on entry (and exit).
6396 kq_remove_knote(struct kqueue
*kq
, struct knote
*kn
, struct proc
*p
,
6397 struct knote_lock_ctx
*knlc
)
6399 struct filedesc
*fdp
= p
->p_fd
;
6400 struct klist
*list
= NULL
;
6402 bool is_fd
= kn
->kn_is_fd
;
6411 assert((u_int
)fdp
->fd_knlistsize
> kn
->kn_id
);
6412 list
= &fdp
->fd_knlist
[kn
->kn_id
];
6414 list
= &fdp
->fd_knhash
[KN_HASH(kn
->kn_id
, fdp
->fd_knhashmask
)];
6416 SLIST_REMOVE(list
, kn
, knote
, kn_link
);
6419 kq_state
= kq
->kq_state
;
6421 knote_unlock_cancel(kq
, kn
, knlc
);
6431 if (kq_state
& KQ_DYNAMIC
) {
6432 kqworkloop_release((struct kqworkloop
*)kq
);
6437 * kq_find_knote_and_kq_lock - lookup a knote in the fd table for process
6438 * and, if the knote is found, acquires the kqlock while holding the fd table lock/spinlock.
6440 * fd_knhashlock or fdlock unheld on entry (and exit)
6443 static struct knote
*
6444 kq_find_knote_and_kq_lock(struct kqueue
*kq
, struct kevent_qos_s
*kev
,
6445 bool is_fd
, struct proc
*p
)
6447 struct filedesc
*fdp
= p
->p_fd
;
6457 * Temporary horrible hack:
6458 * this cast is gross and will go away in a future change.
6459 * It is OK to do because we don't look at xflags/s_fflags,
6460 * and that when we cast down the kev this way,
6461 * the truncated filter field works.
6463 kn
= knote_fdfind(kq
, (struct kevent_internal_s
*)kev
, is_fd
, p
);
6467 assert(knote_get_kq(kn
) == kq
);
6479 __attribute__((noinline
))
6481 kqfile_wakeup(struct kqfile
*kqf
, __unused kq_index_t qos
)
6483 /* flag wakeups during processing */
6484 if (kqf
->kqf_state
& KQ_PROCESSING
) {
6485 kqf
->kqf_state
|= KQ_WAKEUP
;
6488 /* wakeup a thread waiting on this queue */
6489 if (kqf
->kqf_state
& (KQ_SLEEP
| KQ_SEL
)) {
6490 kqf
->kqf_state
&= ~(KQ_SLEEP
| KQ_SEL
);
6491 waitq_wakeup64_all((struct waitq
*)&kqf
->kqf_wqs
, KQ_EVENT
,
6492 THREAD_AWAKENED
, WAITQ_ALL_PRIORITIES
);
6495 /* wakeup other kqueues/select sets we're inside */
6496 KNOTE(&kqf
->kqf_sel
.si_note
, 0);
6499 static struct kqtailq
*
6500 knote_get_tailq(kqueue_t kqu
, struct knote
*kn
)
6502 kq_index_t qos_index
= kn
->kn_qos_index
;
6504 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
6505 assert(qos_index
< KQWL_NBUCKETS
);
6506 } else if (kqu
.kq
->kq_state
& KQ_WORKQ
) {
6507 assert(qos_index
< KQWQ_NBUCKETS
);
6509 assert(qos_index
== QOS_INDEX_KQFILE
);
6511 static_assert(offsetof(struct kqueue
, kq_queue
) == sizeof(struct kqueue
),
6512 "struct kqueue::kq_queue must be exactly at the end");
6513 return &kqu
.kq
->kq_queue
[qos_index
];
6517 knote_enqueue(kqueue_t kqu
, struct knote
*kn
, kn_status_t wakeup_mask
)
6521 if ((kn
->kn_status
& (KN_ACTIVE
| KN_STAYACTIVE
)) == 0) {
6525 if (kn
->kn_status
& (KN_DISABLED
| KN_SUPPRESSED
| KN_DROPPING
)) {
6529 if ((kn
->kn_status
& KN_QUEUED
) == 0) {
6530 struct kqtailq
*queue
= knote_get_tailq(kqu
, kn
);
6532 TAILQ_INSERT_TAIL(queue
, kn
, kn_tqe
);
6533 kn
->kn_status
|= KN_QUEUED
;
6535 } else if ((kn
->kn_status
& KN_STAYACTIVE
) == 0) {
6539 if (kn
->kn_status
& wakeup_mask
) {
6540 if (kqu
.kq
->kq_state
& KQ_WORKLOOP
) {
6541 kqworkloop_wakeup(kqu
.kqwl
, kn
->kn_qos_index
);
6542 } else if (kqu
.kq
->kq_state
& KQ_WORKQ
) {
6543 kqworkq_wakeup(kqu
.kqwq
, kn
->kn_qos_index
);
6545 kqfile_wakeup(kqu
.kqf
, kn
->kn_qos_index
);
6550 __attribute__((always_inline
))
6552 knote_dequeue(kqueue_t kqu
, struct knote
*kn
)
6554 if (kn
->kn_status
& KN_QUEUED
) {
6555 struct kqtailq
*queue
= knote_get_tailq(kqu
, kn
);
6557 // attaching the knote calls knote_reset_priority() without
6558 // the kqlock which is fine, so we can't call kqlock_held()
6559 // if we're not queued.
6562 TAILQ_REMOVE(queue
, kn
, kn_tqe
);
6563 kn
->kn_status
&= ~KN_QUEUED
;
6568 /* called with kqueue lock held */
6570 knote_suppress(kqueue_t kqu
, struct knote
*kn
)
6572 struct kqtailq
*suppressq
;
6576 assert((kn
->kn_status
& KN_SUPPRESSED
) == 0);
6577 assert(kn
->kn_status
& KN_QUEUED
);
6579 knote_dequeue(kqu
, kn
);
6580 /* deactivate - so new activations indicate a wakeup */
6581 kn
->kn_status
&= ~KN_ACTIVE
;
6582 kn
->kn_status
|= KN_SUPPRESSED
;
6583 suppressq
= kqueue_get_suppressed_queue(kqu
, kn
);
6584 TAILQ_INSERT_TAIL(suppressq
, kn
, kn_tqe
);
6587 __attribute__((always_inline
))
6589 knote_unsuppress_noqueue(kqueue_t kqu
, struct knote
*kn
)
6591 struct kqtailq
*suppressq
;
6595 assert(kn
->kn_status
& KN_SUPPRESSED
);
6597 kn
->kn_status
&= ~KN_SUPPRESSED
;
6598 suppressq
= kqueue_get_suppressed_queue(kqu
, kn
);
6599 TAILQ_REMOVE(suppressq
, kn
, kn_tqe
);
6602 * If the knote is no longer active, reset its push,
6603 * and resynchronize kn_qos_index with kn_qos_override
6604 * for knotes with a real qos.
6606 if ((kn
->kn_status
& KN_ACTIVE
) == 0 && knote_has_qos(kn
)) {
6607 kn
->kn_qos_override
= _pthread_priority_thread_qos_fast(kn
->kn_qos
);
6609 kn
->kn_qos_index
= kn
->kn_qos_override
;
6612 /* called with kqueue lock held */
6614 knote_unsuppress(kqueue_t kqu
, struct knote
*kn
)
6616 if (kn
->kn_status
& KN_SUPPRESSED
) {
6617 knote_unsuppress_noqueue(kqu
, kn
);
6619 /* don't wakeup if unsuppressing just a stay-active knote */
6620 knote_enqueue(kqu
, kn
, KN_ACTIVE
);
6624 __attribute__((always_inline
))
6626 knote_mark_active(struct knote
*kn
)
6628 if ((kn
->kn_status
& KN_ACTIVE
) == 0) {
6629 KDBG_DEBUG(KEV_EVTID(BSD_KEVENT_KNOTE_ACTIVATE
),
6630 kn
->kn_udata
, kn
->kn_status
| (kn
->kn_id
<< 32),
6634 kn
->kn_status
|= KN_ACTIVE
;
6637 /* called with kqueue lock held */
6639 knote_activate(kqueue_t kqu
, struct knote
*kn
, int result
)
6641 assert(result
& FILTER_ACTIVE
);
6642 if (result
& FILTER_ADJUST_EVENT_QOS_BIT
) {
6643 // may dequeue the knote
6644 knote_adjust_qos(kqu
.kq
, kn
, result
);
6646 knote_mark_active(kn
);
6647 knote_enqueue(kqu
, kn
, KN_ACTIVE
| KN_STAYACTIVE
);
6651 * This function applies changes requested by f_attach or f_touch for
6652 * a given filter. It proceeds in a carefully chosen order to help
6653 * every single transition do the minimal amount of work possible.
6656 knote_apply_touch(kqueue_t kqu
, struct knote
*kn
, struct kevent_qos_s
*kev
,
6659 kn_status_t wakeup_mask
= KN_ACTIVE
;
6661 if ((kev
->flags
& EV_ENABLE
) && (kn
->kn_status
& KN_DISABLED
)) {
6663 * When a stayactive knote is reenabled, we may have missed wakeups
6664 * while it was disabled, so we need to poll it. To do so, ask
6665 * knote_enqueue() below to reenqueue it.
6667 wakeup_mask
|= KN_STAYACTIVE
;
6668 kn
->kn_status
&= ~KN_DISABLED
;
6671 * it is possible for userland to have knotes registered for a given
6672 * workloop `wl_orig` but really handled on another workloop `wl_new`.
6674 * In that case, rearming will happen from the servicer thread of
6675 * `wl_new` which if `wl_orig` is no longer being serviced, would cause
6676 * this knote to stay suppressed forever if we only relied on
6677 * kqworkloop_acknowledge_events to be called by `wl_orig`.
6679 * However if we see the KQ_PROCESSING bit on `wl_orig` set, we can't
6680 * unsuppress because that would mess with the processing phase of
6681 * `wl_orig`, however it also means kqworkloop_acknowledge_events()
6684 if (__improbable(kn
->kn_status
& KN_SUPPRESSED
)) {
6685 if ((kqu
.kq
->kq_state
& KQ_PROCESSING
) == 0) {
6686 knote_unsuppress_noqueue(kqu
, kn
);
6691 if ((result
& FILTER_UPDATE_REQ_QOS
) && kev
->qos
&& kev
->qos
!= kn
->kn_qos
) {
6692 // may dequeue the knote
6693 knote_reset_priority(kqu
, kn
, kev
->qos
);
6697 * When we unsuppress above, or because of knote_reset_priority(),
6698 * the knote may have been dequeued, we need to restore the invariant
6699 * that if the knote is active it needs to be queued now that
6700 * we're done applying changes.
6702 if (result
& FILTER_ACTIVE
) {
6703 knote_activate(kqu
, kn
, result
);
6705 knote_enqueue(kqu
, kn
, wakeup_mask
);
6708 if ((result
& FILTER_THREADREQ_NODEFEER
) &&
6709 act_clear_astkevent(current_thread(), AST_KEVENT_REDRIVE_THREADREQ
)) {
6710 workq_kern_threadreq_redrive(kqu
.kq
->kq_p
, WORKQ_THREADREQ_NONE
);
6715 * knote_drop - disconnect and drop the knote
6717 * Called with the kqueue locked, returns with the kqueue unlocked.
6719 * If a knote locking context is passed, it is canceled.
6721 * The knote may have already been detached from
6722 * (or not yet attached to) its source object.
6725 knote_drop(struct kqueue
*kq
, struct knote
*kn
, struct knote_lock_ctx
*knlc
)
6727 struct proc
*p
= kq
->kq_p
;
6731 assert((kn
->kn_status
& KN_DROPPING
) == 0);
6733 assert((kn
->kn_status
& KN_LOCKED
) == 0);
6735 kn
->kn_status
|= KN_DROPPING
;
6737 if (kn
->kn_status
& KN_SUPPRESSED
) {
6738 knote_unsuppress_noqueue(kq
, kn
);
6740 knote_dequeue(kq
, kn
);
6742 knote_wait_for_post(kq
, kn
);
6744 knote_fops(kn
)->f_detach(kn
);
6746 /* kq may be freed when kq_remove_knote() returns */
6747 kq_remove_knote(kq
, kn
, p
, knlc
);
6748 if (kn
->kn_is_fd
&& ((kn
->kn_status
& KN_VANISHED
) == 0)) {
6749 fp_drop(p
, kn
->kn_id
, kn
->kn_fp
, 0);
6758 knote_zone
= zinit(sizeof(struct knote
), 8192 * sizeof(struct knote
),
6759 8192, "knote zone");
6760 zone_change(knote_zone
, Z_CACHING_ENABLED
, TRUE
);
6762 kqfile_zone
= zinit(sizeof(struct kqfile
), 8192 * sizeof(struct kqfile
),
6763 8192, "kqueue file zone");
6765 kqworkq_zone
= zinit(sizeof(struct kqworkq
), 8192 * sizeof(struct kqworkq
),
6766 8192, "kqueue workq zone");
6768 kqworkloop_zone
= zinit(sizeof(struct kqworkloop
), 8192 * sizeof(struct kqworkloop
),
6769 8192, "kqueue workloop zone");
6770 zone_change(kqworkloop_zone
, Z_CACHING_ENABLED
, TRUE
);
6772 /* allocate kq lock group attribute and group */
6773 kq_lck_grp_attr
= lck_grp_attr_alloc_init();
6775 kq_lck_grp
= lck_grp_alloc_init("kqueue", kq_lck_grp_attr
);
6777 /* Allocate kq lock attribute */
6778 kq_lck_attr
= lck_attr_alloc_init();
6780 #if CONFIG_MEMORYSTATUS
6781 /* Initialize the memorystatus list lock */
6782 memorystatus_kevent_init(kq_lck_grp
, kq_lck_attr
);
6785 SYSINIT(knote
, SI_SUB_PSEUDO
, SI_ORDER_ANY
, knote_init
, NULL
);
6787 const struct filterops
*
6788 knote_fops(struct knote
*kn
)
6790 return sysfilt_ops
[kn
->kn_filtid
];
6793 static struct knote
*
6796 struct knote
*kn
= ((struct knote
*)zalloc(knote_zone
));
6797 bzero(kn
, sizeof(struct knote
));
6802 knote_free(struct knote
*kn
)
6804 assert((kn
->kn_status
& (KN_LOCKED
| KN_POSTING
)) == 0);
6805 zfree(knote_zone
, kn
);
6808 #pragma mark - syscalls: kevent, kevent64, kevent_qos, kevent_id
6811 kevent_get_context(thread_t thread
)
6813 uthread_t ut
= get_bsdthread_info(thread
);
6814 return &ut
->uu_save
.uus_kevent
;
6818 kevent_args_requesting_events(unsigned int flags
, int nevents
)
6820 return !(flags
& KEVENT_FLAG_ERROR_EVENTS
) && nevents
> 0;
6824 kevent_adjust_flags_for_proc(proc_t p
, int flags
)
6826 __builtin_assume(p
);
6827 return flags
| (IS_64BIT_PROCESS(p
) ? KEVENT_FLAG_PROC64
: 0);
6831 * @function kevent_get_kqfile
6834 * Lookup a kqfile by fd.
6837 * Callers: kevent, kevent64, kevent_qos
6839 * This is not assumed to be a fastpath (kqfile interfaces are legacy)
6843 kevent_get_kqfile(struct proc
*p
, int fd
, int flags
,
6844 struct fileproc
**fp
, struct kqueue
**kqp
)
6849 error
= fp_getfkq(p
, fd
, fp
, &kq
);
6850 if (__improbable(error
)) {
6854 uint16_t kq_state
= os_atomic_load(&kq
->kq_state
, relaxed
);
6855 if (__improbable((kq_state
& (KQ_KEV32
| KQ_KEV64
| KQ_KEV_QOS
)) == 0)) {
6857 kq_state
= kq
->kq_state
;
6858 if (!(kq_state
& (KQ_KEV32
| KQ_KEV64
| KQ_KEV_QOS
))) {
6859 if (flags
& KEVENT_FLAG_LEGACY32
) {
6860 kq_state
|= KQ_KEV32
;
6861 } else if (flags
& KEVENT_FLAG_LEGACY64
) {
6862 kq_state
|= KQ_KEV64
;
6864 kq_state
|= KQ_KEV_QOS
;
6866 kq
->kq_state
= kq_state
;
6872 * kqfiles can't be used through the legacy kevent()
6873 * and other interfaces at the same time.
6875 if (__improbable((bool)(flags
& KEVENT_FLAG_LEGACY32
) !=
6876 (bool)(kq_state
& KQ_KEV32
))) {
6877 fp_drop(p
, fd
, *fp
, 0);
6886 * @function kevent_get_kqwq
6889 * Lookup or create the process kqwq (faspath).
6892 * Callers: kevent64, kevent_qos
6896 kevent_get_kqwq(proc_t p
, int flags
, int nevents
, struct kqueue
**kqp
)
6898 struct kqworkq
*kqwq
= p
->p_fd
->fd_wqkqueue
;
6900 if (__improbable(kevent_args_requesting_events(flags
, nevents
))) {
6903 if (__improbable(kqwq
== NULL
)) {
6904 kqwq
= kqworkq_alloc(p
, flags
);
6905 if (__improbable(kqwq
== NULL
)) {
6910 *kqp
= &kqwq
->kqwq_kqueue
;
6914 #pragma mark kevent copyio
6917 * @function kevent_get_data_size
6920 * Copies in the extra data size from user-space.
6923 kevent_get_data_size(int flags
, user_addr_t data_avail
, user_addr_t data_out
,
6926 if (!data_avail
|| !data_out
) {
6927 kectx
->kec_data_size
= 0;
6928 kectx
->kec_data_resid
= 0;
6929 } else if (flags
& KEVENT_FLAG_PROC64
) {
6930 user64_size_t usize
= 0;
6931 int error
= copyin((user_addr_t
)data_avail
, &usize
, sizeof(usize
));
6932 if (__improbable(error
)) {
6935 kectx
->kec_data_resid
= kectx
->kec_data_size
= (user_size_t
)usize
;
6937 user32_size_t usize
= 0;
6938 int error
= copyin((user_addr_t
)data_avail
, &usize
, sizeof(usize
));
6939 if (__improbable(error
)) {
6942 kectx
->kec_data_avail
= data_avail
;
6943 kectx
->kec_data_resid
= kectx
->kec_data_size
= (user_size_t
)usize
;
6945 kectx
->kec_data_out
= data_out
;
6946 kectx
->kec_data_avail
= data_avail
;
6951 * @function kevent_put_data_size
6954 * Copies out the residual data size to user-space if any has been used.
6957 kevent_put_data_size(unsigned int flags
, kevent_ctx_t kectx
)
6959 if (kectx
->kec_data_resid
== kectx
->kec_data_size
) {
6962 if (flags
& KEVENT_FLAG_KERNEL
) {
6963 *(user_size_t
*)(uintptr_t)kectx
->kec_data_avail
= kectx
->kec_data_resid
;
6966 if (flags
& KEVENT_FLAG_PROC64
) {
6967 user64_size_t usize
= (user64_size_t
)kectx
->kec_data_resid
;
6968 return copyout(&usize
, (user_addr_t
)kectx
->kec_data_avail
, sizeof(usize
));
6970 user32_size_t usize
= (user32_size_t
)kectx
->kec_data_resid
;
6971 return copyout(&usize
, (user_addr_t
)kectx
->kec_data_avail
, sizeof(usize
));
6976 * @function kevent_legacy_copyin
6979 * Handles the copyin of a kevent/kevent64 event.
6982 kevent_legacy_copyin(user_addr_t
*addrp
, struct kevent_qos_s
*kevp
, unsigned int flags
)
6986 assert((flags
& (KEVENT_FLAG_LEGACY32
| KEVENT_FLAG_LEGACY64
)) != 0);
6988 if (flags
& KEVENT_FLAG_LEGACY64
) {
6989 struct kevent64_s kev64
;
6991 error
= copyin(*addrp
, (caddr_t
)&kev64
, sizeof(kev64
));
6992 if (__improbable(error
)) {
6995 *addrp
+= sizeof(kev64
);
6996 *kevp
= (struct kevent_qos_s
){
6997 .ident
= kev64
.ident
,
6998 .filter
= kev64
.filter
,
6999 /* Make sure user doesn't pass in any system flags */
7000 .flags
= kev64
.flags
& ~EV_SYSFLAGS
,
7001 .udata
= kev64
.udata
,
7002 .fflags
= kev64
.fflags
,
7004 .ext
[0] = kev64
.ext
[0],
7005 .ext
[1] = kev64
.ext
[1],
7007 } else if (flags
& KEVENT_FLAG_PROC64
) {
7008 struct user64_kevent kev64
;
7010 error
= copyin(*addrp
, (caddr_t
)&kev64
, sizeof(kev64
));
7011 if (__improbable(error
)) {
7014 *addrp
+= sizeof(kev64
);
7015 *kevp
= (struct kevent_qos_s
){
7016 .ident
= kev64
.ident
,
7017 .filter
= kev64
.filter
,
7018 /* Make sure user doesn't pass in any system flags */
7019 .flags
= kev64
.flags
& ~EV_SYSFLAGS
,
7020 .udata
= kev64
.udata
,
7021 .fflags
= kev64
.fflags
,
7025 struct user32_kevent kev32
;
7027 error
= copyin(*addrp
, (caddr_t
)&kev32
, sizeof(kev32
));
7028 if (__improbable(error
)) {
7031 *addrp
+= sizeof(kev32
);
7032 *kevp
= (struct kevent_qos_s
){
7033 .ident
= (uintptr_t)kev32
.ident
,
7034 .filter
= kev32
.filter
,
7035 /* Make sure user doesn't pass in any system flags */
7036 .flags
= kev32
.flags
& ~EV_SYSFLAGS
,
7037 .udata
= CAST_USER_ADDR_T(kev32
.udata
),
7038 .fflags
= kev32
.fflags
,
7039 .data
= (intptr_t)kev32
.data
,
7047 * @function kevent_modern_copyin
7050 * Handles the copyin of a kevent_qos/kevent_id event.
7053 kevent_modern_copyin(user_addr_t
*addrp
, struct kevent_qos_s
*kevp
)
7055 int error
= copyin(*addrp
, (caddr_t
)kevp
, sizeof(struct kevent_qos_s
));
7056 if (__probable(!error
)) {
7057 /* Make sure user doesn't pass in any system flags */
7058 *addrp
+= sizeof(struct kevent_qos_s
);
7059 kevp
->flags
&= ~EV_SYSFLAGS
;
7065 * @function kevent_legacy_copyout
7068 * Handles the copyout of a kevent/kevent64 event.
7071 kevent_legacy_copyout(struct kevent_qos_s
*kevp
, user_addr_t
*addrp
, unsigned int flags
)
7076 assert((flags
& (KEVENT_FLAG_LEGACY32
| KEVENT_FLAG_LEGACY64
)) != 0);
7079 * fully initialize the differnt output event structure
7080 * types from the internal kevent (and some universal
7081 * defaults for fields not represented in the internal
7084 * Note: these structures have no padding hence the C99
7085 * initializers below do not leak kernel info.
7087 if (flags
& KEVENT_FLAG_LEGACY64
) {
7088 struct kevent64_s kev64
= {
7089 .ident
= kevp
->ident
,
7090 .filter
= kevp
->filter
,
7091 .flags
= kevp
->flags
,
7092 .fflags
= kevp
->fflags
,
7093 .data
= (int64_t)kevp
->data
,
7094 .udata
= kevp
->udata
,
7095 .ext
[0] = kevp
->ext
[0],
7096 .ext
[1] = kevp
->ext
[1],
7098 advance
= sizeof(struct kevent64_s
);
7099 error
= copyout((caddr_t
)&kev64
, *addrp
, advance
);
7100 } else if (flags
& KEVENT_FLAG_PROC64
) {
7102 * deal with the special case of a user-supplied
7103 * value of (uintptr_t)-1.
7105 uint64_t ident
= (kevp
->ident
== (uintptr_t)-1) ?
7106 (uint64_t)-1LL : (uint64_t)kevp
->ident
;
7107 struct user64_kevent kev64
= {
7109 .filter
= kevp
->filter
,
7110 .flags
= kevp
->flags
,
7111 .fflags
= kevp
->fflags
,
7112 .data
= (int64_t) kevp
->data
,
7113 .udata
= kevp
->udata
,
7115 advance
= sizeof(kev64
);
7116 error
= copyout((caddr_t
)&kev64
, *addrp
, advance
);
7118 struct user32_kevent kev32
= {
7119 .ident
= (uint32_t)kevp
->ident
,
7120 .filter
= kevp
->filter
,
7121 .flags
= kevp
->flags
,
7122 .fflags
= kevp
->fflags
,
7123 .data
= (int32_t)kevp
->data
,
7124 .udata
= kevp
->udata
,
7126 advance
= sizeof(kev32
);
7127 error
= copyout((caddr_t
)&kev32
, *addrp
, advance
);
7129 if (__probable(!error
)) {
7136 * @function kevent_modern_copyout
7139 * Handles the copyout of a kevent_qos/kevent_id event.
7143 kevent_modern_copyout(struct kevent_qos_s
*kevp
, user_addr_t
*addrp
)
7145 int error
= copyout((caddr_t
)kevp
, *addrp
, sizeof(struct kevent_qos_s
));
7146 if (__probable(!error
)) {
7147 *addrp
+= sizeof(struct kevent_qos_s
);
7152 #pragma mark kevent core implementation
7155 * @function kevent_callback_inline
7158 * Callback for each individual event
7161 * This is meant to be inlined in kevent_modern_callback and
7162 * kevent_legacy_callback.
7166 kevent_callback_inline(struct kevent_qos_s
*kevp
, kevent_ctx_t kectx
, bool legacy
)
7170 assert(kectx
->kec_process_noutputs
< kectx
->kec_process_nevents
);
7173 * Copy out the appropriate amount of event data for this user.
7176 error
= kevent_legacy_copyout(kevp
, &kectx
->kec_process_eventlist
,
7177 kectx
->kec_process_flags
);
7179 error
= kevent_modern_copyout(kevp
, &kectx
->kec_process_eventlist
);
7183 * If there isn't space for additional events, return
7184 * a harmless error to stop the processing here
7186 if (error
== 0 && ++kectx
->kec_process_noutputs
== kectx
->kec_process_nevents
) {
7187 error
= EWOULDBLOCK
;
7193 * @function kevent_modern_callback
7196 * Callback for each individual modern event.
7199 * This callback handles kevent_qos/kevent_id events.
7202 kevent_modern_callback(struct kevent_qos_s
*kevp
, kevent_ctx_t kectx
)
7204 return kevent_callback_inline(kevp
, kectx
, /*legacy*/ false);
7208 * @function kevent_legacy_callback
7211 * Callback for each individual legacy event.
7214 * This callback handles kevent/kevent64 events.
7217 kevent_legacy_callback(struct kevent_qos_s
*kevp
, kevent_ctx_t kectx
)
7219 return kevent_callback_inline(kevp
, kectx
, /*legacy*/ true);
7223 * @function kevent_cleanup
7226 * Handles the cleanup returning from a kevent call.
7229 * kevent entry points will take a reference on workloops,
7230 * and a usecount on the fileglob of kqfiles.
7232 * This function undoes this on the exit paths of kevents.
7235 * The error to return to userspace.
7238 kevent_cleanup(kqueue_t kqu
, int flags
, int error
, kevent_ctx_t kectx
)
7240 // poll should not call any codepath leading to this
7241 assert((flags
& KEVENT_FLAG_POLL
) == 0);
7243 if (flags
& KEVENT_FLAG_WORKLOOP
) {
7244 kqworkloop_release(kqu
.kqwl
);
7245 } else if (flags
& KEVENT_FLAG_WORKQ
) {
7248 fp_drop(kqu
.kqf
->kqf_p
, kectx
->kec_fd
, kectx
->kec_fp
, 0);
7251 /* don't restart after signals... */
7252 if (error
== ERESTART
) {
7254 } else if (error
== 0) {
7255 /* don't abandon other output just because of residual copyout failures */
7256 (void)kevent_put_data_size(flags
, kectx
);
7259 if (flags
& KEVENT_FLAG_PARKING
) {
7260 thread_t th
= current_thread();
7261 struct uthread
*uth
= get_bsdthread_info(th
);
7262 if (uth
->uu_kqr_bound
) {
7263 thread_unfreeze_base_pri(th
);
7270 * @function kqueue_process
7273 * Process the triggered events in a kqueue.
7276 * Walk the queued knotes and validate that they are really still triggered
7277 * events by calling the filter routines (if necessary).
7279 * For each event that is still considered triggered, invoke the callback
7282 * caller holds a reference on the kqueue.
7283 * kqueue locked on entry and exit - but may be dropped
7284 * kqueue list locked (held for duration of call)
7286 * This is only called by kqueue_scan() so that the compiler can inline it.
7289 * - 0: no event was returned, no other error occured
7290 * - EBADF: the kqueue is being destroyed (KQ_DRAIN is set)
7291 * - EWOULDBLOCK: (not an error) events have been found and we should return
7292 * - EFAULT: copyout failed
7293 * - filter specific errors
7296 kqueue_process(kqueue_t kqu
, int flags
, kevent_ctx_t kectx
,
7297 kevent_callback_t callback
)
7299 workq_threadreq_t kqr
= current_uthread()->uu_kqr_bound
;
7301 int error
= 0, rc
= 0;
7302 struct kqtailq
*base_queue
, *queue
;
7303 #if DEBUG || DEVELOPMENT
7306 uint16_t kq_type
= (kqu
.kq
->kq_state
& (KQ_WORKQ
| KQ_WORKLOOP
));
7308 if (kq_type
& KQ_WORKQ
) {
7309 rc
= kqworkq_begin_processing(kqu
.kqwq
, kqr
, flags
);
7310 } else if (kq_type
& KQ_WORKLOOP
) {
7311 rc
= kqworkloop_begin_processing(kqu
.kqwl
, flags
);
7314 rc
= kqfile_begin_processing(kqu
.kqf
);
7321 /* Nothing to process */
7326 * loop through the enqueued knotes associated with this request,
7327 * processing each one. Each request may have several queues
7328 * of knotes to process (depending on the type of kqueue) so we
7329 * have to loop through all the queues as long as we have additional
7334 if (kq_type
& KQ_WORKQ
) {
7335 base_queue
= queue
= &kqu
.kqwq
->kqwq_queue
[kqr
->tr_kq_qos_index
];
7336 } else if (kq_type
& KQ_WORKLOOP
) {
7337 base_queue
= &kqu
.kqwl
->kqwl_queue
[0];
7338 queue
= &kqu
.kqwl
->kqwl_queue
[KQWL_NBUCKETS
- 1];
7340 base_queue
= queue
= &kqu
.kqf
->kqf_queue
;
7344 while ((kn
= TAILQ_FIRST(queue
)) != NULL
) {
7345 error
= knote_process(kn
, kectx
, callback
);
7346 if (error
== EJUSTRETURN
) {
7348 } else if (__improbable(error
)) {
7349 /* error is EWOULDBLOCK when the out event array is full */
7350 goto stop_processing
;
7353 } while (queue
-- > base_queue
);
7355 if (kectx
->kec_process_noutputs
) {
7356 /* callers will transform this into no error */
7357 error
= EWOULDBLOCK
;
7362 * If KEVENT_FLAG_PARKING is set, and no kevents have been returned,
7363 * we want to unbind the kqrequest from the thread.
7365 * However, because the kq locks are dropped several times during process,
7366 * new knotes may have fired again, in which case, we want to fail the end
7367 * processing and process again, until it converges.
7369 * If we have an error or returned events, end processing never fails.
7372 flags
&= ~KEVENT_FLAG_PARKING
;
7374 if (kq_type
& KQ_WORKQ
) {
7375 rc
= kqworkq_end_processing(kqu
.kqwq
, kqr
, flags
);
7376 } else if (kq_type
& KQ_WORKLOOP
) {
7377 rc
= kqworkloop_end_processing(kqu
.kqwl
, KQ_PROCESSING
, flags
);
7379 rc
= kqfile_end_processing(kqu
.kqf
);
7382 if (__probable(error
)) {
7386 if (__probable(rc
>= 0)) {
7387 assert(rc
== 0 || rc
== EBADF
);
7391 #if DEBUG || DEVELOPMENT
7392 if (retries
-- == 0) {
7393 panic("kevent: way too many knote_process retries, kq: %p (0x%04x)",
7394 kqu
.kq
, kqu
.kq
->kq_state
);
7397 if (kq_type
& (KQ_WORKQ
| KQ_WORKLOOP
)) {
7398 assert(flags
& KEVENT_FLAG_PARKING
);
7406 * @function kqueue_scan_continue
7409 * The continuation used by kqueue_scan for kevent entry points.
7412 * Assumes we inherit a use/ref count on the kq or its fileglob.
7414 * This is called by kqueue_scan if neither KEVENT_FLAG_POLL nor
7415 * KEVENT_FLAG_KERNEL was set, and the caller had to wait.
7417 OS_NORETURN OS_NOINLINE
7419 kqueue_scan_continue(void *data
, wait_result_t wait_result
)
7421 uthread_t ut
= current_uthread();
7422 kevent_ctx_t kectx
= &ut
->uu_save
.uus_kevent
;
7423 int error
= 0, flags
= kectx
->kec_process_flags
;
7424 struct kqueue
*kq
= data
;
7427 * only kevent variants call in here, so we know the callback is
7428 * kevent_legacy_callback or kevent_modern_callback.
7430 assert((flags
& (KEVENT_FLAG_POLL
| KEVENT_FLAG_KERNEL
)) == 0);
7432 switch (wait_result
) {
7433 case THREAD_AWAKENED
:
7434 if (__improbable(flags
& (KEVENT_FLAG_LEGACY32
| KEVENT_FLAG_LEGACY64
))) {
7435 error
= kqueue_scan(kq
, flags
, kectx
, kevent_legacy_callback
);
7437 error
= kqueue_scan(kq
, flags
, kectx
, kevent_modern_callback
);
7440 case THREAD_TIMED_OUT
:
7443 case THREAD_INTERRUPTED
:
7446 case THREAD_RESTART
:
7450 panic("%s: - invalid wait_result (%d)", __func__
, wait_result
);
7454 error
= kevent_cleanup(kq
, flags
, error
, kectx
);
7455 *(int32_t *)&ut
->uu_rval
= kectx
->kec_process_noutputs
;
7456 unix_syscall_return(error
);
7460 * @function kqueue_scan
7463 * Scan and wait for events in a kqueue (used by poll & kevent).
7466 * Process the triggered events in a kqueue.
7468 * If there are no events triggered arrange to wait for them:
7469 * - unless KEVENT_FLAG_IMMEDIATE is set in kectx->kec_process_flags
7470 * - possibly until kectx->kec_deadline expires
7472 * When it waits, and that neither KEVENT_FLAG_POLL nor KEVENT_FLAG_KERNEL
7473 * are set, then it will wait in the kqueue_scan_continue continuation.
7475 * poll() will block in place, and KEVENT_FLAG_KERNEL calls
7476 * all pass KEVENT_FLAG_IMMEDIATE and will not wait.
7479 * The kqueue being scanned.
7482 * The KEVENT_FLAG_* flags for this call.
7485 * The context used for this scan.
7486 * The uthread_t::uu_save.uus_kevent storage is used for this purpose.
7489 * The callback to be called on events sucessfully processed.
7490 * (Either kevent_legacy_callback, kevent_modern_callback or poll_callback)
7493 kqueue_scan(struct kqueue
*kq
, int flags
, kevent_ctx_t kectx
,
7494 kevent_callback_t callback
)
7500 error
= kqueue_process(kq
, flags
, kectx
, callback
);
7503 * If we got an error, events returned (EWOULDBLOCK)
7504 * or blocking was disallowed (KEVENT_FLAG_IMMEDIATE),
7507 if (__probable(error
|| (flags
& KEVENT_FLAG_IMMEDIATE
))) {
7509 return error
== EWOULDBLOCK
? 0 : error
;
7512 waitq_assert_wait64_leeway((struct waitq
*)&kq
->kq_wqs
,
7513 KQ_EVENT
, THREAD_ABORTSAFE
, TIMEOUT_URGENCY_USER_NORMAL
,
7514 kectx
->kec_deadline
, TIMEOUT_NO_LEEWAY
);
7515 kq
->kq_state
|= KQ_SLEEP
;
7519 if (__probable((flags
& (KEVENT_FLAG_POLL
| KEVENT_FLAG_KERNEL
)) == 0)) {
7520 thread_block_parameter(kqueue_scan_continue
, kq
);
7521 __builtin_unreachable();
7524 wait_result_t wr
= thread_block(THREAD_CONTINUE_NULL
);
7526 case THREAD_AWAKENED
:
7528 case THREAD_TIMED_OUT
:
7530 case THREAD_INTERRUPTED
:
7532 case THREAD_RESTART
:
7535 panic("%s: - bad wait_result (%d)", __func__
, wr
);
7541 * @function kevent_internal
7544 * Common kevent code.
7547 * Needs to be inlined to specialize for legacy or modern and
7548 * eliminate dead code.
7550 * This is the core logic of kevent entry points, that will:
7551 * - register kevents
7552 * - optionally scan the kqueue for events
7554 * The caller is giving kevent_internal a reference on the kqueue
7555 * or its fileproc that needs to be cleaned up by kevent_cleanup().
7559 kevent_internal(kqueue_t kqu
,
7560 user_addr_t changelist
, int nchanges
,
7561 user_addr_t ueventlist
, int nevents
,
7562 int flags
, kevent_ctx_t kectx
, int32_t *retval
,
7565 int error
= 0, noutputs
= 0, register_rc
;
7567 /* only bound threads can receive events on workloops */
7568 if (!legacy
&& (flags
& KEVENT_FLAG_WORKLOOP
)) {
7569 #if CONFIG_WORKLOOP_DEBUG
7570 UU_KEVENT_HISTORY_WRITE_ENTRY(current_uthread(), {
7571 .uu_kqid
= kqu
.kqwl
->kqwl_dynamicid
,
7572 .uu_kq
= error
? NULL
: kqu
.kq
,
7574 .uu_nchanges
= nchanges
,
7575 .uu_nevents
= nevents
,
7578 #endif // CONFIG_WORKLOOP_DEBUG
7580 if (flags
& KEVENT_FLAG_KERNEL
) {
7581 /* see kevent_workq_internal */
7582 error
= copyout(&kqu
.kqwl
->kqwl_dynamicid
,
7583 ueventlist
- sizeof(kqueue_id_t
), sizeof(kqueue_id_t
));
7584 kectx
->kec_data_resid
-= sizeof(kqueue_id_t
);
7585 if (__improbable(error
)) {
7590 if (kevent_args_requesting_events(flags
, nevents
)) {
7592 * Disable the R2K notification while doing a register, if the
7593 * caller wants events too, we don't want the AST to be set if we
7594 * will process these events soon.
7597 kqu
.kq
->kq_state
&= ~KQ_R2K_ARMED
;
7599 flags
|= KEVENT_FLAG_NEEDS_END_PROCESSING
;
7603 /* register all the change requests the user provided... */
7604 while (nchanges
> 0 && error
== 0) {
7605 struct kevent_qos_s kev
;
7606 struct knote
*kn
= NULL
;
7609 error
= kevent_legacy_copyin(&changelist
, &kev
, flags
);
7611 error
= kevent_modern_copyin(&changelist
, &kev
);
7617 register_rc
= kevent_register(kqu
.kq
, &kev
, &kn
);
7618 if (__improbable(!legacy
&& (register_rc
& FILTER_REGISTER_WAIT
))) {
7619 thread_t thread
= current_thread();
7623 if (act_clear_astkevent(thread
, AST_KEVENT_REDRIVE_THREADREQ
)) {
7624 workq_kern_threadreq_redrive(kqu
.kq
->kq_p
, WORKQ_THREADREQ_NONE
);
7627 // f_post_register_wait is meant to call a continuation and not to
7628 // return, which is why we don't support FILTER_REGISTER_WAIT if
7629 // KEVENT_FLAG_ERROR_EVENTS is not passed, or if the event that
7630 // waits isn't the last.
7632 // It is implementable, but not used by any userspace code at the
7633 // moment, so for now return ENOTSUP if someone tries to do it.
7634 if (nchanges
== 1 && noutputs
< nevents
&&
7635 (flags
& KEVENT_FLAG_KERNEL
) == 0 &&
7636 (flags
& KEVENT_FLAG_PARKING
) == 0 &&
7637 (flags
& KEVENT_FLAG_ERROR_EVENTS
) &&
7638 (flags
& KEVENT_FLAG_WORKLOOP
)) {
7639 uthread_t ut
= get_bsdthread_info(thread
);
7642 * store the continuation/completion data in the uthread
7644 * Note: the kectx aliases with this,
7645 * and is destroyed in the process.
7647 ut
->uu_save
.uus_kevent_register
= (struct _kevent_register
){
7650 .eventout
= noutputs
,
7651 .ueventlist
= ueventlist
,
7653 knote_fops(kn
)->f_post_register_wait(ut
, kn
,
7654 &ut
->uu_save
.uus_kevent_register
);
7655 __builtin_unreachable();
7659 kev
.flags
|= EV_ERROR
;
7662 assert((register_rc
& FILTER_REGISTER_WAIT
) == 0);
7665 // keep in sync with kevent_register_wait_return()
7666 if (noutputs
< nevents
&& (kev
.flags
& (EV_ERROR
| EV_RECEIPT
))) {
7667 if ((kev
.flags
& EV_ERROR
) == 0) {
7668 kev
.flags
|= EV_ERROR
;
7672 error
= kevent_legacy_copyout(&kev
, &ueventlist
, flags
);
7674 error
= kevent_modern_copyout(&kev
, &ueventlist
);
7679 } else if (kev
.flags
& EV_ERROR
) {
7685 if ((flags
& KEVENT_FLAG_ERROR_EVENTS
) == 0 &&
7686 nevents
> 0 && noutputs
== 0 && error
== 0) {
7687 kectx
->kec_process_flags
= flags
;
7688 kectx
->kec_process_nevents
= nevents
;
7689 kectx
->kec_process_noutputs
= 0;
7690 kectx
->kec_process_eventlist
= ueventlist
;
7693 error
= kqueue_scan(kqu
.kq
, flags
, kectx
, kevent_legacy_callback
);
7695 error
= kqueue_scan(kqu
.kq
, flags
, kectx
, kevent_modern_callback
);
7698 noutputs
= kectx
->kec_process_noutputs
;
7699 } else if (!legacy
&& (flags
& KEVENT_FLAG_NEEDS_END_PROCESSING
)) {
7701 * If we didn't through kqworkloop_end_processing(),
7702 * we need to do it here.
7704 * kqueue_scan will call kqworkloop_end_processing(),
7705 * so we only need to do it if we didn't scan.
7708 kqworkloop_end_processing(kqu
.kqwl
, 0, 0);
7714 return kevent_cleanup(kqu
.kq
, flags
, error
, kectx
);
7717 #pragma mark modern syscalls: kevent_qos, kevent_id, kevent_workq_internal
7720 * @function kevent_modern_internal
7723 * The backend of the kevent_id and kevent_workq_internal entry points.
7726 * Needs to be inline due to the number of arguments.
7730 kevent_modern_internal(kqueue_t kqu
,
7731 user_addr_t changelist
, int nchanges
,
7732 user_addr_t ueventlist
, int nevents
,
7733 int flags
, kevent_ctx_t kectx
, int32_t *retval
)
7735 return kevent_internal(kqu
.kq
, changelist
, nchanges
,
7736 ueventlist
, nevents
, flags
, kectx
, retval
, /*legacy*/ false);
7740 * @function kevent_id
7743 * The kevent_id() syscall.
7746 kevent_id(struct proc
*p
, struct kevent_id_args
*uap
, int32_t *retval
)
7748 int error
, flags
= uap
->flags
& KEVENT_FLAG_USER
;
7749 uthread_t uth
= current_uthread();
7750 workq_threadreq_t kqr
= uth
->uu_kqr_bound
;
7751 kevent_ctx_t kectx
= &uth
->uu_save
.uus_kevent
;
7754 flags
= kevent_adjust_flags_for_proc(p
, flags
);
7755 flags
|= KEVENT_FLAG_DYNAMIC_KQUEUE
;
7757 if (__improbable((flags
& (KEVENT_FLAG_WORKQ
| KEVENT_FLAG_WORKLOOP
)) !=
7758 KEVENT_FLAG_WORKLOOP
)) {
7762 error
= kevent_get_data_size(flags
, uap
->data_available
, uap
->data_out
, kectx
);
7763 if (__improbable(error
)) {
7767 kectx
->kec_deadline
= 0;
7768 kectx
->kec_fp
= NULL
;
7770 /* the kec_process_* fields are filled if kqueue_scann is called only */
7773 * Get the kq we are going to be working on
7774 * As a fastpath, look at the currently bound workloop.
7776 kqu
.kqwl
= kqr
? kqr_kqworkloop(kqr
) : NULL
;
7777 if (kqu
.kqwl
&& kqu
.kqwl
->kqwl_dynamicid
== uap
->id
) {
7778 if (__improbable(flags
& KEVENT_FLAG_DYNAMIC_KQ_MUST_NOT_EXIST
)) {
7781 kqworkloop_retain(kqu
.kqwl
);
7782 } else if (__improbable(kevent_args_requesting_events(flags
, uap
->nevents
))) {
7785 error
= kqworkloop_get_or_create(p
, uap
->id
, NULL
, flags
, &kqu
.kqwl
);
7786 if (__improbable(error
)) {
7791 return kevent_modern_internal(kqu
, uap
->changelist
, uap
->nchanges
,
7792 uap
->eventlist
, uap
->nevents
, flags
, kectx
, retval
);
7796 * @function kevent_workq_internal
7799 * This function is exported for the sake of the workqueue subsystem.
7801 * It is called in two ways:
7802 * - when a thread is about to go to userspace to ask for pending event
7803 * - when a thread is returning from userspace with events back
7805 * the workqueue subsystem will only use the following flags:
7806 * - KEVENT_FLAG_STACK_DATA (always)
7807 * - KEVENT_FLAG_IMMEDIATE (always)
7808 * - KEVENT_FLAG_PARKING (depending on whether it is going to or returning from
7811 * It implicitly acts on the bound kqueue, and for the case of workloops
7812 * will copyout the kqueue ID before anything else.
7815 * Pthread will have setup the various arguments to fit this stack layout:
7817 * +-------....----+--------------+-----------+--------------------+
7818 * | user stack | data avail | nevents | pthread_self() |
7819 * +-------....----+--------------+-----------+--------------------+
7821 * data_out eventlist
7823 * When a workloop is used, the workloop ID is copied out right before
7824 * the eventlist and is taken from the data buffer.
7827 * This function is carefuly tailored to not make any call except the final tail
7828 * call into kevent_modern_internal. (LTO inlines current_uthread()).
7830 * This function is performance sensitive due to the workq subsystem.
7833 kevent_workq_internal(struct proc
*p
,
7834 user_addr_t changelist
, int nchanges
,
7835 user_addr_t eventlist
, int nevents
,
7836 user_addr_t data_out
, user_size_t
*data_available
,
7837 unsigned int flags
, int32_t *retval
)
7839 uthread_t uth
= current_uthread();
7840 workq_threadreq_t kqr
= uth
->uu_kqr_bound
;
7841 kevent_ctx_t kectx
= &uth
->uu_save
.uus_kevent
;
7844 assert(flags
== (KEVENT_FLAG_STACK_DATA
| KEVENT_FLAG_IMMEDIATE
) ||
7845 flags
== (KEVENT_FLAG_STACK_DATA
| KEVENT_FLAG_IMMEDIATE
| KEVENT_FLAG_PARKING
));
7847 kectx
->kec_data_out
= data_out
;
7848 kectx
->kec_data_avail
= (uint64_t)data_available
;
7849 kectx
->kec_data_size
= *data_available
;
7850 kectx
->kec_data_resid
= *data_available
;
7851 kectx
->kec_deadline
= 0;
7852 kectx
->kec_fp
= NULL
;
7854 /* the kec_process_* fields are filled if kqueue_scann is called only */
7856 flags
= kevent_adjust_flags_for_proc(p
, flags
);
7858 if (kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
) {
7859 kqu
.kqwl
= __container_of(kqr
, struct kqworkloop
, kqwl_request
);
7860 kqworkloop_retain(kqu
.kqwl
);
7862 flags
|= KEVENT_FLAG_WORKLOOP
| KEVENT_FLAG_DYNAMIC_KQUEUE
|
7865 kqu
.kqwq
= p
->p_fd
->fd_wqkqueue
;
7867 flags
|= KEVENT_FLAG_WORKQ
| KEVENT_FLAG_KERNEL
;
7870 return kevent_modern_internal(kqu
, changelist
, nchanges
,
7871 eventlist
, nevents
, flags
, kectx
, retval
);
7875 * @function kevent_qos
7878 * The kevent_qos() syscall.
7881 kevent_qos(struct proc
*p
, struct kevent_qos_args
*uap
, int32_t *retval
)
7883 uthread_t uth
= current_uthread();
7884 kevent_ctx_t kectx
= &uth
->uu_save
.uus_kevent
;
7885 int error
, flags
= uap
->flags
& KEVENT_FLAG_USER
;
7888 if (__improbable(flags
& KEVENT_ID_FLAG_USER
)) {
7892 flags
= kevent_adjust_flags_for_proc(p
, flags
);
7894 error
= kevent_get_data_size(flags
, uap
->data_available
, uap
->data_out
, kectx
);
7895 if (__improbable(error
)) {
7899 kectx
->kec_deadline
= 0;
7900 kectx
->kec_fp
= NULL
;
7901 kectx
->kec_fd
= uap
->fd
;
7902 /* the kec_process_* fields are filled if kqueue_scann is called only */
7904 /* get the kq we are going to be working on */
7905 if (__probable(flags
& KEVENT_FLAG_WORKQ
)) {
7906 error
= kevent_get_kqwq(p
, flags
, uap
->nevents
, &kq
);
7908 error
= kevent_get_kqfile(p
, uap
->fd
, flags
, &kectx
->kec_fp
, &kq
);
7910 if (__improbable(error
)) {
7914 return kevent_modern_internal(kq
, uap
->changelist
, uap
->nchanges
,
7915 uap
->eventlist
, uap
->nevents
, flags
, kectx
, retval
);
7918 #pragma mark legacy syscalls: kevent, kevent64
7921 * @function kevent_legacy_get_deadline
7924 * Compute the deadline for the legacy kevent syscalls.
7927 * This is not necessary if KEVENT_FLAG_IMMEDIATE is specified,
7928 * as this takes precedence over the deadline.
7930 * This function will fail if utimeout is USER_ADDR_NULL
7931 * (the caller should check).
7934 kevent_legacy_get_deadline(int flags
, user_addr_t utimeout
, uint64_t *deadline
)
7938 if (flags
& KEVENT_FLAG_PROC64
) {
7939 struct user64_timespec ts64
;
7940 int error
= copyin(utimeout
, &ts64
, sizeof(ts64
));
7941 if (__improbable(error
)) {
7944 ts
.tv_sec
= ts64
.tv_sec
;
7945 ts
.tv_nsec
= ts64
.tv_nsec
;
7947 struct user32_timespec ts32
;
7948 int error
= copyin(utimeout
, &ts32
, sizeof(ts32
));
7949 if (__improbable(error
)) {
7952 ts
.tv_sec
= ts32
.tv_sec
;
7953 ts
.tv_nsec
= ts32
.tv_nsec
;
7955 if (!timespec_is_valid(&ts
)) {
7959 clock_absolutetime_interval_to_deadline(tstoabstime(&ts
), deadline
);
7964 * @function kevent_legacy_internal
7967 * The core implementation for kevent and kevent64
7971 kevent_legacy_internal(struct proc
*p
, struct kevent64_args
*uap
,
7972 int32_t *retval
, int flags
)
7974 uthread_t uth
= current_uthread();
7975 kevent_ctx_t kectx
= &uth
->uu_save
.uus_kevent
;
7979 if (__improbable(uap
->flags
& KEVENT_ID_FLAG_USER
)) {
7983 flags
= kevent_adjust_flags_for_proc(p
, flags
);
7985 kectx
->kec_data_out
= 0;
7986 kectx
->kec_data_avail
= 0;
7987 kectx
->kec_data_size
= 0;
7988 kectx
->kec_data_resid
= 0;
7989 kectx
->kec_deadline
= 0;
7990 kectx
->kec_fp
= NULL
;
7991 kectx
->kec_fd
= uap
->fd
;
7992 /* the kec_process_* fields are filled if kqueue_scann is called only */
7994 /* convert timeout to absolute - if we have one (and not immediate) */
7995 if (__improbable(uap
->timeout
&& !(flags
& KEVENT_FLAG_IMMEDIATE
))) {
7996 error
= kevent_legacy_get_deadline(flags
, uap
->timeout
,
7997 &kectx
->kec_deadline
);
7998 if (__improbable(error
)) {
8003 /* get the kq we are going to be working on */
8004 if (flags
& KEVENT_FLAG_WORKQ
) {
8005 error
= kevent_get_kqwq(p
, flags
, uap
->nevents
, &kq
);
8007 error
= kevent_get_kqfile(p
, uap
->fd
, flags
, &kectx
->kec_fp
, &kq
);
8009 if (__improbable(error
)) {
8013 return kevent_internal(kq
, uap
->changelist
, uap
->nchanges
,
8014 uap
->eventlist
, uap
->nevents
, flags
, kectx
, retval
,
8022 * The legacy kevent() syscall.
8025 kevent(struct proc
*p
, struct kevent_args
*uap
, int32_t *retval
)
8027 struct kevent64_args args
= {
8029 .changelist
= uap
->changelist
,
8030 .nchanges
= uap
->nchanges
,
8031 .eventlist
= uap
->eventlist
,
8032 .nevents
= uap
->nevents
,
8033 .timeout
= uap
->timeout
,
8036 return kevent_legacy_internal(p
, &args
, retval
, KEVENT_FLAG_LEGACY32
);
8040 * @function kevent64
8043 * The legacy kevent64() syscall.
8046 kevent64(struct proc
*p
, struct kevent64_args
*uap
, int32_t *retval
)
8048 int flags
= (uap
->flags
& KEVENT_FLAG_USER
) | KEVENT_FLAG_LEGACY64
;
8049 return kevent_legacy_internal(p
, uap
, retval
, flags
);
8052 #pragma mark - socket interface
8055 #include <sys/param.h>
8056 #include <sys/socket.h>
8057 #include <sys/protosw.h>
8058 #include <sys/domain.h>
8059 #include <sys/mbuf.h>
8060 #include <sys/kern_event.h>
8061 #include <sys/malloc.h>
8062 #include <sys/sys_domain.h>
8063 #include <sys/syslog.h>
8066 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
8070 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
8073 static lck_grp_attr_t
*kev_lck_grp_attr
;
8074 static lck_attr_t
*kev_lck_attr
;
8075 static lck_grp_t
*kev_lck_grp
;
8076 static decl_lck_rw_data(, kev_lck_data
);
8077 static lck_rw_t
*kev_rwlock
= &kev_lck_data
;
8079 static int kev_attach(struct socket
*so
, int proto
, struct proc
*p
);
8080 static int kev_detach(struct socket
*so
);
8081 static int kev_control(struct socket
*so
, u_long cmd
, caddr_t data
,
8082 struct ifnet
*ifp
, struct proc
*p
);
8083 static lck_mtx_t
* event_getlock(struct socket
*, int);
8084 static int event_lock(struct socket
*, int, void *);
8085 static int event_unlock(struct socket
*, int, void *);
8087 static int event_sofreelastref(struct socket
*);
8088 static void kev_delete(struct kern_event_pcb
*);
8090 static struct pr_usrreqs event_usrreqs
= {
8091 .pru_attach
= kev_attach
,
8092 .pru_control
= kev_control
,
8093 .pru_detach
= kev_detach
,
8094 .pru_soreceive
= soreceive
,
8097 static struct protosw eventsw
[] = {
8099 .pr_type
= SOCK_RAW
,
8100 .pr_protocol
= SYSPROTO_EVENT
,
8101 .pr_flags
= PR_ATOMIC
,
8102 .pr_usrreqs
= &event_usrreqs
,
8103 .pr_lock
= event_lock
,
8104 .pr_unlock
= event_unlock
,
8105 .pr_getlock
= event_getlock
,
8109 __private_extern__
int kevt_getstat SYSCTL_HANDLER_ARGS
;
8110 __private_extern__
int kevt_pcblist SYSCTL_HANDLER_ARGS
;
8112 SYSCTL_NODE(_net_systm
, OID_AUTO
, kevt
,
8113 CTLFLAG_RW
| CTLFLAG_LOCKED
, 0, "Kernel event family");
8115 struct kevtstat kevtstat
;
8116 SYSCTL_PROC(_net_systm_kevt
, OID_AUTO
, stats
,
8117 CTLTYPE_STRUCT
| CTLFLAG_RD
| CTLFLAG_LOCKED
, 0, 0,
8118 kevt_getstat
, "S,kevtstat", "");
8120 SYSCTL_PROC(_net_systm_kevt
, OID_AUTO
, pcblist
,
8121 CTLTYPE_STRUCT
| CTLFLAG_RD
| CTLFLAG_LOCKED
, 0, 0,
8122 kevt_pcblist
, "S,xkevtpcb", "");
8125 event_getlock(struct socket
*so
, int flags
)
8127 #pragma unused(flags)
8128 struct kern_event_pcb
*ev_pcb
= (struct kern_event_pcb
*)so
->so_pcb
;
8130 if (so
->so_pcb
!= NULL
) {
8131 if (so
->so_usecount
< 0) {
8132 panic("%s: so=%p usecount=%d lrh= %s\n", __func__
,
8133 so
, so
->so_usecount
, solockhistory_nr(so
));
8137 panic("%s: so=%p NULL NO so_pcb %s\n", __func__
,
8138 so
, solockhistory_nr(so
));
8141 return &ev_pcb
->evp_mtx
;
8145 event_lock(struct socket
*so
, int refcount
, void *lr
)
8150 lr_saved
= __builtin_return_address(0);
8155 if (so
->so_pcb
!= NULL
) {
8156 lck_mtx_lock(&((struct kern_event_pcb
*)so
->so_pcb
)->evp_mtx
);
8158 panic("%s: so=%p NO PCB! lr=%p lrh= %s\n", __func__
,
8159 so
, lr_saved
, solockhistory_nr(so
));
8163 if (so
->so_usecount
< 0) {
8164 panic("%s: so=%p so_pcb=%p lr=%p ref=%d lrh= %s\n", __func__
,
8165 so
, so
->so_pcb
, lr_saved
, so
->so_usecount
,
8166 solockhistory_nr(so
));
8174 so
->lock_lr
[so
->next_lock_lr
] = lr_saved
;
8175 so
->next_lock_lr
= (so
->next_lock_lr
+ 1) % SO_LCKDBG_MAX
;
8180 event_unlock(struct socket
*so
, int refcount
, void *lr
)
8183 lck_mtx_t
*mutex_held
;
8186 lr_saved
= __builtin_return_address(0);
8194 if (so
->so_usecount
< 0) {
8195 panic("%s: so=%p usecount=%d lrh= %s\n", __func__
,
8196 so
, so
->so_usecount
, solockhistory_nr(so
));
8199 if (so
->so_pcb
== NULL
) {
8200 panic("%s: so=%p NO PCB usecount=%d lr=%p lrh= %s\n", __func__
,
8201 so
, so
->so_usecount
, (void *)lr_saved
,
8202 solockhistory_nr(so
));
8205 mutex_held
= (&((struct kern_event_pcb
*)so
->so_pcb
)->evp_mtx
);
8207 LCK_MTX_ASSERT(mutex_held
, LCK_MTX_ASSERT_OWNED
);
8208 so
->unlock_lr
[so
->next_unlock_lr
] = lr_saved
;
8209 so
->next_unlock_lr
= (so
->next_unlock_lr
+ 1) % SO_LCKDBG_MAX
;
8211 if (so
->so_usecount
== 0) {
8212 VERIFY(so
->so_flags
& SOF_PCBCLEARING
);
8213 event_sofreelastref(so
);
8215 lck_mtx_unlock(mutex_held
);
8222 event_sofreelastref(struct socket
*so
)
8224 struct kern_event_pcb
*ev_pcb
= (struct kern_event_pcb
*)so
->so_pcb
;
8226 LCK_MTX_ASSERT(&(ev_pcb
->evp_mtx
), LCK_MTX_ASSERT_OWNED
);
8231 * Disable upcall in the event another thread is in kev_post_msg()
8232 * appending record to the receive socket buffer, since sbwakeup()
8233 * may release the socket lock otherwise.
8235 so
->so_rcv
.sb_flags
&= ~SB_UPCALL
;
8236 so
->so_snd
.sb_flags
&= ~SB_UPCALL
;
8237 so
->so_event
= sonullevent
;
8238 lck_mtx_unlock(&(ev_pcb
->evp_mtx
));
8240 LCK_MTX_ASSERT(&(ev_pcb
->evp_mtx
), LCK_MTX_ASSERT_NOTOWNED
);
8241 lck_rw_lock_exclusive(kev_rwlock
);
8242 LIST_REMOVE(ev_pcb
, evp_link
);
8243 kevtstat
.kes_pcbcount
--;
8244 kevtstat
.kes_gencnt
++;
8245 lck_rw_done(kev_rwlock
);
8248 sofreelastref(so
, 1);
8252 static int event_proto_count
= (sizeof(eventsw
) / sizeof(struct protosw
));
8255 struct kern_event_head kern_event_head
;
8257 static u_int32_t static_event_id
= 0;
8259 #define EVPCB_ZONE_MAX 65536
8260 #define EVPCB_ZONE_NAME "kerneventpcb"
8261 static struct zone
*ev_pcb_zone
;
8264 * Install the protosw's for the NKE manager. Invoked at extension load time
8267 kern_event_init(struct domain
*dp
)
8272 VERIFY(!(dp
->dom_flags
& DOM_INITIALIZED
));
8273 VERIFY(dp
== systemdomain
);
8275 kev_lck_grp_attr
= lck_grp_attr_alloc_init();
8276 if (kev_lck_grp_attr
== NULL
) {
8277 panic("%s: lck_grp_attr_alloc_init failed\n", __func__
);
8281 kev_lck_grp
= lck_grp_alloc_init("Kernel Event Protocol",
8283 if (kev_lck_grp
== NULL
) {
8284 panic("%s: lck_grp_alloc_init failed\n", __func__
);
8288 kev_lck_attr
= lck_attr_alloc_init();
8289 if (kev_lck_attr
== NULL
) {
8290 panic("%s: lck_attr_alloc_init failed\n", __func__
);
8294 lck_rw_init(kev_rwlock
, kev_lck_grp
, kev_lck_attr
);
8295 if (kev_rwlock
== NULL
) {
8296 panic("%s: lck_mtx_alloc_init failed\n", __func__
);
8300 for (i
= 0, pr
= &eventsw
[0]; i
< event_proto_count
; i
++, pr
++) {
8301 net_add_proto(pr
, dp
, 1);
8304 ev_pcb_zone
= zinit(sizeof(struct kern_event_pcb
),
8305 EVPCB_ZONE_MAX
* sizeof(struct kern_event_pcb
), 0, EVPCB_ZONE_NAME
);
8306 if (ev_pcb_zone
== NULL
) {
8307 panic("%s: failed allocating ev_pcb_zone", __func__
);
8310 zone_change(ev_pcb_zone
, Z_EXPAND
, TRUE
);
8311 zone_change(ev_pcb_zone
, Z_CALLERACCT
, TRUE
);
8315 kev_attach(struct socket
*so
, __unused
int proto
, __unused
struct proc
*p
)
8318 struct kern_event_pcb
*ev_pcb
;
8320 error
= soreserve(so
, KEV_SNDSPACE
, KEV_RECVSPACE
);
8325 if ((ev_pcb
= (struct kern_event_pcb
*)zalloc(ev_pcb_zone
)) == NULL
) {
8328 bzero(ev_pcb
, sizeof(struct kern_event_pcb
));
8329 lck_mtx_init(&ev_pcb
->evp_mtx
, kev_lck_grp
, kev_lck_attr
);
8331 ev_pcb
->evp_socket
= so
;
8332 ev_pcb
->evp_vendor_code_filter
= 0xffffffff;
8334 so
->so_pcb
= (caddr_t
) ev_pcb
;
8335 lck_rw_lock_exclusive(kev_rwlock
);
8336 LIST_INSERT_HEAD(&kern_event_head
, ev_pcb
, evp_link
);
8337 kevtstat
.kes_pcbcount
++;
8338 kevtstat
.kes_gencnt
++;
8339 lck_rw_done(kev_rwlock
);
8345 kev_delete(struct kern_event_pcb
*ev_pcb
)
8347 VERIFY(ev_pcb
!= NULL
);
8348 lck_mtx_destroy(&ev_pcb
->evp_mtx
, kev_lck_grp
);
8349 zfree(ev_pcb_zone
, ev_pcb
);
8353 kev_detach(struct socket
*so
)
8355 struct kern_event_pcb
*ev_pcb
= (struct kern_event_pcb
*) so
->so_pcb
;
8357 if (ev_pcb
!= NULL
) {
8358 soisdisconnected(so
);
8359 so
->so_flags
|= SOF_PCBCLEARING
;
8366 * For now, kev_vendor_code and mbuf_tags use the same
8370 kev_vendor_code_find(
8372 u_int32_t
*out_vendor_code
)
8374 if (strlen(string
) >= KEV_VENDOR_CODE_MAX_STR_LEN
) {
8377 return net_str_id_find_internal(string
, out_vendor_code
,
8378 NSI_VENDOR_CODE
, 1);
8382 kev_msg_post(struct kev_msg
*event_msg
)
8384 mbuf_tag_id_t min_vendor
, max_vendor
;
8386 net_str_id_first_last(&min_vendor
, &max_vendor
, NSI_VENDOR_CODE
);
8388 if (event_msg
== NULL
) {
8393 * Limit third parties to posting events for registered vendor codes
8396 if (event_msg
->vendor_code
< min_vendor
||
8397 event_msg
->vendor_code
> max_vendor
) {
8398 os_atomic_inc(&kevtstat
.kes_badvendor
, relaxed
);
8401 return kev_post_msg(event_msg
);
8405 kev_post_msg(struct kev_msg
*event_msg
)
8407 struct mbuf
*m
, *m2
;
8408 struct kern_event_pcb
*ev_pcb
;
8409 struct kern_event_msg
*ev
;
8411 u_int32_t total_size
;
8414 /* Verify the message is small enough to fit in one mbuf w/o cluster */
8415 total_size
= KEV_MSG_HEADER_SIZE
;
8417 for (i
= 0; i
< 5; i
++) {
8418 if (event_msg
->dv
[i
].data_length
== 0) {
8421 total_size
+= event_msg
->dv
[i
].data_length
;
8424 if (total_size
> MLEN
) {
8425 os_atomic_inc(&kevtstat
.kes_toobig
, relaxed
);
8429 m
= m_get(M_WAIT
, MT_DATA
);
8431 os_atomic_inc(&kevtstat
.kes_nomem
, relaxed
);
8434 ev
= mtod(m
, struct kern_event_msg
*);
8435 total_size
= KEV_MSG_HEADER_SIZE
;
8437 tmp
= (char *) &ev
->event_data
[0];
8438 for (i
= 0; i
< 5; i
++) {
8439 if (event_msg
->dv
[i
].data_length
== 0) {
8443 total_size
+= event_msg
->dv
[i
].data_length
;
8444 bcopy(event_msg
->dv
[i
].data_ptr
, tmp
,
8445 event_msg
->dv
[i
].data_length
);
8446 tmp
+= event_msg
->dv
[i
].data_length
;
8449 ev
->id
= ++static_event_id
;
8450 ev
->total_size
= total_size
;
8451 ev
->vendor_code
= event_msg
->vendor_code
;
8452 ev
->kev_class
= event_msg
->kev_class
;
8453 ev
->kev_subclass
= event_msg
->kev_subclass
;
8454 ev
->event_code
= event_msg
->event_code
;
8456 m
->m_len
= total_size
;
8457 lck_rw_lock_shared(kev_rwlock
);
8458 for (ev_pcb
= LIST_FIRST(&kern_event_head
);
8460 ev_pcb
= LIST_NEXT(ev_pcb
, evp_link
)) {
8461 lck_mtx_lock(&ev_pcb
->evp_mtx
);
8462 if (ev_pcb
->evp_socket
->so_pcb
== NULL
) {
8463 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8466 if (ev_pcb
->evp_vendor_code_filter
!= KEV_ANY_VENDOR
) {
8467 if (ev_pcb
->evp_vendor_code_filter
!= ev
->vendor_code
) {
8468 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8472 if (ev_pcb
->evp_class_filter
!= KEV_ANY_CLASS
) {
8473 if (ev_pcb
->evp_class_filter
!= ev
->kev_class
) {
8474 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8478 if ((ev_pcb
->evp_subclass_filter
!=
8479 KEV_ANY_SUBCLASS
) &&
8480 (ev_pcb
->evp_subclass_filter
!=
8481 ev
->kev_subclass
)) {
8482 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8488 m2
= m_copym(m
, 0, m
->m_len
, M_WAIT
);
8490 os_atomic_inc(&kevtstat
.kes_nomem
, relaxed
);
8492 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8493 lck_rw_done(kev_rwlock
);
8496 if (sbappendrecord(&ev_pcb
->evp_socket
->so_rcv
, m2
)) {
8498 * We use "m" for the socket stats as it would be
8499 * unsafe to use "m2"
8501 so_inc_recv_data_stat(ev_pcb
->evp_socket
,
8502 1, m
->m_len
, MBUF_TC_BE
);
8504 sorwakeup(ev_pcb
->evp_socket
);
8505 os_atomic_inc(&kevtstat
.kes_posted
, relaxed
);
8507 os_atomic_inc(&kevtstat
.kes_fullsock
, relaxed
);
8509 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8512 lck_rw_done(kev_rwlock
);
8518 kev_control(struct socket
*so
,
8521 __unused
struct ifnet
*ifp
,
8522 __unused
struct proc
*p
)
8524 struct kev_request
*kev_req
= (struct kev_request
*) data
;
8525 struct kern_event_pcb
*ev_pcb
;
8526 struct kev_vendor_code
*kev_vendor
;
8527 u_int32_t
*id_value
= (u_int32_t
*) data
;
8531 *id_value
= static_event_id
;
8534 ev_pcb
= (struct kern_event_pcb
*) so
->so_pcb
;
8535 ev_pcb
->evp_vendor_code_filter
= kev_req
->vendor_code
;
8536 ev_pcb
->evp_class_filter
= kev_req
->kev_class
;
8537 ev_pcb
->evp_subclass_filter
= kev_req
->kev_subclass
;
8540 ev_pcb
= (struct kern_event_pcb
*) so
->so_pcb
;
8541 kev_req
->vendor_code
= ev_pcb
->evp_vendor_code_filter
;
8542 kev_req
->kev_class
= ev_pcb
->evp_class_filter
;
8543 kev_req
->kev_subclass
= ev_pcb
->evp_subclass_filter
;
8545 case SIOCGKEVVENDOR
:
8546 kev_vendor
= (struct kev_vendor_code
*)data
;
8547 /* Make sure string is NULL terminated */
8548 kev_vendor
->vendor_string
[KEV_VENDOR_CODE_MAX_STR_LEN
- 1] = 0;
8549 return net_str_id_find_internal(kev_vendor
->vendor_string
,
8550 &kev_vendor
->vendor_code
, NSI_VENDOR_CODE
, 0);
8559 kevt_getstat SYSCTL_HANDLER_ARGS
8561 #pragma unused(oidp, arg1, arg2)
8564 lck_rw_lock_shared(kev_rwlock
);
8566 if (req
->newptr
!= USER_ADDR_NULL
) {
8570 if (req
->oldptr
== USER_ADDR_NULL
) {
8571 req
->oldidx
= sizeof(struct kevtstat
);
8575 error
= SYSCTL_OUT(req
, &kevtstat
,
8576 MIN(sizeof(struct kevtstat
), req
->oldlen
));
8578 lck_rw_done(kev_rwlock
);
8583 __private_extern__
int
8584 kevt_pcblist SYSCTL_HANDLER_ARGS
8586 #pragma unused(oidp, arg1, arg2)
8589 struct xsystmgen xsg
;
8591 size_t item_size
= ROUNDUP64(sizeof(struct xkevtpcb
)) +
8592 ROUNDUP64(sizeof(struct xsocket_n
)) +
8593 2 * ROUNDUP64(sizeof(struct xsockbuf_n
)) +
8594 ROUNDUP64(sizeof(struct xsockstat_n
));
8595 struct kern_event_pcb
*ev_pcb
;
8597 buf
= _MALLOC(item_size
, M_TEMP
, M_WAITOK
| M_ZERO
);
8602 lck_rw_lock_shared(kev_rwlock
);
8604 n
= kevtstat
.kes_pcbcount
;
8606 if (req
->oldptr
== USER_ADDR_NULL
) {
8607 req
->oldidx
= (n
+ n
/ 8) * item_size
;
8610 if (req
->newptr
!= USER_ADDR_NULL
) {
8614 bzero(&xsg
, sizeof(xsg
));
8615 xsg
.xg_len
= sizeof(xsg
);
8617 xsg
.xg_gen
= kevtstat
.kes_gencnt
;
8618 xsg
.xg_sogen
= so_gencnt
;
8619 error
= SYSCTL_OUT(req
, &xsg
, sizeof(xsg
));
8624 * We are done if there is no pcb
8631 for (i
= 0, ev_pcb
= LIST_FIRST(&kern_event_head
);
8632 i
< n
&& ev_pcb
!= NULL
;
8633 i
++, ev_pcb
= LIST_NEXT(ev_pcb
, evp_link
)) {
8634 struct xkevtpcb
*xk
= (struct xkevtpcb
*)buf
;
8635 struct xsocket_n
*xso
= (struct xsocket_n
*)
8636 ADVANCE64(xk
, sizeof(*xk
));
8637 struct xsockbuf_n
*xsbrcv
= (struct xsockbuf_n
*)
8638 ADVANCE64(xso
, sizeof(*xso
));
8639 struct xsockbuf_n
*xsbsnd
= (struct xsockbuf_n
*)
8640 ADVANCE64(xsbrcv
, sizeof(*xsbrcv
));
8641 struct xsockstat_n
*xsostats
= (struct xsockstat_n
*)
8642 ADVANCE64(xsbsnd
, sizeof(*xsbsnd
));
8644 bzero(buf
, item_size
);
8646 lck_mtx_lock(&ev_pcb
->evp_mtx
);
8648 xk
->kep_len
= sizeof(struct xkevtpcb
);
8649 xk
->kep_kind
= XSO_EVT
;
8650 xk
->kep_evtpcb
= (uint64_t)VM_KERNEL_ADDRPERM(ev_pcb
);
8651 xk
->kep_vendor_code_filter
= ev_pcb
->evp_vendor_code_filter
;
8652 xk
->kep_class_filter
= ev_pcb
->evp_class_filter
;
8653 xk
->kep_subclass_filter
= ev_pcb
->evp_subclass_filter
;
8655 sotoxsocket_n(ev_pcb
->evp_socket
, xso
);
8656 sbtoxsockbuf_n(ev_pcb
->evp_socket
?
8657 &ev_pcb
->evp_socket
->so_rcv
: NULL
, xsbrcv
);
8658 sbtoxsockbuf_n(ev_pcb
->evp_socket
?
8659 &ev_pcb
->evp_socket
->so_snd
: NULL
, xsbsnd
);
8660 sbtoxsockstat_n(ev_pcb
->evp_socket
, xsostats
);
8662 lck_mtx_unlock(&ev_pcb
->evp_mtx
);
8664 error
= SYSCTL_OUT(req
, buf
, item_size
);
8669 * Give the user an updated idea of our state.
8670 * If the generation differs from what we told
8671 * her before, she knows that something happened
8672 * while we were processing this request, and it
8673 * might be necessary to retry.
8675 bzero(&xsg
, sizeof(xsg
));
8676 xsg
.xg_len
= sizeof(xsg
);
8678 xsg
.xg_gen
= kevtstat
.kes_gencnt
;
8679 xsg
.xg_sogen
= so_gencnt
;
8680 error
= SYSCTL_OUT(req
, &xsg
, sizeof(xsg
));
8687 lck_rw_done(kev_rwlock
);
8692 #endif /* SOCKETS */
8696 fill_kqueueinfo(struct kqueue
*kq
, struct kqueue_info
* kinfo
)
8698 struct vinfo_stat
* st
;
8700 st
= &kinfo
->kq_stat
;
8702 st
->vst_size
= kq
->kq_count
;
8703 if (kq
->kq_state
& KQ_KEV_QOS
) {
8704 st
->vst_blksize
= sizeof(struct kevent_qos_s
);
8705 } else if (kq
->kq_state
& KQ_KEV64
) {
8706 st
->vst_blksize
= sizeof(struct kevent64_s
);
8708 st
->vst_blksize
= sizeof(struct kevent
);
8710 st
->vst_mode
= S_IFIFO
;
8711 st
->vst_ino
= (kq
->kq_state
& KQ_DYNAMIC
) ?
8712 ((struct kqworkloop
*)kq
)->kqwl_dynamicid
: 0;
8714 /* flags exported to libproc as PROC_KQUEUE_* (sys/proc_info.h) */
8715 #define PROC_KQUEUE_MASK (KQ_SEL|KQ_SLEEP|KQ_KEV32|KQ_KEV64|KQ_KEV_QOS|KQ_WORKQ|KQ_WORKLOOP)
8716 kinfo
->kq_state
= kq
->kq_state
& PROC_KQUEUE_MASK
;
8722 fill_kqueue_dyninfo(struct kqworkloop
*kqwl
, struct kqueue_dyninfo
*kqdi
)
8724 workq_threadreq_t kqr
= &kqwl
->kqwl_request
;
8725 workq_threadreq_param_t trp
= {};
8728 if ((kqwl
->kqwl_state
& KQ_WORKLOOP
) == 0) {
8732 if ((err
= fill_kqueueinfo(&kqwl
->kqwl_kqueue
, &kqdi
->kqdi_info
))) {
8738 kqdi
->kqdi_servicer
= thread_tid(kqr_thread(kqr
));
8739 kqdi
->kqdi_owner
= thread_tid(kqwl
->kqwl_owner
);
8740 kqdi
->kqdi_request_state
= kqr
->tr_state
;
8741 kqdi
->kqdi_async_qos
= kqr
->tr_kq_qos_index
;
8742 kqdi
->kqdi_events_qos
= kqr
->tr_kq_override_index
;
8743 kqdi
->kqdi_sync_waiters
= 0;
8744 kqdi
->kqdi_sync_waiter_qos
= 0;
8746 trp
.trp_value
= kqwl
->kqwl_params
;
8747 if (trp
.trp_flags
& TRP_PRIORITY
) {
8748 kqdi
->kqdi_pri
= trp
.trp_pri
;
8753 if (trp
.trp_flags
& TRP_POLICY
) {
8754 kqdi
->kqdi_pol
= trp
.trp_pol
;
8759 if (trp
.trp_flags
& TRP_CPUPERCENT
) {
8760 kqdi
->kqdi_cpupercent
= trp
.trp_cpupercent
;
8762 kqdi
->kqdi_cpupercent
= 0;
8772 knote_markstayactive(struct knote
*kn
)
8774 struct kqueue
*kq
= knote_get_kq(kn
);
8778 kn
->kn_status
|= KN_STAYACTIVE
;
8781 * Making a knote stay active is a property of the knote that must be
8782 * established before it is fully attached.
8784 assert((kn
->kn_status
& (KN_QUEUED
| KN_SUPPRESSED
)) == 0);
8786 /* handle all stayactive knotes on the (appropriate) manager */
8787 if (kq
->kq_state
& KQ_WORKLOOP
) {
8788 struct kqworkloop
*kqwl
= (struct kqworkloop
*)kq
;
8790 qos
= _pthread_priority_thread_qos(kn
->kn_qos
);
8791 assert(qos
&& qos
< THREAD_QOS_LAST
);
8792 kqworkloop_update_threads_qos(kqwl
, KQWL_UTQ_UPDATE_STAYACTIVE_QOS
, qos
);
8793 qos
= KQWL_BUCKET_STAYACTIVE
;
8794 } else if (kq
->kq_state
& KQ_WORKQ
) {
8795 qos
= KQWQ_QOS_MANAGER
;
8797 qos
= THREAD_QOS_UNSPECIFIED
;
8800 kn
->kn_qos_override
= qos
;
8801 kn
->kn_qos_index
= qos
;
8803 knote_activate(kq
, kn
, FILTER_ACTIVE
);
8808 knote_clearstayactive(struct knote
*kn
)
8810 struct kqueue
*kq
= knote_get_kq(kn
);
8812 kn
->kn_status
&= ~(KN_STAYACTIVE
| KN_ACTIVE
);
8813 knote_dequeue(kq
, kn
);
8817 static unsigned long
8818 kevent_extinfo_emit(struct kqueue
*kq
, struct knote
*kn
, struct kevent_extinfo
*buf
,
8819 unsigned long buflen
, unsigned long nknotes
)
8821 for (; kn
; kn
= SLIST_NEXT(kn
, kn_link
)) {
8822 if (kq
== knote_get_kq(kn
)) {
8823 if (nknotes
< buflen
) {
8824 struct kevent_extinfo
*info
= &buf
[nknotes
];
8828 info
->kqext_kev
= *(struct kevent_qos_s
*)&kn
->kn_kevent
;
8829 if (knote_has_qos(kn
)) {
8830 info
->kqext_kev
.qos
=
8831 _pthread_priority_thread_qos_fast(kn
->kn_qos
);
8833 info
->kqext_kev
.qos
= kn
->kn_qos_override
;
8835 info
->kqext_kev
.filter
|= 0xff00; /* sign extend filter */
8836 info
->kqext_kev
.xflags
= 0; /* this is where sfflags lives */
8837 info
->kqext_kev
.data
= 0; /* this is where sdata lives */
8838 info
->kqext_sdata
= kn
->kn_sdata
;
8839 info
->kqext_status
= kn
->kn_status
;
8840 info
->kqext_sfflags
= kn
->kn_sfflags
;
8845 /* we return total number of knotes, which may be more than requested */
8854 kevent_copyout_proc_dynkqids(void *proc
, user_addr_t ubuf
, uint32_t ubufsize
,
8855 int32_t *nkqueues_out
)
8857 proc_t p
= (proc_t
)proc
;
8858 struct filedesc
*fdp
= p
->p_fd
;
8859 unsigned int nkqueues
= 0;
8860 unsigned long ubuflen
= ubufsize
/ sizeof(kqueue_id_t
);
8861 size_t buflen
, bufsize
;
8862 kqueue_id_t
*kq_ids
= NULL
;
8867 if (ubuf
== USER_ADDR_NULL
&& ubufsize
!= 0) {
8872 buflen
= min(ubuflen
, PROC_PIDDYNKQUEUES_MAX
);
8875 if (os_mul_overflow(sizeof(kqueue_id_t
), buflen
, &bufsize
)) {
8879 kq_ids
= kalloc(bufsize
);
8884 bzero(kq_ids
, bufsize
);
8889 if (fdp
->fd_kqhashmask
> 0) {
8890 for (uint32_t i
= 0; i
< fdp
->fd_kqhashmask
+ 1; i
++) {
8891 struct kqworkloop
*kqwl
;
8893 LIST_FOREACH(kqwl
, &fdp
->fd_kqhash
[i
], kqwl_hashlink
) {
8894 /* report the number of kqueues, even if they don't all fit */
8895 if (nkqueues
< buflen
) {
8896 kq_ids
[nkqueues
] = kqwl
->kqwl_dynamicid
;
8907 if (os_mul_overflow(sizeof(kqueue_id_t
), min(buflen
, nkqueues
), ©size
)) {
8912 assert(ubufsize
>= copysize
);
8913 err
= copyout(kq_ids
, ubuf
, copysize
);
8918 kfree(kq_ids
, bufsize
);
8922 *nkqueues_out
= (int)min(nkqueues
, PROC_PIDDYNKQUEUES_MAX
);
8928 kevent_copyout_dynkqinfo(void *proc
, kqueue_id_t kq_id
, user_addr_t ubuf
,
8929 uint32_t ubufsize
, int32_t *size_out
)
8931 proc_t p
= (proc_t
)proc
;
8932 struct kqworkloop
*kqwl
;
8934 struct kqueue_dyninfo kqdi
= { };
8938 if (ubufsize
< sizeof(struct kqueue_info
)) {
8942 kqwl
= kqworkloop_hash_lookup_and_retain(p
->p_fd
, kq_id
);
8948 * backward compatibility: allow the argument to this call to only be
8949 * a struct kqueue_info
8951 if (ubufsize
>= sizeof(struct kqueue_dyninfo
)) {
8952 ubufsize
= sizeof(struct kqueue_dyninfo
);
8953 err
= fill_kqueue_dyninfo(kqwl
, &kqdi
);
8955 ubufsize
= sizeof(struct kqueue_info
);
8956 err
= fill_kqueueinfo(&kqwl
->kqwl_kqueue
, &kqdi
.kqdi_info
);
8958 if (err
== 0 && (err
= copyout(&kqdi
, ubuf
, ubufsize
)) == 0) {
8959 *size_out
= ubufsize
;
8961 kqworkloop_release(kqwl
);
8966 kevent_copyout_dynkqextinfo(void *proc
, kqueue_id_t kq_id
, user_addr_t ubuf
,
8967 uint32_t ubufsize
, int32_t *nknotes_out
)
8969 proc_t p
= (proc_t
)proc
;
8970 struct kqworkloop
*kqwl
;
8973 kqwl
= kqworkloop_hash_lookup_and_retain(p
->p_fd
, kq_id
);
8978 err
= pid_kqueue_extinfo(p
, &kqwl
->kqwl_kqueue
, ubuf
, ubufsize
, nknotes_out
);
8979 kqworkloop_release(kqwl
);
8984 pid_kqueue_extinfo(proc_t p
, struct kqueue
*kq
, user_addr_t ubuf
,
8985 uint32_t bufsize
, int32_t *retval
)
8990 struct filedesc
*fdp
= p
->p_fd
;
8991 unsigned long nknotes
= 0;
8992 unsigned long buflen
= bufsize
/ sizeof(struct kevent_extinfo
);
8993 struct kevent_extinfo
*kqext
= NULL
;
8995 /* arbitrary upper limit to cap kernel memory usage, copyout size, etc. */
8996 buflen
= min(buflen
, PROC_PIDFDKQUEUE_KNOTES_MAX
);
8998 kqext
= kalloc(buflen
* sizeof(struct kevent_extinfo
));
8999 if (kqext
== NULL
) {
9003 bzero(kqext
, buflen
* sizeof(struct kevent_extinfo
));
9006 for (i
= 0; i
< fdp
->fd_knlistsize
; i
++) {
9007 kn
= SLIST_FIRST(&fdp
->fd_knlist
[i
]);
9008 nknotes
= kevent_extinfo_emit(kq
, kn
, kqext
, buflen
, nknotes
);
9012 if (fdp
->fd_knhashmask
!= 0) {
9013 for (i
= 0; i
< (int)fdp
->fd_knhashmask
+ 1; i
++) {
9015 kn
= SLIST_FIRST(&fdp
->fd_knhash
[i
]);
9016 nknotes
= kevent_extinfo_emit(kq
, kn
, kqext
, buflen
, nknotes
);
9021 assert(bufsize
>= sizeof(struct kevent_extinfo
) * min(buflen
, nknotes
));
9022 err
= copyout(kqext
, ubuf
, sizeof(struct kevent_extinfo
) * min(buflen
, nknotes
));
9026 kfree(kqext
, buflen
* sizeof(struct kevent_extinfo
));
9031 *retval
= min(nknotes
, PROC_PIDFDKQUEUE_KNOTES_MAX
);
9037 klist_copy_udata(struct klist
*list
, uint64_t *buf
,
9038 unsigned int buflen
, unsigned int nknotes
)
9041 SLIST_FOREACH(kn
, list
, kn_link
) {
9042 if (nknotes
< buflen
) {
9044 * kevent_register will always set kn_udata atomically
9045 * so that we don't have to take any kqlock here.
9047 buf
[nknotes
] = os_atomic_load_wide(&kn
->kn_udata
, relaxed
);
9049 /* we return total number of knotes, which may be more than requested */
9057 kevent_proc_copy_uptrs(void *proc
, uint64_t *buf
, int bufsize
)
9059 proc_t p
= (proc_t
)proc
;
9060 struct filedesc
*fdp
= p
->p_fd
;
9061 unsigned int nuptrs
= 0;
9062 unsigned long buflen
= bufsize
/ sizeof(uint64_t);
9063 struct kqworkloop
*kqwl
;
9066 assert(buf
!= NULL
);
9070 for (int i
= 0; i
< fdp
->fd_knlistsize
; i
++) {
9071 nuptrs
= klist_copy_udata(&fdp
->fd_knlist
[i
], buf
, buflen
, nuptrs
);
9076 if (fdp
->fd_knhashmask
!= 0) {
9077 for (size_t i
= 0; i
< fdp
->fd_knhashmask
+ 1; i
++) {
9078 nuptrs
= klist_copy_udata(&fdp
->fd_knhash
[i
], buf
, buflen
, nuptrs
);
9084 if (fdp
->fd_kqhashmask
!= 0) {
9085 for (size_t i
= 0; i
< fdp
->fd_kqhashmask
+ 1; i
++) {
9086 LIST_FOREACH(kqwl
, &fdp
->fd_kqhash
[i
], kqwl_hashlink
) {
9087 if (nuptrs
< buflen
) {
9088 buf
[nuptrs
] = kqwl
->kqwl_dynamicid
;
9100 kevent_set_return_to_kernel_user_tsd(proc_t p
, thread_t thread
)
9103 bool proc_is_64bit
= !!(p
->p_flag
& P_LP64
);
9104 size_t user_addr_size
= proc_is_64bit
? 8 : 4;
9105 uint32_t ast_flags32
= 0;
9106 uint64_t ast_flags64
= 0;
9107 struct uthread
*ut
= get_bsdthread_info(thread
);
9109 if (ut
->uu_kqr_bound
!= NULL
) {
9110 ast_flags64
|= R2K_WORKLOOP_PENDING_EVENTS
;
9113 if (ast_flags64
== 0) {
9117 if (!(p
->p_flag
& P_LP64
)) {
9118 ast_flags32
= (uint32_t)ast_flags64
;
9119 assert(ast_flags64
< 0x100000000ull
);
9122 ast_addr
= thread_rettokern_addr(thread
);
9123 if (ast_addr
== 0) {
9127 if (copyout((proc_is_64bit
? (void *)&ast_flags64
: (void *)&ast_flags32
),
9128 (user_addr_t
)ast_addr
,
9129 user_addr_size
) != 0) {
9130 printf("pid %d (tid:%llu): copyout of return_to_kernel ast flags failed with "
9131 "ast_addr = %llu\n", p
->p_pid
, thread_tid(current_thread()), ast_addr
);
9136 kevent_ast(thread_t thread
, uint16_t bits
)
9138 proc_t p
= current_proc();
9140 if (bits
& AST_KEVENT_REDRIVE_THREADREQ
) {
9141 workq_kern_threadreq_redrive(p
, WORKQ_THREADREQ_CAN_CREATE_THREADS
);
9143 if (bits
& AST_KEVENT_RETURN_TO_KERNEL
) {
9144 kevent_set_return_to_kernel_user_tsd(p
, thread
);
9148 #if DEVELOPMENT || DEBUG
9150 #define KEVENT_SYSCTL_BOUND_ID 1
9153 kevent_sysctl SYSCTL_HANDLER_ARGS
9155 #pragma unused(oidp, arg2)
9156 uintptr_t type
= (uintptr_t)arg1
;
9157 uint64_t bound_id
= 0;
9159 if (type
!= KEVENT_SYSCTL_BOUND_ID
) {
9167 struct uthread
*ut
= get_bsdthread_info(current_thread());
9172 workq_threadreq_t kqr
= ut
->uu_kqr_bound
;
9174 if (kqr
->tr_flags
& WORKQ_TR_FLAG_WORKLOOP
) {
9175 bound_id
= kqr_kqworkloop(kqr
)->kqwl_dynamicid
;
9181 return sysctl_io_number(req
, bound_id
, sizeof(bound_id
), NULL
, NULL
);
9184 SYSCTL_NODE(_kern
, OID_AUTO
, kevent
, CTLFLAG_RW
| CTLFLAG_LOCKED
, 0,
9185 "kevent information");
9187 SYSCTL_PROC(_kern_kevent
, OID_AUTO
, bound_id
,
9188 CTLTYPE_QUAD
| CTLFLAG_RD
| CTLFLAG_LOCKED
| CTLFLAG_MASKED
,
9189 (void *)KEVENT_SYSCTL_BOUND_ID
,
9190 sizeof(kqueue_id_t
), kevent_sysctl
, "Q",
9191 "get the ID of the bound kqueue");
9193 #endif /* DEVELOPMENT || DEBUG */