bsd/sys/eventvar.h

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*-
  29  * Copyright (c) 1999,2000 Jonathan Lemon <jlemon@FreeBSD.org>
  30  * All rights reserved.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  *      $FreeBSD: src/sys/sys/eventvar.h,v 1.1.2.2 2000/07/18 21:49:12 jlemon Exp $
  54  */
  55
  56 #ifndef _SYS_EVENTVAR_H_
  57 #define _SYS_EVENTVAR_H_
  58
  59 #include <sys/event.h>
  60 #include <sys/select.h>
  61 #include <kern/kern_types.h>
  62 #include <kern/waitq.h>
  63
  64 #if defined(XNU_KERNEL_PRIVATE)
  65
  66 typedef int (*kevent_callback_t)(struct kqueue *, struct kevent_internal_s *, void *);
  67 typedef void (*kqueue_continue_t)(struct kqueue *, void *, int);
  68
  69 #include <stdint.h>
  70 #include <kern/locks.h>
  71 #include <sys/pthread_shims.h>
  72 #include <mach/thread_policy.h>
  73
  74 /*
  75  * Lock ordering:
  76  *
  77  * The kqueue locking order can follow a few different patterns:
  78  *
  79  * Standard file-based kqueues (from above):
  80  *     proc fd lock -> kq lock -> kq-waitq-set lock -> thread lock
  81  *
  82  * WorkQ/WorkLoop kqueues (from above):
  83  *     proc fd lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock
  84  *
  85  * Whenever kqueues interact with source locks, it drops all of its own
  86  * locks in exchange for a use-reference on the knote used to synchronize
  87  * with the source code. When those sources post events from below, they
  88  * have the following lock hierarchy.
  89  *
  90  * Standard file-based kqueues (from below):
  91  *     XXX lock -> kq lock -> kq-waitq-set lock -> thread lock
  92  * Standard file-based kqueues with non-kq-aware sources (from below):
  93  *     XXX lock -> kq-waitq-set lock -> thread lock
  94  *
  95  * WorkQ/WorkLoop kqueues (from below):
  96  *     XXX lock -> kq lock -> kq-request lock -> pthread kext locks -> thread lock
  97  * WorkQ/WorkLoop kqueues with non-kq-aware sources (from below):
  98  *     XXX -> kq-waitq-set lock -> kq-request lock -> pthread kext locks -> thread lock
  99  */
 100
 101 #define KQEXTENT        256             /* linear growth by this amount */
 102
 103 /*
 104  * kqueue - common core definition of a kqueue
 105  *
 106  *          No real structures are allocated of this type. They are
 107  *          either kqfile objects or kqworkq objects - each of which is
 108  *          derived from this definition.
 109  */
 110 struct kqueue {
 111         struct waitq_set    kq_wqs;       /* private waitq set */
 112         lck_spin_t          kq_lock;      /* kqueue lock */
 113         uint16_t            kq_state;     /* state of the kq */
 114         uint16_t            kq_level;     /* nesting level of the kq */
 115         uint32_t            kq_count;     /* number of queued events */
 116         struct proc         *kq_p;        /* process containing kqueue */
 117         struct kqtailq      kq_queue[1];  /* variable array of kqtailq structs */
 118 };
 119
 120 #define KQ_SEL            0x001  /* select was recorded for kq */
 121 #define KQ_SLEEP          0x002  /* thread is waiting for events */
 122 #define KQ_PROCWAIT       0x004  /* thread waiting for processing */
 123 #define KQ_KEV32          0x008  /* kq is used with 32-bit events */
 124 #define KQ_KEV64          0x010  /* kq is used with 64-bit events */
 125 #define KQ_KEV_QOS        0x020  /* kq events carry QoS info */
 126 #define KQ_WORKQ          0x040  /* KQ is bound to process workq */
 127 #define KQ_WORKLOOP       0x080  /* KQ is part of a workloop */
 128 #define KQ_PROCESSING     0x100  /* KQ is being processed */
 129 #define KQ_DRAIN          0x200  /* kq is draining */
 130 #define KQ_WAKEUP         0x400  /* kq awakened while processing */
 131 #define KQ_DYNAMIC        0x800  /* kqueue is dynamically managed */
 132 #define KQ_NO_WQ_THREAD   0x1000 /* kq will not have workqueue threads dynamically created */
 133 /*
 134  * kqfile - definition of a typical kqueue opened as a file descriptor
 135  *          via the kqueue() system call.
 136  *
 137  *          Adds selinfo support to the base kqueue definition, as these
 138  *          fds can be fed into select().
 139  */
 140 struct kqfile {
 141         struct kqueue       kqf_kqueue;     /* common kqueue core */
 142         struct kqtailq      kqf_suppressed; /* suppression queue */
 143         struct selinfo      kqf_sel;        /* parent select/kqueue info */
 144 };
 145
 146 #define kqf_wqs      kqf_kqueue.kq_wqs
 147 #define kqf_lock     kqf_kqueue.kq_lock
 148 #define kqf_state    kqf_kqueue.kq_state
 149 #define kqf_level    kqf_kqueue.kq_level
 150 #define kqf_count    kqf_kqueue.kq_count
 151 #define kqf_p        kqf_kqueue.kq_p
 152 #define kqf_queue    kqf_kqueue.kq_queue
 153
 154 #define QOS_INDEX_KQFILE   0          /* number of qos levels in a file kq */
 155
 156 struct kqr_bound {
 157         struct kqtailq   kqrb_suppressed;     /* Per-QoS suppression queues */
 158         thread_t         kqrb_thread;         /* thread to satisfy request */
 159 };
 160
 161 /*
 162  * kqrequest - per-QoS thread request status
 163  */
 164 struct kqrequest {
 165 #if 0
 166         union {
 167                 struct kqr_bound kqru_bound;       /* used when thread is bound */
 168                 struct workq_threadreq_s kqru_req; /* used when request oustanding */
 169         } kqr_u;
 170 #define kqr_suppressed kqr_u.kqru_bound.kqrb_suppressed
 171 #define kqr_thread     kqr_u.kqru_bound.kqrb_thread
 172 #define kqr_req        kqr_u.kqru_req
 173 #else
 174         struct kqr_bound kqr_bound;            /* used when thread is bound */
 175         struct workq_threadreq_s kqr_req;      /* used when request oustanding */
 176 #define kqr_suppressed kqr_bound.kqrb_suppressed
 177 #define kqr_thread     kqr_bound.kqrb_thread
 178 #endif
 179         uint8_t          kqr_state;                    /* KQ/workq interaction state */
 180         uint8_t          kqr_wakeup_indexes;           /* QoS/override levels that woke */
 181         uint16_t         kqr_dsync_waiters:13,         /* number of dispatch sync waiters */
 182                          kqr_dsync_owner_qos:3;        /* Qos override on dispatch sync owner */
 183         uint16_t         kqr_sync_suppress_count;      /* number of suppressed sync ipc knotes */
 184         kq_index_t       kqr_stayactive_qos:3,         /* max QoS of statyactive knotes */
 185                          kqr_owner_override_is_sync:1, /* sync owner has sync ipc override */
 186                          kqr_override_index:3,         /* highest wakeup override index */
 187                          kqr_has_sync_override:1;      /* Qos/override at UI is sync ipc override */
 188
 189         /* set under both the kqlock and the filt_wllock */
 190         kq_index_t       :0;                           /* prevent bitfields coalescing <rdar://problem/31854115> */
 191         kq_index_t       kqr_qos_index:4,              /* QoS for the thread request */
 192                          kqr_dsync_waiters_qos:4;      /* override from dispatch sync waiters */
 193 };
 194
 195
 196 #define KQR_PROCESSING               0x01       /* requested thread is running the q */
 197 #define KQR_THREQUESTED              0x02       /* thread has been requested from workq */
 198 #define KQR_WAKEUP                   0x04       /* wakeup called during processing */
 199 #define KQR_BOUND                    0x08       /* servicing thread is bound */
 200 #define KQR_THOVERCOMMIT             0x20       /* overcommit needed for thread requests */
 201 #define KQR_DRAIN                    0x40       /* cancel initiated - drain fulfill */
 202 #define KQR_R2K_NOTIF_ARMED          0x80       /* ast notifications armed */
 203 /*
 204  * WorkQ kqueues need to request threads to service the triggered
 205  * knotes in the queue.  These threads are brought up on a
 206  * effective-requested-QoS basis. Knotes are segregated based on
 207  * that value - calculated by computing max(event-QoS, kevent-QoS).
 208  * Only one servicing thread is requested at a time for all the
 209  * knotes at a given effective-requested-QoS.
 210  */
 211
 212 #if !defined(KQWQ_QOS_MANAGER)
 213 #define KQWQ_QOS_MANAGER (THREAD_QOS_LAST)
 214 #endif
 215
 216 #if !defined(KQWQ_NQOS)
 217 #define KQWQ_NQOS    (KQWQ_QOS_MANAGER + 1)
 218 #endif
 219
 220 /*
 221  * Workq thread start out a particular effective-requested-QoS, but
 222  * additional events processed by the filters may represent
 223  * backlogged events that may themselves have a higher requested-QoS.
 224  * To represent this, the filter may apply an override to a knote's
 225  * requested QoS.
 226  *
 227  * We further segregate these overridden knotes into different buckets
 228  * by <requested, override> grouping. This allows easy matching of
 229  * knotes to process vs. the highest workq thread override applied.
 230  *
 231  * Only certain override patterns need to be supported. A knote
 232  * cannot have an effective-requested-QoS of UNSPECIFIED - because
 233  * the kevent->qos (when canonicalized) will always be above that
 234  * or indicate manager.  And we don't allow an override to specify
 235  * manager.  This results in the following buckets being needed:
 236  *
 237  *                  Effective-Requested QoS
 238  *           MAINT  BG    UTIL  DEFAULT UINIT UINTER MANAGER
 239  * override:
 240  * MAINT      0
 241  * BG         1      6
 242  * UTILITY    2      7     11
 243  * DEFAULT    3      8     12    15
 244  * UINIT      4      9     13    16     18
 245  * UINTER     5     10     14    17     19     20
 246  *                                                    21
 247  */
 248 #if !defined(KQWQ_NBUCKETS)
 249 #define KQWQ_NBUCKETS 22
 250 #endif
 251
 252 /*
 253  * kqworkq - definition of a private kqueue used to coordinate event
 254  *           handling for pthread work queues.
 255  *
 256  *           These have per-qos processing queues and state to coordinate with
 257  *           the pthread kext to ask for threads at corresponding pthread priority
 258  *           values.
 259  */
 260 struct kqworkq {
 261         struct kqueue    kqwq_kqueue;
 262         struct kqtailq   kqwq_queuecont[KQWQ_NBUCKETS-1]; /* continue array of queues */
 263         struct kqrequest kqwq_request[KQWQ_NQOS];         /* per-QoS request states */
 264         lck_spin_t       kqwq_reqlock;                    /* kqueue request lock */
 265 };
 266
 267 #define kqwq_wqs     kqwq_kqueue.kq_wqs
 268 #define kqwq_lock    kqwq_kqueue.kq_lock
 269 #define kqwq_state   kqwq_kqueue.kq_state
 270 #define kqwq_level   kqwq_kqueue.kq_level
 271 #define kqwq_count   kqwq_kqueue.kq_count
 272 #define kqwq_p       kqwq_kqueue.kq_p
 273 #define kqwq_queue   kqwq_kqueue.kq_queue
 274
 275 #define kqwq_req_lock(kqwq)    lck_spin_lock(&kqwq->kqwq_reqlock)
 276 #define kqwq_req_unlock(kqwq)  lck_spin_unlock(&kqwq->kqwq_reqlock)
 277 #define kqwq_req_held(kqwq)    LCK_SPIN_ASSERT(&kqwq->kqwq_reqlock, LCK_ASSERT_OWNED)
 278
 279 #define KQWQ_THMANAGER    0x10      /* expect manager thread to run the queue */
 280
 281 /*
 282  * WorkLoop kqueues need to request a thread to service the triggered
 283  * knotes in the queue.  The thread is brought up on a
 284  * effective-requested-QoS basis. Knotes are segregated based on
 285  * that value. Once a request is made, it cannot be undone.  If
 286  * events with higher QoS arrive after, they are stored in their
 287  * own queues and an override applied to the original request based
 288  * on the delta between the two QoS values.
 289  */
 290
 291 /*
 292  * "Stay-active" knotes are held in a separate bucket that indicates
 293  * special handling required. They are kept separate because the
 294  * wakeups issued to them don't have context to tell us where to go
 295  * to find and process them. All processing of them happens at the
 296  * highest QoS. Unlike WorkQ kqueues, there is no special singular
 297  * "manager thread" for a process. We simply request a servicing
 298  * thread at the higest known QoS when these are woken (or override
 299  * an existing request to that).
 300  */
 301 #define KQWL_BUCKET_STAYACTIVE (THREAD_QOS_LAST)
 302
 303 #if !defined(KQWL_NBUCKETS)
 304 #define KQWL_NBUCKETS    (KQWL_BUCKET_STAYACTIVE + 1)
 305 #endif
 306
 307 /*
 308  * kqworkloop - definition of a private kqueue used to coordinate event
 309  *              handling for pthread workloops.
 310  *
 311  *              Workloops vary from workqs in that only a single thread is ever
 312  *              requested to service a workloop at a time.  But unlike workqs,
 313  *              workloops may be "owned" by user-space threads that are
 314  *              synchronously draining an event off the workloop. In those cases,
 315  *              any overrides have to be applied to the owner until it relinqueshes
 316  *              ownership.
 317  *
 318  *      NOTE:   "lane" support is TBD.
 319  */
 320 struct kqworkloop {
 321         struct kqueue    kqwl_kqueue;                     /* queue of events */
 322         struct kqtailq   kqwl_queuecont[KQWL_NBUCKETS-1]; /* continue array of queues */
 323         struct kqrequest kqwl_request;                    /* thread request state */
 324         lck_spin_t       kqwl_reqlock;                    /* kqueue request lock */
 325         lck_mtx_t        kqwl_statelock;                  /* state/debounce lock */
 326         thread_t         kqwl_owner;                      /* current [sync] owner thread */
 327         uint32_t         kqwl_retains;                    /* retain references */
 328         kqueue_id_t      kqwl_dynamicid;                  /* dynamic identity */
 329         SLIST_ENTRY(kqworkloop) kqwl_hashlink;            /* linkage for search list */
 330 };
 331
 332 SLIST_HEAD(kqlist, kqworkloop);
 333
 334 #define kqwl_wqs     kqwl_kqueue.kq_wqs
 335 #define kqwl_lock    kqwl_kqueue.kq_lock
 336 #define kqwl_state   kqwl_kqueue.kq_state
 337 #define kqwl_level   kqwl_kqueue.kq_level
 338 #define kqwl_count   kqwl_kqueue.kq_count
 339 #define kqwl_p       kqwl_kqueue.kq_p
 340 #define kqwl_queue   kqwl_kqueue.kq_queue
 341
 342 #define kqwl_req_lock(kqwl)    lck_spin_lock(&kqwl->kqwl_reqlock)
 343 #define kqwl_req_unlock(kqwl)  lck_spin_unlock(&kqwl->kqwl_reqlock)
 344 #define kqwl_req_held(kqwl)    LCK_SPIN_ASSERT(&kqwl->kqwl_reqlock, LCK_ASSERT_OWNED)
 345
 346 #define KQ_WORKLOOP_RETAINS_MAX UINT32_MAX
 347
 348 extern int workloop_fulfill_threadreq(struct proc *p, workq_threadreq_t req, thread_t thread, int flags);
 349
 350 extern struct kqueue *kqueue_alloc(struct proc *, unsigned int);
 351 extern void kqueue_dealloc(struct kqueue *);
 352
 353 extern void knotes_dealloc(struct proc *);
 354
 355 extern void kevent_register(struct kqueue *, struct kevent_internal_s *, struct proc *);
 356 extern int kqueue_scan(struct kqueue *, kevent_callback_t, kqueue_continue_t,
 357                        void *, struct filt_process_s *, struct timeval *, struct proc *);
 358 extern int kqueue_stat(struct kqueue *, void *, int, proc_t);
 359
 360 #endif /* XNU_KERNEL_PRIVATE */
 361
 362 #endif /* !_SYS_EVENTVAR_H_ */
 363
 364
 365
 366