xnu-7195.81.3.tar.gz

[apple/xnu.git] / osfmk / kern / waitq.h

diff --git a/osfmk/kern/waitq.h b/osfmk/kern/waitq.h
index 9eb863a7b9e152d0851f420ca8ab24c7dc9d0ed6..dd90a09ba612a34bc5fa889e3837f251df52ad69 100644 (file)
--- a/osfmk/kern/waitq.h
+++ b/osfmk/kern/waitq.h
@@ -49,7 +49,7 @@
  */
 #define WAITQ_ALL_PRIORITIES   (-1)
 #define WAITQ_PROMOTE_PRIORITY (-2)
-#define WAITQ_SELECT_MAX_PRI   (-3)
+#define WAITQ_PROMOTE_ON_WAKE  (-3)
 
 typedef enum e_waitq_lock_state {
        WAITQ_KEEP_LOCKED    = 0x01,
@@ -101,10 +101,16 @@ typedef enum e_waitq_lock_state {
 
 enum waitq_type {
        WQT_INVALID = 0,
+       WQT_TSPROXY = 0x1,
        WQT_QUEUE   = 0x2,
        WQT_SET     = 0x3,
 };
 
+__options_decl(waitq_options_t, uint32_t, {
+       WQ_OPTION_NONE                 = 0,
+       WQ_OPTION_HANDOFF              = 1,
+});
+
 #if CONFIG_WAITQ_STATS
 #define NWAITQ_BTFRAMES 5
 struct wq_stats {
@@ -141,7 +147,7 @@ struct waitq {
            waitq_prepost:1,     /* waitq supports prepost? */
            waitq_irq:1,         /* waitq requires interrupts disabled */
            waitq_isvalid:1,     /* waitq structure is valid */
-           waitq_turnstile_or_port:1,     /* waitq is embedded in a turnstile (if irq safe), or port (if not irq safe) */
+           waitq_turnstile:1,   /* waitq is embedded in a turnstile */
            waitq_eventmask:_EVENT_MASK_BITS;
        /* the wait queue set (set-of-sets) to which this queue belongs */
 #if __arm64__
@@ -153,8 +159,17 @@ struct waitq {
        uint64_t waitq_set_id;
        uint64_t waitq_prepost_id;
        union {
-               queue_head_t            waitq_queue;            /* queue of elements */
-               struct priority_queue   waitq_prio_queue;       /* priority ordered queue of elements */
+               queue_head_t            waitq_queue;               /* queue of elements - used for waitq not embedded in turnstile or ports */
+               struct priority_queue_sched_max waitq_prio_queue;  /* priority ordered queue of elements - used for waitqs embedded in turnstiles */
+               struct {                                           /* used for waitqs embedded in ports */
+                       struct turnstile   *waitq_ts;              /* used to store receive turnstile of the port */
+                       union {
+                               void               *waitq_tspriv;  /* non special-reply port, used to store the watchport element for port used to store
+                                                                   * receive turnstile of the port */
+                               int                waitq_priv_pid; /* special-reply port, used to store the pid that copies out the send once right of the
+                                                                   * special-reply port. */
+                       };
+               };
        };
 };
 
@@ -175,7 +190,7 @@ struct waitq_set {
        };
 };
 
-#define WQSET_NOT_LINKED ((uint64_t)(~0))
+#define WQSET_NOT_LINKED       ((uint64_t)(~0))
 static_assert(sizeof(struct waitq_set) == WQS_OPAQUE_SIZE, "waitq_set structure size mismatch");
 static_assert(__alignof(struct waitq_set) == WQS_OPAQUE_ALIGN, "waitq_set structure alignment mismatch");
 
@@ -184,11 +199,11 @@ extern void waitq_bootstrap(void);
 #define waitq_is_queue(wq) \
        ((wq)->waitq_type == WQT_QUEUE)
 
-#define waitq_is_turnstile_queue(wq) \
-       (((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_proxy(wq) \
+       ((wq)->waitq_type == WQT_TSPROXY)
 
-#define waitq_is_port_queue(wq) \
-       (!((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_queue(wq) \
+       (((wq)->waitq_irq) && (wq)->waitq_turnstile)
 
 #define waitq_is_set(wq) \
        ((wq)->waitq_type == WQT_SET && ((struct waitq_set *)(wq))->wqset_id != 0)
@@ -209,16 +224,6 @@ extern void waitq_bootstrap(void);
  */
 extern void waitq_invalidate_locked(struct waitq *wq);
 
-static inline boolean_t
-waitq_empty(struct waitq *wq)
-{
-       if (waitq_is_turnstile_queue(wq)) {
-               return priority_queue_empty(&(wq->waitq_prio_queue));
-       } else {
-               return queue_empty(&(wq->waitq_queue));
-       }
-}
-
 extern lck_grp_t waitq_lck_grp;
 
 #if __arm64__
@@ -281,7 +286,8 @@ extern kern_return_t waitq_wakeup64_one_locked(struct waitq *waitq,
     wait_result_t result,
     uint64_t *reserved_preposts,
     int priority,
-    waitq_lock_state_t lock_state);
+    waitq_lock_state_t lock_state,
+    waitq_options_t options);
 
 /* return identity of a thread awakened for a particular <wait_queue,event> */
 extern thread_t
@@ -388,14 +394,17 @@ extern struct waitq *_global_eventq(char *event, size_t event_length);
 
 extern struct waitq *global_waitq(int index);
 
+typedef uint16_t waitq_set_prepost_hook_t;
+
 /*
  * set alloc/init/free
  */
-extern struct waitq_set *waitq_set_alloc(int policy, void *prepost_hook);
+extern struct waitq_set *waitq_set_alloc(int policy,
+    waitq_set_prepost_hook_t *prepost_hook);
 
 extern kern_return_t waitq_set_init(struct waitq_set *wqset,
     int policy, uint64_t *reserved_link,
-    void *prepost_hook);
+    waitq_set_prepost_hook_t *prepost_hook);
 
 extern void waitq_set_deinit(struct waitq_set *wqset);
 
@@ -463,8 +472,6 @@ extern int waitq_is_global(struct waitq *waitq);
 
 extern int waitq_irq_safe(struct waitq *waitq);
 
-extern struct waitq * waitq_get_safeq(struct waitq *waitq);
-
 #if CONFIG_WAITQ_STATS
 /*
  * waitq statistics