X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..a991bd8d3e7fe02dbca0644054bab73c5b75324a:/osfmk/kern/waitq.h?ds=inline

diff --git a/osfmk/kern/waitq.h b/osfmk/kern/waitq.h
index 9eb863a7b..dd90a09ba 100644
--- a/osfmk/kern/waitq.h
+++ b/osfmk/kern/waitq.h
@@ -49,7 +49,7 @@
  */
 #define WAITQ_ALL_PRIORITIES   (-1)
 #define WAITQ_PROMOTE_PRIORITY (-2)
-#define WAITQ_SELECT_MAX_PRI   (-3)
+#define WAITQ_PROMOTE_ON_WAKE  (-3)
 
 typedef enum e_waitq_lock_state {
 	WAITQ_KEEP_LOCKED    = 0x01,
@@ -101,10 +101,16 @@ typedef enum e_waitq_lock_state {
 
 enum waitq_type {
 	WQT_INVALID = 0,
+	WQT_TSPROXY = 0x1,
 	WQT_QUEUE   = 0x2,
 	WQT_SET     = 0x3,
 };
 
+__options_decl(waitq_options_t, uint32_t, {
+	WQ_OPTION_NONE                 = 0,
+	WQ_OPTION_HANDOFF              = 1,
+});
+
 #if CONFIG_WAITQ_STATS
 #define NWAITQ_BTFRAMES 5
 struct wq_stats {
@@ -141,7 +147,7 @@ struct waitq {
 	    waitq_prepost:1,     /* waitq supports prepost? */
 	    waitq_irq:1,         /* waitq requires interrupts disabled */
 	    waitq_isvalid:1,     /* waitq structure is valid */
-	    waitq_turnstile_or_port:1,     /* waitq is embedded in a turnstile (if irq safe), or port (if not irq safe) */
+	    waitq_turnstile:1,   /* waitq is embedded in a turnstile */
 	    waitq_eventmask:_EVENT_MASK_BITS;
 	/* the wait queue set (set-of-sets) to which this queue belongs */
 #if __arm64__
@@ -153,8 +159,17 @@ struct waitq {
 	uint64_t waitq_set_id;
 	uint64_t waitq_prepost_id;
 	union {
-		queue_head_t            waitq_queue;            /* queue of elements */
-		struct priority_queue   waitq_prio_queue;       /* priority ordered queue of elements */
+		queue_head_t            waitq_queue;               /* queue of elements - used for waitq not embedded in turnstile or ports */
+		struct priority_queue_sched_max waitq_prio_queue;  /* priority ordered queue of elements - used for waitqs embedded in turnstiles */
+		struct {                                           /* used for waitqs embedded in ports */
+			struct turnstile   *waitq_ts;              /* used to store receive turnstile of the port */
+			union {
+				void               *waitq_tspriv;  /* non special-reply port, used to store the watchport element for port used to store
+				                                    * receive turnstile of the port */
+				int                waitq_priv_pid; /* special-reply port, used to store the pid that copies out the send once right of the
+				                                    * special-reply port. */
+			};
+		};
 	};
 };
 
@@ -175,7 +190,7 @@ struct waitq_set {
 	};
 };
 
-#define WQSET_NOT_LINKED ((uint64_t)(~0))
+#define WQSET_NOT_LINKED       ((uint64_t)(~0))
 static_assert(sizeof(struct waitq_set) == WQS_OPAQUE_SIZE, "waitq_set structure size mismatch");
 static_assert(__alignof(struct waitq_set) == WQS_OPAQUE_ALIGN, "waitq_set structure alignment mismatch");
 
@@ -184,11 +199,11 @@ extern void waitq_bootstrap(void);
 #define waitq_is_queue(wq) \
 	((wq)->waitq_type == WQT_QUEUE)
 
-#define waitq_is_turnstile_queue(wq) \
-	(((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_proxy(wq) \
+	((wq)->waitq_type == WQT_TSPROXY)
 
-#define waitq_is_port_queue(wq) \
-	(!((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_queue(wq) \
+	(((wq)->waitq_irq) && (wq)->waitq_turnstile)
 
 #define waitq_is_set(wq) \
 	((wq)->waitq_type == WQT_SET && ((struct waitq_set *)(wq))->wqset_id != 0)
@@ -209,16 +224,6 @@ extern void waitq_bootstrap(void);
  */
 extern void waitq_invalidate_locked(struct waitq *wq);
 
-static inline boolean_t
-waitq_empty(struct waitq *wq)
-{
-	if (waitq_is_turnstile_queue(wq)) {
-		return priority_queue_empty(&(wq->waitq_prio_queue));
-	} else {
-		return queue_empty(&(wq->waitq_queue));
-	}
-}
-
 extern lck_grp_t waitq_lck_grp;
 
 #if __arm64__
@@ -281,7 +286,8 @@ extern kern_return_t waitq_wakeup64_one_locked(struct waitq *waitq,
     wait_result_t result,
     uint64_t *reserved_preposts,
     int priority,
-    waitq_lock_state_t lock_state);
+    waitq_lock_state_t lock_state,
+    waitq_options_t options);
 
 /* return identity of a thread awakened for a particular <wait_queue,event> */
 extern thread_t
@@ -388,14 +394,17 @@ extern struct waitq *_global_eventq(char *event, size_t event_length);
 
 extern struct waitq *global_waitq(int index);
 
+typedef uint16_t waitq_set_prepost_hook_t;
+
 /*
  * set alloc/init/free
  */
-extern struct waitq_set *waitq_set_alloc(int policy, void *prepost_hook);
+extern struct waitq_set *waitq_set_alloc(int policy,
+    waitq_set_prepost_hook_t *prepost_hook);
 
 extern kern_return_t waitq_set_init(struct waitq_set *wqset,
     int policy, uint64_t *reserved_link,
-    void *prepost_hook);
+    waitq_set_prepost_hook_t *prepost_hook);
 
 extern void waitq_set_deinit(struct waitq_set *wqset);
 
@@ -463,8 +472,6 @@ extern int waitq_is_global(struct waitq *waitq);
 
 extern int waitq_irq_safe(struct waitq *waitq);
 
-extern struct waitq * waitq_get_safeq(struct waitq *waitq);
-
 #if CONFIG_WAITQ_STATS
 /*
  * waitq statistics