xnu-7195.81.3.tar.gz

[apple/xnu.git] / osfmk / kern / waitq.h

diff --git a/osfmk/kern/waitq.h b/osfmk/kern/waitq.h
index 9eb863a7b9e152d0851f420ca8ab24c7dc9d0ed6..dd90a09ba612a34bc5fa889e3837f251df52ad69 100644 (file)
--- a/osfmk/kern/waitq.h
+++ b/osfmk/kern/waitq.h
@@ -49,7 +49,7 @@
  */
 #define WAITQ_ALL_PRIORITIES   (-1)
 #define WAITQ_PROMOTE_PRIORITY (-2)
  */
 #define WAITQ_ALL_PRIORITIES   (-1)
 #define WAITQ_PROMOTE_PRIORITY (-2)

-#define WAITQ_SELECT_MAX_PRI   (-3)
+#define WAITQ_PROMOTE_ON_WAKE  (-3)

 
 typedef enum e_waitq_lock_state {
        WAITQ_KEEP_LOCKED    = 0x01,
 
 typedef enum e_waitq_lock_state {
        WAITQ_KEEP_LOCKED    = 0x01,


@@ -101,10 +101,16 @@ typedef enum e_waitq_lock_state {
 
 enum waitq_type {
        WQT_INVALID = 0,
 
 enum waitq_type {
        WQT_INVALID = 0,

+       WQT_TSPROXY = 0x1,

        WQT_QUEUE   = 0x2,
        WQT_SET     = 0x3,
 };
 
        WQT_QUEUE   = 0x2,
        WQT_SET     = 0x3,
 };
 

+__options_decl(waitq_options_t, uint32_t, {
+       WQ_OPTION_NONE                 = 0,
+       WQ_OPTION_HANDOFF              = 1,
+});
+

 #if CONFIG_WAITQ_STATS
 #define NWAITQ_BTFRAMES 5
 struct wq_stats {
 #if CONFIG_WAITQ_STATS
 #define NWAITQ_BTFRAMES 5
 struct wq_stats {


@@ -141,7 +147,7 @@ struct waitq {
            waitq_prepost:1,     /* waitq supports prepost? */
            waitq_irq:1,         /* waitq requires interrupts disabled */
            waitq_isvalid:1,     /* waitq structure is valid */
            waitq_prepost:1,     /* waitq supports prepost? */
            waitq_irq:1,         /* waitq requires interrupts disabled */
            waitq_isvalid:1,     /* waitq structure is valid */

-           waitq_turnstile_or_port:1,     /* waitq is embedded in a turnstile (if irq safe), or port (if not irq safe) */
+           waitq_turnstile:1,   /* waitq is embedded in a turnstile */

            waitq_eventmask:_EVENT_MASK_BITS;
        /* the wait queue set (set-of-sets) to which this queue belongs */
 #if __arm64__
            waitq_eventmask:_EVENT_MASK_BITS;
        /* the wait queue set (set-of-sets) to which this queue belongs */
 #if __arm64__


@@ -153,8 +159,17 @@ struct waitq {
        uint64_t waitq_set_id;
        uint64_t waitq_prepost_id;
        union {
        uint64_t waitq_set_id;
        uint64_t waitq_prepost_id;
        union {

-               queue_head_t            waitq_queue;            /* queue of elements */
-               struct priority_queue   waitq_prio_queue;       /* priority ordered queue of elements */
+               queue_head_t            waitq_queue;               /* queue of elements - used for waitq not embedded in turnstile or ports */
+               struct priority_queue_sched_max waitq_prio_queue;  /* priority ordered queue of elements - used for waitqs embedded in turnstiles */
+               struct {                                           /* used for waitqs embedded in ports */
+                       struct turnstile   *waitq_ts;              /* used to store receive turnstile of the port */
+                       union {
+                               void               *waitq_tspriv;  /* non special-reply port, used to store the watchport element for port used to store
+                                                                   * receive turnstile of the port */
+                               int                waitq_priv_pid; /* special-reply port, used to store the pid that copies out the send once right of the
+                                                                   * special-reply port. */
+                       };
+               };

        };
 };
 
        };
 };
 


@@ -175,7 +190,7 @@ struct waitq_set {
        };
 };
 
        };
 };
 

-#define WQSET_NOT_LINKED ((uint64_t)(~0))
+#define WQSET_NOT_LINKED       ((uint64_t)(~0))

 static_assert(sizeof(struct waitq_set) == WQS_OPAQUE_SIZE, "waitq_set structure size mismatch");
 static_assert(__alignof(struct waitq_set) == WQS_OPAQUE_ALIGN, "waitq_set structure alignment mismatch");
 
 static_assert(sizeof(struct waitq_set) == WQS_OPAQUE_SIZE, "waitq_set structure size mismatch");
 static_assert(__alignof(struct waitq_set) == WQS_OPAQUE_ALIGN, "waitq_set structure alignment mismatch");
 


@@ -184,11 +199,11 @@ extern void waitq_bootstrap(void);
 #define waitq_is_queue(wq) \
        ((wq)->waitq_type == WQT_QUEUE)
 
 #define waitq_is_queue(wq) \
        ((wq)->waitq_type == WQT_QUEUE)
 

-#define waitq_is_turnstile_queue(wq) \
-       (((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_proxy(wq) \
+       ((wq)->waitq_type == WQT_TSPROXY)

 
 

-#define waitq_is_port_queue(wq) \
-       (!((wq)->waitq_irq) && (wq)->waitq_turnstile_or_port)
+#define waitq_is_turnstile_queue(wq) \
+       (((wq)->waitq_irq) && (wq)->waitq_turnstile)

 
 #define waitq_is_set(wq) \
        ((wq)->waitq_type == WQT_SET && ((struct waitq_set *)(wq))->wqset_id != 0)
 
 #define waitq_is_set(wq) \
        ((wq)->waitq_type == WQT_SET && ((struct waitq_set *)(wq))->wqset_id != 0)


@@ -209,16 +224,6 @@ extern void waitq_bootstrap(void);
  */
 extern void waitq_invalidate_locked(struct waitq *wq);
 
  */
 extern void waitq_invalidate_locked(struct waitq *wq);
 

-static inline boolean_t
-waitq_empty(struct waitq *wq)
-{
-       if (waitq_is_turnstile_queue(wq)) {
-               return priority_queue_empty(&(wq->waitq_prio_queue));
-       } else {
-               return queue_empty(&(wq->waitq_queue));
-       }
-}
-

 extern lck_grp_t waitq_lck_grp;
 
 #if __arm64__
 extern lck_grp_t waitq_lck_grp;
 
 #if __arm64__


@@ -281,7 +286,8 @@ extern kern_return_t waitq_wakeup64_one_locked(struct waitq *waitq,
     wait_result_t result,
     uint64_t *reserved_preposts,
     int priority,
     wait_result_t result,
     uint64_t *reserved_preposts,
     int priority,

-    waitq_lock_state_t lock_state);
+    waitq_lock_state_t lock_state,
+    waitq_options_t options);

 
 /* return identity of a thread awakened for a particular <wait_queue,event> */
 extern thread_t
 
 /* return identity of a thread awakened for a particular <wait_queue,event> */
 extern thread_t


@@ -388,14 +394,17 @@ extern struct waitq *_global_eventq(char *event, size_t event_length);
 
 extern struct waitq *global_waitq(int index);
 
 
 extern struct waitq *global_waitq(int index);
 

+typedef uint16_t waitq_set_prepost_hook_t;
+

 /*
  * set alloc/init/free
  */
 /*
  * set alloc/init/free
  */

-extern struct waitq_set *waitq_set_alloc(int policy, void *prepost_hook);
+extern struct waitq_set *waitq_set_alloc(int policy,
+    waitq_set_prepost_hook_t *prepost_hook);

 
 extern kern_return_t waitq_set_init(struct waitq_set *wqset,
     int policy, uint64_t *reserved_link,
 
 extern kern_return_t waitq_set_init(struct waitq_set *wqset,
     int policy, uint64_t *reserved_link,

-    void *prepost_hook);
+    waitq_set_prepost_hook_t *prepost_hook);

 
 extern void waitq_set_deinit(struct waitq_set *wqset);
 
 
 extern void waitq_set_deinit(struct waitq_set *wqset);
 


@@ -463,8 +472,6 @@ extern int waitq_is_global(struct waitq *waitq);
 
 extern int waitq_irq_safe(struct waitq *waitq);
 
 
 extern int waitq_irq_safe(struct waitq *waitq);
 

-extern struct waitq * waitq_get_safeq(struct waitq *waitq);
-

 #if CONFIG_WAITQ_STATS
 /*
  * waitq statistics
 #if CONFIG_WAITQ_STATS
 /*
  * waitq statistics