X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0b4e3aa066abc0728aacb4bbeb86f53f9737156e..9bccf70c0258c7cac2dcb80011b2a964d884c552:/osfmk/kern/wait_queue.h?ds=sidebyside diff --git a/osfmk/kern/wait_queue.h b/osfmk/kern/wait_queue.h index edfb02799..960a07b8d 100644 --- a/osfmk/kern/wait_queue.h +++ b/osfmk/kern/wait_queue.h @@ -22,14 +22,21 @@ #ifndef _KERN_WAIT_QUEUE_H_ #define _KERN_WAIT_QUEUE_H_ -#include /* for wait_queue_t */ +#include + +#ifdef __APPLE_API_PRIVATE + #include #include /* for kern_return_t */ +#include /* for wait_queue_t */ + +#ifdef MACH_KERNEL_PRIVATE #include #include + /* * wait_queue_t * This is the definition of the common event wait queue @@ -44,42 +51,34 @@ * NOTE: Hardware locks are used to protect event wait * queues since interrupt code is free to post events to * them. - * WARNING: Cannot change this data structure without updating SIZEOF_WAITQUEUE */ typedef struct wait_queue { + unsigned int /* flags */ + /* boolean_t */ wq_type:16, /* only public field */ + wq_fifo:1, /* fifo wakeup policy? */ + wq_isprepost:1, /* is waitq preposted? set only */ + :0; /* force to long boundary */ hw_lock_data_t wq_interlock; /* interlock */ - unsigned int /* flags */ - /* boolean_t */ wq_fifo:1, /* fifo wakeup policy? */ - wq_issub:1, /* is waitq linked? */ - wq_isprepost:1, /* is waitq preposted? sub only */ - :0; /* force to long boundary */ - queue_head_t wq_queue; /* queue of elements */ + queue_head_t wq_queue; /* queue of elements */ } WaitQueue; -#define SIZEOF_WAITQUEUE 16 /* 16 bytes for wq */ -#define SIZEOF_WAITQUEUE_SUB 28 /* 24 byets for wqs */ -#define SIZEOF_WAITQUEUE_ELEMENT 16 /* 16 byets per wqe */ -#define SIZEOF_WAITQUEUE_LINK 28 /* 28 byets per wqe */ - -#ifdef MACH_KERNEL_PRIVATE - /* - * wait_queue_sub_t - * This is the common definition for a subordinate wait queue. + * wait_queue_set_t + * This is the common definition for a set wait queue. * These can be linked as members/elements of multiple regular * wait queues. They have an additional set of linkages to * identify the linkage structures that point to them. - * WARNING: Cannot change this data structure without updating SIZEOF_WAITQUEUE_SUB */ -typedef struct wait_queue_sub { - WaitQueue wqs_wait_queue; /* our wait queue */ - queue_head_t wqs_sublinks; /* links from sub perspective */ +typedef struct wait_queue_set { + WaitQueue wqs_wait_queue; /* our wait queue */ + queue_head_t wqs_setlinks; /* links from set perspective */ unsigned int wqs_refcount; /* refcount for preposting */ -} WaitQueueSub; - - -#define WAIT_QUEUE_SUB_NULL ((wait_queue_sub_t)0) +} WaitQueueSet; +#define wqs_type wqs_wait_queue.wq_type +#define wqs_fifo wqs_wait_queue.wq_fifo +#define wqs_isprepost wqs_wait_queue.wq_isprepost +#define wqs_queue wqs_wait_queue.wq_queue /* * wait_queue_element_t @@ -87,23 +86,23 @@ typedef struct wait_queue_sub { * queue. It is the common first fields in a thread shuttle * and wait_queue_link_t. In that way, a wait queue can * consist of both thread shuttle elements and links off of - * to other (subordinate) wait queues. + * to other (set) wait queues. * - * WARNING: The first three fields of the thread shuttle - * definition does not use this definition yet. Any change in + * WARNING: These fields correspond to fields in the thread + * shuttle (run queue links and run queue pointer). Any change in * the layout here will have to be matched with a change there. - * WARNING: Cannot change this data structure without updating SIZEOF_WAITQUEUE_ELEMENT */ typedef struct wait_queue_element { queue_chain_t wqe_links; /* link of elements on this queue */ + void * wqe_type; /* Identifies link vs. thread */ wait_queue_t wqe_queue; /* queue this element is on */ - event_t wqe_event; /* event this element is waiting for */ -} *wait_queue_element_t; +} WaitQueueElement; +typedef WaitQueueElement *wait_queue_element_t; /* * wait_queue_link_t - * Specialized wait queue element type for linking subordinate + * Specialized wait queue element type for linking set * event waits queues onto a wait queue. In this way, an event * can be constructed so that any thread waiting on any number * of associated wait queues can handle the event, while letting @@ -115,162 +114,207 @@ typedef struct wait_queue_element { * event queues of which it is a member. An IPC event post associated * with that port may wake up any thread from any of those portsets, * or one that was waiting locally on the port itself. - * WARNING: Cannot change this data structure without updating SIZEOF_WAITQUEUE_LINK */ typedef struct wait_queue_link { - struct wait_queue_element wql_element; /* element on master */ - queue_chain_t wql_sublinks; /* element on sub */ - wait_queue_sub_t wql_subqueue; /* sub queue */ + WaitQueueElement wql_element; /* element on master */ + queue_chain_t wql_setlinks; /* element on set */ + wait_queue_set_t wql_setqueue; /* set queue */ } WaitQueueLink; - -#define WAIT_QUEUE_LINK_NULL ((wait_queue_link_t)0) - #define wql_links wql_element.wqe_links +#define wql_type wql_element.wqe_type #define wql_queue wql_element.wqe_queue -#define wql_event wql_element.wqe_event - -#define wait_queue_empty(wq) (queue_empty(&(wq)->wq_queue)) - -#define wait_queue_held(wq) (hw_lock_held(&(wq)->wq_interlock)) -#define wait_queue_is_sub(wqs) ((wqs)->wqs_wait_queue.wq_issub) -#define wqs_lock(wqs) wait_queue_lock(&(wqs)->wqs_wait_queue) -#define wqs_unlock(wqs) wait_queue_unlock(&(wqs)->wqs_wait_queue) -#define wqs_lock_try(wqs) wait_queue__try_lock(&(wqs)->wqs_wait_queue) +#define _WAIT_QUEUE_inited 0xf1d0 +#define _WAIT_QUEUE_SET_inited 0xf1d1 -extern int wait_queue_subordinate; -#define WAIT_QUEUE_SUBORDINATE &_wait_queue_subordinate +#define wait_queue_is_queue(wq) \ + ((wq)->wq_type == _WAIT_QUEUE_inited) -extern void wait_queue_init( - wait_queue_t wait_queue, - int policy); +#define wait_queue_is_set(wqs) \ + ((wqs)->wqs_type == _WAIT_QUEUE_SET_inited) -extern kern_return_t wait_queue_link( - wait_queue_t wait_queue, - wait_queue_sub_t subordinate_queue); +#define wait_queue_is_valid(wq) \ + (((wq)->wq_type & ~1) == _WAIT_QUEUE_inited) -extern kern_return_t wait_queue_unlink( - wait_queue_t wait_queue, - wait_queue_sub_t subordinate_queue); -extern void wait_queue_unlink_one( - wait_queue_t wait_queue, - wait_queue_sub_t *subordinate_queue_pointer); +#define wait_queue_empty(wq) (queue_empty(&(wq)->wq_queue)) +#define wait_queue_held(wq) (hw_lock_held(&(wq)->wq_interlock)) +#define wait_queue_lock_try(wq) (hw_lock_try(&(wq)->wq_interlock)) -extern boolean_t wait_queue_member_queue( - wait_queue_t wait_queue, - wait_queue_sub_t subordinate_queue); +/* + * Double the standard lock timeout, because wait queues tend + * to iterate over a number of threads - locking each. If there is + * a problem with a thread lock, it normally times out at the wait + * queue level first, hiding the real problem. + */ +#define wait_queue_lock(wq) \ + ((void) (!hw_lock_to(&(wq)->wq_interlock, LockTimeOut * 2) ? \ + panic("wait queue deadlock - wq=0x%x, cpu=%d\n", \ + wq, cpu_number()) : 0)) -extern kern_return_t clear_wait_queue_internal( - thread_t thread, - int result); +#define wait_queue_unlock(wq) \ + (assert(wait_queue_held(wq)), hw_lock_unlock(&(wq)->wq_interlock)) -extern kern_return_t wait_queue_remove( - thread_t thread); +#define wqs_lock(wqs) wait_queue_lock(&(wqs)->wqs_wait_queue) +#define wqs_unlock(wqs) wait_queue_unlock(&(wqs)->wqs_wait_queue) +#define wqs_lock_try(wqs) wait_queue__try_lock(&(wqs)->wqs_wait_queue) #define wait_queue_assert_possible(thread) \ ((thread)->wait_queue == WAIT_QUEUE_NULL) - - /******** Decomposed interfaces (to build higher level constructs) ***********/ -extern void wait_queue_lock( - wait_queue_t wait_queue); - -extern void wait_queue_unlock( - wait_queue_t wait_queue); - -extern boolean_t wait_queue_lock_try( - wait_queue_t wait_queue); - /* assert intent to wait on a locked wait queue */ -extern boolean_t wait_queue_assert_wait_locked( +__private_extern__ wait_result_t wait_queue_assert_wait64_locked( wait_queue_t wait_queue, - event_t wait_event, - int interruptible, + event64_t wait_event, + wait_interrupt_t interruptible, boolean_t unlock); /* peek to see which thread would be chosen for a wakeup - but keep on queue */ -extern void wait_queue_peek_locked( +__private_extern__ void wait_queue_peek64_locked( wait_queue_t wait_queue, - event_t event, + event64_t event, thread_t *thread, wait_queue_t *found_queue); /* peek to see which thread would be chosen for a wakeup - but keep on queue */ -extern void wait_queue_pull_thread_locked( +__private_extern__ void wait_queue_pull_thread_locked( wait_queue_t wait_queue, thread_t thread, boolean_t unlock); /* wakeup all threads waiting for a particular event on locked queue */ -extern kern_return_t wait_queue_wakeup_one_locked( +__private_extern__ kern_return_t wait_queue_wakeup64_all_locked( wait_queue_t wait_queue, - event_t wake_event, - int result, + event64_t wake_event, + wait_result_t result, boolean_t unlock); /* wakeup one thread waiting for a particular event on locked queue */ -extern kern_return_t wait_queue_wakeup_one_locked( +__private_extern__ kern_return_t wait_queue_wakeup64_one_locked( wait_queue_t wait_queue, - event_t wake_event, - int result, + event64_t wake_event, + wait_result_t result, boolean_t unlock); -/* return the identity of a thread that is waiting for */ -extern thread_t wait_queue_recommend_locked( - wait_queue_t wait_queue, - event_t wake_event); - /* return identity of a thread awakened for a particular */ -extern thread_t wait_queue_wakeup_identity_locked( +__private_extern__ thread_t wait_queue_wakeup64_identity_locked( wait_queue_t wait_queue, - event_t wake_event, - int result, + event64_t wake_event, + wait_result_t result, boolean_t unlock); /* wakeup thread iff its still waiting for a particular event on locked queue */ -extern kern_return_t wait_queue_wakeup_thread_locked( +__private_extern__ kern_return_t wait_queue_wakeup64_thread_locked( wait_queue_t wait_queue, - event_t wake_event, + event64_t wake_event, thread_t thread, - int result, + wait_result_t result, boolean_t unlock); #endif /* MACH_KERNEL_PRIVATE */ +#ifdef __APPLE_API_UNSTABLE +/******** Semi-Public interfaces (not a part of a higher construct) ************/ + +extern kern_return_t wait_queue_init( + wait_queue_t wait_queue, + int policy); + +extern wait_queue_set_t wait_queue_set_alloc( + int policy); + +extern kern_return_t wait_queue_set_free( + wait_queue_set_t set_queue); + +extern wait_queue_link_t wait_queue_link_alloc( + int policy); + +extern kern_return_t wait_queue_link_free( + wait_queue_link_t link_element); + +#endif /* __APPLE_API_UNSTABLE */ + +#ifdef __APPLE_API_EVOLVING + extern wait_queue_t wait_queue_alloc( - int policy); + int policy); + +extern kern_return_t wait_queue_free( + wait_queue_t wait_queue); -extern void wait_queue_free( +extern kern_return_t wait_queue_link( + wait_queue_t wait_queue, + wait_queue_set_t set_queue); + +extern kern_return_t wait_queue_unlink( + wait_queue_t wait_queue, + wait_queue_set_t set_queue); + +extern kern_return_t wait_queue_unlink_all( wait_queue_t wait_queue); -/******** Standalone interfaces (not a part of a higher construct) ************/ +extern kern_return_t wait_queue_set_unlink_all( + wait_queue_set_t set_queue); + +/* assert intent to wait on pair */ +extern wait_result_t wait_queue_assert_wait64( + wait_queue_t wait_queue, + event64_t wait_event, + wait_interrupt_t interruptible); + +/* wakeup the most appropriate thread waiting on pair */ +extern kern_return_t wait_queue_wakeup64_one( + wait_queue_t wait_queue, + event64_t wake_event, + wait_result_t result); + +/* wakeup all the threads waiting on pair */ +extern kern_return_t wait_queue_wakeup64_all( + wait_queue_t wait_queue, + event64_t wake_event, + wait_result_t result); + +/* wakeup a specified thread waiting iff waiting on pair */ +extern kern_return_t wait_queue_wakeup64_thread( + wait_queue_t wait_queue, + event64_t wake_event, + thread_t thread, + wait_result_t result); + +#endif /* __APPLE_API_EVOLVING */ + +/* + * Compatibility Wait Queue APIs based on pointer events instead of 64bit + * integer events. + */ /* assert intent to wait on pair */ -extern boolean_t wait_queue_assert_wait( +extern wait_result_t wait_queue_assert_wait( wait_queue_t wait_queue, event_t wait_event, - int interruptible); + wait_interrupt_t interruptible); /* wakeup the most appropriate thread waiting on pair */ -extern kern_return_t wait_queue_wakeup_one( +extern kern_return_t wait_queue_wakeup_one( wait_queue_t wait_queue, event_t wake_event, - int result); + wait_result_t result); /* wakeup all the threads waiting on pair */ -extern kern_return_t wait_queue_wakeup_all( +extern kern_return_t wait_queue_wakeup_all( wait_queue_t wait_queue, event_t wake_event, - int result); + wait_result_t result); /* wakeup a specified thread waiting iff waiting on pair */ -extern kern_return_t wait_queue_wakeup_thread( +extern kern_return_t wait_queue_wakeup_thread( wait_queue_t wait_queue, event_t wake_event, thread_t thread, - int result); + wait_result_t result); + +#endif /* __APPLE_API_PRIVATE */ #endif /* _KERN_WAIT_QUEUE_H_ */