X-Git-Url: https://git.saurik.com/apple/libdispatch.git/blobdiff_plain/fa22f35b3ccab0081bb7090c32773dcd7463a045..refs/heads/master:/src/queue_internal.h diff --git a/src/queue_internal.h b/src/queue_internal.h index 2435b13..f70356a 100644 --- a/src/queue_internal.h +++ b/src/queue_internal.h @@ -44,21 +44,25 @@ #define DISPATCH_CACHELINE_ALIGN \ __attribute__((__aligned__(DISPATCH_CACHELINE_SIZE))) +#define DISPATCH_CACHELINE_PAD_SIZE(type) \ + (roundup(sizeof(type), DISPATCH_CACHELINE_SIZE) - sizeof(type)) + #pragma mark - #pragma mark dispatch_queue_t DISPATCH_ENUM(dispatch_queue_flags, uint32_t, - DQF_NONE = 0x0000, - DQF_AUTORELEASE_ALWAYS = 0x0001, - DQF_AUTORELEASE_NEVER = 0x0002, -#define _DQF_AUTORELEASE_MASK 0x0003 - DQF_THREAD_BOUND = 0x0004, // queue is bound to a thread - DQF_BARRIER_BIT = 0x0008, // queue is a barrier on its target - DQF_TARGETED = 0x0010, // queue is targeted by another object - DQF_LABEL_NEEDS_FREE = 0x0020, // queue label was strduped; need to free it - DQF_CANNOT_TRYSYNC = 0x0040, - DQF_RELEASED = 0x0080, // xref_cnt == -1 + DQF_NONE = 0x00000000, + DQF_AUTORELEASE_ALWAYS = 0x00010000, + DQF_AUTORELEASE_NEVER = 0x00020000, +#define _DQF_AUTORELEASE_MASK 0x00030000 + DQF_THREAD_BOUND = 0x00040000, // queue is bound to a thread + DQF_BARRIER_BIT = 0x00080000, // queue is a barrier on its target + DQF_TARGETED = 0x00100000, // queue is targeted by another object + DQF_LABEL_NEEDS_FREE = 0x00200000, // queue label was strduped; need to free it + DQF_CANNOT_TRYSYNC = 0x00400000, + DQF_RELEASED = 0x00800000, // xref_cnt == -1 + DQF_LEGACY = 0x01000000, // only applies to sources // @@ -77,81 +81,71 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, // will be -p-. // // -pd - // Received EV_DELETE (from ap-), needs to free `ds_dkev`, the knote is - // gone from the kernel, but ds_dkev lives. Next state will be --d. + // Received EV_DELETE (from ap-), needs to unregister ds_refs, the muxnote + // is gone from the kernel. Next state will be --d. // // -p- // Received an EV_ONESHOT event (from a--), or the delivery of an event // causing the cancellation to fail with EINPROGRESS was delivered - // (from ap-). The knote still lives, next state will be --d. + // (from ap-). The muxnote still lives, next state will be --d. // // --d - // Final state of the source, the knote is gone from the kernel and - // ds_dkev is freed. The source can safely be released. + // Final state of the source, the muxnote is gone from the kernel and + // ds_refs is unregistered. The source can safely be released. // // a-d (INVALID) // apd (INVALID) // Setting DSF_DELETED should also always atomically clear DSF_ARMED. If - // the knote is gone from the kernel, it makes no sense whatsoever to + // the muxnote is gone from the kernel, it makes no sense whatsoever to // have it armed. And generally speaking, once `d` or `p` has been set, // `a` cannot do a cleared -> set transition anymore // (see _dispatch_source_try_set_armed). // - DSF_CANCEL_WAITER = 0x0800, // synchronous waiters for cancel - DSF_CANCELED = 0x1000, // cancellation has been requested - DSF_ARMED = 0x2000, // source is armed - DSF_DEFERRED_DELETE = 0x4000, // source is pending delete - DSF_DELETED = 0x8000, // source knote is deleted + DSF_WLH_CHANGED = 0x04000000, + DSF_CANCEL_WAITER = 0x08000000, // synchronous waiters for cancel + DSF_CANCELED = 0x10000000, // cancellation has been requested + DSF_ARMED = 0x20000000, // source is armed + DSF_DEFERRED_DELETE = 0x40000000, // source is pending delete + DSF_DELETED = 0x80000000, // source muxnote is deleted #define DSF_STATE_MASK (DSF_ARMED | DSF_DEFERRED_DELETE | DSF_DELETED) - DQF_WIDTH_MASK = 0xffff0000, -#define DQF_WIDTH_SHIFT 16 +#define DQF_FLAGS_MASK ((dispatch_queue_flags_t)0xffff0000) +#define DQF_WIDTH_MASK ((dispatch_queue_flags_t)0x0000ffff) +#define DQF_WIDTH(n) ((dispatch_queue_flags_t)(uint16_t)(n)) ); #define _DISPATCH_QUEUE_HEADER(x) \ struct os_mpsc_queue_s _as_oq[0]; \ DISPATCH_OBJECT_HEADER(x); \ _OS_MPSC_QUEUE_FIELDS(dq, dq_state); \ - dispatch_queue_t dq_specific_q; \ - union { \ - uint32_t volatile dq_atomic_flags; \ - DISPATCH_STRUCT_LITTLE_ENDIAN_2( \ - uint16_t dq_atomic_bits, \ - uint16_t dq_width \ - ); \ - }; \ uint32_t dq_side_suspend_cnt; \ - DISPATCH_INTROSPECTION_QUEUE_HEADER; \ - dispatch_unfair_lock_s dq_sidelock - /* LP64: 32bit hole on LP64 */ + dispatch_unfair_lock_s dq_sidelock; \ + union { \ + dispatch_queue_t dq_specific_q; \ + struct dispatch_source_refs_s *ds_refs; \ + struct dispatch_timer_source_refs_s *ds_timer_refs; \ + struct dispatch_mach_recv_refs_s *dm_recv_refs; \ + }; \ + DISPATCH_UNION_LE(uint32_t volatile dq_atomic_flags, \ + const uint16_t dq_width, \ + const uint16_t __dq_opaque \ + ); \ + DISPATCH_INTROSPECTION_QUEUE_HEADER + /* LP64: 32bit hole */ #define DISPATCH_QUEUE_HEADER(x) \ struct dispatch_queue_s _as_dq[0]; \ _DISPATCH_QUEUE_HEADER(x) -#define DISPATCH_QUEUE_ALIGN __attribute__((aligned(8))) +struct _dispatch_unpadded_queue_s { + _DISPATCH_QUEUE_HEADER(dummy); +}; -#define DISPATCH_QUEUE_WIDTH_POOL 0x7fff -#define DISPATCH_QUEUE_WIDTH_MAX 0x7ffe -#define DISPATCH_QUEUE_USES_REDIRECTION(width) \ - ({ uint16_t _width = (width); \ - _width > 1 && _width < DISPATCH_QUEUE_WIDTH_POOL; }) +#define DISPATCH_QUEUE_CACHELINE_PAD \ + DISPATCH_CACHELINE_PAD_SIZE(struct _dispatch_unpadded_queue_s) #define DISPATCH_QUEUE_CACHELINE_PADDING \ char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD] -#ifdef __LP64__ -#define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (sizeof(uint32_t) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \ - + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) -#elif OS_OBJECT_HAVE_OBJC1 -#define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (11*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \ - + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) -#else -#define DISPATCH_QUEUE_CACHELINE_PAD (( \ - (12*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \ - + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE) -#endif /* * dispatch queues `dq_state` demystified @@ -161,27 +155,27 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, * Most Significant 32 bit Word * ---------------------------- * - * sc: suspend count (bits 63 - 57) + * sc: suspend count (bits 63 - 58) * The suspend count unsurprisingly holds the suspend count of the queue * Only 7 bits are stored inline. Extra counts are transfered in a side * suspend count and when that has happened, the ssc: bit is set. */ -#define DISPATCH_QUEUE_SUSPEND_INTERVAL 0x0200000000000000ull -#define DISPATCH_QUEUE_SUSPEND_HALF 0x40u +#define DISPATCH_QUEUE_SUSPEND_INTERVAL 0x0400000000000000ull +#define DISPATCH_QUEUE_SUSPEND_HALF 0x20u /* - * ssc: side suspend count (bit 56) + * ssc: side suspend count (bit 57) * This bit means that the total suspend count didn't fit in the inline * suspend count, and that there are additional suspend counts stored in the * `dq_side_suspend_cnt` field. */ -#define DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT 0x0100000000000000ull +#define DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT 0x0200000000000000ull /* - * i: inactive bit (bit 55) + * i: inactive bit (bit 56) * This bit means that the object is inactive (see dispatch_activate) */ -#define DISPATCH_QUEUE_INACTIVE 0x0080000000000000ull +#define DISPATCH_QUEUE_INACTIVE 0x0100000000000000ull /* - * na: needs activation (bit 54) + * na: needs activation (bit 55) * This bit is set if the object is created inactive. It tells * dispatch_queue_wakeup to perform various tasks at first wakeup. * @@ -189,27 +183,32 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, * the object from being woken up (because _dq_state_should_wakeup will say * no), except in the dispatch_activate/dispatch_resume codepath. */ -#define DISPATCH_QUEUE_NEEDS_ACTIVATION 0x0040000000000000ull +#define DISPATCH_QUEUE_NEEDS_ACTIVATION 0x0080000000000000ull /* * This mask covers the suspend count (sc), side suspend count bit (ssc), * inactive (i) and needs activation (na) bits */ -#define DISPATCH_QUEUE_SUSPEND_BITS_MASK 0xffc0000000000000ull +#define DISPATCH_QUEUE_SUSPEND_BITS_MASK 0xff80000000000000ull /* - * ib: in barrier (bit 53) + * ib: in barrier (bit 54) * This bit is set when the queue is currently executing a barrier */ -#define DISPATCH_QUEUE_IN_BARRIER 0x0020000000000000ull +#define DISPATCH_QUEUE_IN_BARRIER 0x0040000000000000ull /* - * qf: queue full (bit 52) + * qf: queue full (bit 53) * This bit is a subtle hack that allows to check for any queue width whether * the full width of the queue is used or reserved (depending on the context) * In other words that the queue has reached or overflown its capacity. */ -#define DISPATCH_QUEUE_WIDTH_FULL_BIT 0x0010000000000000ull -#define DISPATCH_QUEUE_WIDTH_FULL 0x8000ull +#define DISPATCH_QUEUE_WIDTH_FULL_BIT 0x0020000000000000ull +#define DISPATCH_QUEUE_WIDTH_FULL 0x1000ull +#define DISPATCH_QUEUE_WIDTH_POOL (DISPATCH_QUEUE_WIDTH_FULL - 1) +#define DISPATCH_QUEUE_WIDTH_MAX (DISPATCH_QUEUE_WIDTH_FULL - 2) +#define DISPATCH_QUEUE_USES_REDIRECTION(width) \ + ({ uint16_t _width = (width); \ + _width > 1 && _width < DISPATCH_QUEUE_WIDTH_POOL; }) /* - * w: width (bits 51 - 37) + * w: width (bits 52 - 41) * This encodes how many work items are in flight. Barriers hold `dq_width` * of them while they run. This is encoded as a signed offset with respect, * to full use, where the negative values represent how many available slots @@ -218,19 +217,19 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, * * When this value is positive, then `wo` is always set to 1. */ -#define DISPATCH_QUEUE_WIDTH_INTERVAL 0x0000002000000000ull -#define DISPATCH_QUEUE_WIDTH_MASK 0x001fffe000000000ull -#define DISPATCH_QUEUE_WIDTH_SHIFT 37 +#define DISPATCH_QUEUE_WIDTH_INTERVAL 0x0000020000000000ull +#define DISPATCH_QUEUE_WIDTH_MASK 0x003ffe0000000000ull +#define DISPATCH_QUEUE_WIDTH_SHIFT 41 /* - * pb: pending barrier (bit 36) + * pb: pending barrier (bit 40) * Drainers set this bit when they couldn't run the next work item and it is * a barrier. When this bit is set, `dq_width - 1` work item slots are * reserved so that no wakeup happens until the last work item in flight * completes. */ -#define DISPATCH_QUEUE_PENDING_BARRIER 0x0000001000000000ull +#define DISPATCH_QUEUE_PENDING_BARRIER 0x0000010000000000ull /* - * d: dirty bit (bit 35) + * d: dirty bit (bit 39) * This bit is set when a queue transitions from empty to not empty. * This bit is set before dq_items_head is set, with appropriate barriers. * Any thread looking at a queue head is responsible for unblocking any @@ -342,68 +341,70 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, * * So on the async "acquire" side, there is no subtlety at all. */ -#define DISPATCH_QUEUE_DIRTY 0x0000000800000000ull +#define DISPATCH_QUEUE_DIRTY 0x0000008000000000ull /* - * qo: (bit 34) - * Set when a queue has a useful override set. - * This bit is only cleared when the final drain_try_unlock() succeeds. - * - * When the queue dq_override is touched (overrides or-ed in), usually with - * _dispatch_queue_override_priority(), then the HAS_OVERRIDE bit is set - * with a release barrier and one of these three things happen next: - * - * - the queue is enqueued, which will cause it to be drained, and the - * override to be handled by _dispatch_queue_drain_try_unlock(). - * In rare cases it could cause the queue to be queued while empty though. + * md: enqueued/draining on manager (bit 38) + * Set when enqueued and draining on the manager hierarchy. * - * - the DIRTY bit is also set with a release barrier, which pairs with - * the handling of these bits by _dispatch_queue_drain_try_unlock(), - * so that dq_override is reset properly. + * Unlike the ENQUEUED bit, it is kept until the queue is unlocked from its + * invoke call on the manager. This is used to prevent stealing, and + * overrides to be applied down the target queue chain. + */ +#define DISPATCH_QUEUE_ENQUEUED_ON_MGR 0x0000004000000000ull +/* + * r: queue graph role (bits 37 - 36) + * Queue role in the target queue graph * - * - the queue was suspended, and _dispatch_queue_resume() will handle the - * override as part of its wakeup sequence. + * 11: unused + * 10: WLH base + * 01: non wlh base + * 00: inner queue */ -#define DISPATCH_QUEUE_HAS_OVERRIDE 0x0000000400000000ull +#define DISPATCH_QUEUE_ROLE_MASK 0x0000003000000000ull +#define DISPATCH_QUEUE_ROLE_BASE_WLH 0x0000002000000000ull +#define DISPATCH_QUEUE_ROLE_BASE_ANON 0x0000001000000000ull +#define DISPATCH_QUEUE_ROLE_INNER 0x0000000000000000ull /* - * p: pended bit (bit 33) - * Set when a drain lock has been pended. When this bit is set, - * the drain lock is taken and ENQUEUED is never set. + * o: has override (bit 35, if role is DISPATCH_QUEUE_ROLE_BASE_ANON) + * Set when a queue has received a QOS override and needs to reset it. + * This bit is only cleared when the final drain_try_unlock() succeeds. * - * This bit marks a queue that needs further processing but was kept pended - * by an async drainer (not reenqueued) in the hope of being able to drain - * it further later. + * sw: has received sync wait (bit 35, if role DISPATCH_QUEUE_ROLE_BASE_WLH) + * Set when a queue owner has been exposed to the kernel because of + * dispatch_sync() contention. */ -#define DISPATCH_QUEUE_DRAIN_PENDED 0x0000000200000000ull +#define DISPATCH_QUEUE_RECEIVED_OVERRIDE 0x0000000800000000ull +#define DISPATCH_QUEUE_RECEIVED_SYNC_WAIT 0x0000000800000000ull /* - * e: enqueued bit (bit 32) - * Set when a queue is enqueued on its target queue + * max_qos: max qos (bits 34 - 32) + * This is the maximum qos that has been enqueued on the queue */ -#define DISPATCH_QUEUE_ENQUEUED 0x0000000100000000ull +#define DISPATCH_QUEUE_MAX_QOS_MASK 0x0000000700000000ull +#define DISPATCH_QUEUE_MAX_QOS_SHIFT 32 /* * dl: drain lock (bits 31-0) * This is used by the normal drain to drain exlusively relative to other * drain stealers (like the QoS Override codepath). It holds the identity * (thread port) of the current drainer. + * + * st: sync transfer (bit 1 or 30) + * Set when a dispatch_sync() is transferred to + * + * e: enqueued bit (bit 0 or 31) + * Set when a queue is enqueued on its target queue */ -#define DISPATCH_QUEUE_DRAIN_UNLOCK_MASK 0x00000002ffffffffull -#ifdef DLOCK_NOWAITERS_BIT -#define DISPATCH_QUEUE_DRAIN_OWNER_MASK \ - ((uint64_t)(DLOCK_OWNER_MASK | DLOCK_NOFAILED_TRYLOCK_BIT)) -#define DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT(v) \ - (((v) & ~(DISPATCH_QUEUE_DRAIN_PENDED|DISPATCH_QUEUE_DRAIN_OWNER_MASK))\ - ^ DLOCK_NOWAITERS_BIT) -#define DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK \ - (DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_HAS_OVERRIDE | \ - DLOCK_NOWAITERS_BIT) -#else -#define DISPATCH_QUEUE_DRAIN_OWNER_MASK \ - ((uint64_t)(DLOCK_OWNER_MASK | DLOCK_FAILED_TRYLOCK_BIT)) -#define DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT(v) \ - ((v) & ~(DISPATCH_QUEUE_DRAIN_PENDED|DISPATCH_QUEUE_DRAIN_OWNER_MASK)) +#define DISPATCH_QUEUE_DRAIN_OWNER_MASK ((uint64_t)DLOCK_OWNER_MASK) +#define DISPATCH_QUEUE_SYNC_TRANSFER ((uint64_t)DLOCK_FAILED_TRYLOCK_BIT) +#define DISPATCH_QUEUE_ENQUEUED ((uint64_t)DLOCK_WAITERS_BIT) + #define DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK \ - (DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_HAS_OVERRIDE | \ - DLOCK_WAITERS_BIT) -#endif + (DISPATCH_QUEUE_ENQUEUED_ON_MGR | DISPATCH_QUEUE_ENQUEUED | \ + DISPATCH_QUEUE_ROLE_MASK | DISPATCH_QUEUE_MAX_QOS_MASK) + +#define DISPATCH_QUEUE_DRAIN_UNLOCK_MASK \ + (DISPATCH_QUEUE_DRAIN_OWNER_MASK | DISPATCH_QUEUE_RECEIVED_OVERRIDE | \ + DISPATCH_QUEUE_RECEIVED_SYNC_WAIT | DISPATCH_QUEUE_SYNC_TRANSFER) + /* ******************************************************************************* * @@ -425,8 +426,6 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, * that right. To do so, prior to taking any decision, they also try to own * the full "barrier" width on the given queue. * - * see _dispatch_try_lock_transfer_or_wakeup - * ******************************************************************************* * * Enqueuing and wakeup rules @@ -497,12 +496,17 @@ DISPATCH_ENUM(dispatch_queue_flags, uint32_t, (DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_INTERVAL) DISPATCH_CLASS_DECL(queue); -#if !(defined(__cplusplus) && DISPATCH_INTROSPECTION) + +#if !defined(__cplusplus) || !DISPATCH_INTROSPECTION struct dispatch_queue_s { _DISPATCH_QUEUE_HEADER(queue); DISPATCH_QUEUE_CACHELINE_PADDING; // for static queues only -} DISPATCH_QUEUE_ALIGN; -#endif // !(defined(__cplusplus) && DISPATCH_INTROSPECTION) +} DISPATCH_ATOMIC64_ALIGN; + +#if __has_feature(c_static_assert) && !DISPATCH_INTROSPECTION +_Static_assert(sizeof(struct dispatch_queue_s) <= 128, "dispatch queue size"); +#endif +#endif // !defined(__cplusplus) || !DISPATCH_INTROSPECTION DISPATCH_INTERNAL_SUBCLASS_DECL(queue_serial, queue); DISPATCH_INTERNAL_SUBCLASS_DECL(queue_concurrent, queue); @@ -520,16 +524,14 @@ typedef union { struct dispatch_source_s *_ds; struct dispatch_mach_s *_dm; struct dispatch_queue_specific_queue_s *_dqsq; - struct dispatch_timer_aggregate_s *_dta; #if USE_OBJC os_mpsc_queue_t _ojbc_oq; dispatch_queue_t _objc_dq; dispatch_source_t _objc_ds; dispatch_mach_t _objc_dm; dispatch_queue_specific_queue_t _objc_dqsq; - dispatch_timer_aggregate_t _objc_dta; #endif -} dispatch_queue_class_t __attribute__((__transparent_union__)); +} dispatch_queue_class_t DISPATCH_TRANSPARENT_UNION; typedef struct dispatch_thread_context_s *dispatch_thread_context_t; typedef struct dispatch_thread_context_s { @@ -546,52 +548,59 @@ typedef struct dispatch_thread_frame_s { // must be in the same order as our TSD keys! dispatch_queue_t dtf_queue; dispatch_thread_frame_t dtf_prev; - struct dispatch_object_s *dtf_deferred; } dispatch_thread_frame_s; -DISPATCH_ENUM(dispatch_queue_wakeup_target, long, - DISPATCH_QUEUE_WAKEUP_NONE = 0, - DISPATCH_QUEUE_WAKEUP_TARGET, - DISPATCH_QUEUE_WAKEUP_MGR, -); +typedef dispatch_queue_t dispatch_queue_wakeup_target_t; +#define DISPATCH_QUEUE_WAKEUP_NONE ((dispatch_queue_wakeup_target_t)0) +#define DISPATCH_QUEUE_WAKEUP_TARGET ((dispatch_queue_wakeup_target_t)1) +#define DISPATCH_QUEUE_WAKEUP_MGR (&_dispatch_mgr_q) +#define DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT ((dispatch_queue_wakeup_target_t)-1) -void _dispatch_queue_class_override_drainer(dispatch_queue_t dqu, - pthread_priority_t pp, dispatch_wakeup_flags_t flags); -void _dispatch_queue_class_wakeup(dispatch_queue_t dqu, pthread_priority_t pp, +void _dispatch_queue_class_wakeup(dispatch_queue_t dqu, dispatch_qos_t qos, dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target); - -void _dispatch_queue_destroy(dispatch_queue_t dq); -void _dispatch_queue_dispose(dispatch_queue_t dq); +dispatch_priority_t _dispatch_queue_compute_priority_and_wlh( + dispatch_queue_t dq, dispatch_wlh_t *wlh_out); +void _dispatch_queue_destroy(dispatch_queue_t dq, bool *allow_free); +void _dispatch_queue_dispose(dispatch_queue_t dq, bool *allow_free); +void _dispatch_queue_xref_dispose(struct dispatch_queue_s *dq); void _dispatch_queue_set_target_queue(dispatch_queue_t dq, dispatch_queue_t tq); void _dispatch_queue_suspend(dispatch_queue_t dq); void _dispatch_queue_resume(dispatch_queue_t dq, bool activate); -void _dispatch_queue_finalize_activation(dispatch_queue_t dq); -void _dispatch_queue_invoke(dispatch_queue_t dq, dispatch_invoke_flags_t flags); -void _dispatch_queue_push_list_slow(dispatch_queue_t dq, unsigned int n); +void _dispatch_queue_finalize_activation(dispatch_queue_t dq, + bool *allow_resume); +void _dispatch_queue_invoke(dispatch_queue_t dq, + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags); +void _dispatch_global_queue_poke(dispatch_queue_t dq, int n, int floor); void _dispatch_queue_push(dispatch_queue_t dq, dispatch_object_t dou, - pthread_priority_t pp); -void _dispatch_try_lock_transfer_or_wakeup(dispatch_queue_t dq); -void _dispatch_queue_wakeup(dispatch_queue_t dq, pthread_priority_t pp, + dispatch_qos_t qos); +void _dispatch_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos, dispatch_wakeup_flags_t flags); -dispatch_queue_t _dispatch_queue_serial_drain(dispatch_queue_t dq, - dispatch_invoke_flags_t flags, uint64_t *owned, - struct dispatch_object_s **dc_ptr); -void _dispatch_queue_drain_deferred_invoke(dispatch_queue_t dq, - dispatch_invoke_flags_t flags, uint64_t to_unlock, - struct dispatch_object_s *dc); -void _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t - dqsq); -void _dispatch_root_queue_wakeup(dispatch_queue_t dq, pthread_priority_t pp, +dispatch_queue_wakeup_target_t _dispatch_queue_serial_drain(dispatch_queue_t dq, + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags, + uint64_t *owned); +void _dispatch_queue_drain_sync_waiter(dispatch_queue_t dq, + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags, + uint64_t owned); +void _dispatch_queue_specific_queue_dispose( + dispatch_queue_specific_queue_t dqsq, bool *allow_free); +void _dispatch_root_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos, dispatch_wakeup_flags_t flags); -void _dispatch_root_queue_drain_deferred_item(dispatch_queue_t dq, - struct dispatch_object_s *dou, pthread_priority_t pp); -void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq); -void _dispatch_main_queue_wakeup(dispatch_queue_t dq, pthread_priority_t pp, +void _dispatch_root_queue_push(dispatch_queue_t dq, dispatch_object_t dou, + dispatch_qos_t qos); +#if DISPATCH_USE_KEVENT_WORKQUEUE +void _dispatch_root_queue_drain_deferred_item(dispatch_deferred_items_t ddi + DISPATCH_PERF_MON_ARGS_PROTO); +void _dispatch_root_queue_drain_deferred_wlh(dispatch_deferred_items_t ddi + DISPATCH_PERF_MON_ARGS_PROTO); +#endif +void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq, + bool *allow_free); +void _dispatch_main_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos, dispatch_wakeup_flags_t flags); -void _dispatch_runloop_queue_wakeup(dispatch_queue_t dq, pthread_priority_t pp, +void _dispatch_runloop_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos, dispatch_wakeup_flags_t flags); void _dispatch_runloop_queue_xref_dispose(dispatch_queue_t dq); -void _dispatch_runloop_queue_dispose(dispatch_queue_t dq); +void _dispatch_runloop_queue_dispose(dispatch_queue_t dq, bool *allow_free); void _dispatch_mgr_queue_drain(void); #if DISPATCH_USE_MGR_THREAD && DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES void _dispatch_mgr_priority_init(void); @@ -603,13 +612,13 @@ void _dispatch_kevent_workqueue_init(void); #else static inline void _dispatch_kevent_workqueue_init(void) {} #endif -void _dispatch_sync_recurse_invoke(void *ctxt); void _dispatch_apply_invoke(void *ctxt); void _dispatch_apply_redirect_invoke(void *ctxt); void _dispatch_barrier_async_detached_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func); +#define DISPATCH_BARRIER_TRYSYNC_SUSPEND 0x1 void _dispatch_barrier_trysync_or_async_f(dispatch_queue_t dq, void *ctxt, - dispatch_function_t func); + dispatch_function_t func, uint32_t flags); void _dispatch_queue_atfork_child(void); #if DISPATCH_DEBUG @@ -623,10 +632,9 @@ size_t dispatch_queue_debug(dispatch_queue_t dq, char* buf, size_t bufsiz); size_t _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf, size_t bufsiz); -#define DISPATCH_QUEUE_QOS_COUNT 6 -#define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QUEUE_QOS_COUNT * 2) +#define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QOS_MAX * 2) -// must be in lowest to highest qos order (as encoded in pthread_priority_t) +// must be in lowest to highest qos order (as encoded in dispatch_qos_t) // overcommit qos index values need bit 1 set enum { DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS = 0, @@ -644,18 +652,25 @@ enum { _DISPATCH_ROOT_QUEUE_IDX_COUNT, }; +// skip zero +// 1 - main_q +// 2 - mgr_q +// 3 - mgr_root_q +// 4,5,6,7,8,9,10,11,12,13,14,15 - global queues +// we use 'xadd' on Intel, so the initial value == next assigned +#define DISPATCH_QUEUE_SERIAL_NUMBER_INIT 16 extern unsigned long volatile _dispatch_queue_serial_numbers; extern struct dispatch_queue_s _dispatch_root_queues[]; extern struct dispatch_queue_s _dispatch_mgr_q; void _dispatch_root_queues_init(void); -#if HAVE_PTHREAD_WORKQUEUE_QOS -extern pthread_priority_t _dispatch_background_priority; -extern pthread_priority_t _dispatch_user_initiated_priority; +#if DISPATCH_DEBUG +#define DISPATCH_ASSERT_ON_MANAGER_QUEUE() \ + dispatch_assert_queue(&_dispatch_mgr_q) +#else +#define DISPATCH_ASSERT_ON_MANAGER_QUEUE() #endif -typedef uint8_t _dispatch_qos_class_t; - #pragma mark - #pragma mark dispatch_queue_attr_t @@ -668,8 +683,7 @@ typedef enum { DISPATCH_CLASS_DECL(queue_attr); struct dispatch_queue_attr_s { OS_OBJECT_STRUCT_HEADER(dispatch_queue_attr); - _dispatch_qos_class_t dqa_qos_class; - int8_t dqa_relative_priority; + dispatch_priority_requested_t dqa_qos_and_relpri; uint16_t dqa_overcommit:2; uint16_t dqa_autorelease_frequency:2; uint16_t dqa_concurrent:1; @@ -753,7 +767,6 @@ dispatch_queue_attr_t _dispatch_get_default_queue_attr(void); void *dc_ctxt; \ void *dc_data; \ void *dc_other -#define _DISPATCH_SIZEOF_PTR 8 #elif OS_OBJECT_HAVE_OBJC1 #define DISPATCH_CONTINUATION_HEADER(x) \ dispatch_function_t dc_func; \ @@ -771,7 +784,6 @@ dispatch_queue_attr_t _dispatch_get_default_queue_attr(void); void *dc_ctxt; \ void *dc_data; \ void *dc_other -#define _DISPATCH_SIZEOF_PTR 4 #else #define DISPATCH_CONTINUATION_HEADER(x) \ union { \ @@ -789,24 +801,23 @@ dispatch_queue_attr_t _dispatch_get_default_queue_attr(void); void *dc_ctxt; \ void *dc_data; \ void *dc_other -#define _DISPATCH_SIZEOF_PTR 4 #endif #define _DISPATCH_CONTINUATION_PTRS 8 #if DISPATCH_HW_CONFIG_UP // UP devices don't contend on continuations so we don't need to force them to // occupy a whole cacheline (which is intended to avoid contention) #define DISPATCH_CONTINUATION_SIZE \ - (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR) + (_DISPATCH_CONTINUATION_PTRS * DISPATCH_SIZEOF_PTR) #else #define DISPATCH_CONTINUATION_SIZE ROUND_UP_TO_CACHELINE_SIZE( \ - (_DISPATCH_CONTINUATION_PTRS * _DISPATCH_SIZEOF_PTR)) + (_DISPATCH_CONTINUATION_PTRS * DISPATCH_SIZEOF_PTR)) #endif #define ROUND_UP_TO_CONTINUATION_SIZE(x) \ (((x) + (DISPATCH_CONTINUATION_SIZE - 1u)) & \ ~(DISPATCH_CONTINUATION_SIZE - 1u)) // continuation is a dispatch_sync or dispatch_barrier_sync -#define DISPATCH_OBJ_SYNC_SLOW_BIT 0x001ul +#define DISPATCH_OBJ_SYNC_WAITER_BIT 0x001ul // continuation acts as a barrier #define DISPATCH_OBJ_BARRIER_BIT 0x002ul // continuation resources are freed on run @@ -822,17 +833,35 @@ dispatch_queue_attr_t _dispatch_get_default_queue_attr(void); #define DISPATCH_OBJ_CTXT_FETCH_BIT 0x040ul // use the voucher from the continuation even if the queue has voucher set #define DISPATCH_OBJ_ENFORCE_VOUCHER 0x080ul +// never set on continuations, used by mach.c only +#define DISPATCH_OBJ_MACH_BARRIER 0x1000000ul -struct dispatch_continuation_s { +typedef struct dispatch_continuation_s { struct dispatch_object_s _as_do[0]; DISPATCH_CONTINUATION_HEADER(continuation); -}; -typedef struct dispatch_continuation_s *dispatch_continuation_t; +} *dispatch_continuation_t; + +typedef struct dispatch_sync_context_s { + struct dispatch_object_s _as_do[0]; + struct dispatch_continuation_s _as_dc[0]; + DISPATCH_CONTINUATION_HEADER(continuation); + dispatch_function_t dsc_func; + void *dsc_ctxt; +#if DISPATCH_COCOA_COMPAT + dispatch_thread_frame_s dsc_dtf; +#endif + dispatch_thread_event_s dsc_event; + dispatch_tid dsc_waiter; + dispatch_qos_t dsc_override_qos_floor; + dispatch_qos_t dsc_override_qos; + bool dsc_wlh_was_first; + bool dsc_release_storage; +} *dispatch_sync_context_t; typedef struct dispatch_continuation_vtable_s { _OS_OBJECT_CLASS_HEADER(); DISPATCH_INVOKABLE_VTABLE_HEADER(dispatch_continuation); -} *dispatch_continuation_vtable_t; +} const *dispatch_continuation_vtable_t; #ifndef DISPATCH_CONTINUATION_CACHE_LIMIT #if TARGET_OS_EMBEDDED @@ -848,8 +877,9 @@ dispatch_continuation_t _dispatch_continuation_alloc_from_heap(void); void _dispatch_continuation_free_to_heap(dispatch_continuation_t c); void _dispatch_continuation_async(dispatch_queue_t dq, dispatch_continuation_t dc); -void _dispatch_continuation_pop(dispatch_object_t dou, dispatch_queue_t dq, - dispatch_invoke_flags_t flags); +void _dispatch_continuation_pop(dispatch_object_t dou, + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags, + dispatch_queue_t dq); void _dispatch_continuation_invoke(dispatch_object_t dou, voucher_t override_voucher, dispatch_invoke_flags_t flags); @@ -871,6 +901,7 @@ enum { DC_MACH_SEND_BARRRIER_DRAIN_TYPE, DC_MACH_SEND_BARRIER_TYPE, DC_MACH_RECV_BARRIER_TYPE, + DC_MACH_ASYNC_REPLY_TYPE, #if HAVE_PTHREAD_WORKQUEUE_QOS DC_OVERRIDE_STEALING_TYPE, DC_OVERRIDE_OWNING_TYPE, @@ -897,12 +928,12 @@ extern const struct dispatch_continuation_vtable_s void _dispatch_async_redirect_invoke(dispatch_continuation_t dc, - dispatch_invoke_flags_t flags); + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags); #if HAVE_PTHREAD_WORKQUEUE_QOS void _dispatch_queue_override_invoke(dispatch_continuation_t dc, - dispatch_invoke_flags_t flags); + dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags); #endif #define DC_VTABLE(name) (&_dispatch_continuation_vtables[DC_##name##_TYPE]) @@ -920,8 +951,14 @@ _dispatch_queue_override_invoke(dispatch_continuation_t dc, void _dispatch_set_priority_and_mach_voucher_slow(pthread_priority_t pri, mach_voucher_t kv); voucher_t _dispatch_set_priority_and_voucher_slow(pthread_priority_t pri, - voucher_t voucher, _dispatch_thread_set_self_t flags); - + voucher_t voucher, dispatch_thread_set_self_t flags); +#else +static inline void +_dispatch_set_priority_and_mach_voucher_slow(pthread_priority_t pri, + mach_voucher_t kv) +{ + (void)pri; (void)kv; +} #endif #pragma mark - #pragma mark dispatch_apply_t @@ -932,7 +969,7 @@ struct dispatch_apply_s { dispatch_continuation_t da_dc; dispatch_thread_event_s da_event; dispatch_invoke_flags_t da_flags; - uint32_t da_thr_cnt; + int32_t da_thr_cnt; }; typedef struct dispatch_apply_s *dispatch_apply_t; @@ -941,7 +978,7 @@ typedef struct dispatch_apply_s *dispatch_apply_t; #ifdef __BLOCKS__ -#define DISPATCH_BLOCK_API_MASK (0x80u - 1) +#define DISPATCH_BLOCK_API_MASK (0x100u - 1) #define DISPATCH_BLOCK_HAS_VOUCHER (1u << 31) #define DISPATCH_BLOCK_HAS_PRIORITY (1u << 30) @@ -988,15 +1025,13 @@ void _dispatch_block_sync_invoke(void *block); void _dispatch_continuation_init_slow(dispatch_continuation_t dc, dispatch_queue_class_t dqu, dispatch_block_flags_t flags); -void _dispatch_continuation_update_bits(dispatch_continuation_t dc, - uintptr_t dc_flags); -bool _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, +long _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func); /* exported for tests in dispatch_trysync.c */ DISPATCH_EXPORT DISPATCH_NOTHROW -bool _dispatch_trysync_f(dispatch_queue_t dq, void *ctxt, +long _dispatch_trysync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t f); #endif /* __BLOCKS__ */