From c0fd1a46ca556db3ab6fd5fef441de241a8a5a3f Mon Sep 17 00:00:00 2001 From: Apple Date: Thu, 15 Dec 2016 17:53:17 +0000 Subject: [PATCH] libdispatch-703.30.5.tar.gz --- src/inline_internal.h | 5 ++++- src/object_internal.h | 3 +++ src/queue.c | 41 +++++++++++++++++++++++++++-------------- src/source.c | 2 +- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/inline_internal.h b/src/inline_internal.h index d1c73dd..daa0e9d 100644 --- a/src/inline_internal.h +++ b/src/inline_internal.h @@ -946,7 +946,10 @@ static inline void _dispatch_queue_try_wakeup(dispatch_queue_t dq, uint64_t dq_state, dispatch_wakeup_flags_t flags) { - if (_dq_state_should_wakeup(dq_state)) { + if (_dq_state_is_runnable(dq_state) && + !_dq_state_drain_locked(dq_state) && + (!_dq_state_is_enqueued(dq_state) || + (flags & DISPATCH_WAKEUP_WAITER_HANDOFF))) { if (slowpath(_dq_state_is_dirty(dq_state))) { // // seq_cst wrt state changes that were flushed and not acted upon diff --git a/src/object_internal.h b/src/object_internal.h index 80bb102..63f6cf5 100644 --- a/src/object_internal.h +++ b/src/object_internal.h @@ -253,6 +253,9 @@ DISPATCH_ENUM(dispatch_wakeup_flags, uint32_t, // preserved (used to solve a race with _dispatch_queue_drain_try_unlock()) DISPATCH_WAKEUP_WAS_OVERRIDDEN = 0x00000010, + // This wakeup is caused by a handoff from a slow waiter. + DISPATCH_WAKEUP_WAITER_HANDOFF = 0x00000020, + #define _DISPATCH_WAKEUP_OVERRIDE_BITS \ ((dispatch_wakeup_flags_t)(DISPATCH_WAKEUP_OVERRIDING | \ DISPATCH_WAKEUP_WAS_OVERRIDDEN)) diff --git a/src/queue.c b/src/queue.c index 58c545b..aeef1e8 100644 --- a/src/queue.c +++ b/src/queue.c @@ -1686,18 +1686,18 @@ _dispatch_queue_resume(dispatch_queue_t dq, bool activate) } if ((dq_state ^ value) & DISPATCH_QUEUE_IN_BARRIER) { - _dispatch_release(dq); - return _dispatch_try_lock_transfer_or_wakeup(dq); - } - - if (_dq_state_should_wakeup(value)) { + _dispatch_try_lock_transfer_or_wakeup(dq); + } else if (_dq_state_should_wakeup(value)) { // // seq_cst wrt state changes that were flushed and not acted upon os_atomic_thread_fence(acquire); pthread_priority_t pp = _dispatch_queue_reset_override_priority(dq, _dispatch_queue_is_thread_bound(dq)); + // Balancing the retain() done in suspend() for rdar://8181908 return dx_wakeup(dq, pp, DISPATCH_WAKEUP_CONSUME); } + + // Balancing the retain() done in suspend() for rdar://8181908 return _dispatch_release_tailcall(dq); over_resume: @@ -3797,7 +3797,8 @@ _dispatch_non_barrier_complete(dispatch_queue_t dq) return _dispatch_try_lock_transfer_or_wakeup(dq); } if (!_dq_state_is_runnable(old_state)) { - _dispatch_queue_try_wakeup(dq, new_state, 0); + _dispatch_queue_try_wakeup(dq, new_state, + DISPATCH_WAKEUP_WAITER_HANDOFF); } } @@ -4249,6 +4250,9 @@ _dispatch_global_queue_poke_slow(dispatch_queue_t dq, unsigned int n) uint32_t i = n; int r; + dispatch_once_f(&_dispatch_root_queues_pred, NULL, + _dispatch_root_queues_init_once); + _dispatch_debug_root_queue(dq, __func__); #if HAVE_PTHREAD_WORKQUEUES #if DISPATCH_USE_PTHREAD_POOL @@ -4689,7 +4693,8 @@ attempt_running_slow_head: // for sources and mach channels in the first place. owned = _dispatch_queue_adjust_owned(dq, owned, dc); dq_state = _dispatch_queue_drain_unlock(dq, owned, NULL); - return _dispatch_queue_try_wakeup(dq, dq_state, 0); + return _dispatch_queue_try_wakeup(dq, dq_state, + DISPATCH_WAKEUP_WAITER_HANDOFF); } else if (!fastpath(_dispatch_queue_drain_try_unlock(dq, owned))) { // someone enqueued a slow item at the head // looping may be its last chance @@ -4880,7 +4885,7 @@ _dispatch_queue_override_invoke(dispatch_continuation_t dc, DISPATCH_ALWAYS_INLINE static inline bool -_dispatch_need_global_root_queue_push_override(dispatch_queue_t rq, +_dispatch_need_global_root_queue_override(dispatch_queue_t rq, pthread_priority_t pp) { pthread_priority_t rqp = rq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; @@ -4894,14 +4899,20 @@ _dispatch_need_global_root_queue_push_override(dispatch_queue_t rq, DISPATCH_ALWAYS_INLINE static inline bool -_dispatch_need_global_root_queue_push_override_stealer(dispatch_queue_t rq, - pthread_priority_t pp) +_dispatch_need_global_root_queue_override_stealer(dispatch_queue_t rq, + pthread_priority_t pp, dispatch_wakeup_flags_t wflags) { pthread_priority_t rqp = rq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; bool defaultqueue = rq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; if (unlikely(!rqp)) return false; + if (wflags & DISPATCH_WAKEUP_WAITER_HANDOFF) { + if (!(wflags & _DISPATCH_WAKEUP_OVERRIDE_BITS)) { + return false; + } + } + pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; return defaultqueue || pp > rqp; } @@ -5041,9 +5052,11 @@ _dispatch_queue_class_wakeup_with_override(dispatch_queue_t dq, apply_again: if (dx_type(tq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) { - if (_dispatch_need_global_root_queue_push_override_stealer(tq, pp)) { + if (_dispatch_need_global_root_queue_override_stealer(tq, pp, flags)) { _dispatch_root_queue_push_override_stealer(tq, dq, pp); } + } else if (flags & DISPATCH_WAKEUP_WAITER_HANDOFF) { + dx_wakeup(tq, pp, flags); } else if (_dispatch_queue_need_override(tq, pp)) { dx_wakeup(tq, pp, DISPATCH_WAKEUP_OVERRIDING); } @@ -5131,7 +5144,7 @@ _dispatch_trystash_to_deferred_items(dispatch_queue_t dq, dispatch_object_t dou, dq = old_dq; dou._do = old_dou; } - if (_dispatch_need_global_root_queue_push_override(dq, pp)) { + if (_dispatch_need_global_root_queue_override(dq, pp)) { return _dispatch_root_queue_push_override(dq, dou, pp); } // bit of cheating: we should really pass `pp` but we know that we are @@ -5172,7 +5185,7 @@ _dispatch_queue_push(dispatch_queue_t dq, dispatch_object_t dou, if (unlikely(_dispatch_root_queues_pred != DLOCK_ONCE_DONE)) { return _dispatch_queue_push_slow(dq, dou, pp); } - if (_dispatch_need_global_root_queue_push_override(dq, pp)) { + if (_dispatch_need_global_root_queue_override(dq, pp)) { return _dispatch_root_queue_push_override(dq, dou, pp); } #endif @@ -5314,7 +5327,7 @@ _dispatch_queue_class_wakeup(dispatch_queue_t dq, pthread_priority_t pp, } #if HAVE_PTHREAD_WORKQUEUE_QOS - if ((flags & DISPATCH_WAKEUP_OVERRIDING) + if ((flags & (DISPATCH_WAKEUP_OVERRIDING | DISPATCH_WAKEUP_WAITER_HANDOFF)) && target == DISPATCH_QUEUE_WAKEUP_TARGET) { return _dispatch_queue_class_wakeup_with_override(dq, pp, flags, new_state); diff --git a/src/source.c b/src/source.c index a5a2c94..3d0eee8 100644 --- a/src/source.c +++ b/src/source.c @@ -5149,7 +5149,7 @@ out: pp = _dm_state_get_override(new_state); _dispatch_mach_send_barrier_drain_push(dm, pp); } else { - if (needs_mgr) { + if (needs_mgr || dr->dm_needs_mgr) { pp = _dm_state_get_override(new_state); } else { pp = 0; -- 2.45.2