libdispatch-913.1.6.tar.gz

[apple/libdispatch.git] / src / mach.c

/*
 * Copyright (c) 2008-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

#include "internal.h"
#if HAVE_MACH

#define DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT 0x1
#define DISPATCH_MACH_REGISTER_FOR_REPLY 0x2
#define DISPATCH_MACH_WAIT_FOR_REPLY 0x4
#define DISPATCH_MACH_OWNED_REPLY_PORT 0x8
#define DISPATCH_MACH_ASYNC_REPLY 0x10
#define DISPATCH_MACH_OPTIONS_MASK 0xffff

#define DM_SEND_STATUS_SUCCESS 0x1
#define DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT 0x2

DISPATCH_ENUM(dispatch_mach_send_invoke_flags, uint32_t,
        DM_SEND_INVOKE_NONE            = 0x0,
        DM_SEND_INVOKE_MAKE_DIRTY      = 0x1,
        DM_SEND_INVOKE_NEEDS_BARRIER   = 0x2,
        DM_SEND_INVOKE_CANCEL          = 0x4,
        DM_SEND_INVOKE_CAN_RUN_BARRIER = 0x8,
        DM_SEND_INVOKE_IMMEDIATE_SEND  = 0x10,
);
#define DM_SEND_INVOKE_IMMEDIATE_SEND_MASK \
                ((dispatch_mach_send_invoke_flags_t)DM_SEND_INVOKE_IMMEDIATE_SEND)

static inline mach_msg_option_t _dispatch_mach_checkin_options(void);
static mach_port_t _dispatch_mach_msg_get_remote_port(dispatch_object_t dou);
static mach_port_t _dispatch_mach_msg_get_reply_port(dispatch_object_t dou);
static void _dispatch_mach_msg_disconnected(dispatch_mach_t dm,
                mach_port_t local_port, mach_port_t remote_port);
static inline void _dispatch_mach_msg_reply_received(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, mach_port_t local_port);
static dispatch_mach_msg_t _dispatch_mach_msg_create_reply_disconnected(
                dispatch_object_t dou, dispatch_mach_reply_refs_t dmr,
                dispatch_mach_reason_t reason);
static bool _dispatch_mach_reconnect_invoke(dispatch_mach_t dm,
                dispatch_object_t dou);
static inline mach_msg_header_t* _dispatch_mach_msg_get_msg(
                dispatch_mach_msg_t dmsg);
static void _dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou,
                dispatch_qos_t qos);
static void _dispatch_mach_cancel(dispatch_mach_t dm);
static void _dispatch_mach_push_send_barrier_drain(dispatch_mach_t dm,
                dispatch_qos_t qos);
static void _dispatch_mach_handle_or_push_received_msg(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg);
static void _dispatch_mach_push_async_reply_msg(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, dispatch_queue_t drq);
static dispatch_queue_t _dispatch_mach_msg_context_async_reply_queue(
                void *ctxt);
static dispatch_continuation_t _dispatch_mach_msg_async_reply_wrap(
                dispatch_mach_msg_t dmsg, dispatch_mach_t dm);
static void _dispatch_mach_notification_kevent_unregister(dispatch_mach_t dm);
static void _dispatch_mach_notification_kevent_register(dispatch_mach_t dm,
                mach_port_t send);

// For tests only.
DISPATCH_EXPORT void _dispatch_mach_hooks_install_default(void);

dispatch_source_t
_dispatch_source_create_mach_msg_direct_recv(mach_port_t recvp,
                const struct dispatch_continuation_s *dc)
{
        dispatch_source_t ds;
        ds = dispatch_source_create(&_dispatch_source_type_mach_recv_direct,
                        recvp, 0, &_dispatch_mgr_q);
        os_atomic_store(&ds->ds_refs->ds_handler[DS_EVENT_HANDLER],
                        (dispatch_continuation_t)dc, relaxed);
        return ds;
}

#pragma mark -
#pragma mark dispatch to XPC callbacks

static dispatch_mach_xpc_hooks_t _dispatch_mach_xpc_hooks;

// Default dmxh_direct_message_handler callback that does not handle
// messages inline.
static bool
_dispatch_mach_xpc_no_handle_message(
                void *_Nullable context DISPATCH_UNUSED,
                dispatch_mach_reason_t reason DISPATCH_UNUSED,
                dispatch_mach_msg_t message DISPATCH_UNUSED,
                mach_error_t error DISPATCH_UNUSED)
{
        return false;
}

// Default dmxh_msg_context_reply_queue callback that returns a NULL queue.
static dispatch_queue_t
_dispatch_mach_msg_context_no_async_reply_queue(
                void *_Nonnull msg_context DISPATCH_UNUSED)
{
        return NULL;
}

// Default dmxh_async_reply_handler callback that crashes when called.
DISPATCH_NORETURN
static void
_dispatch_mach_default_async_reply_handler(void *context DISPATCH_UNUSED,
                dispatch_mach_reason_t reason DISPATCH_UNUSED,
                dispatch_mach_msg_t message DISPATCH_UNUSED)
{
        DISPATCH_CLIENT_CRASH(_dispatch_mach_xpc_hooks,
                        "_dispatch_mach_default_async_reply_handler called");
}

// Default dmxh_enable_sigterm_notification callback that enables delivery of
// SIGTERM notifications (for backwards compatibility).
static bool
_dispatch_mach_enable_sigterm(void *_Nullable context DISPATCH_UNUSED)
{
        return true;
}

// Callbacks from dispatch to XPC. The default is to not support any callbacks.
static const struct dispatch_mach_xpc_hooks_s _dispatch_mach_xpc_hooks_default
                = {
        .version = DISPATCH_MACH_XPC_HOOKS_VERSION,
        .dmxh_direct_message_handler = &_dispatch_mach_xpc_no_handle_message,
        .dmxh_msg_context_reply_queue =
                        &_dispatch_mach_msg_context_no_async_reply_queue,
        .dmxh_async_reply_handler = &_dispatch_mach_default_async_reply_handler,
        .dmxh_enable_sigterm_notification = &_dispatch_mach_enable_sigterm,
};

static dispatch_mach_xpc_hooks_t _dispatch_mach_xpc_hooks
                = &_dispatch_mach_xpc_hooks_default;

void
dispatch_mach_hooks_install_4libxpc(dispatch_mach_xpc_hooks_t hooks)
{
        if (!os_atomic_cmpxchg(&_dispatch_mach_xpc_hooks,
                        &_dispatch_mach_xpc_hooks_default, hooks, relaxed)) {
                DISPATCH_CLIENT_CRASH(_dispatch_mach_xpc_hooks,
                                "dispatch_mach_hooks_install_4libxpc called twice");
        }
}

void
_dispatch_mach_hooks_install_default(void)
{
        os_atomic_store(&_dispatch_mach_xpc_hooks,
                        &_dispatch_mach_xpc_hooks_default, relaxed);
}

#pragma mark -
#pragma mark dispatch_mach_t

static dispatch_mach_t
_dispatch_mach_create(const char *label, dispatch_queue_t q, void *context,
                dispatch_mach_handler_function_t handler, bool handler_is_block,
                bool is_xpc)
{
        dispatch_mach_recv_refs_t dmrr;
        dispatch_mach_send_refs_t dmsr;
        dispatch_mach_t dm;
        dm = _dispatch_object_alloc(DISPATCH_VTABLE(mach),
                        sizeof(struct dispatch_mach_s));
        _dispatch_queue_init(dm->_as_dq, DQF_LEGACY, 1,
                        DISPATCH_QUEUE_INACTIVE | DISPATCH_QUEUE_ROLE_INNER);

        dm->dq_label = label;
        dm->do_ref_cnt++; // the reference _dispatch_mach_cancel_invoke holds
        dm->dm_is_xpc = is_xpc;

        dmrr = dux_create(&_dispatch_mach_type_recv, 0, 0)._dmrr;
        dispatch_assert(dmrr->du_is_direct);
        dmrr->du_owner_wref = _dispatch_ptr2wref(dm);
        dmrr->dmrr_handler_func = handler;
        dmrr->dmrr_handler_ctxt = context;
        dmrr->dmrr_handler_is_block = handler_is_block;
        dm->dm_recv_refs = dmrr;

        dmsr = dux_create(&_dispatch_mach_type_send, 0,
                        DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD)._dmsr;
        dmsr->du_owner_wref = _dispatch_ptr2wref(dm);
        dm->dm_send_refs = dmsr;

        if (slowpath(!q)) {
                q = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, true);
        } else {
                _dispatch_retain(q);
        }
        dm->do_targetq = q;
        _dispatch_object_debug(dm, "%s", __func__);
        return dm;
}

dispatch_mach_t
dispatch_mach_create(const char *label, dispatch_queue_t q,
                dispatch_mach_handler_t handler)
{
        dispatch_block_t bb = _dispatch_Block_copy((void*)handler);
        return _dispatch_mach_create(label, q, bb,
                        (dispatch_mach_handler_function_t)_dispatch_Block_invoke(bb), true,
                        false);
}

dispatch_mach_t
dispatch_mach_create_f(const char *label, dispatch_queue_t q, void *context,
                dispatch_mach_handler_function_t handler)
{
        return _dispatch_mach_create(label, q, context, handler, false, false);
}

dispatch_mach_t
dispatch_mach_create_4libxpc(const char *label, dispatch_queue_t q,
                void *context, dispatch_mach_handler_function_t handler)
{
        return _dispatch_mach_create(label, q, context, handler, false, true);
}

void
_dispatch_mach_dispose(dispatch_mach_t dm, bool *allow_free)
{
        _dispatch_object_debug(dm, "%s", __func__);
        _dispatch_unote_dispose(dm->dm_recv_refs);
        dm->dm_recv_refs = NULL;
        _dispatch_unote_dispose(dm->dm_send_refs);
        dm->dm_send_refs = NULL;
        if (dm->dm_xpc_term_refs) {
                _dispatch_unote_dispose(dm->dm_xpc_term_refs);
                dm->dm_xpc_term_refs = NULL;
        }
        _dispatch_queue_destroy(dm->_as_dq, allow_free);
}

void
dispatch_mach_connect(dispatch_mach_t dm, mach_port_t receive,
                mach_port_t send, dispatch_mach_msg_t checkin)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        uint32_t disconnect_cnt;

        if (MACH_PORT_VALID(receive)) {
                dm->dm_recv_refs->du_ident = receive;
                _dispatch_retain(dm); // the reference the manager queue holds
        }
        dmsr->dmsr_send = send;
        if (MACH_PORT_VALID(send)) {
                if (checkin) {
                        dispatch_mach_msg_t dmsg = checkin;
                        dispatch_retain(dmsg);
                        dmsg->dmsg_options = _dispatch_mach_checkin_options();
                        dmsr->dmsr_checkin_port = _dispatch_mach_msg_get_remote_port(dmsg);
                }
                dmsr->dmsr_checkin = checkin;
        }
        dispatch_assert(DISPATCH_MACH_NEVER_CONNECTED - 1 ==
                        DISPATCH_MACH_NEVER_INSTALLED);
        disconnect_cnt = os_atomic_dec2o(dmsr, dmsr_disconnect_cnt, release);
        if (unlikely(disconnect_cnt != DISPATCH_MACH_NEVER_INSTALLED)) {
                DISPATCH_CLIENT_CRASH(disconnect_cnt, "Channel already connected");
        }
        _dispatch_object_debug(dm, "%s", __func__);
        return dispatch_activate(dm);
}

static inline bool
_dispatch_mach_reply_tryremove(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr)
{
        bool removed;
        _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
        if ((removed = _TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
                TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
                _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
        }
        _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
        return removed;
}

DISPATCH_NOINLINE
static void
_dispatch_mach_reply_waiter_unregister(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, uint32_t options)
{
        dispatch_mach_msg_t dmsgr = NULL;
        bool disconnected = (options & DU_UNREGISTER_DISCONNECTED);
        if (options & DU_UNREGISTER_REPLY_REMOVE) {
                _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
                if (unlikely(!_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
                        DISPATCH_INTERNAL_CRASH(0, "Could not find reply registration");
                }
                TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
                _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
                _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
        }
        if (disconnected) {
                dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr,
                                DISPATCH_MACH_DISCONNECTED);
        } else if (dmr->dmr_voucher) {
                _voucher_release(dmr->dmr_voucher);
                dmr->dmr_voucher = NULL;
        }
        _dispatch_debug("machport[0x%08x]: unregistering for sync reply%s, ctxt %p",
                        _dispatch_mach_reply_get_reply_port((mach_port_t)dmr->du_ident),
                        disconnected ? " (disconnected)" : "", dmr->dmr_ctxt);
        if (dmsgr) {
                return _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
        }
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_reply_list_remove(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr) {
        // dmsr_replies_lock must be held by the caller.
        bool removed = false;
        if (likely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
                TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
                _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
                removed = true;
        }
        return removed;
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, uint32_t options)
{
        dispatch_assert(!_TAILQ_IS_ENQUEUED(dmr, dmr_list));

        bool disconnected = (options & DU_UNREGISTER_DISCONNECTED);
        _dispatch_debug("machport[0x%08x]: unregistering for reply%s, ctxt %p",
                        (mach_port_t)dmr->du_ident, disconnected ? " (disconnected)" : "",
                        dmr->dmr_ctxt);
        if (!_dispatch_unote_unregister(dmr, options)) {
                _dispatch_debug("machport[0x%08x]: deferred delete kevent[%p]",
                                                (mach_port_t)dmr->du_ident, dmr);
                dispatch_assert(options == DU_UNREGISTER_DISCONNECTED);
                return false;
        }

        dispatch_mach_msg_t dmsgr = NULL;
        dispatch_queue_t drq = NULL;
        if (disconnected) {
                // The next call is guaranteed to always transfer or consume the voucher
                // in the dmr, if there is one.
                dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr,
                        dmr->dmr_async_reply ? DISPATCH_MACH_ASYNC_WAITER_DISCONNECTED
                        : DISPATCH_MACH_DISCONNECTED);
                if (dmr->dmr_ctxt) {
                        drq = _dispatch_mach_msg_context_async_reply_queue(dmr->dmr_ctxt);
                }
                dispatch_assert(dmr->dmr_voucher == NULL);
        } else if (dmr->dmr_voucher) {
                _voucher_release(dmr->dmr_voucher);
                dmr->dmr_voucher = NULL;
        }
        _dispatch_unote_dispose(dmr);

        if (dmsgr) {
                if (drq) {
                        _dispatch_mach_push_async_reply_msg(dm, dmsgr, drq);
                } else {
                        _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
                }
        }
        return true;
}

DISPATCH_NOINLINE
static void
_dispatch_mach_reply_waiter_register(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, mach_port_t reply_port,
                dispatch_mach_msg_t dmsg, mach_msg_option_t msg_opts)
{
        dmr->du_owner_wref = _dispatch_ptr2wref(dm);
        dmr->du_wlh = NULL;
        dmr->du_filter = EVFILT_MACHPORT;
        dmr->du_ident = reply_port;
        if (msg_opts & DISPATCH_MACH_OWNED_REPLY_PORT) {
                _dispatch_mach_reply_mark_reply_port_owned(dmr);
        } else {
                if (dmsg->dmsg_voucher) {
                        dmr->dmr_voucher = _voucher_retain(dmsg->dmsg_voucher);
                }
                dmr->dmr_priority = _dispatch_priority_from_pp(dmsg->dmsg_priority);
                // make reply context visible to leaks rdar://11777199
                dmr->dmr_ctxt = dmsg->do_ctxt;
        }

        _dispatch_debug("machport[0x%08x]: registering for sync reply, ctxt %p",
                        reply_port, dmsg->do_ctxt);
        _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
        if (unlikely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
                DISPATCH_INTERNAL_CRASH(dmr->dmr_list.tqe_prev,
                                "Reply already registered");
        }
        TAILQ_INSERT_TAIL(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
        _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
}

DISPATCH_NOINLINE
static void
_dispatch_mach_reply_kevent_register(dispatch_mach_t dm, mach_port_t reply_port,
                dispatch_mach_msg_t dmsg)
{
        dispatch_mach_reply_refs_t dmr;
        dispatch_priority_t mpri, pri, overcommit;
        dispatch_wlh_t wlh;

        dmr = dux_create(&_dispatch_mach_type_reply, reply_port, 0)._dmr;
        dispatch_assert(dmr->du_is_direct);
        dmr->du_owner_wref = _dispatch_ptr2wref(dm);
        if (dmsg->dmsg_voucher) {
                dmr->dmr_voucher = _voucher_retain(dmsg->dmsg_voucher);
        }
        dmr->dmr_priority = _dispatch_priority_from_pp(dmsg->dmsg_priority);
        // make reply context visible to leaks rdar://11777199
        dmr->dmr_ctxt = dmsg->do_ctxt;

        dispatch_queue_t drq = NULL;
        if (dmsg->dmsg_options & DISPATCH_MACH_ASYNC_REPLY) {
                dmr->dmr_async_reply = true;
                drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
        }

        if (!drq) {
                pri = dm->dq_priority;
                wlh = dm->dm_recv_refs->du_wlh;
        } else if (dx_type(drq) == DISPATCH_QUEUE_NETWORK_EVENT_TYPE) {
                pri = DISPATCH_PRIORITY_FLAG_MANAGER;
                wlh = (dispatch_wlh_t)drq;
        } else if (dx_hastypeflag(drq, QUEUE_ROOT)) {
                pri = drq->dq_priority;
                wlh = DISPATCH_WLH_ANON;
        } else if (drq == dm->do_targetq) {
                pri = dm->dq_priority;
                wlh = dm->dm_recv_refs->du_wlh;
        } else if (!(pri = _dispatch_queue_compute_priority_and_wlh(drq, &wlh))) {
                pri = drq->dq_priority;
                wlh = DISPATCH_WLH_ANON;
        }
        if (pri & DISPATCH_PRIORITY_REQUESTED_MASK) {
                overcommit = pri & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
                pri &= DISPATCH_PRIORITY_REQUESTED_MASK;
                mpri = _dispatch_priority_from_pp_strip_flags(dmsg->dmsg_priority);
                if (pri < mpri) pri = mpri;
                pri |= overcommit;
        } else {
                pri = DISPATCH_PRIORITY_FLAG_MANAGER;
        }

        _dispatch_debug("machport[0x%08x]: registering for reply, ctxt %p",
                        reply_port, dmsg->do_ctxt);
        _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
        if (unlikely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
                DISPATCH_INTERNAL_CRASH(dmr->dmr_list.tqe_prev,
                                "Reply already registered");
        }
        TAILQ_INSERT_TAIL(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
        _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);

        if (!_dispatch_unote_register(dmr, wlh, pri)) {
                _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
                _dispatch_mach_reply_list_remove(dm, dmr);
                _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
                _dispatch_mach_reply_kevent_unregister(dm, dmr,
                                DU_UNREGISTER_DISCONNECTED);
        }
}

#pragma mark -
#pragma mark dispatch_mach_msg

DISPATCH_ALWAYS_INLINE DISPATCH_CONST
static inline bool
_dispatch_use_mach_special_reply_port(void)
{
#if DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE
        return true;
#else
#define thread_get_special_reply_port() ({__builtin_trap(); MACH_PORT_NULL;})
        return false;
#endif
}

static mach_port_t
_dispatch_get_thread_reply_port(void)
{
        mach_port_t reply_port, mrp;
        if (_dispatch_use_mach_special_reply_port()) {
                mrp = _dispatch_get_thread_special_reply_port();
        } else {
                mrp = _dispatch_get_thread_mig_reply_port();
        }
        if (mrp) {
                reply_port = mrp;
                _dispatch_debug("machport[0x%08x]: borrowed thread sync reply port",
                                reply_port);
        } else {
                if (_dispatch_use_mach_special_reply_port()) {
                        reply_port = thread_get_special_reply_port();
                        _dispatch_set_thread_special_reply_port(reply_port);
                } else {
                        reply_port = mach_reply_port();
                        _dispatch_set_thread_mig_reply_port(reply_port);
                }
                if (unlikely(!MACH_PORT_VALID(reply_port))) {
                        DISPATCH_CLIENT_CRASH(_dispatch_use_mach_special_reply_port(),
                                "Unable to allocate reply port, possible port leak");
                }
                _dispatch_debug("machport[0x%08x]: allocated thread sync reply port",
                                reply_port);
        }
        _dispatch_debug_machport(reply_port);
        return reply_port;
}

static void
_dispatch_clear_thread_reply_port(mach_port_t reply_port)
{
        mach_port_t mrp;
        if (_dispatch_use_mach_special_reply_port()) {
                mrp = _dispatch_get_thread_special_reply_port();
        } else {
                mrp = _dispatch_get_thread_mig_reply_port();
        }
        if (reply_port != mrp) {
                if (mrp) {
                        _dispatch_debug("machport[0x%08x]: did not clear thread sync reply "
                                        "port (found 0x%08x)", reply_port, mrp);
                }
                return;
        }
        if (_dispatch_use_mach_special_reply_port()) {
                _dispatch_set_thread_special_reply_port(MACH_PORT_NULL);
        } else {
                _dispatch_set_thread_mig_reply_port(MACH_PORT_NULL);
        }
        _dispatch_debug_machport(reply_port);
        _dispatch_debug("machport[0x%08x]: cleared thread sync reply port",
                        reply_port);
}

static void
_dispatch_set_thread_reply_port(mach_port_t reply_port)
{
        _dispatch_debug_machport(reply_port);
        mach_port_t mrp;
        if (_dispatch_use_mach_special_reply_port()) {
                mrp = _dispatch_get_thread_special_reply_port();
        } else {
                mrp = _dispatch_get_thread_mig_reply_port();
        }
        if (mrp) {
                kern_return_t kr = mach_port_mod_refs(mach_task_self(), reply_port,
                                MACH_PORT_RIGHT_RECEIVE, -1);
                DISPATCH_VERIFY_MIG(kr);
                dispatch_assume_zero(kr);
                _dispatch_debug("machport[0x%08x]: deallocated sync reply port "
                                "(found 0x%08x)", reply_port, mrp);
        } else {
                if (_dispatch_use_mach_special_reply_port()) {
                        _dispatch_set_thread_special_reply_port(reply_port);
                } else {
                        _dispatch_set_thread_mig_reply_port(reply_port);
                }
                _dispatch_debug("machport[0x%08x]: restored thread sync reply port",
                                reply_port);
        }
}

static inline mach_port_t
_dispatch_mach_msg_get_remote_port(dispatch_object_t dou)
{
        mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dou._dmsg);
        mach_port_t remote = hdr->msgh_remote_port;
        return remote;
}

static inline mach_port_t
_dispatch_mach_msg_get_reply_port(dispatch_object_t dou)
{
        mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dou._dmsg);
        mach_port_t local = hdr->msgh_local_port;
        if (!MACH_PORT_VALID(local) || MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) !=
                        MACH_MSG_TYPE_MAKE_SEND_ONCE) return MACH_PORT_NULL;
        return local;
}

static inline void
_dispatch_mach_msg_set_reason(dispatch_mach_msg_t dmsg, mach_error_t err,
                unsigned long reason)
{
        dispatch_assert_zero(reason & ~(unsigned long)code_emask);
        dmsg->dmsg_error = ((err || !reason) ? err :
                         err_local|err_sub(0x3e0)|(mach_error_t)reason);
}

static inline unsigned long
_dispatch_mach_msg_get_reason(dispatch_mach_msg_t dmsg, mach_error_t *err_ptr)
{
        mach_error_t err = dmsg->dmsg_error;

        if ((err & system_emask) == err_local && err_get_sub(err) == 0x3e0) {
                *err_ptr = 0;
                return err_get_code(err);
        }
        *err_ptr = err;
        return err ? DISPATCH_MACH_MESSAGE_SEND_FAILED : DISPATCH_MACH_MESSAGE_SENT;
}

static inline dispatch_mach_msg_t
_dispatch_mach_msg_create_recv(mach_msg_header_t *hdr, mach_msg_size_t siz,
                dispatch_mach_reply_refs_t dmr, uint32_t flags)
{
        dispatch_mach_msg_destructor_t destructor;
        dispatch_mach_msg_t dmsg;
        voucher_t voucher;
        pthread_priority_t pp;

        if (dmr) {
                _voucher_mach_msg_clear(hdr, false); // deallocate reply message voucher
                pp = _dispatch_priority_to_pp(dmr->dmr_priority);
                voucher = dmr->dmr_voucher;
                dmr->dmr_voucher = NULL; // transfer reference
        } else {
                voucher = voucher_create_with_mach_msg(hdr);
                pp = _dispatch_priority_compute_propagated(
                                _voucher_get_priority(voucher), 0);
        }

        destructor = (flags & DISPATCH_EV_MSG_NEEDS_FREE) ?
                        DISPATCH_MACH_MSG_DESTRUCTOR_FREE :
                        DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT;
        dmsg = dispatch_mach_msg_create(hdr, siz, destructor, NULL);
        if (!(flags & DISPATCH_EV_MSG_NEEDS_FREE)) {
                _dispatch_ktrace2(DISPATCH_MACH_MSG_hdr_move,
                                (uint64_t)hdr, (uint64_t)dmsg->dmsg_buf);
        }
        dmsg->dmsg_voucher = voucher;
        dmsg->dmsg_priority = pp;
        dmsg->do_ctxt = dmr ? dmr->dmr_ctxt : NULL;
        _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_MESSAGE_RECEIVED);
        _dispatch_voucher_debug("mach-msg[%p] create", voucher, dmsg);
        _dispatch_voucher_ktrace_dmsg_push(dmsg);
        return dmsg;
}

void
_dispatch_mach_merge_msg(dispatch_unote_t du, uint32_t flags,
                mach_msg_header_t *hdr, mach_msg_size_t siz)
{
        // this function is very similar with what _dispatch_source_merge_evt does
        // but can't reuse it as handling the message must be protected by the
        // internal refcount between the first half and the trailer of what
        // _dispatch_source_merge_evt does.

        dispatch_mach_recv_refs_t dmrr = du._dmrr;
        dispatch_mach_t dm = _dispatch_wref2ptr(dmrr->du_owner_wref);
        dispatch_queue_flags_t dqf;
        dispatch_mach_msg_t dmsg;

        dispatch_assert(_dispatch_unote_needs_rearm(du));

        if (flags & EV_VANISHED) {
                DISPATCH_CLIENT_CRASH(du._du->du_ident,
                                "Unexpected EV_VANISHED (do not destroy random mach ports)");
        }

        // once we modify the queue atomic flags below, it will allow concurrent
        // threads running _dispatch_mach_invoke2 to dispose of the source,
        // so we can't safely borrow the reference we get from the muxnote udata
        // anymore, and need our own
        dispatch_wakeup_flags_t wflags = DISPATCH_WAKEUP_CONSUME_2;
        _dispatch_retain_2(dm); // rdar://20382435

        if (unlikely((flags & EV_ONESHOT) && !(flags & EV_DELETE))) {
                dqf = _dispatch_queue_atomic_flags_set_and_clear(dm->_as_dq,
                                DSF_DEFERRED_DELETE, DSF_ARMED);
                _dispatch_debug("kevent-source[%p]: deferred delete oneshot kevent[%p]",
                                dm, dmrr);
        } else if (unlikely(flags & (EV_ONESHOT | EV_DELETE))) {
                _dispatch_source_refs_unregister(dm->_as_ds,
                                DU_UNREGISTER_ALREADY_DELETED);
                dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
                _dispatch_debug("kevent-source[%p]: deleted kevent[%p]", dm, dmrr);
        } else {
                dqf = _dispatch_queue_atomic_flags_clear(dm->_as_dq, DSF_ARMED);
                _dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", dm, dmrr);
        }

        _dispatch_debug_machport(hdr->msgh_remote_port);
        _dispatch_debug("machport[0x%08x]: received msg id 0x%x, reply on 0x%08x",
                        hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);

        if (dqf & DSF_CANCELED) {
                _dispatch_debug("machport[0x%08x]: drop msg id 0x%x, reply on 0x%08x",
                                hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
                mach_msg_destroy(hdr);
                if (flags & DISPATCH_EV_MSG_NEEDS_FREE) {
                        free(hdr);
                }
                return dx_wakeup(dm, 0, wflags | DISPATCH_WAKEUP_MAKE_DIRTY);
        }

        // Once the mach channel disarming is visible, cancellation will switch to
        // immediate deletion.  If we're preempted here, then the whole cancellation
        // sequence may be complete by the time we really enqueue the message.
        //
        // _dispatch_mach_msg_invoke_with_mach() is responsible for filtering it out
        // to keep the promise that DISPATCH_MACH_DISCONNECTED is the last
        // event sent.

        dmsg = _dispatch_mach_msg_create_recv(hdr, siz, NULL, flags);
        _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
        return _dispatch_release_2_tailcall(dm);
}

void
_dispatch_mach_reply_merge_msg(dispatch_unote_t du, uint32_t flags,
                mach_msg_header_t *hdr, mach_msg_size_t siz)
{
        dispatch_mach_reply_refs_t dmr = du._dmr;
        dispatch_mach_t dm = _dispatch_wref2ptr(dmr->du_owner_wref);
        bool canceled = (_dispatch_queue_atomic_flags(dm->_as_dq) & DSF_CANCELED);
        dispatch_mach_msg_t dmsg = NULL;

        _dispatch_debug_machport(hdr->msgh_remote_port);
        _dispatch_debug("machport[0x%08x]: received msg id 0x%x, reply on 0x%08x",
                        hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);

        if (!canceled) {
                dmsg = _dispatch_mach_msg_create_recv(hdr, siz, dmr, flags);
        }

        if (dmsg) {
                dispatch_queue_t drq = NULL;
                if (dmsg->do_ctxt) {
                        drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
                }
                if (drq) {
                        _dispatch_mach_push_async_reply_msg(dm, dmsg, drq);
                } else {
                        _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
                }
        } else {
                _dispatch_debug("machport[0x%08x]: drop msg id 0x%x, reply on 0x%08x",
                                hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
                mach_msg_destroy(hdr);
                if (flags & DISPATCH_EV_MSG_NEEDS_FREE) {
                        free(hdr);
                }
        }

        dispatch_wakeup_flags_t wflags = 0;
        uint32_t options = DU_UNREGISTER_IMMEDIATE_DELETE;
        if (canceled) {
                options |= DU_UNREGISTER_DISCONNECTED;
        }

        _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
        bool removed = _dispatch_mach_reply_list_remove(dm, dmr);
        dispatch_assert(removed);
        if (TAILQ_EMPTY(&dm->dm_send_refs->dmsr_replies) &&
                        (dm->dm_send_refs->dmsr_disconnect_cnt ||
                        (dm->dq_atomic_flags & DSF_CANCELED))) {
                // When the list is empty, _dispatch_mach_disconnect() may release the
                // last reference count on the Mach channel. To avoid this, take our
                // own reference before releasing the lock.
                wflags = DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2;
                _dispatch_retain_2(dm);
        }
        _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);

        bool result = _dispatch_mach_reply_kevent_unregister(dm, dmr, options);
        dispatch_assert(result);
        if (wflags) dx_wakeup(dm, 0, wflags);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_mach_msg_t
_dispatch_mach_msg_reply_recv(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, mach_port_t reply_port,
                mach_port_t send)
{
        if (slowpath(!MACH_PORT_VALID(reply_port))) {
                DISPATCH_CLIENT_CRASH(reply_port, "Invalid reply port");
        }
        void *ctxt = dmr->dmr_ctxt;
        mach_msg_header_t *hdr, *hdr2 = NULL;
        void *hdr_copyout_addr;
        mach_msg_size_t siz, msgsiz = 0;
        mach_msg_return_t kr;
        mach_msg_option_t options;
        mach_port_t notify = MACH_PORT_NULL;
        siz = mach_vm_round_page(DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
                        DISPATCH_MACH_TRAILER_SIZE);
        hdr = alloca(siz);
        for (mach_vm_address_t p = mach_vm_trunc_page(hdr + vm_page_size);
                        p < (mach_vm_address_t)hdr + siz; p += vm_page_size) {
                *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
        }
        options = DISPATCH_MACH_RCV_OPTIONS & (~MACH_RCV_VOUCHER);
        if (MACH_PORT_VALID(send)) {
                notify = send;
                options |= MACH_RCV_SYNC_WAIT;
        }

retry:
        _dispatch_debug_machport(reply_port);
        _dispatch_debug("machport[0x%08x]: MACH_RCV_MSG %s", reply_port,
                        (options & MACH_RCV_TIMEOUT) ? "poll" : "wait");
        kr = mach_msg(hdr, options, 0, siz, reply_port, MACH_MSG_TIMEOUT_NONE,
                        notify);
        hdr_copyout_addr = hdr;
        _dispatch_debug_machport(reply_port);
        _dispatch_debug("machport[0x%08x]: MACH_RCV_MSG (size %u, opts 0x%x) "
                        "returned: %s - 0x%x", reply_port, siz, options,
                        mach_error_string(kr), kr);
        switch (kr) {
        case MACH_RCV_TOO_LARGE:
                if (!fastpath(hdr->msgh_size <= UINT_MAX -
                                DISPATCH_MACH_TRAILER_SIZE)) {
                        DISPATCH_CLIENT_CRASH(hdr->msgh_size, "Overlarge message");
                }
                if (options & MACH_RCV_LARGE) {
                        msgsiz = hdr->msgh_size + DISPATCH_MACH_TRAILER_SIZE;
                        hdr2 = malloc(msgsiz);
                        if (dispatch_assume(hdr2)) {
                                hdr = hdr2;
                                siz = msgsiz;
                        }
                        options |= MACH_RCV_TIMEOUT;
                        options &= ~MACH_RCV_LARGE;
                        goto retry;
                }
                _dispatch_log("BUG in libdispatch client: "
                                "dispatch_mach_send_and_wait_for_reply: dropped message too "
                                "large to fit in memory: id = 0x%x, size = %u", hdr->msgh_id,
                                hdr->msgh_size);
                break;
        case MACH_RCV_INVALID_NAME: // rdar://problem/21963848
        case MACH_RCV_PORT_CHANGED: // rdar://problem/21885327
        case MACH_RCV_PORT_DIED:
                // channel was disconnected/canceled and reply port destroyed
                _dispatch_debug("machport[0x%08x]: sync reply port destroyed, ctxt %p: "
                                "%s - 0x%x", reply_port, ctxt, mach_error_string(kr), kr);
                goto out;
        case MACH_MSG_SUCCESS:
                if (hdr->msgh_remote_port) {
                        _dispatch_debug_machport(hdr->msgh_remote_port);
                }
                _dispatch_debug("machport[0x%08x]: received msg id 0x%x, size = %u, "
                                "reply on 0x%08x", hdr->msgh_local_port, hdr->msgh_id,
                                hdr->msgh_size, hdr->msgh_remote_port);
                siz = hdr->msgh_size + DISPATCH_MACH_TRAILER_SIZE;
                if (hdr2 && siz < msgsiz) {
                        void *shrink = realloc(hdr2, msgsiz);
                        if (shrink) hdr = hdr2 = shrink;
                }
                break;
        case MACH_RCV_INVALID_NOTIFY:
        default:
                DISPATCH_INTERNAL_CRASH(kr, "Unexpected error from mach_msg_receive");
                break;
        }
        _dispatch_mach_msg_reply_received(dm, dmr, hdr->msgh_local_port);
        hdr->msgh_local_port = MACH_PORT_NULL;
        if (slowpath((dm->dq_atomic_flags & DSF_CANCELED) || kr)) {
                if (!kr) mach_msg_destroy(hdr);
                goto out;
        }
        dispatch_mach_msg_t dmsg;
        dispatch_mach_msg_destructor_t destructor = (!hdr2) ?
                        DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT :
                        DISPATCH_MACH_MSG_DESTRUCTOR_FREE;
        dmsg = dispatch_mach_msg_create(hdr, siz, destructor, NULL);
        if (!hdr2 || hdr != hdr_copyout_addr) {
                _dispatch_ktrace2(DISPATCH_MACH_MSG_hdr_move,
                                (uint64_t)hdr_copyout_addr,
                                (uint64_t)_dispatch_mach_msg_get_msg(dmsg));
        }
        dmsg->do_ctxt = ctxt;
        return dmsg;
out:
        free(hdr2);
        return NULL;
}

static inline void
_dispatch_mach_msg_reply_received(dispatch_mach_t dm,
                dispatch_mach_reply_refs_t dmr, mach_port_t local_port)
{
        bool removed = _dispatch_mach_reply_tryremove(dm, dmr);
        if (!MACH_PORT_VALID(local_port) || !removed) {
                // port moved/destroyed during receive, or reply waiter was never
                // registered or already removed (disconnected)
                return;
        }
        mach_port_t reply_port = _dispatch_mach_reply_get_reply_port(
                        (mach_port_t)dmr->du_ident);
        _dispatch_debug("machport[0x%08x]: unregistered for sync reply, ctxt %p",
                        reply_port, dmr->dmr_ctxt);
        if (_dispatch_mach_reply_is_reply_port_owned(dmr)) {
                _dispatch_set_thread_reply_port(reply_port);
                if (local_port != reply_port) {
                        DISPATCH_CLIENT_CRASH(local_port,
                                        "Reply received on unexpected port");
                }
                return;
        }
        mach_msg_header_t *hdr;
        dispatch_mach_msg_t dmsg;
        dmsg = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
                        DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
        hdr->msgh_local_port = local_port;
        dmsg->dmsg_voucher = dmr->dmr_voucher;
        dmr->dmr_voucher = NULL;  // transfer reference
        dmsg->dmsg_priority = _dispatch_priority_to_pp(dmr->dmr_priority);
        dmsg->do_ctxt = dmr->dmr_ctxt;
        _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_REPLY_RECEIVED);
        return _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
}

static inline void
_dispatch_mach_msg_disconnected(dispatch_mach_t dm, mach_port_t local_port,
                mach_port_t remote_port)
{
        mach_msg_header_t *hdr;
        dispatch_mach_msg_t dmsg;
        dmsg = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
                        DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
        if (local_port) hdr->msgh_local_port = local_port;
        if (remote_port) hdr->msgh_remote_port = remote_port;
        _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_DISCONNECTED);
        _dispatch_debug("machport[0x%08x]: %s right disconnected", local_port ?
                        local_port : remote_port, local_port ? "receive" : "send");
        return _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
}

static inline dispatch_mach_msg_t
_dispatch_mach_msg_create_reply_disconnected(dispatch_object_t dou,
                dispatch_mach_reply_refs_t dmr, dispatch_mach_reason_t reason)
{
        dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr;
        mach_port_t reply_port = dmsg ? dmsg->dmsg_reply :
                        _dispatch_mach_reply_get_reply_port((mach_port_t)dmr->du_ident);
        voucher_t v;

        if (!reply_port) {
                if (!dmsg) {
                        v = dmr->dmr_voucher;
                        dmr->dmr_voucher = NULL; // transfer reference
                        if (v) _voucher_release(v);
                }
                return NULL;
        }

        if (dmsg) {
                v = dmsg->dmsg_voucher;
                if (v) _voucher_retain(v);
        } else {
                v = dmr->dmr_voucher;
                dmr->dmr_voucher = NULL; // transfer reference
        }

        if ((dmsg && (dmsg->dmsg_options & DISPATCH_MACH_WAIT_FOR_REPLY) &&
                        (dmsg->dmsg_options & DISPATCH_MACH_OWNED_REPLY_PORT)) ||
                        (dmr && !_dispatch_unote_registered(dmr) &&
                        _dispatch_mach_reply_is_reply_port_owned(dmr))) {
                if (v) _voucher_release(v);
                // deallocate owned reply port to break _dispatch_mach_msg_reply_recv
                // out of waiting in mach_msg(MACH_RCV_MSG)
                kern_return_t kr = mach_port_mod_refs(mach_task_self(), reply_port,
                                MACH_PORT_RIGHT_RECEIVE, -1);
                DISPATCH_VERIFY_MIG(kr);
                dispatch_assume_zero(kr);
                return NULL;
        }

        mach_msg_header_t *hdr;
        dmsgr = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
                        DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
        dmsgr->dmsg_voucher = v;
        hdr->msgh_local_port = reply_port;
        if (dmsg) {
                dmsgr->dmsg_priority = dmsg->dmsg_priority;
                dmsgr->do_ctxt = dmsg->do_ctxt;
        } else {
                dmsgr->dmsg_priority = _dispatch_priority_to_pp(dmr->dmr_priority);
                dmsgr->do_ctxt = dmr->dmr_ctxt;
        }
        _dispatch_mach_msg_set_reason(dmsgr, 0, reason);
        _dispatch_debug("machport[0x%08x]: reply disconnected, ctxt %p",
                        hdr->msgh_local_port, dmsgr->do_ctxt);
        return dmsgr;
}

DISPATCH_NOINLINE
static void
_dispatch_mach_msg_not_sent(dispatch_mach_t dm, dispatch_object_t dou)
{
        dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr;
        dispatch_queue_t drq = NULL;
        mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
        mach_msg_option_t msg_opts = dmsg->dmsg_options;
        _dispatch_debug("machport[0x%08x]: not sent msg id 0x%x, ctxt %p, "
                        "msg_opts 0x%x, kvoucher 0x%08x, reply on 0x%08x",
                        msg->msgh_remote_port, msg->msgh_id, dmsg->do_ctxt,
                        msg_opts, msg->msgh_voucher_port, dmsg->dmsg_reply);
        unsigned long reason = (msg_opts & DISPATCH_MACH_REGISTER_FOR_REPLY) ?
                        0 : DISPATCH_MACH_MESSAGE_NOT_SENT;
        dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL,
                        msg_opts & DISPATCH_MACH_ASYNC_REPLY
                        ? DISPATCH_MACH_ASYNC_WAITER_DISCONNECTED
                        : DISPATCH_MACH_DISCONNECTED);
        if (dmsg->do_ctxt) {
                drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
        }
        _dispatch_mach_msg_set_reason(dmsg, 0, reason);
        _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
        if (dmsgr) {
                if (drq) {
                        _dispatch_mach_push_async_reply_msg(dm, dmsgr, drq);
                } else {
                        _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
                }
        }
}

DISPATCH_NOINLINE
static uint32_t
_dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou,
                dispatch_mach_reply_refs_t dmr, dispatch_qos_t qos,
                dispatch_mach_send_invoke_flags_t send_flags)
{
        dispatch_mach_send_refs_t dsrr = dm->dm_send_refs;
        dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr = NULL;
        voucher_t voucher = dmsg->dmsg_voucher;
        dispatch_queue_t drq = NULL;
        mach_voucher_t ipc_kvoucher = MACH_VOUCHER_NULL;
        uint32_t send_status = 0;
        bool clear_voucher = false, kvoucher_move_send = false;
        mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
        bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) ==
                        MACH_MSG_TYPE_MOVE_SEND_ONCE);
        mach_port_t reply_port = dmsg->dmsg_reply;
        if (!is_reply) {
                dm->dm_needs_mgr = 0;
                if (unlikely(dsrr->dmsr_checkin && dmsg != dsrr->dmsr_checkin)) {
                        // send initial checkin message
                        if (unlikely(_dispatch_unote_registered(dsrr) &&
                                        _dispatch_queue_get_current() != &_dispatch_mgr_q)) {
                                // send kevent must be uninstalled on the manager queue
                                dm->dm_needs_mgr = 1;
                                goto out;
                        }
                        if (unlikely(!_dispatch_mach_msg_send(dm,
                                        dsrr->dmsr_checkin, NULL, qos, DM_SEND_INVOKE_NONE))) {
                                goto out;
                        }
                        dsrr->dmsr_checkin = NULL;
                }
        }
        mach_msg_return_t kr = 0;
        mach_msg_option_t opts = 0, msg_opts = dmsg->dmsg_options;
        if (!(msg_opts & DISPATCH_MACH_REGISTER_FOR_REPLY)) {
                mach_msg_priority_t msg_priority = MACH_MSG_PRIORITY_UNSPECIFIED;
                opts = MACH_SEND_MSG | (msg_opts & ~DISPATCH_MACH_OPTIONS_MASK);
                if (!is_reply) {
                        if (dmsg != dsrr->dmsr_checkin) {
                                msg->msgh_remote_port = dsrr->dmsr_send;
                        }
                        if (_dispatch_queue_get_current() == &_dispatch_mgr_q) {
                                if (unlikely(!_dispatch_unote_registered(dsrr))) {
                                        _dispatch_mach_notification_kevent_register(dm,
                                                        msg->msgh_remote_port);
                                }
                                if (likely(_dispatch_unote_registered(dsrr))) {
                                        if (os_atomic_load2o(dsrr, dmsr_notification_armed,
                                                        relaxed)) {
                                                goto out;
                                        }
                                        opts |= MACH_SEND_NOTIFY;
                                }
                        }
                        opts |= MACH_SEND_TIMEOUT;
                        if (dmsg->dmsg_priority != _voucher_get_priority(voucher)) {
                                ipc_kvoucher = _voucher_create_mach_voucher_with_priority(
                                                voucher, dmsg->dmsg_priority);
                        }
                        _dispatch_voucher_debug("mach-msg[%p] msg_set", voucher, dmsg);
                        if (ipc_kvoucher) {
                                kvoucher_move_send = true;
                                clear_voucher = _voucher_mach_msg_set_mach_voucher(msg,
                                                ipc_kvoucher, kvoucher_move_send);
                        } else {
                                clear_voucher = _voucher_mach_msg_set(msg, voucher);
                        }
                        if (qos) {
                                opts |= MACH_SEND_OVERRIDE;
                                msg_priority = (mach_msg_priority_t)
                                                _dispatch_priority_compute_propagated(
                                                _dispatch_qos_to_pp(qos), 0);
                        }
                }
                _dispatch_debug_machport(msg->msgh_remote_port);
                if (reply_port) _dispatch_debug_machport(reply_port);
                if (msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) {
                        if (msg_opts & DISPATCH_MACH_OWNED_REPLY_PORT) {
                                if (_dispatch_use_mach_special_reply_port()) {
                                        opts |= MACH_SEND_SYNC_OVERRIDE;
                                }
                                _dispatch_clear_thread_reply_port(reply_port);
                        }
                        _dispatch_mach_reply_waiter_register(dm, dmr, reply_port, dmsg,
                                        msg_opts);
                }
                kr = mach_msg(msg, opts, msg->msgh_size, 0, MACH_PORT_NULL, 0,
                                msg_priority);
                _dispatch_debug("machport[0x%08x]: sent msg id 0x%x, ctxt %p, "
                                "opts 0x%x, msg_opts 0x%x, kvoucher 0x%08x, reply on 0x%08x: "
                                "%s - 0x%x", msg->msgh_remote_port, msg->msgh_id, dmsg->do_ctxt,
                                opts, msg_opts, msg->msgh_voucher_port, reply_port,
                                mach_error_string(kr), kr);
                if (unlikely(kr && (msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY))) {
                        _dispatch_mach_reply_waiter_unregister(dm, dmr,
                                        DU_UNREGISTER_REPLY_REMOVE);
                }
                if (clear_voucher) {
                        if (kr == MACH_SEND_INVALID_VOUCHER && msg->msgh_voucher_port) {
                                DISPATCH_CLIENT_CRASH(kr, "Voucher port corruption");
                        }
                        mach_voucher_t kv;
                        kv = _voucher_mach_msg_clear(msg, kvoucher_move_send);
                        if (kvoucher_move_send) ipc_kvoucher = kv;
                }
        }
        if (kr == MACH_SEND_TIMED_OUT && (opts & MACH_SEND_TIMEOUT)) {
                if (opts & MACH_SEND_NOTIFY) {
                        _dispatch_debug("machport[0x%08x]: send-possible notification "
                                        "armed", (mach_port_t)dsrr->du_ident);
                        _dispatch_mach_notification_set_armed(dsrr);
                } else {
                        // send kevent must be installed on the manager queue
                        dm->dm_needs_mgr = 1;
                }
                if (ipc_kvoucher) {
                        _dispatch_kvoucher_debug("reuse on re-send", ipc_kvoucher);
                        voucher_t ipc_voucher;
                        ipc_voucher = _voucher_create_with_priority_and_mach_voucher(
                                        voucher, dmsg->dmsg_priority, ipc_kvoucher);
                        _dispatch_voucher_debug("mach-msg[%p] replace voucher[%p]",
                                        ipc_voucher, dmsg, voucher);
                        if (dmsg->dmsg_voucher) _voucher_release(dmsg->dmsg_voucher);
                        dmsg->dmsg_voucher = ipc_voucher;
                }
                goto out;
        } else if (ipc_kvoucher && (kr || !kvoucher_move_send)) {
                _voucher_dealloc_mach_voucher(ipc_kvoucher);
        }
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
        if (!(msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) && !kr && reply_port &&
                        !(_dispatch_unote_registered(dmrr) &&
                        dmrr->du_ident == reply_port)) {
                _dispatch_mach_reply_kevent_register(dm, reply_port, dmsg);
        }
        if (unlikely(!is_reply && dmsg == dsrr->dmsr_checkin &&
                        _dispatch_unote_registered(dsrr))) {
                _dispatch_mach_notification_kevent_unregister(dm);
        }
        if (slowpath(kr)) {
                // Send failed, so reply was never registered <rdar://problem/14309159>
                dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL,
                                msg_opts & DISPATCH_MACH_ASYNC_REPLY
                                ? DISPATCH_MACH_ASYNC_WAITER_DISCONNECTED
                                : DISPATCH_MACH_DISCONNECTED);
                if (dmsg->do_ctxt) {
                        drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
                }
        }
        _dispatch_mach_msg_set_reason(dmsg, kr, 0);
        if ((send_flags & DM_SEND_INVOKE_IMMEDIATE_SEND) &&
                        (msg_opts & DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT)) {
                // Return sent message synchronously <rdar://problem/25947334>
                send_status |= DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT;
        } else {
                _dispatch_mach_handle_or_push_received_msg(dm, dmsg);
        }
        if (dmsgr) {
                if (drq) {
                        _dispatch_mach_push_async_reply_msg(dm, dmsgr, drq);
                } else {
                        _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
                }
        }
        send_status |= DM_SEND_STATUS_SUCCESS;
out:
        return send_status;
}

#pragma mark -
#pragma mark dispatch_mach_send_refs_t

#define _dmsr_state_needs_lock_override(dq_state, qos) \
                unlikely(qos < _dq_state_max_qos(dq_state))

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dmsr_state_max_qos(uint64_t dmsr_state)
{
        return _dq_state_max_qos(dmsr_state);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dmsr_state_needs_override(uint64_t dmsr_state, dispatch_qos_t qos)
{
        dmsr_state &= DISPATCH_MACH_STATE_MAX_QOS_MASK;
        return dmsr_state < _dq_state_from_qos(qos);
}

DISPATCH_ALWAYS_INLINE
static inline uint64_t
_dmsr_state_merge_override(uint64_t dmsr_state, dispatch_qos_t qos)
{
        if (_dmsr_state_needs_override(dmsr_state, qos)) {
                dmsr_state &= ~DISPATCH_MACH_STATE_MAX_QOS_MASK;
                dmsr_state |= _dq_state_from_qos(qos);
                dmsr_state |= DISPATCH_MACH_STATE_DIRTY;
                dmsr_state |= DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
        }
        return dmsr_state;
}

#define _dispatch_mach_send_push_update_tail(dmsr, tail) \
                os_mpsc_push_update_tail(dmsr, dmsr, tail, do_next)
#define _dispatch_mach_send_push_update_head(dmsr, head) \
                os_mpsc_push_update_head(dmsr, dmsr, head)
#define _dispatch_mach_send_get_head(dmsr) \
                os_mpsc_get_head(dmsr, dmsr)
#define _dispatch_mach_send_unpop_head(dmsr, dc, dc_next) \
                os_mpsc_undo_pop_head(dmsr, dmsr, dc, dc_next, do_next)
#define _dispatch_mach_send_pop_head(dmsr, head) \
                os_mpsc_pop_head(dmsr, dmsr, head, do_next)

#define dm_push(dm, dc, qos) \
                _dispatch_queue_push((dm)->_as_dq, dc, qos)

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_mach_send_push_inline(dispatch_mach_send_refs_t dmsr,
                dispatch_object_t dou)
{
        if (_dispatch_mach_send_push_update_tail(dmsr, dou._do)) {
                _dispatch_mach_send_push_update_head(dmsr, dou._do);
                return true;
        }
        return false;
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_send_drain(dispatch_mach_t dm, dispatch_invoke_flags_t flags,
                dispatch_mach_send_invoke_flags_t send_flags)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_mach_reply_refs_t dmr;
        dispatch_mach_msg_t dmsg;
        struct dispatch_object_s *dc = NULL, *next_dc = NULL;
        dispatch_qos_t qos = _dmsr_state_max_qos(dmsr->dmsr_state);
        uint64_t old_state, new_state;
        uint32_t send_status;
        bool needs_mgr, disconnecting, returning_send_result = false;

again:
        needs_mgr = false; disconnecting = false;
        while (dmsr->dmsr_tail) {
                dc = _dispatch_mach_send_get_head(dmsr);
                do {
                        dispatch_mach_send_invoke_flags_t sf = send_flags;
                        // Only request immediate send result for the first message
                        send_flags &= ~DM_SEND_INVOKE_IMMEDIATE_SEND_MASK;
                        next_dc = _dispatch_mach_send_pop_head(dmsr, dc);
                        if (_dispatch_object_has_type(dc,
                                        DISPATCH_CONTINUATION_TYPE(MACH_SEND_BARRIER))) {
                                if (!(send_flags & DM_SEND_INVOKE_CAN_RUN_BARRIER)) {
                                        goto partial_drain;
                                }
                                _dispatch_continuation_pop(dc, NULL, flags, dm->_as_dq);
                                continue;
                        }
                        if (_dispatch_object_is_sync_waiter(dc)) {
                                dmsg = ((dispatch_continuation_t)dc)->dc_data;
                                dmr = ((dispatch_continuation_t)dc)->dc_other;
                        } else if (_dispatch_object_has_vtable(dc)) {
                                dmsg = (dispatch_mach_msg_t)dc;
                                dmr = NULL;
                        } else {
                                if (_dispatch_unote_registered(dmsr) &&
                                                (_dispatch_queue_get_current() != &_dispatch_mgr_q)) {
                                        // send kevent must be uninstalled on the manager queue
                                        needs_mgr = true;
                                        goto partial_drain;
                                }
                                if (unlikely(!_dispatch_mach_reconnect_invoke(dm, dc))) {
                                        disconnecting = true;
                                        goto partial_drain;
                                }
                                _dispatch_perfmon_workitem_inc();
                                continue;
                        }
                        _dispatch_voucher_ktrace_dmsg_pop(dmsg);
                        if (unlikely(dmsr->dmsr_disconnect_cnt ||
                                        (dm->dq_atomic_flags & DSF_CANCELED))) {
                                _dispatch_mach_msg_not_sent(dm, dmsg);
                                _dispatch_perfmon_workitem_inc();
                                continue;
                        }
                        send_status = _dispatch_mach_msg_send(dm, dmsg, dmr, qos, sf);
                        if (unlikely(!send_status)) {
                                goto partial_drain;
                        }
                        if (send_status & DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT) {
                                returning_send_result = true;
                        }
                        _dispatch_perfmon_workitem_inc();
                } while ((dc = next_dc));
        }

        os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, release, {
                if (old_state & DISPATCH_MACH_STATE_DIRTY) {
                        new_state = old_state;
                        new_state &= ~DISPATCH_MACH_STATE_DIRTY;
                        new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                        new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
                } else {
                        // unlock
                        new_state = 0;
                }
        });
        goto out;

partial_drain:
        // if this is not a complete drain, we must undo some things
        _dispatch_mach_send_unpop_head(dmsr, dc, next_dc);

        if (_dispatch_object_has_type(dc,
                        DISPATCH_CONTINUATION_TYPE(MACH_SEND_BARRIER))) {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, release, {
                        new_state = old_state;
                        new_state |= DISPATCH_MACH_STATE_DIRTY;
                        new_state |= DISPATCH_MACH_STATE_PENDING_BARRIER;
                        new_state &= ~DISPATCH_MACH_STATE_UNLOCK_MASK;
                        new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                });
        } else {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, release, {
                        new_state = old_state;
                        if (old_state & (DISPATCH_MACH_STATE_DIRTY |
                                        DISPATCH_MACH_STATE_RECEIVED_OVERRIDE)) {
                                new_state &= ~DISPATCH_MACH_STATE_DIRTY;
                                new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                                new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
                        } else {
                                new_state |= DISPATCH_MACH_STATE_DIRTY;
                                new_state &= ~DISPATCH_MACH_STATE_UNLOCK_MASK;
                        }
                });
        }

out:
        if (old_state & DISPATCH_MACH_STATE_RECEIVED_OVERRIDE) {
                // Ensure that the root queue sees that this thread was overridden.
                _dispatch_set_basepri_override_qos(_dmsr_state_max_qos(old_state));
        }

        if (unlikely(new_state & DISPATCH_MACH_STATE_UNLOCK_MASK)) {
                qos = _dmsr_state_max_qos(new_state);
                os_atomic_thread_fence(dependency);
                dmsr = os_atomic_force_dependency_on(dmsr, new_state);
                goto again;
        }

        if (new_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
                qos = _dmsr_state_max_qos(new_state);
                _dispatch_mach_push_send_barrier_drain(dm, qos);
        } else {
                if (needs_mgr || dm->dm_needs_mgr) {
                        qos = _dmsr_state_max_qos(new_state);
                } else {
                        qos = 0;
                }
                if (!disconnecting) dx_wakeup(dm, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
        }
        return returning_send_result;
}

DISPATCH_NOINLINE
static void
_dispatch_mach_send_invoke(dispatch_mach_t dm, dispatch_invoke_flags_t flags,
                dispatch_mach_send_invoke_flags_t send_flags)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_lock owner_self = _dispatch_lock_value_for_self();
        uint64_t old_state, new_state;

        uint64_t canlock_mask = DISPATCH_MACH_STATE_UNLOCK_MASK;
        uint64_t canlock_state = 0;

        if (send_flags & DM_SEND_INVOKE_NEEDS_BARRIER) {
                canlock_mask |= DISPATCH_MACH_STATE_PENDING_BARRIER;
                canlock_state = DISPATCH_MACH_STATE_PENDING_BARRIER;
        } else if (!(send_flags & DM_SEND_INVOKE_CAN_RUN_BARRIER)) {
                canlock_mask |= DISPATCH_MACH_STATE_PENDING_BARRIER;
        }

        dispatch_qos_t oq_floor = _dispatch_get_basepri_override_qos_floor();
retry:
        os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, acquire, {
                new_state = old_state;
                if (unlikely((old_state & canlock_mask) != canlock_state)) {
                        if (!(send_flags & DM_SEND_INVOKE_MAKE_DIRTY)) {
                                os_atomic_rmw_loop_give_up(break);
                        }
                        new_state |= DISPATCH_MACH_STATE_DIRTY;
                } else {
                        if (_dmsr_state_needs_lock_override(old_state, oq_floor)) {
                                os_atomic_rmw_loop_give_up({
                                        oq_floor = _dispatch_queue_override_self(old_state);
                                        goto retry;
                                });
                        }
                        new_state |= owner_self;
                        new_state &= ~DISPATCH_MACH_STATE_DIRTY;
                        new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                        new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
                }
        });

        if (unlikely((old_state & canlock_mask) != canlock_state)) {
                return;
        }
        if (send_flags & DM_SEND_INVOKE_CANCEL) {
                _dispatch_mach_cancel(dm);
        }
        _dispatch_mach_send_drain(dm, flags, send_flags);
}

DISPATCH_NOINLINE
void
_dispatch_mach_send_barrier_drain_invoke(dispatch_continuation_t dc,
                DISPATCH_UNUSED dispatch_invoke_context_t dic,
                dispatch_invoke_flags_t flags)
{
        dispatch_mach_t dm = (dispatch_mach_t)_dispatch_queue_get_current();
        uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
        dispatch_thread_frame_s dtf;

        DISPATCH_COMPILER_CAN_ASSUME(dc->dc_priority == DISPATCH_NO_PRIORITY);
        DISPATCH_COMPILER_CAN_ASSUME(dc->dc_voucher == DISPATCH_NO_VOUCHER);
        // hide the mach channel (see _dispatch_mach_barrier_invoke comment)
        _dispatch_thread_frame_stash(&dtf);
        _dispatch_continuation_pop_forwarded(dc, DISPATCH_NO_VOUCHER, dc_flags,{
                _dispatch_mach_send_invoke(dm, flags,
                                DM_SEND_INVOKE_NEEDS_BARRIER | DM_SEND_INVOKE_CAN_RUN_BARRIER);
        });
        _dispatch_thread_frame_unstash(&dtf);
}

DISPATCH_NOINLINE
static void
_dispatch_mach_push_send_barrier_drain(dispatch_mach_t dm, dispatch_qos_t qos)
{
        dispatch_continuation_t dc = _dispatch_continuation_alloc();

        dc->do_vtable = DC_VTABLE(MACH_SEND_BARRRIER_DRAIN);
        dc->dc_func = NULL;
        dc->dc_ctxt = NULL;
        dc->dc_voucher = DISPATCH_NO_VOUCHER;
        dc->dc_priority = DISPATCH_NO_PRIORITY;
        dm_push(dm, dc, qos);
}

DISPATCH_NOINLINE
static void
_dispatch_mach_send_push(dispatch_mach_t dm, dispatch_continuation_t dc,
                dispatch_qos_t qos)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        uint64_t old_state, new_state, state_flags = 0;
        dispatch_tid owner;
        bool wakeup;

        // <rdar://problem/25896179> when pushing a send barrier that destroys
        // the last reference to this channel, and the send queue is already
        // draining on another thread, the send barrier may run as soon as
        // _dispatch_mach_send_push_inline() returns.
        _dispatch_retain_2(dm);

        wakeup = _dispatch_mach_send_push_inline(dmsr, dc);
        if (wakeup) {
                state_flags = DISPATCH_MACH_STATE_DIRTY;
                if (dc->do_vtable == DC_VTABLE(MACH_SEND_BARRIER)) {
                        state_flags |= DISPATCH_MACH_STATE_PENDING_BARRIER;
                }
        }

        if (state_flags) {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, release, {
                        new_state = _dmsr_state_merge_override(old_state, qos);
                        new_state |= state_flags;
                });
        } else {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, relaxed, {
                        new_state = _dmsr_state_merge_override(old_state, qos);
                        if (old_state == new_state) {
                                os_atomic_rmw_loop_give_up(break);
                        }
                });
        }

        qos = _dmsr_state_max_qos(new_state);
        owner = _dispatch_lock_owner((dispatch_lock)old_state);
        if (owner) {
                if (_dmsr_state_needs_override(old_state, qos)) {
                        _dispatch_wqthread_override_start_check_owner(owner, qos,
                                        &dmsr->dmsr_state_lock.dul_lock);
                }
                return _dispatch_release_2_tailcall(dm);
        }

        dispatch_wakeup_flags_t wflags = 0;
        if (state_flags & DISPATCH_MACH_STATE_PENDING_BARRIER) {
                _dispatch_mach_push_send_barrier_drain(dm, qos);
        } else if (wakeup || dmsr->dmsr_disconnect_cnt ||
                        (dm->dq_atomic_flags & DSF_CANCELED)) {
                wflags = DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2;
        } else if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
                wflags = DISPATCH_WAKEUP_CONSUME_2;
        }
        if (wflags) {
                return dx_wakeup(dm, qos, wflags);
        }
        return _dispatch_release_2_tailcall(dm);
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_send_push_and_trydrain(dispatch_mach_t dm,
                dispatch_object_t dou, dispatch_qos_t qos,
                dispatch_mach_send_invoke_flags_t send_flags)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_lock owner_self = _dispatch_lock_value_for_self();
        uint64_t old_state, new_state, canlock_mask, state_flags = 0;
        dispatch_tid owner;

        bool wakeup = _dispatch_mach_send_push_inline(dmsr, dou);
        if (wakeup) {
                state_flags = DISPATCH_MACH_STATE_DIRTY;
        }

        if (unlikely(dmsr->dmsr_disconnect_cnt ||
                        (dm->dq_atomic_flags & DSF_CANCELED))) {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, release, {
                        new_state = _dmsr_state_merge_override(old_state, qos);
                        new_state |= state_flags;
                });
                dx_wakeup(dm, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
                return false;
        }

        canlock_mask = DISPATCH_MACH_STATE_UNLOCK_MASK |
                        DISPATCH_MACH_STATE_PENDING_BARRIER;
        if (state_flags) {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, seq_cst, {
                        new_state = _dmsr_state_merge_override(old_state, qos);
                        new_state |= state_flags;
                        if (likely((old_state & canlock_mask) == 0)) {
                                new_state |= owner_self;
                                new_state &= ~DISPATCH_MACH_STATE_DIRTY;
                                new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                                new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
                        }
                });
        } else {
                os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, acquire, {
                        new_state = _dmsr_state_merge_override(old_state, qos);
                        if (new_state == old_state) {
                                os_atomic_rmw_loop_give_up(return false);
                        }
                        if (likely((old_state & canlock_mask) == 0)) {
                                new_state |= owner_self;
                                new_state &= ~DISPATCH_MACH_STATE_DIRTY;
                                new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
                                new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
                        }
                });
        }

        owner = _dispatch_lock_owner((dispatch_lock)old_state);
        if (owner) {
                if (_dmsr_state_needs_override(old_state, qos)) {
                        _dispatch_wqthread_override_start_check_owner(owner, qos,
                                        &dmsr->dmsr_state_lock.dul_lock);
                }
                return false;
        }

        if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
                dx_wakeup(dm, qos, 0);
                return false;
        }

        // Ensure our message is still at the head of the queue and has not already
        // been dequeued by another thread that raced us to the send queue lock.
        // A plain load of the head and comparison against our object pointer is
        // sufficient.
        if (unlikely(!(wakeup && dou._do == dmsr->dmsr_head))) {
                // Don't request immediate send result for messages we don't own
                send_flags &= ~DM_SEND_INVOKE_IMMEDIATE_SEND_MASK;
        }
        return _dispatch_mach_send_drain(dm, DISPATCH_INVOKE_NONE, send_flags);
}

#pragma mark -
#pragma mark dispatch_mach

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_mach_notification_kevent_unregister(dispatch_mach_t dm)
{
        DISPATCH_ASSERT_ON_MANAGER_QUEUE();
        if (_dispatch_unote_registered(dm->dm_send_refs)) {
                dispatch_assume(_dispatch_unote_unregister(dm->dm_send_refs, 0));
        }
        dm->dm_send_refs->du_ident = 0;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_mach_notification_kevent_register(dispatch_mach_t dm,mach_port_t send)
{
        DISPATCH_ASSERT_ON_MANAGER_QUEUE();
        dm->dm_send_refs->du_ident = send;
        dispatch_assume(_dispatch_unote_register(dm->dm_send_refs,
                        DISPATCH_WLH_ANON, 0));
}

void
_dispatch_mach_merge_notification(dispatch_unote_t du,
                uint32_t flags DISPATCH_UNUSED, uintptr_t data,
                uintptr_t status DISPATCH_UNUSED,
                pthread_priority_t pp DISPATCH_UNUSED)
{
        dispatch_mach_send_refs_t dmsr = du._dmsr;
        dispatch_mach_t dm = _dispatch_wref2ptr(dmsr->du_owner_wref);

        if (data & dmsr->du_fflags) {
                _dispatch_mach_send_invoke(dm, DISPATCH_INVOKE_MANAGER_DRAIN,
                                DM_SEND_INVOKE_MAKE_DIRTY);
        }
}

DISPATCH_NOINLINE
static void
_dispatch_mach_handle_or_push_received_msg(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg)
{
        mach_error_t error;
        dispatch_mach_reason_t reason = _dispatch_mach_msg_get_reason(dmsg, &error);
        if (reason == DISPATCH_MACH_MESSAGE_RECEIVED || !dm->dm_is_xpc ||
                        !_dispatch_mach_xpc_hooks->dmxh_direct_message_handler(
                        dm->dm_recv_refs->dmrr_handler_ctxt, reason, dmsg, error)) {
                // Not XPC client or not a message that XPC can handle inline - push
                // it onto the channel queue.
                dm_push(dm, dmsg, _dispatch_qos_from_pp(dmsg->dmsg_priority));
        } else {
                // XPC handled the message inline. Do the cleanup that would otherwise
                // have happened in _dispatch_mach_msg_invoke(), leaving out steps that
                // are not required in this context.
                dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
                dispatch_release(dmsg);
        }
}

DISPATCH_ALWAYS_INLINE
static void
_dispatch_mach_push_async_reply_msg(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, dispatch_queue_t drq) {
        // Push the message onto the given queue. This function is only used for
        // replies to messages sent by
        // dispatch_mach_send_with_result_and_async_reply_4libxpc().
        dispatch_continuation_t dc = _dispatch_mach_msg_async_reply_wrap(dmsg, dm);
        _dispatch_trace_continuation_push(drq, dc);
        dx_push(drq, dc, _dispatch_qos_from_pp(dmsg->dmsg_priority));
}

#pragma mark -
#pragma mark dispatch_mach_t

static inline mach_msg_option_t
_dispatch_mach_checkin_options(void)
{
        mach_msg_option_t options = 0;
#if DISPATCH_USE_CHECKIN_NOIMPORTANCE
        options = MACH_SEND_NOIMPORTANCE; // <rdar://problem/16996737>
#endif
        return options;
}


static inline mach_msg_option_t
_dispatch_mach_send_options(void)
{
        mach_msg_option_t options = 0;
        return options;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_mach_priority_propagate(mach_msg_option_t options,
                pthread_priority_t *msg_pp)
{
#if DISPATCH_USE_NOIMPORTANCE_QOS
        if (options & MACH_SEND_NOIMPORTANCE) {
                *msg_pp = 0;
                return 0;
        }
#endif
        unsigned int flags = DISPATCH_PRIORITY_PROPAGATE_CURRENT;
        if ((options & DISPATCH_MACH_WAIT_FOR_REPLY) &&
                        (options & DISPATCH_MACH_OWNED_REPLY_PORT) &&
                        _dispatch_use_mach_special_reply_port()) {
                flags |= DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC;
        }
        *msg_pp = _dispatch_priority_compute_propagated(0, flags);
        // TODO: remove QoS contribution of sync IPC messages to send queue
        // rdar://31848737
        return _dispatch_qos_from_pp(*msg_pp);
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_send_msg(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
                dispatch_continuation_t dc_wait, mach_msg_option_t options)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        if (slowpath(dmsg->do_next != DISPATCH_OBJECT_LISTLESS)) {
                DISPATCH_CLIENT_CRASH(dmsg->do_next, "Message already enqueued");
        }
        dispatch_retain(dmsg);
        pthread_priority_t msg_pp;
        dispatch_qos_t qos = _dispatch_mach_priority_propagate(options, &msg_pp);
        options |= _dispatch_mach_send_options();
        dmsg->dmsg_options = options;
        mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
        dmsg->dmsg_reply = _dispatch_mach_msg_get_reply_port(dmsg);
        bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) ==
                        MACH_MSG_TYPE_MOVE_SEND_ONCE);
        dmsg->dmsg_priority = msg_pp;
        dmsg->dmsg_voucher = _voucher_copy();
        _dispatch_voucher_debug("mach-msg[%p] set", dmsg->dmsg_voucher, dmsg);

        uint32_t send_status;
        bool returning_send_result = false;
        dispatch_mach_send_invoke_flags_t send_flags = DM_SEND_INVOKE_NONE;
        if (options & DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT) {
                send_flags = DM_SEND_INVOKE_IMMEDIATE_SEND;
        }
        if (is_reply && !dmsg->dmsg_reply && !dmsr->dmsr_disconnect_cnt &&
                        !(dm->dq_atomic_flags & DSF_CANCELED)) {
                // replies are sent to a send-once right and don't need the send queue
                dispatch_assert(!dc_wait);
                send_status = _dispatch_mach_msg_send(dm, dmsg, NULL, 0, send_flags);
                dispatch_assert(send_status);
                returning_send_result = !!(send_status &
                                DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT);
        } else {
                _dispatch_voucher_ktrace_dmsg_push(dmsg);
                dispatch_object_t dou = { ._dmsg = dmsg };
                if (dc_wait) dou._dc = dc_wait;
                returning_send_result = _dispatch_mach_send_push_and_trydrain(dm, dou,
                                qos, send_flags);
        }
        if (returning_send_result) {
                _dispatch_voucher_debug("mach-msg[%p] clear", dmsg->dmsg_voucher, dmsg);
                if (dmsg->dmsg_voucher) _voucher_release(dmsg->dmsg_voucher);
                dmsg->dmsg_voucher = NULL;
                dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
                dispatch_release(dmsg);
        }
        return returning_send_result;
}

DISPATCH_NOINLINE
void
dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
                mach_msg_option_t options)
{
        dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
        options &= ~DISPATCH_MACH_OPTIONS_MASK;
        bool returned_send_result = _dispatch_mach_send_msg(dm, dmsg, NULL,options);
        dispatch_assert(!returned_send_result);
}

DISPATCH_NOINLINE
void
dispatch_mach_send_with_result(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
                mach_msg_option_t options, dispatch_mach_send_flags_t send_flags,
                dispatch_mach_reason_t *send_result, mach_error_t *send_error)
{
        if (unlikely(send_flags != DISPATCH_MACH_SEND_DEFAULT)) {
                DISPATCH_CLIENT_CRASH(send_flags, "Invalid send flags");
        }
        dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
        options &= ~DISPATCH_MACH_OPTIONS_MASK;
        options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
        bool returned_send_result = _dispatch_mach_send_msg(dm, dmsg, NULL,options);
        unsigned long reason = DISPATCH_MACH_NEEDS_DEFERRED_SEND;
        mach_error_t err = 0;
        if (returned_send_result) {
                reason = _dispatch_mach_msg_get_reason(dmsg, &err);
        }
        *send_result = reason;
        *send_error = err;
}

static inline
dispatch_mach_msg_t
_dispatch_mach_send_and_wait_for_reply(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, mach_msg_option_t options,
                bool *returned_send_result)
{
        mach_port_t send = MACH_PORT_NULL;
        mach_port_t reply_port = _dispatch_mach_msg_get_reply_port(dmsg);
        if (!reply_port) {
                // use per-thread mach reply port <rdar://24597802>
                reply_port = _dispatch_get_thread_reply_port();
                mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg);
                dispatch_assert(MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) ==
                                MACH_MSG_TYPE_MAKE_SEND_ONCE);
                hdr->msgh_local_port = reply_port;
                options |= DISPATCH_MACH_OWNED_REPLY_PORT;
        }
        options |= DISPATCH_MACH_WAIT_FOR_REPLY;

        dispatch_mach_reply_refs_t dmr;
#if DISPATCH_DEBUG
        dmr = _dispatch_calloc(1, sizeof(*dmr));
#else
        struct dispatch_mach_reply_refs_s dmr_buf = { };
        dmr = &dmr_buf;
#endif
        struct dispatch_continuation_s dc_wait = {
                .dc_flags = DISPATCH_OBJ_SYNC_WAITER_BIT,
                .dc_data = dmsg,
                .dc_other = dmr,
                .dc_priority = DISPATCH_NO_PRIORITY,
                .dc_voucher = DISPATCH_NO_VOUCHER,
        };
        dmr->dmr_ctxt = dmsg->do_ctxt;
        dmr->dmr_waiter_tid = _dispatch_tid_self();
        *returned_send_result = _dispatch_mach_send_msg(dm, dmsg, &dc_wait,options);
        if (options & DISPATCH_MACH_OWNED_REPLY_PORT) {
                _dispatch_clear_thread_reply_port(reply_port);
                if (_dispatch_use_mach_special_reply_port()) {
                        // link special reply port to send right for remote receive right
                        // TODO: extend to pre-connect phase <rdar://problem/31823384>
                        send = dm->dm_send_refs->dmsr_send;
                }
        }
        dmsg = _dispatch_mach_msg_reply_recv(dm, dmr, reply_port, send);
#if DISPATCH_DEBUG
        free(dmr);
#endif
        return dmsg;
}

DISPATCH_NOINLINE
dispatch_mach_msg_t
dispatch_mach_send_and_wait_for_reply(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, mach_msg_option_t options)
{
        bool returned_send_result;
        dispatch_mach_msg_t reply;
        dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
        options &= ~DISPATCH_MACH_OPTIONS_MASK;
        reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
                        &returned_send_result);
        dispatch_assert(!returned_send_result);
        return reply;
}

DISPATCH_NOINLINE
dispatch_mach_msg_t
dispatch_mach_send_with_result_and_wait_for_reply(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, mach_msg_option_t options,
                dispatch_mach_send_flags_t send_flags,
                dispatch_mach_reason_t *send_result, mach_error_t *send_error)
{
        if (unlikely(send_flags != DISPATCH_MACH_SEND_DEFAULT)) {
                DISPATCH_CLIENT_CRASH(send_flags, "Invalid send flags");
        }
        bool returned_send_result;
        dispatch_mach_msg_t reply;
        dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
        options &= ~DISPATCH_MACH_OPTIONS_MASK;
        options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
        reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
                        &returned_send_result);
        unsigned long reason = DISPATCH_MACH_NEEDS_DEFERRED_SEND;
        mach_error_t err = 0;
        if (returned_send_result) {
                reason = _dispatch_mach_msg_get_reason(dmsg, &err);
        }
        *send_result = reason;
        *send_error = err;
        return reply;
}

DISPATCH_NOINLINE
void
dispatch_mach_send_with_result_and_async_reply_4libxpc(dispatch_mach_t dm,
                dispatch_mach_msg_t dmsg, mach_msg_option_t options,
                dispatch_mach_send_flags_t send_flags,
                dispatch_mach_reason_t *send_result, mach_error_t *send_error)
{
        if (unlikely(send_flags != DISPATCH_MACH_SEND_DEFAULT)) {
                DISPATCH_CLIENT_CRASH(send_flags, "Invalid send flags");
        }
        if (unlikely(!dm->dm_is_xpc)) {
                DISPATCH_CLIENT_CRASH(0,
                        "dispatch_mach_send_with_result_and_wait_for_reply is XPC only");
        }

        dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
        options &= ~DISPATCH_MACH_OPTIONS_MASK;
        options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
        mach_port_t reply_port = _dispatch_mach_msg_get_reply_port(dmsg);
        if (!reply_port) {
                DISPATCH_CLIENT_CRASH(0, "Reply port needed for async send with reply");
        }
        options |= DISPATCH_MACH_ASYNC_REPLY;
        bool returned_send_result = _dispatch_mach_send_msg(dm, dmsg, NULL,options);
        unsigned long reason = DISPATCH_MACH_NEEDS_DEFERRED_SEND;
        mach_error_t err = 0;
        if (returned_send_result) {
                reason = _dispatch_mach_msg_get_reason(dmsg, &err);
        }
        *send_result = reason;
        *send_error = err;
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_disconnect(dispatch_mach_t dm)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        bool disconnected;
        if (_dispatch_unote_registered(dmsr)) {
                _dispatch_mach_notification_kevent_unregister(dm);
        }
        if (MACH_PORT_VALID(dmsr->dmsr_send)) {
                _dispatch_mach_msg_disconnected(dm, MACH_PORT_NULL, dmsr->dmsr_send);
                dmsr->dmsr_send = MACH_PORT_NULL;
        }
        if (dmsr->dmsr_checkin) {
                _dispatch_mach_msg_not_sent(dm, dmsr->dmsr_checkin);
                dmsr->dmsr_checkin = NULL;
        }
        _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
        dispatch_mach_reply_refs_t dmr, tmp;
        TAILQ_FOREACH_SAFE(dmr, &dm->dm_send_refs->dmsr_replies, dmr_list, tmp) {
                TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
                _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
                if (_dispatch_unote_registered(dmr)) {
                        if (!_dispatch_mach_reply_kevent_unregister(dm, dmr,
                                        DU_UNREGISTER_DISCONNECTED)) {
                                TAILQ_INSERT_HEAD(&dm->dm_send_refs->dmsr_replies, dmr,
                                        dmr_list);
                        }
                } else {
                        _dispatch_mach_reply_waiter_unregister(dm, dmr,
                                DU_UNREGISTER_DISCONNECTED);
                }
        }
        disconnected = TAILQ_EMPTY(&dm->dm_send_refs->dmsr_replies);
        _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
        return disconnected;
}

static void
_dispatch_mach_cancel(dispatch_mach_t dm)
{
        _dispatch_object_debug(dm, "%s", __func__);
        if (!_dispatch_mach_disconnect(dm)) return;

        bool uninstalled = true;
        dispatch_assert(!dm->dm_uninstalled);

        if (dm->dm_xpc_term_refs) {
                uninstalled = _dispatch_unote_unregister(dm->dm_xpc_term_refs, 0);
        }

        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
        mach_port_t local_port = (mach_port_t)dmrr->du_ident;
        if (local_port) {
                // handle the deferred delete case properly, similar to what
                // _dispatch_source_invoke2() does
                dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
                if ((dqf & DSF_DEFERRED_DELETE) && !(dqf & DSF_ARMED)) {
                        _dispatch_source_refs_unregister(dm->_as_ds,
                                        DU_UNREGISTER_IMMEDIATE_DELETE);
                        dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
                } else if (!(dqf & DSF_DEFERRED_DELETE) && !(dqf & DSF_DELETED)) {
                        _dispatch_source_refs_unregister(dm->_as_ds, 0);
                        dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
                }
                if ((dqf & DSF_STATE_MASK) == DSF_DELETED) {
                        _dispatch_mach_msg_disconnected(dm, local_port, MACH_PORT_NULL);
                        dmrr->du_ident = 0;
                } else {
                        uninstalled = false;
                }
        } else {
                _dispatch_queue_atomic_flags_set_and_clear(dm->_as_dq, DSF_DELETED,
                                DSF_ARMED | DSF_DEFERRED_DELETE);
        }

        if (dm->dm_send_refs->dmsr_disconnect_cnt) {
                uninstalled = false; // <rdar://problem/31233110>
        }
        if (uninstalled) dm->dm_uninstalled = uninstalled;
}

DISPATCH_NOINLINE
static bool
_dispatch_mach_reconnect_invoke(dispatch_mach_t dm, dispatch_object_t dou)
{
        if (!_dispatch_mach_disconnect(dm)) return false;
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dmsr->dmsr_checkin = dou._dc->dc_data;
        dmsr->dmsr_send = (mach_port_t)dou._dc->dc_other;
        _dispatch_continuation_free(dou._dc);
        (void)os_atomic_dec2o(dmsr, dmsr_disconnect_cnt, relaxed);
        _dispatch_object_debug(dm, "%s", __func__);
        _dispatch_release(dm); // <rdar://problem/26266265>
        return true;
}

DISPATCH_NOINLINE
void
dispatch_mach_reconnect(dispatch_mach_t dm, mach_port_t send,
                dispatch_mach_msg_t checkin)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        (void)os_atomic_inc2o(dmsr, dmsr_disconnect_cnt, relaxed);
        if (MACH_PORT_VALID(send) && checkin) {
                dispatch_mach_msg_t dmsg = checkin;
                dispatch_retain(dmsg);
                dmsg->dmsg_options = _dispatch_mach_checkin_options();
                dmsr->dmsr_checkin_port = _dispatch_mach_msg_get_remote_port(dmsg);
        } else {
                checkin = NULL;
                dmsr->dmsr_checkin_port = MACH_PORT_NULL;
        }
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        dc->dc_flags = DISPATCH_OBJ_CONSUME_BIT;
        // actually called manually in _dispatch_mach_send_drain
        dc->dc_func = (void*)_dispatch_mach_reconnect_invoke;
        dc->dc_ctxt = dc;
        dc->dc_data = checkin;
        dc->dc_other = (void*)(uintptr_t)send;
        dc->dc_voucher = DISPATCH_NO_VOUCHER;
        dc->dc_priority = DISPATCH_NO_PRIORITY;
        _dispatch_retain(dm); // <rdar://problem/26266265>
        return _dispatch_mach_send_push(dm, dc, 0);
}

DISPATCH_NOINLINE
mach_port_t
dispatch_mach_get_checkin_port(dispatch_mach_t dm)
{
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        if (slowpath(dm->dq_atomic_flags & DSF_CANCELED)) {
                return MACH_PORT_DEAD;
        }
        return dmsr->dmsr_checkin_port;
}

DISPATCH_NOINLINE
static void
_dispatch_mach_connect_invoke(dispatch_mach_t dm)
{
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
        _dispatch_client_callout4(dmrr->dmrr_handler_ctxt,
                        DISPATCH_MACH_CONNECTED, NULL, 0, dmrr->dmrr_handler_func);
        dm->dm_connect_handler_called = 1;
        _dispatch_perfmon_workitem_inc();
}

DISPATCH_ALWAYS_INLINE
static void
_dispatch_mach_msg_invoke_with_mach(dispatch_mach_msg_t dmsg,
                dispatch_invoke_flags_t flags, dispatch_mach_t dm)
{
        dispatch_mach_recv_refs_t dmrr;
        mach_error_t err;
        unsigned long reason = _dispatch_mach_msg_get_reason(dmsg, &err);
        dispatch_thread_set_self_t adopt_flags = DISPATCH_PRIORITY_ENFORCE|
                        DISPATCH_VOUCHER_CONSUME|DISPATCH_VOUCHER_REPLACE;

        dmrr = dm->dm_recv_refs;
        dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
        _dispatch_voucher_ktrace_dmsg_pop(dmsg);
        _dispatch_voucher_debug("mach-msg[%p] adopt", dmsg->dmsg_voucher, dmsg);
        (void)_dispatch_adopt_priority_and_set_voucher(dmsg->dmsg_priority,
                        dmsg->dmsg_voucher, adopt_flags);
        dmsg->dmsg_voucher = NULL;
        dispatch_invoke_with_autoreleasepool(flags, {
                if (flags & DISPATCH_INVOKE_ASYNC_REPLY) {
                        _dispatch_client_callout3(dmrr->dmrr_handler_ctxt, reason, dmsg,
                                        _dispatch_mach_xpc_hooks->dmxh_async_reply_handler);
                } else {
                        if (slowpath(!dm->dm_connect_handler_called)) {
                                _dispatch_mach_connect_invoke(dm);
                        }
                        if (reason == DISPATCH_MACH_MESSAGE_RECEIVED &&
                                        (_dispatch_queue_atomic_flags(dm->_as_dq) & DSF_CANCELED)) {
                                // <rdar://problem/32184699> Do not deliver message received
                                // after cancellation: _dispatch_mach_merge_msg can be preempted
                                // for a long time between clearing DSF_ARMED but before
                                // enqueuing the message, allowing for cancellation to complete,
                                // and then the message event to be delivered.
                                //
                                // This makes XPC unhappy because some of these messages are
                                // port-destroyed notifications that can cause it to try to
                                // reconnect on a channel that is almost fully canceled
                        } else {
                                _dispatch_client_callout4(dmrr->dmrr_handler_ctxt, reason, dmsg,
                                                err, dmrr->dmrr_handler_func);
                        }
                }
                _dispatch_perfmon_workitem_inc();
        });
        _dispatch_introspection_queue_item_complete(dmsg);
        dispatch_release(dmsg);
}

DISPATCH_NOINLINE
void
_dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg,
                DISPATCH_UNUSED dispatch_invoke_context_t dic,
                dispatch_invoke_flags_t flags)
{
        dispatch_thread_frame_s dtf;

        // hide mach channel
        dispatch_mach_t dm = (dispatch_mach_t)_dispatch_thread_frame_stash(&dtf);
        _dispatch_mach_msg_invoke_with_mach(dmsg, flags, dm);
        _dispatch_thread_frame_unstash(&dtf);
}

DISPATCH_NOINLINE
void
_dispatch_mach_barrier_invoke(dispatch_continuation_t dc,
                DISPATCH_UNUSED dispatch_invoke_context_t dic,
                dispatch_invoke_flags_t flags)
{
        dispatch_thread_frame_s dtf;
        dispatch_mach_t dm = dc->dc_other;
        dispatch_mach_recv_refs_t dmrr;
        uintptr_t dc_flags = (uintptr_t)dc->dc_data;
        unsigned long type = dc_type(dc);

        // hide mach channel from clients
        if (type == DISPATCH_CONTINUATION_TYPE(MACH_RECV_BARRIER)) {
                // on the send queue, the mach channel isn't the current queue
                // its target queue is the current one already
                _dispatch_thread_frame_stash(&dtf);
        }
        dmrr = dm->dm_recv_refs;
        DISPATCH_COMPILER_CAN_ASSUME(dc_flags & DISPATCH_OBJ_CONSUME_BIT);
        _dispatch_continuation_pop_forwarded(dc, DISPATCH_NO_VOUCHER, dc_flags, {
                dispatch_invoke_with_autoreleasepool(flags, {
                        if (slowpath(!dm->dm_connect_handler_called)) {
                                _dispatch_mach_connect_invoke(dm);
                        }
                        _dispatch_client_callout(dc->dc_ctxt, dc->dc_func);
                        _dispatch_client_callout4(dmrr->dmrr_handler_ctxt,
                                        DISPATCH_MACH_BARRIER_COMPLETED, NULL, 0,
                                        dmrr->dmrr_handler_func);
                });
        });
        if (type == DISPATCH_CONTINUATION_TYPE(MACH_RECV_BARRIER)) {
                _dispatch_thread_frame_unstash(&dtf);
        }
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_mach_barrier_set_vtable(dispatch_continuation_t dc,
                dispatch_mach_t dm, dispatch_continuation_vtable_t vtable)
{
        dc->dc_data = (void *)dc->dc_flags;
        dc->dc_other = dm;
        dc->do_vtable = vtable; // Must be after dc_flags load, dc_vtable aliases
}

DISPATCH_NOINLINE
void
dispatch_mach_send_barrier_f(dispatch_mach_t dm, void *context,
                dispatch_function_t func)
{
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER;
        dispatch_qos_t qos;

        _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags);
        _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_SEND_BARRIER));
        _dispatch_trace_continuation_push(dm->_as_dq, dc);
        qos = _dispatch_continuation_override_qos(dm->_as_dq, dc);
        return _dispatch_mach_send_push(dm, dc, qos);
}

DISPATCH_NOINLINE
void
dispatch_mach_send_barrier(dispatch_mach_t dm, dispatch_block_t barrier)
{
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER;
        dispatch_qos_t qos;

        _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags);
        _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_SEND_BARRIER));
        _dispatch_trace_continuation_push(dm->_as_dq, dc);
        qos = _dispatch_continuation_override_qos(dm->_as_dq, dc);
        return _dispatch_mach_send_push(dm, dc, qos);
}

DISPATCH_NOINLINE
void
dispatch_mach_receive_barrier_f(dispatch_mach_t dm, void *context,
                dispatch_function_t func)
{
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER;

        _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags);
        _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_RECV_BARRIER));
        return _dispatch_continuation_async(dm->_as_dq, dc);
}

DISPATCH_NOINLINE
void
dispatch_mach_receive_barrier(dispatch_mach_t dm, dispatch_block_t barrier)
{
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER;

        _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags);
        _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_RECV_BARRIER));
        return _dispatch_continuation_async(dm->_as_dq, dc);
}

DISPATCH_NOINLINE
static void
_dispatch_mach_cancel_invoke(dispatch_mach_t dm, dispatch_invoke_flags_t flags)
{
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;

        dispatch_invoke_with_autoreleasepool(flags, {
                if (slowpath(!dm->dm_connect_handler_called)) {
                        _dispatch_mach_connect_invoke(dm);
                }
                _dispatch_client_callout4(dmrr->dmrr_handler_ctxt,
                                DISPATCH_MACH_CANCELED, NULL, 0, dmrr->dmrr_handler_func);
                _dispatch_perfmon_workitem_inc();
        });
        dm->dm_cancel_handler_called = 1;
        _dispatch_release(dm); // the retain is done at creation time
}

DISPATCH_NOINLINE
void
dispatch_mach_cancel(dispatch_mach_t dm)
{
        dispatch_source_cancel(dm->_as_ds);
}

static void
_dispatch_mach_install(dispatch_mach_t dm, dispatch_wlh_t wlh,
                dispatch_priority_t pri)
{
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
        uint32_t disconnect_cnt;

        if (dmrr->du_ident) {
                _dispatch_source_refs_register(dm->_as_ds, wlh, pri);
                dispatch_assert(dmrr->du_is_direct);
        }

        if (dm->dm_is_xpc) {
                bool monitor_sigterm;
                if (_dispatch_mach_xpc_hooks->version < 3) {
                        monitor_sigterm = true;
                } else if (!_dispatch_mach_xpc_hooks->dmxh_enable_sigterm_notification){
                        monitor_sigterm = true;
                } else {
                        monitor_sigterm =
                                        _dispatch_mach_xpc_hooks->dmxh_enable_sigterm_notification(
                                        dm->dm_recv_refs->dmrr_handler_ctxt);
                }
                if (monitor_sigterm) {
                        dispatch_xpc_term_refs_t _dxtr =
                                        dux_create(&_dispatch_xpc_type_sigterm, SIGTERM, 0)._dxtr;
                        _dxtr->du_owner_wref = _dispatch_ptr2wref(dm);
                        dm->dm_xpc_term_refs = _dxtr;
                        _dispatch_unote_register(dm->dm_xpc_term_refs, wlh, pri);
                }
        }
        if (!dm->dq_priority) {
                // _dispatch_mach_reply_kevent_register assumes this has been done
                // which is unlike regular sources or queues, the DEFAULTQUEUE flag
                // is used so that the priority of the channel doesn't act as
                // a QoS floor for incoming messages (26761457)
                dm->dq_priority = pri;
        }
        dm->ds_is_installed = true;
        if (unlikely(!os_atomic_cmpxchgv2o(dm->dm_send_refs, dmsr_disconnect_cnt,
                        DISPATCH_MACH_NEVER_INSTALLED, 0, &disconnect_cnt, release))) {
                DISPATCH_INTERNAL_CRASH(disconnect_cnt, "Channel already installed");
        }
}

void
_dispatch_mach_finalize_activation(dispatch_mach_t dm, bool *allow_resume)
{
        dispatch_priority_t pri;
        dispatch_wlh_t wlh;

        // call "super"
        _dispatch_queue_finalize_activation(dm->_as_dq, allow_resume);

        if (!dm->ds_is_installed) {
                pri = _dispatch_queue_compute_priority_and_wlh(dm->_as_dq, &wlh);
                if (pri) _dispatch_mach_install(dm, wlh, pri);
        }
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_mach_tryarm(dispatch_mach_t dm, dispatch_queue_flags_t *out_dqf)
{
        dispatch_queue_flags_t oqf, nqf;
        bool rc = os_atomic_rmw_loop2o(dm, dq_atomic_flags, oqf, nqf, relaxed, {
                nqf = oqf;
                if (nqf & (DSF_ARMED | DSF_CANCELED | DSF_DEFERRED_DELETE |
                                DSF_DELETED)) {
                        // the test is inside the loop because it's convenient but the
                        // result should not change for the duration of the rmw_loop
                        os_atomic_rmw_loop_give_up(break);
                }
                nqf |= DSF_ARMED;
        });
        if (out_dqf) *out_dqf = nqf;
        return rc;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_wakeup_target_t
_dispatch_mach_invoke2(dispatch_object_t dou,
                dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
                uint64_t *owned)
{
        dispatch_mach_t dm = dou._dm;
        dispatch_queue_wakeup_target_t retq = NULL;
        dispatch_queue_t dq = _dispatch_queue_get_current();
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
        dispatch_queue_flags_t dqf = 0;

        if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN) && dmrr &&
                        _dispatch_unote_wlh_changed(dmrr, _dispatch_get_wlh())) {
                dqf = _dispatch_queue_atomic_flags_set_orig(dm->_as_dq,
                                DSF_WLH_CHANGED);
                if (!(dqf & DSF_WLH_CHANGED)) {
                        if (dm->dm_is_xpc) {
                                _dispatch_bug_deprecated("Changing target queue "
                                                "hierarchy after xpc connection was activated");
                        } else {
                                _dispatch_bug_deprecated("Changing target queue "
                                                "hierarchy after mach channel was activated");
                        }
                }
        }

        // This function performs all mach channel actions. Each action is
        // responsible for verifying that it takes place on the appropriate queue.
        // If the current queue is not the correct queue for this action, the
        // correct queue will be returned and the invoke will be re-driven on that
        // queue.

        // The order of tests here in invoke and in wakeup should be consistent.

        if (unlikely(!dm->ds_is_installed)) {
                // The channel needs to be installed on the kevent queue.
                if (unlikely(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
                        return dm->do_targetq;
                }
                _dispatch_mach_install(dm, _dispatch_get_wlh(),_dispatch_get_basepri());
                _dispatch_perfmon_workitem_inc();
        }

        if (_dispatch_queue_class_probe(dm)) {
                if (dq == dm->do_targetq) {
drain:
                        retq = _dispatch_queue_serial_drain(dm->_as_dq, dic, flags, owned);
                } else {
                        retq = dm->do_targetq;
                }
        }

        if (!retq && _dispatch_unote_registered(dmrr)) {
                if (_dispatch_mach_tryarm(dm, &dqf)) {
                        _dispatch_unote_resume(dmrr);
                        if (dq == dm->do_targetq && !dq->do_targetq && !dmsr->dmsr_tail &&
                                        (dq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT) &&
                                        _dispatch_wlh_should_poll_unote(dmrr)) {
                                // try to redrive the drain from under the lock for channels
                                // targeting an overcommit root queue to avoid parking
                                // when the next message has already fired
                                _dispatch_event_loop_drain(KEVENT_FLAG_IMMEDIATE);
                                if (dm->dq_items_tail) goto drain;
                        }
                }
        } else {
                dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
        }

        if (dmsr->dmsr_tail) {
                bool requires_mgr = dm->dm_needs_mgr || (dmsr->dmsr_disconnect_cnt &&
                                _dispatch_unote_registered(dmsr));
                if (!os_atomic_load2o(dmsr, dmsr_notification_armed, relaxed) ||
                                (dqf & DSF_CANCELED) || dmsr->dmsr_disconnect_cnt) {
                        // The channel has pending messages to send.
                        if (unlikely(requires_mgr && dq != &_dispatch_mgr_q)) {
                                return retq ? retq : &_dispatch_mgr_q;
                        }
                        dispatch_mach_send_invoke_flags_t send_flags = DM_SEND_INVOKE_NONE;
                        if (dq != &_dispatch_mgr_q) {
                                send_flags |= DM_SEND_INVOKE_CAN_RUN_BARRIER;
                        }
                        _dispatch_mach_send_invoke(dm, flags, send_flags);
                }
                if (!retq) retq = DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT;
        } else if (!retq && (dqf & DSF_CANCELED)) {
                // The channel has been cancelled and needs to be uninstalled from the
                // manager queue. After uninstallation, the cancellation handler needs
                // to be delivered to the target queue.
                if (!dm->dm_uninstalled) {
                        if ((dqf & DSF_STATE_MASK) == (DSF_ARMED | DSF_DEFERRED_DELETE)) {
                                // waiting for the delivery of a deferred delete event
                                return retq ? retq : DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT;
                        }
                        if (dq != &_dispatch_mgr_q) {
                                return retq ? retq : &_dispatch_mgr_q;
                        }
                        _dispatch_mach_send_invoke(dm, flags, DM_SEND_INVOKE_CANCEL);
                        if (unlikely(!dm->dm_uninstalled)) {
                                // waiting for the delivery of a deferred delete event
                                // or deletion didn't happen because send_invoke couldn't
                                // acquire the send lock
                                return retq ? retq : DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT;
                        }
                }
                if (!dm->dm_cancel_handler_called) {
                        if (dq != dm->do_targetq) {
                                return retq ? retq : dm->do_targetq;
                        }
                        _dispatch_mach_cancel_invoke(dm, flags);
                }
        }

        return retq;
}

DISPATCH_NOINLINE
void
_dispatch_mach_invoke(dispatch_mach_t dm,
                dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags)
{
        _dispatch_queue_class_invoke(dm, dic, flags,
                        DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS, _dispatch_mach_invoke2);
}

void
_dispatch_mach_wakeup(dispatch_mach_t dm, dispatch_qos_t qos,
                dispatch_wakeup_flags_t flags)
{
        // This function determines whether the mach channel needs to be invoked.
        // The order of tests here in probe and in invoke should be consistent.

        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_queue_wakeup_target_t tq = DISPATCH_QUEUE_WAKEUP_NONE;
        dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);

        if (!dm->ds_is_installed) {
                // The channel needs to be installed on the kevent queue.
                tq = DISPATCH_QUEUE_WAKEUP_TARGET;
                goto done;
        }

        if (_dispatch_queue_class_probe(dm)) {
                tq = DISPATCH_QUEUE_WAKEUP_TARGET;
                goto done;
        }

        if (_dispatch_lock_is_locked(dmsr->dmsr_state_lock.dul_lock)) {
                // Sending and uninstallation below require the send lock, the channel
                // will be woken up when the lock is dropped <rdar://15132939&15203957>
                goto done;
        }

        if (dmsr->dmsr_tail) {
                bool requires_mgr = dm->dm_needs_mgr || (dmsr->dmsr_disconnect_cnt &&
                                _dispatch_unote_registered(dmsr));
                if (!os_atomic_load2o(dmsr, dmsr_notification_armed, relaxed) ||
                                (dqf & DSF_CANCELED) || dmsr->dmsr_disconnect_cnt) {
                        if (unlikely(requires_mgr)) {
                                tq = DISPATCH_QUEUE_WAKEUP_MGR;
                        } else {
                                tq = DISPATCH_QUEUE_WAKEUP_TARGET;
                        }
                }
        } else if (dqf & DSF_CANCELED) {
                if (!dm->dm_uninstalled) {
                        if ((dqf & DSF_STATE_MASK) == (DSF_ARMED | DSF_DEFERRED_DELETE)) {
                                // waiting for the delivery of a deferred delete event
                        } else {
                                // The channel needs to be uninstalled from the manager queue
                                tq = DISPATCH_QUEUE_WAKEUP_MGR;
                        }
                } else if (!dm->dm_cancel_handler_called) {
                        // the cancellation handler needs to be delivered to the target
                        // queue.
                        tq = DISPATCH_QUEUE_WAKEUP_TARGET;
                }
        }

done:
        if ((tq == DISPATCH_QUEUE_WAKEUP_TARGET) &&
                        dm->do_targetq == &_dispatch_mgr_q) {
                tq = DISPATCH_QUEUE_WAKEUP_MGR;
        }

        return _dispatch_queue_class_wakeup(dm->_as_dq, qos, flags, tq);
}

static void
_dispatch_mach_sigterm_invoke(void *ctx)
{
        dispatch_mach_t dm = ctx;
        if (!(dm->dq_atomic_flags & DSF_CANCELED)) {
                dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
                _dispatch_client_callout4(dmrr->dmrr_handler_ctxt,
                                DISPATCH_MACH_SIGTERM_RECEIVED, NULL, 0,
                                dmrr->dmrr_handler_func);
        }
}

void
_dispatch_xpc_sigterm_merge(dispatch_unote_t du,
                uint32_t flags DISPATCH_UNUSED, uintptr_t data DISPATCH_UNUSED,
                uintptr_t status DISPATCH_UNUSED, pthread_priority_t pp)
{
        dispatch_mach_t dm = _dispatch_wref2ptr(du._du->du_owner_wref);
        uint32_t options = 0;
        if ((flags & EV_UDATA_SPECIFIC) && (flags & EV_ONESHOT) &&
                        !(flags & EV_DELETE)) {
                options = DU_UNREGISTER_IMMEDIATE_DELETE;
        } else {
                dispatch_assert((flags & EV_ONESHOT) && (flags & EV_DELETE));
                options = DU_UNREGISTER_ALREADY_DELETED;
        }
        _dispatch_unote_unregister(du, options);

        if (!(dm->dq_atomic_flags & DSF_CANCELED)) {
                _dispatch_barrier_async_detached_f(dm->_as_dq, dm,
                                _dispatch_mach_sigterm_invoke);
        } else {
                dx_wakeup(dm, _dispatch_qos_from_pp(pp), DISPATCH_WAKEUP_MAKE_DIRTY);
        }
}

#pragma mark -
#pragma mark dispatch_mach_msg_t

dispatch_mach_msg_t
dispatch_mach_msg_create(mach_msg_header_t *msg, size_t size,
                dispatch_mach_msg_destructor_t destructor, mach_msg_header_t **msg_ptr)
{
        if (slowpath(size < sizeof(mach_msg_header_t)) ||
                        slowpath(destructor && !msg)) {
                DISPATCH_CLIENT_CRASH(size, "Empty message");
        }

        dispatch_mach_msg_t dmsg;
        size_t msg_size = sizeof(struct dispatch_mach_msg_s);
        if (!destructor && os_add_overflow(msg_size,
                          (size - sizeof(dmsg->dmsg_msg)), &msg_size)) {
                DISPATCH_CLIENT_CRASH(size, "Message size too large");
        }

        dmsg = _dispatch_object_alloc(DISPATCH_VTABLE(mach_msg), msg_size);
        if (destructor) {
                dmsg->dmsg_msg = msg;
        } else if (msg) {
                memcpy(dmsg->dmsg_buf, msg, size);
        }
        dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
        dmsg->do_targetq = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, false);
        dmsg->dmsg_destructor = destructor;
        dmsg->dmsg_size = size;
        if (msg_ptr) {
                *msg_ptr = _dispatch_mach_msg_get_msg(dmsg);
        }
        return dmsg;
}

void
_dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg,
                DISPATCH_UNUSED bool *allow_free)
{
        if (dmsg->dmsg_voucher) {
                _voucher_release(dmsg->dmsg_voucher);
                dmsg->dmsg_voucher = NULL;
        }
        switch (dmsg->dmsg_destructor) {
        case DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT:
                break;
        case DISPATCH_MACH_MSG_DESTRUCTOR_FREE:
                free(dmsg->dmsg_msg);
                break;
        case DISPATCH_MACH_MSG_DESTRUCTOR_VM_DEALLOCATE: {
                mach_vm_size_t vm_size = dmsg->dmsg_size;
                mach_vm_address_t vm_addr = (uintptr_t)dmsg->dmsg_msg;
                (void)dispatch_assume_zero(mach_vm_deallocate(mach_task_self(),
                                vm_addr, vm_size));
                break;
        }}
}

static inline mach_msg_header_t*
_dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg)
{
        return dmsg->dmsg_destructor ? dmsg->dmsg_msg :
                        (mach_msg_header_t*)dmsg->dmsg_buf;
}

mach_msg_header_t*
dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg, size_t *size_ptr)
{
        if (size_ptr) {
                *size_ptr = dmsg->dmsg_size;
        }
        return _dispatch_mach_msg_get_msg(dmsg);
}

size_t
_dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf, size_t bufsiz)
{
        size_t offset = 0;
        offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
                        dx_kind(dmsg), dmsg);
        offset += _dispatch_object_debug_attr(dmsg, buf + offset, bufsiz - offset);
        offset += dsnprintf(&buf[offset], bufsiz - offset, "opts/err = 0x%x, "
                        "msgh[%p] = { ", dmsg->dmsg_options, dmsg->dmsg_buf);
        mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg);
        if (hdr->msgh_id) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "id 0x%x, ",
                                hdr->msgh_id);
        }
        if (hdr->msgh_size) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "size %u, ",
                                hdr->msgh_size);
        }
        if (hdr->msgh_bits) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "bits <l %u, r %u",
                                MACH_MSGH_BITS_LOCAL(hdr->msgh_bits),
                                MACH_MSGH_BITS_REMOTE(hdr->msgh_bits));
                if (MACH_MSGH_BITS_OTHER(hdr->msgh_bits)) {
                        offset += dsnprintf(&buf[offset], bufsiz - offset, ", o 0x%x",
                                        MACH_MSGH_BITS_OTHER(hdr->msgh_bits));
                }
                offset += dsnprintf(&buf[offset], bufsiz - offset, ">, ");
        }
        if (hdr->msgh_local_port && hdr->msgh_remote_port) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "local 0x%x, "
                                "remote 0x%x", hdr->msgh_local_port, hdr->msgh_remote_port);
        } else if (hdr->msgh_local_port) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "local 0x%x",
                                hdr->msgh_local_port);
        } else if (hdr->msgh_remote_port) {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "remote 0x%x",
                                hdr->msgh_remote_port);
        } else {
                offset += dsnprintf(&buf[offset], bufsiz - offset, "no ports");
        }
        offset += dsnprintf(&buf[offset], bufsiz - offset, " } }");
        return offset;
}

DISPATCH_ALWAYS_INLINE
static dispatch_queue_t
_dispatch_mach_msg_context_async_reply_queue(void *msg_context)
{
        if (DISPATCH_MACH_XPC_SUPPORTS_ASYNC_REPLIES(_dispatch_mach_xpc_hooks)) {
                return _dispatch_mach_xpc_hooks->dmxh_msg_context_reply_queue(
                                msg_context);
        }
        return NULL;
}

static dispatch_continuation_t
_dispatch_mach_msg_async_reply_wrap(dispatch_mach_msg_t dmsg,
                dispatch_mach_t dm)
{
        _dispatch_retain(dm); // Released in _dispatch_mach_msg_async_reply_invoke()
        dispatch_continuation_t dc = _dispatch_continuation_alloc();
        dc->do_vtable = DC_VTABLE(MACH_ASYNC_REPLY);
        dc->dc_data = dmsg;
        dc->dc_other = dm;
        dc->dc_priority = DISPATCH_NO_PRIORITY;
        dc->dc_voucher = DISPATCH_NO_VOUCHER;
        return dc;
}

DISPATCH_NOINLINE
void
_dispatch_mach_msg_async_reply_invoke(dispatch_continuation_t dc,
                DISPATCH_UNUSED dispatch_invoke_context_t dic,
                dispatch_invoke_flags_t flags)
{
        // _dispatch_mach_msg_invoke_with_mach() releases the reference on dmsg
        // taken by _dispatch_mach_msg_async_reply_wrap() after handling it.
        dispatch_mach_msg_t dmsg = dc->dc_data;
        dispatch_mach_t dm = dc->dc_other;
        _dispatch_mach_msg_invoke_with_mach(dmsg,
                        flags | DISPATCH_INVOKE_ASYNC_REPLY, dm);

        // Balances _dispatch_mach_msg_async_reply_wrap
        _dispatch_release(dc->dc_other);

        _dispatch_continuation_free(dc);
}

#pragma mark -
#pragma mark dispatch_mig_server

mach_msg_return_t
dispatch_mig_server(dispatch_source_t ds, size_t maxmsgsz,
                dispatch_mig_callback_t callback)
{
        mach_msg_options_t options = MACH_RCV_MSG | MACH_RCV_TIMEOUT
                | MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX)
                | MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | MACH_RCV_VOUCHER;
        mach_msg_options_t tmp_options;
        mig_reply_error_t *bufTemp, *bufRequest, *bufReply;
        mach_msg_return_t kr = 0;
        uint64_t assertion_token = 0;
        uint32_t cnt = 1000; // do not stall out serial queues
        boolean_t demux_success;
        bool received = false;
        size_t rcv_size = maxmsgsz + MAX_TRAILER_SIZE;
        dispatch_source_refs_t dr = ds->ds_refs;

        bufRequest = alloca(rcv_size);
        bufRequest->RetCode = 0;
        for (mach_vm_address_t p = mach_vm_trunc_page(bufRequest + vm_page_size);
                        p < (mach_vm_address_t)bufRequest + rcv_size; p += vm_page_size) {
                *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
        }

        bufReply = alloca(rcv_size);
        bufReply->Head.msgh_size = 0;
        for (mach_vm_address_t p = mach_vm_trunc_page(bufReply + vm_page_size);
                        p < (mach_vm_address_t)bufReply + rcv_size; p += vm_page_size) {
                *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
        }

#if DISPATCH_DEBUG
        options |= MACH_RCV_LARGE; // rdar://problem/8422992
#endif
        tmp_options = options;
        // XXX FIXME -- change this to not starve out the target queue
        for (;;) {
                if (DISPATCH_QUEUE_IS_SUSPENDED(ds) || (--cnt == 0)) {
                        options &= ~MACH_RCV_MSG;
                        tmp_options &= ~MACH_RCV_MSG;

                        if (!(tmp_options & MACH_SEND_MSG)) {
                                goto out;
                        }
                }
                kr = mach_msg(&bufReply->Head, tmp_options, bufReply->Head.msgh_size,
                                (mach_msg_size_t)rcv_size, (mach_port_t)dr->du_ident, 0, 0);

                tmp_options = options;

                if (slowpath(kr)) {
                        switch (kr) {
                        case MACH_SEND_INVALID_DEST:
                        case MACH_SEND_TIMED_OUT:
                                if (bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) {
                                        mach_msg_destroy(&bufReply->Head);
                                }
                                break;
                        case MACH_RCV_TIMED_OUT:
                                // Don't return an error if a message was sent this time or
                                // a message was successfully received previously
                                // rdar://problems/7363620&7791738
                                if(bufReply->Head.msgh_remote_port || received) {
                                        kr = MACH_MSG_SUCCESS;
                                }
                                break;
                        case MACH_RCV_INVALID_NAME:
                                break;
#if DISPATCH_DEBUG
                        case MACH_RCV_TOO_LARGE:
                                // receive messages that are too large and log their id and size
                                // rdar://problem/8422992
                                tmp_options &= ~MACH_RCV_LARGE;
                                size_t large_size = bufReply->Head.msgh_size + MAX_TRAILER_SIZE;
                                void *large_buf = malloc(large_size);
                                if (large_buf) {
                                        rcv_size = large_size;
                                        bufReply = large_buf;
                                }
                                if (!mach_msg(&bufReply->Head, tmp_options, 0,
                                                (mach_msg_size_t)rcv_size,
                                                (mach_port_t)dr->du_ident, 0, 0)) {
                                        _dispatch_log("BUG in libdispatch client: "
                                                        "dispatch_mig_server received message larger than "
                                                        "requested size %zd: id = 0x%x, size = %d",
                                                        maxmsgsz, bufReply->Head.msgh_id,
                                                        bufReply->Head.msgh_size);
                                }
                                if (large_buf) {
                                        free(large_buf);
                                }
                                // fall through
#endif
                        default:
                                _dispatch_bug_mach_client(
                                                "dispatch_mig_server: mach_msg() failed", kr);
                                break;
                        }
                        goto out;
                }

                if (!(tmp_options & MACH_RCV_MSG)) {
                        goto out;
                }

                if (assertion_token) {
#if DISPATCH_USE_IMPORTANCE_ASSERTION
                        int r = proc_importance_assertion_complete(assertion_token);
                        (void)dispatch_assume_zero(r);
#endif
                        assertion_token = 0;
                }
                received = true;

                bufTemp = bufRequest;
                bufRequest = bufReply;
                bufReply = bufTemp;

#if DISPATCH_USE_IMPORTANCE_ASSERTION
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
                int r = proc_importance_assertion_begin_with_msg(&bufRequest->Head,
                                NULL, &assertion_token);
                if (r && slowpath(r != EIO)) {
                        (void)dispatch_assume_zero(r);
                }
#pragma clang diagnostic pop
#endif
                _voucher_replace(voucher_create_with_mach_msg(&bufRequest->Head));
                demux_success = callback(&bufRequest->Head, &bufReply->Head);

                if (!demux_success) {
                        // destroy the request - but not the reply port
                        bufRequest->Head.msgh_remote_port = 0;
                        mach_msg_destroy(&bufRequest->Head);
                } else if (!(bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
                        // if MACH_MSGH_BITS_COMPLEX is _not_ set, then bufReply->RetCode
                        // is present
                        if (slowpath(bufReply->RetCode)) {
                                if (bufReply->RetCode == MIG_NO_REPLY) {
                                        continue;
                                }

                                // destroy the request - but not the reply port
                                bufRequest->Head.msgh_remote_port = 0;
                                mach_msg_destroy(&bufRequest->Head);
                        }
                }

                if (bufReply->Head.msgh_remote_port) {
                        tmp_options |= MACH_SEND_MSG;
                        if (MACH_MSGH_BITS_REMOTE(bufReply->Head.msgh_bits) !=
                                        MACH_MSG_TYPE_MOVE_SEND_ONCE) {
                                tmp_options |= MACH_SEND_TIMEOUT;
                        }
                }
        }

out:
        if (assertion_token) {
#if DISPATCH_USE_IMPORTANCE_ASSERTION
                int r = proc_importance_assertion_complete(assertion_token);
                (void)dispatch_assume_zero(r);
#endif
        }

        return kr;
}

#pragma mark -
#pragma mark dispatch_mach_debug

static size_t
_dispatch_mach_debug_attr(dispatch_mach_t dm, char *buf, size_t bufsiz)
{
        dispatch_queue_t target = dm->do_targetq;
        dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
        dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;

        return dsnprintf(buf, bufsiz, "target = %s[%p], receive = 0x%x, "
                        "send = 0x%x, send-possible = 0x%x%s, checkin = 0x%x%s, "
                        "send state = %016llx, disconnected = %d, canceled = %d ",
                        target && target->dq_label ? target->dq_label : "", target,
                        (mach_port_t)dmrr->du_ident, dmsr->dmsr_send,
                        (mach_port_t)dmsr->du_ident,
                        dmsr->dmsr_notification_armed ? " (armed)" : "",
                        dmsr->dmsr_checkin_port, dmsr->dmsr_checkin ? " (pending)" : "",
                        dmsr->dmsr_state, dmsr->dmsr_disconnect_cnt,
                        (bool)(dm->dq_atomic_flags & DSF_CANCELED));
}

size_t
_dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz)
{
        size_t offset = 0;
        offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
                        dm->dq_label && !dm->dm_cancel_handler_called ? dm->dq_label :
                        dx_kind(dm), dm);
        offset += _dispatch_object_debug_attr(dm, &buf[offset], bufsiz - offset);
        offset += _dispatch_mach_debug_attr(dm, &buf[offset], bufsiz - offset);
        offset += dsnprintf(&buf[offset], bufsiz - offset, "}");
        return offset;
}

#endif /* HAVE_MACH */