-
-#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
-DISPATCH_NOINLINE
-static void
-_dispatch_kevent_machport_drain(_dispatch_kevent_qos_s *ke)
-{
- mach_port_t name = (mach_port_name_t)ke->data;
- dispatch_kevent_t dk;
-
- _dispatch_debug_machport(name);
- dk = _dispatch_kevent_find(name, EVFILT_MACHPORT);
- if (!dispatch_assume(dk)) {
- return;
- }
- _dispatch_mach_portset_update(dk, MACH_PORT_NULL); // emulate EV_DISPATCH
-
- _dispatch_kevent_qos_s kev = {
- .ident = name,
- .filter = EVFILT_MACHPORT,
- .flags = EV_ADD|EV_ENABLE|EV_DISPATCH,
- .fflags = DISPATCH_MACH_RECV_MESSAGE,
- .udata = (uintptr_t)dk,
- };
- _dispatch_kevent_debug("synthetic", &kev);
- _dispatch_kevent_merge(&kev);
-}
-#endif
-
-DISPATCH_NOINLINE
-static void
-_dispatch_kevent_mach_msg_drain(_dispatch_kevent_qos_s *ke)
-{
- mach_msg_header_t *hdr = _dispatch_kevent_mach_msg_buf(ke);
- mach_msg_size_t siz;
- mach_msg_return_t kr = (mach_msg_return_t)ke->fflags;
-
- if (!fastpath(hdr)) {
- DISPATCH_INTERNAL_CRASH(kr, "EVFILT_MACHPORT with no message");
- }
- if (fastpath(!kr)) {
- _dispatch_kevent_mach_msg_recv(ke, hdr);
- goto out;
- } else if (kr != MACH_RCV_TOO_LARGE) {
- goto out;
- } else if (!ke->data) {
- DISPATCH_INTERNAL_CRASH(0, "MACH_RCV_LARGE_IDENTITY with no identity");
- }
- if (slowpath(ke->ext[1] > (UINT_MAX - dispatch_mach_trailer_size))) {
- DISPATCH_INTERNAL_CRASH(ke->ext[1],
- "EVFILT_MACHPORT with overlarge message");
- }
- siz = _dispatch_kevent_mach_msg_size(ke) + dispatch_mach_trailer_size;
- hdr = malloc(siz);
- if (!dispatch_assume(hdr)) {
- // Kernel will discard message too large to fit
- hdr = NULL;
- siz = 0;
- }
- mach_port_t name = (mach_port_name_t)ke->data;
- const mach_msg_option_t options = ((DISPATCH_MACH_RCV_OPTIONS |
- MACH_RCV_TIMEOUT) & ~MACH_RCV_LARGE);
- kr = mach_msg(hdr, options, 0, siz, name, MACH_MSG_TIMEOUT_NONE,
- MACH_PORT_NULL);
- if (fastpath(!kr)) {
- _dispatch_kevent_mach_msg_recv(ke, hdr);
- goto out;
- } else if (kr == MACH_RCV_TOO_LARGE) {
- _dispatch_log("BUG in libdispatch client: "
- "_dispatch_kevent_mach_msg_drain: dropped message too "
- "large to fit in memory: id = 0x%x, size = %u",
- hdr->msgh_id, _dispatch_kevent_mach_msg_size(ke));
- kr = MACH_MSG_SUCCESS;
- }
- if (hdr != _dispatch_kevent_mach_msg_buf(ke)) {
- free(hdr);
- }
-out:
- if (slowpath(kr)) {
- _dispatch_bug_mach_client("_dispatch_kevent_mach_msg_drain: "
- "message reception failed", kr);
- }
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_kevent_merge(_dispatch_kevent_qos_s *ke)
-{
- if (unlikely(!(ke->flags & EV_UDATA_SPECIFIC))) {
-#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
- if (ke->ident == _dispatch_mach_recv_portset) {
- _dispatch_kevent_mach_msg_drain(ke);
- return _dispatch_kq_deferred_update(&_dispatch_mach_recv_kevent);
- } else if (ke->ident == _dispatch_mach_portset) {
- return _dispatch_kevent_machport_drain(ke);
- }
-#endif
- return _dispatch_kevent_error(ke);
- }
-
- dispatch_kevent_t dk = (dispatch_kevent_t)ke->udata;
- dispatch_source_refs_t dr = TAILQ_FIRST(&dk->dk_sources);
- bool is_reply = (dk->dk_kevent.flags & EV_ONESHOT);
- dispatch_source_t ds = _dispatch_source_from_refs(dr);
-
- if (_dispatch_kevent_mach_msg_size(ke)) {
- _dispatch_kevent_mach_msg_drain(ke);
- if (is_reply) {
- // _dispatch_kevent_mach_msg_drain() should have deleted this event
- dispatch_assert(ke->flags & EV_DELETE);
- return;
- }
-
- if (!(ds->dq_atomic_flags & DSF_CANCELED)) {
- // re-arm the mach channel
- ke->fflags = DISPATCH_MACH_RCV_OPTIONS;
- ke->data = 0;
- ke->ext[0] = 0;
- ke->ext[1] = 0;
- return _dispatch_kq_deferred_update(ke);
- }
- } else if (is_reply) {
- DISPATCH_INTERNAL_CRASH(ke->flags, "Unexpected EVFILT_MACHPORT event");
- }
- if (unlikely((ke->flags & EV_VANISHED) &&
- (dx_type(ds) == DISPATCH_MACH_CHANNEL_TYPE))) {
- DISPATCH_CLIENT_CRASH(ke->flags,
- "Unexpected EV_VANISHED (do not destroy random mach ports)");
- }
- return _dispatch_kevent_merge(ke);
-}
-
-static void
-_dispatch_kevent_mach_msg_recv(_dispatch_kevent_qos_s *ke,
- mach_msg_header_t *hdr)
-{
- dispatch_source_refs_t dri;
- dispatch_kevent_t dk;
- mach_port_t name = hdr->msgh_local_port;
- mach_msg_size_t siz = hdr->msgh_size + dispatch_mach_trailer_size;
-
- if (!dispatch_assume(hdr->msgh_size <= UINT_MAX -
- dispatch_mach_trailer_size)) {
- _dispatch_bug_client("_dispatch_kevent_mach_msg_recv: "
- "received overlarge message");
- return _dispatch_kevent_mach_msg_destroy(ke, hdr);
- }
- if (!dispatch_assume(name)) {
- _dispatch_bug_client("_dispatch_kevent_mach_msg_recv: "
- "received message with MACH_PORT_NULL port");
- return _dispatch_kevent_mach_msg_destroy(ke, hdr);
- }
- _dispatch_debug_machport(name);
- if (ke->flags & EV_UDATA_SPECIFIC) {
- dk = (void*)ke->udata;
- } else {
- dk = _dispatch_kevent_find(name, EVFILT_MACHPORT);
- }
- if (!dispatch_assume(dk)) {
- _dispatch_bug_client("_dispatch_kevent_mach_msg_recv: "
- "received message with unknown kevent");
- return _dispatch_kevent_mach_msg_destroy(ke, hdr);
- }
- TAILQ_FOREACH(dri, &dk->dk_sources, dr_list) {
- dispatch_source_t dsi = _dispatch_source_from_refs(dri);
- if (dsi->ds_pending_data_mask & _DISPATCH_MACH_RECV_DIRECT_FLAGS) {
- return _dispatch_source_merge_mach_msg(dsi, dri, dk, ke, hdr, siz);
- }
- }
- _dispatch_bug_client("_dispatch_kevent_mach_msg_recv: "
- "received message with no listeners");
- return _dispatch_kevent_mach_msg_destroy(ke, hdr);
-}
-
-static void
-_dispatch_kevent_mach_msg_destroy(_dispatch_kevent_qos_s *ke,
- mach_msg_header_t *hdr)
-{
- if (hdr) {
- mach_msg_destroy(hdr);
- if (hdr != _dispatch_kevent_mach_msg_buf(ke)) {
- free(hdr);
- }
- }
-}
-
-static void
-_dispatch_source_merge_mach_msg(dispatch_source_t ds, dispatch_source_refs_t dr,
- dispatch_kevent_t dk, _dispatch_kevent_qos_s *ke,
- mach_msg_header_t *hdr, mach_msg_size_t siz)
-{
- if (dx_type(ds) == DISPATCH_SOURCE_KEVENT_TYPE) {
- return _dispatch_source_merge_mach_msg_direct(ds, ke, hdr);
- }
- dispatch_mach_reply_refs_t dmr = NULL;
- if (dk->dk_kevent.flags & EV_ONESHOT) {
- dmr = (dispatch_mach_reply_refs_t)dr;
- }
- return _dispatch_mach_msg_recv((dispatch_mach_t)ds, dmr, ke, hdr, siz);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_notify_merge(mach_port_t name, uint32_t flag, bool final)
-{
- dispatch_source_refs_t dri, dr_next;
- dispatch_kevent_t dk;
- bool unreg;
-
- dk = _dispatch_kevent_find(name, DISPATCH_EVFILT_MACH_NOTIFICATION);
- if (!dk) {
- return;
- }
-
- // Update notification registration state.
- dk->dk_kevent.data &= ~_DISPATCH_MACH_SP_FLAGS;
- _dispatch_kevent_qos_s kev = {
- .ident = name,
- .filter = DISPATCH_EVFILT_MACH_NOTIFICATION,
- .flags = EV_ADD|EV_ENABLE,
- .fflags = flag,
- .udata = (uintptr_t)dk,
- };
- if (final) {
- // This can never happen again
- unreg = true;
- } else {
- // Re-register for notification before delivery
- unreg = _dispatch_kevent_resume(dk, flag, 0);
- }
- DISPATCH_MACH_NOTIFICATION_ARMED(dk) = 0;
- TAILQ_FOREACH_SAFE(dri, &dk->dk_sources, dr_list, dr_next) {
- dispatch_source_t dsi = _dispatch_source_from_refs(dri);
- if (dx_type(dsi) == DISPATCH_MACH_CHANNEL_TYPE) {
- dispatch_mach_t dm = (dispatch_mach_t)dsi;
- _dispatch_mach_merge_notification_kevent(dm, &kev);
- if (unreg && dm->dm_dkev) {
- _dispatch_mach_notification_kevent_unregister(dm);
- }
- } else {
- _dispatch_source_merge_kevent(dsi, &kev);
- if (unreg) {
- _dispatch_source_kevent_unregister(dsi);
- }
- }
- if (!dr_next || DISPATCH_MACH_NOTIFICATION_ARMED(dk)) {
- // current merge is last in list (dk might have been freed)
- // or it re-armed the notification
- return;
- }
- }
-}
-
-static kern_return_t
-_dispatch_mach_notify_update(dispatch_kevent_t dk, uint32_t new_flags,
- uint32_t del_flags, uint32_t mask, mach_msg_id_t notify_msgid,
- mach_port_mscount_t notify_sync)
-{
- mach_port_t previous, port = (mach_port_t)dk->dk_kevent.ident;
- typeof(dk->dk_kevent.data) prev = dk->dk_kevent.data;
- kern_return_t kr, krr = 0;
-
- // Update notification registration state.
- dk->dk_kevent.data |= (new_flags | dk->dk_kevent.fflags) & mask;
- dk->dk_kevent.data &= ~(del_flags & mask);
-
- _dispatch_debug_machport(port);
- if ((dk->dk_kevent.data & mask) && !(prev & mask)) {
- _dispatch_debug("machport[0x%08x]: registering for send-possible "
- "notification", port);
- previous = MACH_PORT_NULL;
- krr = mach_port_request_notification(mach_task_self(), port,
- notify_msgid, notify_sync, _dispatch_get_mach_notify_port(),
- MACH_MSG_TYPE_MAKE_SEND_ONCE, &previous);
- DISPATCH_VERIFY_MIG(krr);
-
- switch(krr) {
- case KERN_INVALID_NAME:
- case KERN_INVALID_RIGHT:
- // Suppress errors & clear registration state
- dk->dk_kevent.data &= ~mask;
- break;
- default:
- // Else, we don't expect any errors from mach. Log any errors
- if (dispatch_assume_zero(krr)) {
- // log the error & clear registration state
- dk->dk_kevent.data &= ~mask;
- } else if (dispatch_assume_zero(previous)) {
- // Another subsystem has beat libdispatch to requesting the
- // specified Mach notification on this port. We should
- // technically cache the previous port and message it when the
- // kernel messages our port. Or we can just say screw those
- // subsystems and deallocate the previous port.
- // They should adopt libdispatch :-P
- kr = mach_port_deallocate(mach_task_self(), previous);
- DISPATCH_VERIFY_MIG(kr);
- (void)dispatch_assume_zero(kr);
- previous = MACH_PORT_NULL;
- }
- }
- } else if (!(dk->dk_kevent.data & mask) && (prev & mask)) {
- _dispatch_debug("machport[0x%08x]: unregistering for send-possible "
- "notification", port);
- previous = MACH_PORT_NULL;
- kr = mach_port_request_notification(mach_task_self(), port,
- notify_msgid, notify_sync, MACH_PORT_NULL,
- MACH_MSG_TYPE_MOVE_SEND_ONCE, &previous);
- DISPATCH_VERIFY_MIG(kr);
-
- switch (kr) {
- case KERN_INVALID_NAME:
- case KERN_INVALID_RIGHT:
- case KERN_INVALID_ARGUMENT:
- break;
- default:
- if (dispatch_assume_zero(kr)) {
- // log the error
- }
- }
- } else {
- return 0;
- }
- if (slowpath(previous)) {
- // the kernel has not consumed the send-once right yet
- (void)dispatch_assume_zero(
- _dispatch_send_consume_send_once_right(previous));
- }
- return krr;
-}
-
-static void
-_dispatch_mach_host_notify_update(void *context DISPATCH_UNUSED)
-{
- static int notify_type = HOST_NOTIFY_CALENDAR_SET;
- kern_return_t kr;
- _dispatch_debug("registering for calendar-change notification");
-retry:
- kr = host_request_notification(_dispatch_get_mach_host_port(),
- notify_type, _dispatch_get_mach_notify_port());
- // Fallback when missing support for newer _SET variant, fires strictly more.
- if (kr == KERN_INVALID_ARGUMENT &&
- notify_type != HOST_NOTIFY_CALENDAR_CHANGE){
- notify_type = HOST_NOTIFY_CALENDAR_CHANGE;
- goto retry;
- }
- DISPATCH_VERIFY_MIG(kr);
- (void)dispatch_assume_zero(kr);
-}
-
-static void
-_dispatch_mach_host_calendar_change_register(void)
-{
- static dispatch_once_t pred;
- dispatch_once_f(&pred, NULL, _dispatch_mach_host_notify_update);
-}
-
-static void
-_dispatch_mach_notify_source_invoke(mach_msg_header_t *hdr)
-{
- mig_reply_error_t reply;
- dispatch_assert(sizeof(mig_reply_error_t) == sizeof(union
- __ReplyUnion___dispatch_libdispatch_internal_protocol_subsystem));
- dispatch_assert(sizeof(mig_reply_error_t) < _dispatch_mach_recv_msg_size);
- boolean_t success = libdispatch_internal_protocol_server(hdr, &reply.Head);
- if (!success && reply.RetCode == MIG_BAD_ID &&
- (hdr->msgh_id == HOST_CALENDAR_SET_REPLYID ||
- hdr->msgh_id == HOST_CALENDAR_CHANGED_REPLYID)) {
- _dispatch_debug("calendar-change notification");
- _dispatch_timers_calendar_change();
- _dispatch_mach_host_notify_update(NULL);
- success = TRUE;
- reply.RetCode = KERN_SUCCESS;
- }
- if (dispatch_assume(success) && reply.RetCode != MIG_NO_REPLY) {
- (void)dispatch_assume_zero(reply.RetCode);
- }
- if (!success || (reply.RetCode && reply.RetCode != MIG_NO_REPLY)) {
- mach_msg_destroy(hdr);
- }
-}
-
-kern_return_t
-_dispatch_mach_notify_port_deleted(mach_port_t notify DISPATCH_UNUSED,
- mach_port_name_t name)
-{
-#if DISPATCH_DEBUG
- _dispatch_log("Corruption: Mach send/send-once/dead-name right 0x%x "
- "deleted prematurely", name);
-#endif
-
- _dispatch_debug_machport(name);
- _dispatch_mach_notify_merge(name, DISPATCH_MACH_SEND_DELETED, true);
-
- return KERN_SUCCESS;
-}
-
-kern_return_t
-_dispatch_mach_notify_dead_name(mach_port_t notify DISPATCH_UNUSED,
- mach_port_name_t name)
-{
- kern_return_t kr;
-
- _dispatch_debug("machport[0x%08x]: dead-name notification", name);
- _dispatch_debug_machport(name);
- _dispatch_mach_notify_merge(name, DISPATCH_MACH_SEND_DEAD, true);
-
- // the act of receiving a dead name notification allocates a dead-name
- // right that must be deallocated
- kr = mach_port_deallocate(mach_task_self(), name);
- DISPATCH_VERIFY_MIG(kr);
- //(void)dispatch_assume_zero(kr);
-
- return KERN_SUCCESS;
-}
-
-kern_return_t
-_dispatch_mach_notify_send_possible(mach_port_t notify DISPATCH_UNUSED,
- mach_port_name_t name)
-{
- _dispatch_debug("machport[0x%08x]: send-possible notification", name);
- _dispatch_debug_machport(name);
- _dispatch_mach_notify_merge(name, DISPATCH_MACH_SEND_POSSIBLE, false);
-
- return KERN_SUCCESS;
-}
-
-#pragma mark -
-#pragma mark dispatch_mach_t
-
-#define DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT 0x1
-#define DISPATCH_MACH_REGISTER_FOR_REPLY 0x2
-#define DISPATCH_MACH_WAIT_FOR_REPLY 0x4
-#define DISPATCH_MACH_OWNED_REPLY_PORT 0x8
-#define DISPATCH_MACH_OPTIONS_MASK 0xffff
-
-#define DM_SEND_STATUS_SUCCESS 0x1
-#define DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT 0x2
-
-DISPATCH_ENUM(dispatch_mach_send_invoke_flags, uint32_t,
- DM_SEND_INVOKE_NONE = 0x0,
- DM_SEND_INVOKE_FLUSH = 0x1,
- DM_SEND_INVOKE_NEEDS_BARRIER = 0x2,
- DM_SEND_INVOKE_CANCEL = 0x4,
- DM_SEND_INVOKE_CAN_RUN_BARRIER = 0x8,
- DM_SEND_INVOKE_IMMEDIATE_SEND = 0x10,
-);
-#define DM_SEND_INVOKE_IMMEDIATE_SEND_MASK \
- ((dispatch_mach_send_invoke_flags_t)DM_SEND_INVOKE_IMMEDIATE_SEND)
-
-static inline pthread_priority_t _dispatch_mach_priority_propagate(
- mach_msg_option_t options);
-static mach_port_t _dispatch_mach_msg_get_remote_port(dispatch_object_t dou);
-static mach_port_t _dispatch_mach_msg_get_reply_port(dispatch_object_t dou);
-static void _dispatch_mach_msg_disconnected(dispatch_mach_t dm,
- mach_port_t local_port, mach_port_t remote_port);
-static inline void _dispatch_mach_msg_reply_received(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, mach_port_t local_port);
-static dispatch_mach_msg_t _dispatch_mach_msg_create_reply_disconnected(
- dispatch_object_t dou, dispatch_mach_reply_refs_t dmr);
-static bool _dispatch_mach_reconnect_invoke(dispatch_mach_t dm,
- dispatch_object_t dou);
-static inline mach_msg_header_t* _dispatch_mach_msg_get_msg(
- dispatch_mach_msg_t dmsg);
-static void _dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou,
- pthread_priority_t pp);
-
-static dispatch_mach_t
-_dispatch_mach_create(const char *label, dispatch_queue_t q, void *context,
- dispatch_mach_handler_function_t handler, bool handler_is_block)
-{
- dispatch_mach_t dm;
- dispatch_mach_refs_t dr;
-
- dm = _dispatch_alloc(DISPATCH_VTABLE(mach),
- sizeof(struct dispatch_mach_s));
- _dispatch_queue_init(dm->_as_dq, DQF_NONE, 1, true);
-
- dm->dq_label = label;
- dm->do_ref_cnt++; // the reference _dispatch_mach_cancel_invoke holds
-
- dr = _dispatch_calloc(1ul, sizeof(struct dispatch_mach_refs_s));
- dr->dr_source_wref = _dispatch_ptr2wref(dm);
- dr->dm_handler_func = handler;
- dr->dm_handler_ctxt = context;
- dm->ds_refs = dr;
- dm->dm_handler_is_block = handler_is_block;
-
- dm->dm_refs = _dispatch_calloc(1ul,
- sizeof(struct dispatch_mach_send_refs_s));
- dm->dm_refs->dr_source_wref = _dispatch_ptr2wref(dm);
- dm->dm_refs->dm_disconnect_cnt = DISPATCH_MACH_NEVER_CONNECTED;
- TAILQ_INIT(&dm->dm_refs->dm_replies);
-
- if (slowpath(!q)) {
- q = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT, true);
- } else {
- _dispatch_retain(q);
- }
- dm->do_targetq = q;
- _dispatch_object_debug(dm, "%s", __func__);
- return dm;
-}
-
-dispatch_mach_t
-dispatch_mach_create(const char *label, dispatch_queue_t q,
- dispatch_mach_handler_t handler)
-{
- dispatch_block_t bb = _dispatch_Block_copy((void*)handler);
- return _dispatch_mach_create(label, q, bb,
- (dispatch_mach_handler_function_t)_dispatch_Block_invoke(bb), true);
-}
-
-dispatch_mach_t
-dispatch_mach_create_f(const char *label, dispatch_queue_t q, void *context,
- dispatch_mach_handler_function_t handler)
-{
- return _dispatch_mach_create(label, q, context, handler, false);
-}
-
-void
-_dispatch_mach_dispose(dispatch_mach_t dm)
-{
- _dispatch_object_debug(dm, "%s", __func__);
- dispatch_mach_refs_t dr = dm->ds_refs;
- if (dm->dm_handler_is_block && dr->dm_handler_ctxt) {
- Block_release(dr->dm_handler_ctxt);
- }
- free(dr);
- free(dm->dm_refs);
- _dispatch_queue_destroy(dm->_as_dq);
-}
-
-void
-dispatch_mach_connect(dispatch_mach_t dm, mach_port_t receive,
- mach_port_t send, dispatch_mach_msg_t checkin)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_kevent_t dk;
- uint32_t disconnect_cnt;
- dispatch_source_type_t type = &_dispatch_source_type_mach_recv_direct;
-
- dm->ds_is_direct_kevent = (bool)_dispatch_evfilt_machport_direct_enabled;
- if (MACH_PORT_VALID(receive)) {
- dk = _dispatch_calloc(1ul, sizeof(struct dispatch_kevent_s));
- dk->dk_kevent = type->ke;
- dk->dk_kevent.ident = receive;
- dk->dk_kevent.flags |= EV_ADD|EV_ENABLE|EV_VANISHED;
- dk->dk_kevent.udata = (uintptr_t)dk;
- TAILQ_INIT(&dk->dk_sources);
- dm->ds_dkev = dk;
- dm->ds_pending_data_mask = DISPATCH_MACH_RECV_MESSAGE_DIRECT;
- dm->ds_needs_rearm = dm->ds_is_direct_kevent;
- if (!dm->ds_is_direct_kevent) {
- dk->dk_kevent.fflags = DISPATCH_MACH_RECV_MESSAGE_DIRECT;
- dk->dk_kevent.flags &= ~(EV_UDATA_SPECIFIC|EV_VANISHED);
- }
- _dispatch_retain(dm); // the reference the manager queue holds
- }
- dr->dm_send = send;
- if (MACH_PORT_VALID(send)) {
- if (checkin) {
- dispatch_retain(checkin);
- checkin->dmsg_options = _dispatch_mach_checkin_options();
- dr->dm_checkin_port = _dispatch_mach_msg_get_remote_port(checkin);
- }
- dr->dm_checkin = checkin;
- }
- // monitor message reply ports
- dm->ds_pending_data_mask |= DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE;
- dispatch_assert(DISPATCH_MACH_NEVER_CONNECTED - 1 ==
- DISPATCH_MACH_NEVER_INSTALLED);
- disconnect_cnt = os_atomic_dec2o(dr, dm_disconnect_cnt, release);
- if (unlikely(disconnect_cnt != DISPATCH_MACH_NEVER_INSTALLED)) {
- DISPATCH_CLIENT_CRASH(disconnect_cnt, "Channel already connected");
- }
- _dispatch_object_debug(dm, "%s", __func__);
- return dispatch_activate(dm);
-}
-
-// assumes low bit of mach port names is always set
-#define DISPATCH_MACH_REPLY_PORT_UNOWNED 0x1u
-
-static inline void
-_dispatch_mach_reply_mark_reply_port_owned(dispatch_mach_reply_refs_t dmr)
-{
- dmr->dmr_reply &= ~DISPATCH_MACH_REPLY_PORT_UNOWNED;
-}
-
-static inline bool
-_dispatch_mach_reply_is_reply_port_owned(dispatch_mach_reply_refs_t dmr)
-{
- mach_port_t reply_port = dmr->dmr_reply;
- return reply_port ? !(reply_port & DISPATCH_MACH_REPLY_PORT_UNOWNED) :false;
-}
-
-static inline mach_port_t
-_dispatch_mach_reply_get_reply_port(dispatch_mach_reply_refs_t dmr)
-{
- mach_port_t reply_port = dmr->dmr_reply;
- return reply_port ? (reply_port | DISPATCH_MACH_REPLY_PORT_UNOWNED) : 0;
-}
-
-static inline bool
-_dispatch_mach_reply_tryremove(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr)
-{
- bool removed;
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- if ((removed = _TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
- }
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
- return removed;
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_reply_waiter_unregister(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, unsigned int options)
-{
- dispatch_mach_msg_t dmsgr = NULL;
- bool disconnected = (options & DKEV_UNREGISTER_DISCONNECTED);
- if (options & DKEV_UNREGISTER_REPLY_REMOVE) {
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- if (unlikely(!_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- DISPATCH_INTERNAL_CRASH(0, "Could not find reply registration");
- }
- TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
- }
- if (disconnected) {
- dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr);
- } else if (dmr->dmr_voucher) {
- _voucher_release(dmr->dmr_voucher);
- dmr->dmr_voucher = NULL;
- }
- _dispatch_debug("machport[0x%08x]: unregistering for sync reply%s, ctxt %p",
- _dispatch_mach_reply_get_reply_port(dmr),
- disconnected ? " (disconnected)" : "", dmr->dmr_ctxt);
- if (dmsgr) {
- return _dispatch_queue_push(dm->_as_dq, dmsgr, dmsgr->dmsg_priority);
- }
- dispatch_assert(!(options & DKEV_UNREGISTER_WAKEUP));
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, unsigned int options)
-{
- dispatch_mach_msg_t dmsgr = NULL;
- bool replies_empty = false;
- bool disconnected = (options & DKEV_UNREGISTER_DISCONNECTED);
- if (options & DKEV_UNREGISTER_REPLY_REMOVE) {
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- if (unlikely(!_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- DISPATCH_INTERNAL_CRASH(0, "Could not find reply registration");
- }
- TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
- replies_empty = TAILQ_EMPTY(&dm->dm_refs->dm_replies);
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
- }
- if (disconnected) {
- dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr);
- } else if (dmr->dmr_voucher) {
- _voucher_release(dmr->dmr_voucher);
- dmr->dmr_voucher = NULL;
- }
- uint32_t flags = DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE;
- dispatch_kevent_t dk = dmr->dmr_dkev;
- _dispatch_debug("machport[0x%08x]: unregistering for reply%s, ctxt %p",
- (mach_port_t)dk->dk_kevent.ident,
- disconnected ? " (disconnected)" : "", dmr->dmr_ctxt);
- if (!dm->ds_is_direct_kevent) {
- dmr->dmr_dkev = NULL;
- TAILQ_REMOVE(&dk->dk_sources, (dispatch_source_refs_t)dmr, dr_list);
- _dispatch_kevent_unregister(dk, flags, 0);
- } else {
- long r = _dispatch_kevent_unregister(dk, flags, options);
- if (r == EINPROGRESS) {
- _dispatch_debug("machport[0x%08x]: deferred delete kevent[%p]",
- (mach_port_t)dk->dk_kevent.ident, dk);
- dispatch_assert(options == DKEV_UNREGISTER_DISCONNECTED);
- // dmr must be put back so that the event delivery finds it, the
- // replies lock is held by the caller.
- TAILQ_INSERT_HEAD(&dm->dm_refs->dm_replies, dmr, dmr_list);
- if (dmsgr) {
- dmr->dmr_voucher = dmsgr->dmsg_voucher;
- dmsgr->dmsg_voucher = NULL;
- dispatch_release(dmsgr);
- }
- return; // deferred unregistration
- }
- dispatch_assume_zero(r);
- dmr->dmr_dkev = NULL;
- _TAILQ_TRASH_ENTRY(dmr, dr_list);
- }
- free(dmr);
- if (dmsgr) {
- return _dispatch_queue_push(dm->_as_dq, dmsgr, dmsgr->dmsg_priority);
- }
- if ((options & DKEV_UNREGISTER_WAKEUP) && replies_empty &&
- (dm->dm_refs->dm_disconnect_cnt ||
- (dm->dq_atomic_flags & DSF_CANCELED))) {
- dx_wakeup(dm, 0, DISPATCH_WAKEUP_FLUSH);
- }
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_reply_waiter_register(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, mach_port_t reply_port,
- dispatch_mach_msg_t dmsg, mach_msg_option_t msg_opts)
-{
- dmr->dr_source_wref = _dispatch_ptr2wref(dm);
- dmr->dmr_dkev = NULL;
- dmr->dmr_reply = reply_port;
- if (msg_opts & DISPATCH_MACH_OWNED_REPLY_PORT) {
- _dispatch_mach_reply_mark_reply_port_owned(dmr);
- } else {
- if (dmsg->dmsg_voucher) {
- dmr->dmr_voucher = _voucher_retain(dmsg->dmsg_voucher);
- }
- dmr->dmr_priority = (dispatch_priority_t)dmsg->dmsg_priority;
- // make reply context visible to leaks rdar://11777199
- dmr->dmr_ctxt = dmsg->do_ctxt;
- }
-
- _dispatch_debug("machport[0x%08x]: registering for sync reply, ctxt %p",
- reply_port, dmsg->do_ctxt);
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- if (unlikely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- DISPATCH_INTERNAL_CRASH(dmr->dmr_list.tqe_prev, "Reply already registered");
- }
- TAILQ_INSERT_TAIL(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_reply_kevent_register(dispatch_mach_t dm, mach_port_t reply_port,
- dispatch_mach_msg_t dmsg)
-{
- dispatch_kevent_t dk;
- dispatch_mach_reply_refs_t dmr;
- dispatch_source_type_t type = &_dispatch_source_type_mach_recv_direct;
- pthread_priority_t mp, pp;
-
- dk = _dispatch_calloc(1ul, sizeof(struct dispatch_kevent_s));
- dk->dk_kevent = type->ke;
- dk->dk_kevent.ident = reply_port;
- dk->dk_kevent.flags |= EV_ADD|EV_ENABLE|EV_ONESHOT;
- dk->dk_kevent.udata = (uintptr_t)dk;
- TAILQ_INIT(&dk->dk_sources);
- if (!dm->ds_is_direct_kevent) {
- dk->dk_kevent.fflags = DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE;
- dk->dk_kevent.flags &= ~(EV_UDATA_SPECIFIC|EV_VANISHED);
- }
-
- dmr = _dispatch_calloc(1ul, sizeof(struct dispatch_mach_reply_refs_s));
- dmr->dr_source_wref = _dispatch_ptr2wref(dm);
- dmr->dmr_dkev = dk;
- dmr->dmr_reply = reply_port;
- if (dmsg->dmsg_voucher) {
- dmr->dmr_voucher = _voucher_retain(dmsg->dmsg_voucher);
- }
- dmr->dmr_priority = (dispatch_priority_t)dmsg->dmsg_priority;
- // make reply context visible to leaks rdar://11777199
- dmr->dmr_ctxt = dmsg->do_ctxt;
-
- pp = dm->dq_priority & ~_PTHREAD_PRIORITY_FLAGS_MASK;
- if (pp && dm->ds_is_direct_kevent) {
- mp = dmsg->dmsg_priority & ~_PTHREAD_PRIORITY_FLAGS_MASK;
- if (pp < mp) pp = mp;
- pp |= dm->dq_priority & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
- } else {
- pp = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
- }
-
- _dispatch_debug("machport[0x%08x]: registering for reply, ctxt %p",
- reply_port, dmsg->do_ctxt);
- uint32_t flags;
- bool do_resume = _dispatch_kevent_register(&dmr->dmr_dkev, pp, &flags);
- TAILQ_INSERT_TAIL(&dmr->dmr_dkev->dk_sources, (dispatch_source_refs_t)dmr,
- dr_list);
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- if (unlikely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- DISPATCH_INTERNAL_CRASH(dmr->dmr_list.tqe_prev, "Reply already registered");
- }
- TAILQ_INSERT_TAIL(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
- if (do_resume && _dispatch_kevent_resume(dmr->dmr_dkev, flags, 0)) {
- return _dispatch_mach_reply_kevent_unregister(dm, dmr,
- DKEV_UNREGISTER_DISCONNECTED|DKEV_UNREGISTER_REPLY_REMOVE);
- }
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_notification_kevent_unregister(dispatch_mach_t dm)
-{
- DISPATCH_ASSERT_ON_MANAGER_QUEUE();
- dispatch_kevent_t dk = dm->dm_dkev;
- dm->dm_dkev = NULL;
- TAILQ_REMOVE(&dk->dk_sources, (dispatch_source_refs_t)dm->dm_refs,
- dr_list);
- dm->ds_pending_data_mask &= ~(unsigned long)
- (DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD);
- _dispatch_kevent_unregister(dk,
- DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD, 0);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_notification_kevent_register(dispatch_mach_t dm,mach_port_t send)
-{
- DISPATCH_ASSERT_ON_MANAGER_QUEUE();
- dispatch_kevent_t dk;
-
- dk = _dispatch_calloc(1ul, sizeof(struct dispatch_kevent_s));
- dk->dk_kevent = _dispatch_source_type_mach_send.ke;
- dk->dk_kevent.ident = send;
- dk->dk_kevent.flags |= EV_ADD|EV_ENABLE;
- dk->dk_kevent.fflags = DISPATCH_MACH_SEND_POSSIBLE|DISPATCH_MACH_SEND_DEAD;
- dk->dk_kevent.udata = (uintptr_t)dk;
- TAILQ_INIT(&dk->dk_sources);
-
- dm->ds_pending_data_mask |= dk->dk_kevent.fflags;
-
- uint32_t flags;
- bool do_resume = _dispatch_kevent_register(&dk,
- _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG, &flags);
- TAILQ_INSERT_TAIL(&dk->dk_sources,
- (dispatch_source_refs_t)dm->dm_refs, dr_list);
- dm->dm_dkev = dk;
- if (do_resume && _dispatch_kevent_resume(dm->dm_dkev, flags, 0)) {
- _dispatch_mach_notification_kevent_unregister(dm);
- }
-}
-
-static mach_port_t
-_dispatch_get_thread_reply_port(void)
-{
- mach_port_t reply_port, mrp = _dispatch_get_thread_mig_reply_port();
- if (mrp) {
- reply_port = mrp;
- _dispatch_debug("machport[0x%08x]: borrowed thread sync reply port",
- reply_port);
- } else {
- reply_port = mach_reply_port();
- _dispatch_set_thread_mig_reply_port(reply_port);
- _dispatch_debug("machport[0x%08x]: allocated thread sync reply port",
- reply_port);
- }
- _dispatch_debug_machport(reply_port);
- return reply_port;
-}
-
-static void
-_dispatch_clear_thread_reply_port(mach_port_t reply_port)
-{
- mach_port_t mrp = _dispatch_get_thread_mig_reply_port();
- if (reply_port != mrp) {
- if (mrp) {
- _dispatch_debug("machport[0x%08x]: did not clear thread sync reply "
- "port (found 0x%08x)", reply_port, mrp);
- }
- return;
- }
- _dispatch_set_thread_mig_reply_port(MACH_PORT_NULL);
- _dispatch_debug_machport(reply_port);
- _dispatch_debug("machport[0x%08x]: cleared thread sync reply port",
- reply_port);
-}
-
-static void
-_dispatch_set_thread_reply_port(mach_port_t reply_port)
-{
- _dispatch_debug_machport(reply_port);
- mach_port_t mrp = _dispatch_get_thread_mig_reply_port();
- if (mrp) {
- kern_return_t kr = mach_port_mod_refs(mach_task_self(), reply_port,
- MACH_PORT_RIGHT_RECEIVE, -1);
- DISPATCH_VERIFY_MIG(kr);
- dispatch_assume_zero(kr);
- _dispatch_debug("machport[0x%08x]: deallocated sync reply port "
- "(found 0x%08x)", reply_port, mrp);
- } else {
- _dispatch_set_thread_mig_reply_port(reply_port);
- _dispatch_debug("machport[0x%08x]: restored thread sync reply port",
- reply_port);
- }
-}
-
-static inline mach_port_t
-_dispatch_mach_msg_get_remote_port(dispatch_object_t dou)
-{
- mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dou._dmsg);
- mach_port_t remote = hdr->msgh_remote_port;
- return remote;
-}
-
-static inline mach_port_t
-_dispatch_mach_msg_get_reply_port(dispatch_object_t dou)
-{
- mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dou._dmsg);
- mach_port_t local = hdr->msgh_local_port;
- if (!MACH_PORT_VALID(local) || MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) !=
- MACH_MSG_TYPE_MAKE_SEND_ONCE) return MACH_PORT_NULL;
- return local;
-}
-
-static inline void
-_dispatch_mach_msg_set_reason(dispatch_mach_msg_t dmsg, mach_error_t err,
- unsigned long reason)
-{
- dispatch_assert_zero(reason & ~(unsigned long)code_emask);
- dmsg->dmsg_error = ((err || !reason) ? err :
- err_local|err_sub(0x3e0)|(mach_error_t)reason);
-}
-
-static inline unsigned long
-_dispatch_mach_msg_get_reason(dispatch_mach_msg_t dmsg, mach_error_t *err_ptr)
-{
- mach_error_t err = dmsg->dmsg_error;
-
- dmsg->dmsg_error = 0;
- if ((err & system_emask) == err_local && err_get_sub(err) == 0x3e0) {
- *err_ptr = 0;
- return err_get_code(err);
- }
- *err_ptr = err;
- return err ? DISPATCH_MACH_MESSAGE_SEND_FAILED : DISPATCH_MACH_MESSAGE_SENT;
-}
-
-static void
-_dispatch_mach_msg_recv(dispatch_mach_t dm, dispatch_mach_reply_refs_t dmr,
- _dispatch_kevent_qos_s *ke, mach_msg_header_t *hdr, mach_msg_size_t siz)
-{
- _dispatch_debug_machport(hdr->msgh_remote_port);
- _dispatch_debug("machport[0x%08x]: received msg id 0x%x, reply on 0x%08x",
- hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
- bool canceled = (dm->dq_atomic_flags & DSF_CANCELED);
- if (!dmr && canceled) {
- // message received after cancellation, _dispatch_mach_kevent_merge is
- // responsible for mach channel source state (e.g. deferred deletion)
- return _dispatch_kevent_mach_msg_destroy(ke, hdr);
- }
- dispatch_mach_msg_t dmsg;
- voucher_t voucher;
- pthread_priority_t priority;
- void *ctxt = NULL;
- if (dmr) {
- _voucher_mach_msg_clear(hdr, false); // deallocate reply message voucher
- voucher = dmr->dmr_voucher;
- dmr->dmr_voucher = NULL; // transfer reference
- priority = dmr->dmr_priority;
- ctxt = dmr->dmr_ctxt;
- unsigned int options = DKEV_DISPOSE_IMMEDIATE_DELETE;
- options |= DKEV_UNREGISTER_REPLY_REMOVE;
- options |= DKEV_UNREGISTER_WAKEUP;
- if (canceled) options |= DKEV_UNREGISTER_DISCONNECTED;
- _dispatch_mach_reply_kevent_unregister(dm, dmr, options);
- ke->flags |= EV_DELETE; // remember that unregister deleted the event
- if (canceled) return;
- } else {
- voucher = voucher_create_with_mach_msg(hdr);
- priority = _voucher_get_priority(voucher);
- }
- dispatch_mach_msg_destructor_t destructor;
- destructor = (hdr == _dispatch_kevent_mach_msg_buf(ke)) ?
- DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT :
- DISPATCH_MACH_MSG_DESTRUCTOR_FREE;
- dmsg = dispatch_mach_msg_create(hdr, siz, destructor, NULL);
- if (hdr == _dispatch_kevent_mach_msg_buf(ke)) {
- _dispatch_ktrace2(DISPATCH_MACH_MSG_hdr_move, (uint64_t)hdr, (uint64_t)dmsg->dmsg_buf);
- }
- dmsg->dmsg_voucher = voucher;
- dmsg->dmsg_priority = priority;
- dmsg->do_ctxt = ctxt;
- _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_MESSAGE_RECEIVED);
- _dispatch_voucher_debug("mach-msg[%p] create", voucher, dmsg);
- _dispatch_voucher_ktrace_dmsg_push(dmsg);
- return _dispatch_queue_push(dm->_as_dq, dmsg, dmsg->dmsg_priority);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_mach_msg_t
-_dispatch_mach_msg_reply_recv(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, mach_port_t reply_port)
-{
- if (slowpath(!MACH_PORT_VALID(reply_port))) {
- DISPATCH_CLIENT_CRASH(reply_port, "Invalid reply port");
- }
- void *ctxt = dmr->dmr_ctxt;
- mach_msg_header_t *hdr, *hdr2 = NULL;
- void *hdr_copyout_addr;
- mach_msg_size_t siz, msgsiz = 0;
- mach_msg_return_t kr;
- mach_msg_option_t options;
- siz = mach_vm_round_page(_dispatch_mach_recv_msg_size +
- dispatch_mach_trailer_size);
- hdr = alloca(siz);
- for (mach_vm_address_t p = mach_vm_trunc_page(hdr + vm_page_size);
- p < (mach_vm_address_t)hdr + siz; p += vm_page_size) {
- *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
- }
- options = DISPATCH_MACH_RCV_OPTIONS & (~MACH_RCV_VOUCHER);
-retry:
- _dispatch_debug_machport(reply_port);
- _dispatch_debug("machport[0x%08x]: MACH_RCV_MSG %s", reply_port,
- (options & MACH_RCV_TIMEOUT) ? "poll" : "wait");
- kr = mach_msg(hdr, options, 0, siz, reply_port, MACH_MSG_TIMEOUT_NONE,
- MACH_PORT_NULL);
- hdr_copyout_addr = hdr;
- _dispatch_debug_machport(reply_port);
- _dispatch_debug("machport[0x%08x]: MACH_RCV_MSG (size %u, opts 0x%x) "
- "returned: %s - 0x%x", reply_port, siz, options,
- mach_error_string(kr), kr);
- switch (kr) {
- case MACH_RCV_TOO_LARGE:
- if (!fastpath(hdr->msgh_size <= UINT_MAX -
- dispatch_mach_trailer_size)) {
- DISPATCH_CLIENT_CRASH(hdr->msgh_size, "Overlarge message");
- }
- if (options & MACH_RCV_LARGE) {
- msgsiz = hdr->msgh_size + dispatch_mach_trailer_size;
- hdr2 = malloc(msgsiz);
- if (dispatch_assume(hdr2)) {
- hdr = hdr2;
- siz = msgsiz;
- }
- options |= MACH_RCV_TIMEOUT;
- options &= ~MACH_RCV_LARGE;
- goto retry;
- }
- _dispatch_log("BUG in libdispatch client: "
- "dispatch_mach_send_and_wait_for_reply: dropped message too "
- "large to fit in memory: id = 0x%x, size = %u", hdr->msgh_id,
- hdr->msgh_size);
- break;
- case MACH_RCV_INVALID_NAME: // rdar://problem/21963848
- case MACH_RCV_PORT_CHANGED: // rdar://problem/21885327
- case MACH_RCV_PORT_DIED:
- // channel was disconnected/canceled and reply port destroyed
- _dispatch_debug("machport[0x%08x]: sync reply port destroyed, ctxt %p: "
- "%s - 0x%x", reply_port, ctxt, mach_error_string(kr), kr);
- goto out;
- case MACH_MSG_SUCCESS:
- if (hdr->msgh_remote_port) {
- _dispatch_debug_machport(hdr->msgh_remote_port);
- }
- _dispatch_debug("machport[0x%08x]: received msg id 0x%x, size = %u, "
- "reply on 0x%08x", hdr->msgh_local_port, hdr->msgh_id,
- hdr->msgh_size, hdr->msgh_remote_port);
- siz = hdr->msgh_size + dispatch_mach_trailer_size;
- if (hdr2 && siz < msgsiz) {
- void *shrink = realloc(hdr2, msgsiz);
- if (shrink) hdr = hdr2 = shrink;
- }
- break;
- default:
- dispatch_assume_zero(kr);
- break;
- }
- _dispatch_mach_msg_reply_received(dm, dmr, hdr->msgh_local_port);
- hdr->msgh_local_port = MACH_PORT_NULL;
- if (slowpath((dm->dq_atomic_flags & DSF_CANCELED) || kr)) {
- if (!kr) mach_msg_destroy(hdr);
- goto out;
- }
- dispatch_mach_msg_t dmsg;
- dispatch_mach_msg_destructor_t destructor = (!hdr2) ?
- DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT :
- DISPATCH_MACH_MSG_DESTRUCTOR_FREE;
- dmsg = dispatch_mach_msg_create(hdr, siz, destructor, NULL);
- if (!hdr2 || hdr != hdr_copyout_addr) {
- _dispatch_ktrace2(DISPATCH_MACH_MSG_hdr_move, (uint64_t)hdr_copyout_addr, (uint64_t)_dispatch_mach_msg_get_msg(dmsg));
- }
- dmsg->do_ctxt = ctxt;
- return dmsg;
-out:
- free(hdr2);
- return NULL;
-}
-
-static inline void
-_dispatch_mach_msg_reply_received(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, mach_port_t local_port)
-{
- bool removed = _dispatch_mach_reply_tryremove(dm, dmr);
- if (!MACH_PORT_VALID(local_port) || !removed) {
- // port moved/destroyed during receive, or reply waiter was never
- // registered or already removed (disconnected)
- return;
- }
- mach_port_t reply_port = _dispatch_mach_reply_get_reply_port(dmr);
- _dispatch_debug("machport[0x%08x]: unregistered for sync reply, ctxt %p",
- reply_port, dmr->dmr_ctxt);
- if (_dispatch_mach_reply_is_reply_port_owned(dmr)) {
- _dispatch_set_thread_reply_port(reply_port);
- if (local_port != reply_port) {
- DISPATCH_CLIENT_CRASH(local_port,
- "Reply received on unexpected port");
- }
- return;
- }
- mach_msg_header_t *hdr;
- dispatch_mach_msg_t dmsg;
- dmsg = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
- DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
- hdr->msgh_local_port = local_port;
- dmsg->dmsg_voucher = dmr->dmr_voucher;
- dmr->dmr_voucher = NULL; // transfer reference
- dmsg->dmsg_priority = dmr->dmr_priority;
- dmsg->do_ctxt = dmr->dmr_ctxt;
- _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_REPLY_RECEIVED);
- return _dispatch_queue_push(dm->_as_dq, dmsg, dmsg->dmsg_priority);
-}
-
-static inline void
-_dispatch_mach_msg_disconnected(dispatch_mach_t dm, mach_port_t local_port,
- mach_port_t remote_port)
-{
- mach_msg_header_t *hdr;
- dispatch_mach_msg_t dmsg;
- dmsg = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
- DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
- if (local_port) hdr->msgh_local_port = local_port;
- if (remote_port) hdr->msgh_remote_port = remote_port;
- _dispatch_mach_msg_set_reason(dmsg, 0, DISPATCH_MACH_DISCONNECTED);
- _dispatch_debug("machport[0x%08x]: %s right disconnected", local_port ?
- local_port : remote_port, local_port ? "receive" : "send");
- return _dispatch_queue_push(dm->_as_dq, dmsg, dmsg->dmsg_priority);
-}
-
-static inline dispatch_mach_msg_t
-_dispatch_mach_msg_create_reply_disconnected(dispatch_object_t dou,
- dispatch_mach_reply_refs_t dmr)
-{
- dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr;
- mach_port_t reply_port = dmsg ? dmsg->dmsg_reply :
- _dispatch_mach_reply_get_reply_port(dmr);
- voucher_t v;
-
- if (!reply_port) {
- if (!dmsg) {
- v = dmr->dmr_voucher;
- dmr->dmr_voucher = NULL; // transfer reference
- if (v) _voucher_release(v);
- }
- return NULL;
- }
-
- if (dmsg) {
- v = dmsg->dmsg_voucher;
- if (v) _voucher_retain(v);
- } else {
- v = dmr->dmr_voucher;
- dmr->dmr_voucher = NULL; // transfer reference
- }
-
- if ((dmsg && (dmsg->dmsg_options & DISPATCH_MACH_WAIT_FOR_REPLY) &&
- (dmsg->dmsg_options & DISPATCH_MACH_OWNED_REPLY_PORT)) ||
- (dmr && !dmr->dmr_dkev &&
- _dispatch_mach_reply_is_reply_port_owned(dmr))) {
- if (v) _voucher_release(v);
- // deallocate owned reply port to break _dispatch_mach_msg_reply_recv
- // out of waiting in mach_msg(MACH_RCV_MSG)
- kern_return_t kr = mach_port_mod_refs(mach_task_self(), reply_port,
- MACH_PORT_RIGHT_RECEIVE, -1);
- DISPATCH_VERIFY_MIG(kr);
- dispatch_assume_zero(kr);
- return NULL;
- }
-
- mach_msg_header_t *hdr;
- dmsgr = dispatch_mach_msg_create(NULL, sizeof(mach_msg_header_t),
- DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT, &hdr);
- dmsgr->dmsg_voucher = v;
- hdr->msgh_local_port = reply_port;
- if (dmsg) {
- dmsgr->dmsg_priority = dmsg->dmsg_priority;
- dmsgr->do_ctxt = dmsg->do_ctxt;
- } else {
- dmsgr->dmsg_priority = dmr->dmr_priority;
- dmsgr->do_ctxt = dmr->dmr_ctxt;
- }
- _dispatch_mach_msg_set_reason(dmsgr, 0, DISPATCH_MACH_DISCONNECTED);
- _dispatch_debug("machport[0x%08x]: reply disconnected, ctxt %p",
- hdr->msgh_local_port, dmsgr->do_ctxt);
- return dmsgr;
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_msg_not_sent(dispatch_mach_t dm, dispatch_object_t dou)
-{
- dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr;
- mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
- mach_msg_option_t msg_opts = dmsg->dmsg_options;
- _dispatch_debug("machport[0x%08x]: not sent msg id 0x%x, ctxt %p, "
- "msg_opts 0x%x, kvoucher 0x%08x, reply on 0x%08x",
- msg->msgh_remote_port, msg->msgh_id, dmsg->do_ctxt,
- msg_opts, msg->msgh_voucher_port, dmsg->dmsg_reply);
- unsigned long reason = (msg_opts & DISPATCH_MACH_REGISTER_FOR_REPLY) ?
- 0 : DISPATCH_MACH_MESSAGE_NOT_SENT;
- dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL);
- _dispatch_mach_msg_set_reason(dmsg, 0, reason);
- _dispatch_queue_push(dm->_as_dq, dmsg, dmsg->dmsg_priority);
- if (dmsgr) _dispatch_queue_push(dm->_as_dq, dmsgr, dmsgr->dmsg_priority);
-}
-
-DISPATCH_NOINLINE
-static uint32_t
-_dispatch_mach_msg_send(dispatch_mach_t dm, dispatch_object_t dou,
- dispatch_mach_reply_refs_t dmr, pthread_priority_t pp,
- dispatch_mach_send_invoke_flags_t send_flags)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_mach_msg_t dmsg = dou._dmsg, dmsgr = NULL;
- voucher_t voucher = dmsg->dmsg_voucher;
- mach_voucher_t ipc_kvoucher = MACH_VOUCHER_NULL;
- uint32_t send_status = 0;
- bool clear_voucher = false, kvoucher_move_send = false;
- mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
- bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) ==
- MACH_MSG_TYPE_MOVE_SEND_ONCE);
- mach_port_t reply_port = dmsg->dmsg_reply;
- if (!is_reply) {
- dr->dm_needs_mgr = 0;
- if (unlikely(dr->dm_checkin && dmsg != dr->dm_checkin)) {
- // send initial checkin message
- if (dm->dm_dkev && slowpath(_dispatch_queue_get_current() !=
- &_dispatch_mgr_q)) {
- // send kevent must be uninstalled on the manager queue
- dr->dm_needs_mgr = 1;
- goto out;
- }
- if (unlikely(!_dispatch_mach_msg_send(dm,
- dr->dm_checkin, NULL, pp, DM_SEND_INVOKE_NONE))) {
- goto out;
- }
- dr->dm_checkin = NULL;
- }
- }
- mach_msg_return_t kr = 0;
- mach_msg_option_t opts = 0, msg_opts = dmsg->dmsg_options;
- if (!(msg_opts & DISPATCH_MACH_REGISTER_FOR_REPLY)) {
- mach_msg_priority_t msg_priority = MACH_MSG_PRIORITY_UNSPECIFIED;
- opts = MACH_SEND_MSG | (msg_opts & ~DISPATCH_MACH_OPTIONS_MASK);
- if (!is_reply) {
- if (dmsg != dr->dm_checkin) {
- msg->msgh_remote_port = dr->dm_send;
- }
- if (_dispatch_queue_get_current() == &_dispatch_mgr_q) {
- if (slowpath(!dm->dm_dkev)) {
- _dispatch_mach_notification_kevent_register(dm,
- msg->msgh_remote_port);
- }
- if (fastpath(dm->dm_dkev)) {
- if (DISPATCH_MACH_NOTIFICATION_ARMED(dm->dm_dkev)) {
- goto out;
- }
- opts |= MACH_SEND_NOTIFY;
- }
- }
- opts |= MACH_SEND_TIMEOUT;
- if (dmsg->dmsg_priority != _voucher_get_priority(voucher)) {
- ipc_kvoucher = _voucher_create_mach_voucher_with_priority(
- voucher, dmsg->dmsg_priority);
- }
- _dispatch_voucher_debug("mach-msg[%p] msg_set", voucher, dmsg);
- if (ipc_kvoucher) {
- kvoucher_move_send = true;
- clear_voucher = _voucher_mach_msg_set_mach_voucher(msg,
- ipc_kvoucher, kvoucher_move_send);
- } else {
- clear_voucher = _voucher_mach_msg_set(msg, voucher);
- }
- if (pp && _dispatch_evfilt_machport_direct_enabled) {
- opts |= MACH_SEND_OVERRIDE;
- msg_priority = (mach_msg_priority_t)pp;
- }
- }
- _dispatch_debug_machport(msg->msgh_remote_port);
- if (reply_port) _dispatch_debug_machport(reply_port);
- if (msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) {
- if (msg_opts & DISPATCH_MACH_OWNED_REPLY_PORT) {
- _dispatch_clear_thread_reply_port(reply_port);
- }
- _dispatch_mach_reply_waiter_register(dm, dmr, reply_port, dmsg,
- msg_opts);
- }
- kr = mach_msg(msg, opts, msg->msgh_size, 0, MACH_PORT_NULL, 0,
- msg_priority);
- _dispatch_debug("machport[0x%08x]: sent msg id 0x%x, ctxt %p, "
- "opts 0x%x, msg_opts 0x%x, kvoucher 0x%08x, reply on 0x%08x: "
- "%s - 0x%x", msg->msgh_remote_port, msg->msgh_id, dmsg->do_ctxt,
- opts, msg_opts, msg->msgh_voucher_port, reply_port,
- mach_error_string(kr), kr);
- if (unlikely(kr && (msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY))) {
- _dispatch_mach_reply_waiter_unregister(dm, dmr,
- DKEV_UNREGISTER_REPLY_REMOVE);
- }
- if (clear_voucher) {
- if (kr == MACH_SEND_INVALID_VOUCHER && msg->msgh_voucher_port) {
- DISPATCH_CLIENT_CRASH(kr, "Voucher port corruption");
- }
- mach_voucher_t kv;
- kv = _voucher_mach_msg_clear(msg, kvoucher_move_send);
- if (kvoucher_move_send) ipc_kvoucher = kv;
- }
- }
- if (kr == MACH_SEND_TIMED_OUT && (opts & MACH_SEND_TIMEOUT)) {
- if (opts & MACH_SEND_NOTIFY) {
- _dispatch_debug("machport[0x%08x]: send-possible notification "
- "armed", (mach_port_t)dm->dm_dkev->dk_kevent.ident);
- DISPATCH_MACH_NOTIFICATION_ARMED(dm->dm_dkev) = 1;
- } else {
- // send kevent must be installed on the manager queue
- dr->dm_needs_mgr = 1;
- }
- if (ipc_kvoucher) {
- _dispatch_kvoucher_debug("reuse on re-send", ipc_kvoucher);
- voucher_t ipc_voucher;
- ipc_voucher = _voucher_create_with_priority_and_mach_voucher(
- voucher, dmsg->dmsg_priority, ipc_kvoucher);
- _dispatch_voucher_debug("mach-msg[%p] replace voucher[%p]",
- ipc_voucher, dmsg, voucher);
- if (dmsg->dmsg_voucher) _voucher_release(dmsg->dmsg_voucher);
- dmsg->dmsg_voucher = ipc_voucher;
- }
- goto out;
- } else if (ipc_kvoucher && (kr || !kvoucher_move_send)) {
- _voucher_dealloc_mach_voucher(ipc_kvoucher);
- }
- if (!(msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) && !kr && reply_port &&
- !(dm->ds_dkev && dm->ds_dkev->dk_kevent.ident == reply_port)) {
- if (!dm->ds_is_direct_kevent &&
- _dispatch_queue_get_current() != &_dispatch_mgr_q) {
- // reply receive kevent must be installed on the manager queue
- dr->dm_needs_mgr = 1;
- dmsg->dmsg_options = msg_opts | DISPATCH_MACH_REGISTER_FOR_REPLY;
- goto out;
- }
- _dispatch_mach_reply_kevent_register(dm, reply_port, dmsg);
- }
- if (unlikely(!is_reply && dmsg == dr->dm_checkin && dm->dm_dkev)) {
- _dispatch_mach_notification_kevent_unregister(dm);
- }
- if (slowpath(kr)) {
- // Send failed, so reply was never registered <rdar://problem/14309159>
- dmsgr = _dispatch_mach_msg_create_reply_disconnected(dmsg, NULL);
- }
- _dispatch_mach_msg_set_reason(dmsg, kr, 0);
- if ((send_flags & DM_SEND_INVOKE_IMMEDIATE_SEND) &&
- (msg_opts & DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT)) {
- // Return sent message synchronously <rdar://problem/25947334>
- send_status |= DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT;
- } else {
- _dispatch_queue_push(dm->_as_dq, dmsg, dmsg->dmsg_priority);
- }
- if (dmsgr) _dispatch_queue_push(dm->_as_dq, dmsgr, dmsgr->dmsg_priority);
- send_status |= DM_SEND_STATUS_SUCCESS;
-out:
- return send_status;
-}
-
-#pragma mark -
-#pragma mark dispatch_mach_send_refs_t
-
-static void _dispatch_mach_cancel(dispatch_mach_t dm);
-static void _dispatch_mach_send_barrier_drain_push(dispatch_mach_t dm,
- pthread_priority_t pp);
-
-DISPATCH_ALWAYS_INLINE
-static inline pthread_priority_t
-_dm_state_get_override(uint64_t dm_state)
-{
- dm_state &= DISPATCH_MACH_STATE_OVERRIDE_MASK;
- return (pthread_priority_t)(dm_state >> 32);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline uint64_t
-_dm_state_override_from_priority(pthread_priority_t pp)
-{
- uint64_t pp_state = pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK;
- return pp_state << 32;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dm_state_needs_override(uint64_t dm_state, uint64_t pp_state)
-{
- return (pp_state > (dm_state & DISPATCH_MACH_STATE_OVERRIDE_MASK));
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline uint64_t
-_dm_state_merge_override(uint64_t dm_state, uint64_t pp_state)
-{
- if (_dm_state_needs_override(dm_state, pp_state)) {
- dm_state &= ~DISPATCH_MACH_STATE_OVERRIDE_MASK;
- dm_state |= pp_state;
- dm_state |= DISPATCH_MACH_STATE_DIRTY;
- dm_state |= DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- }
- return dm_state;
-}
-
-#define _dispatch_mach_send_push_update_tail(dr, tail) \
- os_mpsc_push_update_tail(dr, dm, tail, do_next)
-#define _dispatch_mach_send_push_update_head(dr, head) \
- os_mpsc_push_update_head(dr, dm, head)
-#define _dispatch_mach_send_get_head(dr) \
- os_mpsc_get_head(dr, dm)
-#define _dispatch_mach_send_unpop_head(dr, dc, dc_next) \
- os_mpsc_undo_pop_head(dr, dm, dc, dc_next, do_next)
-#define _dispatch_mach_send_pop_head(dr, head) \
- os_mpsc_pop_head(dr, dm, head, do_next)
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dispatch_mach_send_push_inline(dispatch_mach_send_refs_t dr,
- dispatch_object_t dou)
-{
- if (_dispatch_mach_send_push_update_tail(dr, dou._do)) {
- _dispatch_mach_send_push_update_head(dr, dou._do);
- return true;
- }
- return false;
-}
-
-DISPATCH_NOINLINE
-static bool
-_dispatch_mach_send_drain(dispatch_mach_t dm, dispatch_invoke_flags_t flags,
- dispatch_mach_send_invoke_flags_t send_flags)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_mach_reply_refs_t dmr;
- dispatch_mach_msg_t dmsg;
- struct dispatch_object_s *dc = NULL, *next_dc = NULL;
- pthread_priority_t pp = _dm_state_get_override(dr->dm_state);
- uint64_t old_state, new_state;
- uint32_t send_status;
- bool needs_mgr, disconnecting, returning_send_result = false;
-
-again:
- needs_mgr = false; disconnecting = false;
- while (dr->dm_tail) {
- dc = _dispatch_mach_send_get_head(dr);
- do {
- dispatch_mach_send_invoke_flags_t sf = send_flags;
- // Only request immediate send result for the first message
- send_flags &= ~DM_SEND_INVOKE_IMMEDIATE_SEND_MASK;
- next_dc = _dispatch_mach_send_pop_head(dr, dc);
- if (_dispatch_object_has_type(dc,
- DISPATCH_CONTINUATION_TYPE(MACH_SEND_BARRIER))) {
- if (!(send_flags & DM_SEND_INVOKE_CAN_RUN_BARRIER)) {
- goto partial_drain;
- }
- _dispatch_continuation_pop(dc, dm->_as_dq, flags);
- continue;
- }
- if (_dispatch_object_is_slow_item(dc)) {
- dmsg = ((dispatch_continuation_t)dc)->dc_data;
- dmr = ((dispatch_continuation_t)dc)->dc_other;
- } else if (_dispatch_object_has_vtable(dc)) {
- dmsg = (dispatch_mach_msg_t)dc;
- dmr = NULL;
- } else {
- if ((dm->dm_dkev || !dm->ds_is_direct_kevent) &&
- (_dispatch_queue_get_current() != &_dispatch_mgr_q)) {
- // send kevent must be uninstalled on the manager queue
- needs_mgr = true;
- goto partial_drain;
- }
- if (unlikely(!_dispatch_mach_reconnect_invoke(dm, dc))) {
- disconnecting = true;
- goto partial_drain;
- }
- continue;
- }
- _dispatch_voucher_ktrace_dmsg_pop(dmsg);
- if (unlikely(dr->dm_disconnect_cnt ||
- (dm->dq_atomic_flags & DSF_CANCELED))) {
- _dispatch_mach_msg_not_sent(dm, dmsg);
- continue;
- }
- send_status = _dispatch_mach_msg_send(dm, dmsg, dmr, pp, sf);
- if (unlikely(!send_status)) {
- goto partial_drain;
- }
- if (send_status & DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT) {
- returning_send_result = true;
- }
- } while ((dc = next_dc));
- }
-
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, release, {
- if (old_state & DISPATCH_MACH_STATE_DIRTY) {
- new_state = old_state;
- new_state &= ~DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
- } else {
- // unlock
- new_state = 0;
- }
- });
- goto out;
-
-partial_drain:
- // if this is not a complete drain, we must undo some things
- _dispatch_mach_send_unpop_head(dr, dc, next_dc);
-
- if (_dispatch_object_has_type(dc,
- DISPATCH_CONTINUATION_TYPE(MACH_SEND_BARRIER))) {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, release, {
- new_state = old_state;
- new_state |= DISPATCH_MACH_STATE_DIRTY;
- new_state |= DISPATCH_MACH_STATE_PENDING_BARRIER;
- new_state &= ~DISPATCH_MACH_STATE_UNLOCK_MASK;
- });
- } else {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, release, {
- new_state = old_state;
- if (old_state & (DISPATCH_MACH_STATE_DIRTY |
- DISPATCH_MACH_STATE_RECEIVED_OVERRIDE)) {
- new_state &= ~DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
- } else {
- new_state |= DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_UNLOCK_MASK;
- }
- });
- }
-
-out:
- if (old_state & DISPATCH_MACH_STATE_RECEIVED_OVERRIDE) {
- // Ensure that the root queue sees that this thread was overridden.
- _dispatch_set_defaultpriority_override();
- }
-
- if (unlikely(new_state & DISPATCH_MACH_STATE_UNLOCK_MASK)) {
- os_atomic_thread_fence(acquire);
- pp = _dm_state_get_override(new_state);
- goto again;
- }
-
- if (new_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- pp = _dm_state_get_override(new_state);
- _dispatch_mach_send_barrier_drain_push(dm, pp);
- } else {
- if (needs_mgr || dr->dm_needs_mgr) {
- pp = _dm_state_get_override(new_state);
- } else {
- pp = 0;
- }
- if (!disconnecting) dx_wakeup(dm, pp, DISPATCH_WAKEUP_FLUSH);
- }
- return returning_send_result;
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_send_invoke(dispatch_mach_t dm,
- dispatch_invoke_flags_t flags,
- dispatch_mach_send_invoke_flags_t send_flags)
-{
- dispatch_lock_owner tid_self = _dispatch_tid_self();
- uint64_t old_state, new_state;
- pthread_priority_t pp_floor;
-
- uint64_t canlock_mask = DISPATCH_MACH_STATE_UNLOCK_MASK;
- uint64_t canlock_state = 0;
-
- if (send_flags & DM_SEND_INVOKE_NEEDS_BARRIER) {
- canlock_mask |= DISPATCH_MACH_STATE_PENDING_BARRIER;
- canlock_state = DISPATCH_MACH_STATE_PENDING_BARRIER;
- } else if (!(send_flags & DM_SEND_INVOKE_CAN_RUN_BARRIER)) {
- canlock_mask |= DISPATCH_MACH_STATE_PENDING_BARRIER;
- }
-
- if (flags & DISPATCH_INVOKE_MANAGER_DRAIN) {
- pp_floor = 0;
- } else {
- // _dispatch_queue_class_invoke will have applied the queue override
- // (if any) before we get here. Else use the default base priority
- // as an estimation of the priority we already asked for.
- pp_floor = dm->_as_dq->dq_override;
- if (!pp_floor) {
- pp_floor = _dispatch_get_defaultpriority();
- pp_floor &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
- }
- }
-
-retry:
- os_atomic_rmw_loop2o(dm->dm_refs, dm_state, old_state, new_state, acquire, {
- new_state = old_state;
- if (unlikely((old_state & canlock_mask) != canlock_state)) {
- if (!(send_flags & DM_SEND_INVOKE_FLUSH)) {
- os_atomic_rmw_loop_give_up(break);
- }
- new_state |= DISPATCH_MACH_STATE_DIRTY;
- } else {
- if (likely(pp_floor)) {
- pthread_priority_t pp = _dm_state_get_override(old_state);
- if (unlikely(pp > pp_floor)) {
- os_atomic_rmw_loop_give_up({
- _dispatch_wqthread_override_start(tid_self, pp);
- // Ensure that the root queue sees
- // that this thread was overridden.
- _dispatch_set_defaultpriority_override();
- pp_floor = pp;
- goto retry;
- });
- }
- }
- new_state |= tid_self;
- new_state &= ~DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
- }
- });
-
- if (unlikely((old_state & canlock_mask) != canlock_state)) {
- return;
- }
- if (send_flags & DM_SEND_INVOKE_CANCEL) {
- _dispatch_mach_cancel(dm);
- }
- _dispatch_mach_send_drain(dm, flags, send_flags);
-}
-
-DISPATCH_NOINLINE
-void
-_dispatch_mach_send_barrier_drain_invoke(dispatch_continuation_t dc,
- dispatch_invoke_flags_t flags)
-{
- dispatch_mach_t dm = (dispatch_mach_t)_dispatch_queue_get_current();
- uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
- dispatch_thread_frame_s dtf;
-
- DISPATCH_COMPILER_CAN_ASSUME(dc->dc_priority == DISPATCH_NO_PRIORITY);
- DISPATCH_COMPILER_CAN_ASSUME(dc->dc_voucher == DISPATCH_NO_VOUCHER);
- // hide the mach channel (see _dispatch_mach_barrier_invoke comment)
- _dispatch_thread_frame_stash(&dtf);
- _dispatch_continuation_pop_forwarded(dc, DISPATCH_NO_VOUCHER, dc_flags,{
- _dispatch_mach_send_invoke(dm, flags,
- DM_SEND_INVOKE_NEEDS_BARRIER | DM_SEND_INVOKE_CAN_RUN_BARRIER);
- });
- _dispatch_thread_frame_unstash(&dtf);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_send_barrier_drain_push(dispatch_mach_t dm,
- pthread_priority_t pp)
-{
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
-
- dc->do_vtable = DC_VTABLE(MACH_SEND_BARRRIER_DRAIN);
- dc->dc_func = NULL;
- dc->dc_ctxt = NULL;
- dc->dc_voucher = DISPATCH_NO_VOUCHER;
- dc->dc_priority = DISPATCH_NO_PRIORITY;
- return _dispatch_queue_push(dm->_as_dq, dc, pp);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_send_push(dispatch_mach_t dm, dispatch_continuation_t dc,
- pthread_priority_t pp)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- uint64_t pp_state, old_state, new_state, state_flags = 0;
- dispatch_lock_owner owner;
- bool wakeup;
-
- // <rdar://problem/25896179> when pushing a send barrier that destroys
- // the last reference to this channel, and the send queue is already
- // draining on another thread, the send barrier may run as soon as
- // _dispatch_mach_send_push_inline() returns.
- _dispatch_retain(dm);
- pp_state = _dm_state_override_from_priority(pp);
-
- wakeup = _dispatch_mach_send_push_inline(dr, dc);
- if (wakeup) {
- state_flags = DISPATCH_MACH_STATE_DIRTY;
- if (dc->do_vtable == DC_VTABLE(MACH_SEND_BARRIER)) {
- state_flags |= DISPATCH_MACH_STATE_PENDING_BARRIER;
- }
- }
-
- if (state_flags) {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, release, {
- new_state = _dm_state_merge_override(old_state, pp_state);
- new_state |= state_flags;
- });
- } else {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, relaxed, {
- new_state = _dm_state_merge_override(old_state, pp_state);
- if (old_state == new_state) {
- os_atomic_rmw_loop_give_up(break);
- }
- });
- }
-
- pp = _dm_state_get_override(new_state);
- owner = _dispatch_lock_owner((dispatch_lock)old_state);
- if (owner) {
- if (_dm_state_needs_override(old_state, pp_state)) {
- _dispatch_wqthread_override_start_check_owner(owner, pp,
- &dr->dm_state_lock.dul_lock);
- }
- return _dispatch_release_tailcall(dm);
- }
-
- dispatch_wakeup_flags_t wflags = 0;
- if (state_flags & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- _dispatch_mach_send_barrier_drain_push(dm, pp);
- } else if (wakeup || dr->dm_disconnect_cnt ||
- (dm->dq_atomic_flags & DSF_CANCELED)) {
- wflags = DISPATCH_WAKEUP_FLUSH | DISPATCH_WAKEUP_CONSUME;
- } else if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- wflags = DISPATCH_WAKEUP_OVERRIDING | DISPATCH_WAKEUP_CONSUME;
- }
- if (wflags) {
- return dx_wakeup(dm, pp, wflags);
- }
- return _dispatch_release_tailcall(dm);
-}
-
-DISPATCH_NOINLINE
-static bool
-_dispatch_mach_send_push_and_trydrain(dispatch_mach_t dm,
- dispatch_object_t dou, pthread_priority_t pp,
- dispatch_mach_send_invoke_flags_t send_flags)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_lock_owner tid_self = _dispatch_tid_self();
- uint64_t pp_state, old_state, new_state, canlock_mask, state_flags = 0;
- dispatch_lock_owner owner;
-
- pp_state = _dm_state_override_from_priority(pp);
- bool wakeup = _dispatch_mach_send_push_inline(dr, dou);
- if (wakeup) {
- state_flags = DISPATCH_MACH_STATE_DIRTY;
- }
-
- if (unlikely(dr->dm_disconnect_cnt ||
- (dm->dq_atomic_flags & DSF_CANCELED))) {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, release, {
- new_state = _dm_state_merge_override(old_state, pp_state);
- new_state |= state_flags;
- });
- dx_wakeup(dm, pp, DISPATCH_WAKEUP_FLUSH);
- return false;
- }
-
- canlock_mask = DISPATCH_MACH_STATE_UNLOCK_MASK |
- DISPATCH_MACH_STATE_PENDING_BARRIER;
- if (state_flags) {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, seq_cst, {
- new_state = _dm_state_merge_override(old_state, pp_state);
- new_state |= state_flags;
- if (likely((old_state & canlock_mask) == 0)) {
- new_state |= tid_self;
- new_state &= ~DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
- }
- });
- } else {
- os_atomic_rmw_loop2o(dr, dm_state, old_state, new_state, acquire, {
- new_state = _dm_state_merge_override(old_state, pp_state);
- if (new_state == old_state) {
- os_atomic_rmw_loop_give_up(return false);
- }
- if (likely((old_state & canlock_mask) == 0)) {
- new_state |= tid_self;
- new_state &= ~DISPATCH_MACH_STATE_DIRTY;
- new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
- new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
- }
- });
- }
-
- owner = _dispatch_lock_owner((dispatch_lock)old_state);
- if (owner) {
- if (_dm_state_needs_override(old_state, pp_state)) {
- _dispatch_wqthread_override_start_check_owner(owner, pp,
- &dr->dm_state_lock.dul_lock);
- }
- return false;
- }
-
- if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- dx_wakeup(dm, pp, DISPATCH_WAKEUP_OVERRIDING);
- return false;
- }
-
- // Ensure our message is still at the head of the queue and has not already
- // been dequeued by another thread that raced us to the send queue lock.
- // A plain load of the head and comparison against our object pointer is
- // sufficient.
- if (unlikely(!(wakeup && dou._do == dr->dm_head))) {
- // Don't request immediate send result for messages we don't own
- send_flags &= ~DM_SEND_INVOKE_IMMEDIATE_SEND_MASK;
- }
- return _dispatch_mach_send_drain(dm, DISPATCH_INVOKE_NONE, send_flags);
-}
-
-static void
-_dispatch_mach_merge_notification_kevent(dispatch_mach_t dm,
- const _dispatch_kevent_qos_s *ke)
-{
- if (!(ke->fflags & dm->ds_pending_data_mask)) {
- return;
- }
- _dispatch_mach_send_invoke(dm, DISPATCH_INVOKE_MANAGER_DRAIN,
- DM_SEND_INVOKE_FLUSH);
-}
-
-#pragma mark -
-#pragma mark dispatch_mach_t
-
-static inline mach_msg_option_t
-_dispatch_mach_checkin_options(void)
-{
- mach_msg_option_t options = 0;
-#if DISPATCH_USE_CHECKIN_NOIMPORTANCE
- options = MACH_SEND_NOIMPORTANCE; // <rdar://problem/16996737>
-#endif
- return options;
-}
-
-
-static inline mach_msg_option_t
-_dispatch_mach_send_options(void)
-{
- mach_msg_option_t options = 0;
- return options;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline pthread_priority_t
-_dispatch_mach_priority_propagate(mach_msg_option_t options)
-{
-#if DISPATCH_USE_NOIMPORTANCE_QOS
- if (options & MACH_SEND_NOIMPORTANCE) return 0;
-#else
- (void)options;
-#endif
- return _dispatch_priority_propagate();
-}
-
-DISPATCH_NOINLINE
-static bool
-_dispatch_mach_send_msg(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
- dispatch_continuation_t dc_wait, mach_msg_option_t options)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- if (slowpath(dmsg->do_next != DISPATCH_OBJECT_LISTLESS)) {
- DISPATCH_CLIENT_CRASH(dmsg->do_next, "Message already enqueued");
- }
- dispatch_retain(dmsg);
- pthread_priority_t priority = _dispatch_mach_priority_propagate(options);
- options |= _dispatch_mach_send_options();
- dmsg->dmsg_options = options;
- mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
- dmsg->dmsg_reply = _dispatch_mach_msg_get_reply_port(dmsg);
- bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) ==
- MACH_MSG_TYPE_MOVE_SEND_ONCE);
- dmsg->dmsg_priority = priority;
- dmsg->dmsg_voucher = _voucher_copy();
- _dispatch_voucher_debug("mach-msg[%p] set", dmsg->dmsg_voucher, dmsg);
-
- uint32_t send_status;
- bool returning_send_result = false;
- dispatch_mach_send_invoke_flags_t send_flags = DM_SEND_INVOKE_NONE;
- if (options & DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT) {
- send_flags = DM_SEND_INVOKE_IMMEDIATE_SEND;
- }
- if (is_reply && !dmsg->dmsg_reply && !dr->dm_disconnect_cnt &&
- !(dm->dq_atomic_flags & DSF_CANCELED)) {
- // replies are sent to a send-once right and don't need the send queue
- dispatch_assert(!dc_wait);
- send_status = _dispatch_mach_msg_send(dm, dmsg, NULL, 0, send_flags);
- dispatch_assert(send_status);
- returning_send_result = !!(send_status &
- DM_SEND_STATUS_RETURNING_IMMEDIATE_SEND_RESULT);
- } else {
- _dispatch_voucher_ktrace_dmsg_push(dmsg);
- priority &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
- dispatch_object_t dou = { ._dmsg = dmsg };
- if (dc_wait) dou._dc = dc_wait;
- returning_send_result = _dispatch_mach_send_push_and_trydrain(dm, dou,
- priority, send_flags);
- }
- if (returning_send_result) {
- _dispatch_voucher_debug("mach-msg[%p] clear", dmsg->dmsg_voucher, dmsg);
- if (dmsg->dmsg_voucher) _voucher_release(dmsg->dmsg_voucher);
- dmsg->dmsg_voucher = NULL;
- dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
- dispatch_release(dmsg);
- }
- return returning_send_result;
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_send(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
- mach_msg_option_t options)
-{
- dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
- options &= ~DISPATCH_MACH_OPTIONS_MASK;
- bool returned_send_result = _dispatch_mach_send_msg(dm, dmsg, NULL,options);
- dispatch_assert(!returned_send_result);
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_send_with_result(dispatch_mach_t dm, dispatch_mach_msg_t dmsg,
- mach_msg_option_t options, dispatch_mach_send_flags_t send_flags,
- dispatch_mach_reason_t *send_result, mach_error_t *send_error)
-{
- if (unlikely(send_flags != DISPATCH_MACH_SEND_DEFAULT)) {
- DISPATCH_CLIENT_CRASH(send_flags, "Invalid send flags");
- }
- dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
- options &= ~DISPATCH_MACH_OPTIONS_MASK;
- options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
- bool returned_send_result = _dispatch_mach_send_msg(dm, dmsg, NULL,options);
- unsigned long reason = DISPATCH_MACH_NEEDS_DEFERRED_SEND;
- mach_error_t err = 0;
- if (returned_send_result) {
- reason = _dispatch_mach_msg_get_reason(dmsg, &err);
- }
- *send_result = reason;
- *send_error = err;
-}
-
-static inline
-dispatch_mach_msg_t
-_dispatch_mach_send_and_wait_for_reply(dispatch_mach_t dm,
- dispatch_mach_msg_t dmsg, mach_msg_option_t options,
- bool *returned_send_result)
-{
- mach_port_t reply_port = _dispatch_mach_msg_get_reply_port(dmsg);
- if (!reply_port) {
- // use per-thread mach reply port <rdar://24597802>
- reply_port = _dispatch_get_thread_reply_port();
- mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg);
- dispatch_assert(MACH_MSGH_BITS_LOCAL(hdr->msgh_bits) ==
- MACH_MSG_TYPE_MAKE_SEND_ONCE);
- hdr->msgh_local_port = reply_port;
- options |= DISPATCH_MACH_OWNED_REPLY_PORT;
- }
-
- dispatch_mach_reply_refs_t dmr;
-#if DISPATCH_DEBUG
- dmr = _dispatch_calloc(1, sizeof(*dmr));
-#else
- struct dispatch_mach_reply_refs_s dmr_buf = { };
- dmr = &dmr_buf;
-#endif
- struct dispatch_continuation_s dc_wait = {
- .dc_flags = DISPATCH_OBJ_SYNC_SLOW_BIT,
- .dc_data = dmsg,
- .dc_other = dmr,
- .dc_priority = DISPATCH_NO_PRIORITY,
- .dc_voucher = DISPATCH_NO_VOUCHER,
- };
- dmr->dmr_ctxt = dmsg->do_ctxt;
- *returned_send_result = _dispatch_mach_send_msg(dm, dmsg, &dc_wait,options);
- if (options & DISPATCH_MACH_OWNED_REPLY_PORT) {
- _dispatch_clear_thread_reply_port(reply_port);
- }
- dmsg = _dispatch_mach_msg_reply_recv(dm, dmr, reply_port);
-#if DISPATCH_DEBUG
- free(dmr);
-#endif
- return dmsg;
-}
-
-DISPATCH_NOINLINE
-dispatch_mach_msg_t
-dispatch_mach_send_and_wait_for_reply(dispatch_mach_t dm,
- dispatch_mach_msg_t dmsg, mach_msg_option_t options)
-{
- bool returned_send_result;
- dispatch_mach_msg_t reply;
- dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
- options &= ~DISPATCH_MACH_OPTIONS_MASK;
- options |= DISPATCH_MACH_WAIT_FOR_REPLY;
- reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
- &returned_send_result);
- dispatch_assert(!returned_send_result);
- return reply;
-}
-
-DISPATCH_NOINLINE
-dispatch_mach_msg_t
-dispatch_mach_send_with_result_and_wait_for_reply(dispatch_mach_t dm,
- dispatch_mach_msg_t dmsg, mach_msg_option_t options,
- dispatch_mach_send_flags_t send_flags,
- dispatch_mach_reason_t *send_result, mach_error_t *send_error)
-{
- if (unlikely(send_flags != DISPATCH_MACH_SEND_DEFAULT)) {
- DISPATCH_CLIENT_CRASH(send_flags, "Invalid send flags");
- }
- bool returned_send_result;
- dispatch_mach_msg_t reply;
- dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
- options &= ~DISPATCH_MACH_OPTIONS_MASK;
- options |= DISPATCH_MACH_WAIT_FOR_REPLY;
- options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
- reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
- &returned_send_result);
- unsigned long reason = DISPATCH_MACH_NEEDS_DEFERRED_SEND;
- mach_error_t err = 0;
- if (returned_send_result) {
- reason = _dispatch_mach_msg_get_reason(dmsg, &err);
- }
- *send_result = reason;
- *send_error = err;
- return reply;
-}
-
-DISPATCH_NOINLINE
-static bool
-_dispatch_mach_disconnect(dispatch_mach_t dm)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- bool disconnected;
- if (dm->dm_dkev) {
- _dispatch_mach_notification_kevent_unregister(dm);
- }
- if (MACH_PORT_VALID(dr->dm_send)) {
- _dispatch_mach_msg_disconnected(dm, MACH_PORT_NULL, dr->dm_send);
- }
- dr->dm_send = MACH_PORT_NULL;
- if (dr->dm_checkin) {
- _dispatch_mach_msg_not_sent(dm, dr->dm_checkin);
- dr->dm_checkin = NULL;
- }
- _dispatch_unfair_lock_lock(&dm->dm_refs->dm_replies_lock);
- dispatch_mach_reply_refs_t dmr, tmp;
- TAILQ_FOREACH_SAFE(dmr, &dm->dm_refs->dm_replies, dmr_list, tmp) {
- TAILQ_REMOVE(&dm->dm_refs->dm_replies, dmr, dmr_list);
- _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
- if (dmr->dmr_dkev) {
- _dispatch_mach_reply_kevent_unregister(dm, dmr,
- DKEV_UNREGISTER_DISCONNECTED);
- } else {
- _dispatch_mach_reply_waiter_unregister(dm, dmr,
- DKEV_UNREGISTER_DISCONNECTED);
- }
- }
- disconnected = TAILQ_EMPTY(&dm->dm_refs->dm_replies);
- _dispatch_unfair_lock_unlock(&dm->dm_refs->dm_replies_lock);
- return disconnected;
-}
-
-static void
-_dispatch_mach_cancel(dispatch_mach_t dm)
-{
- _dispatch_object_debug(dm, "%s", __func__);
- if (!_dispatch_mach_disconnect(dm)) return;
- if (dm->ds_dkev) {
- mach_port_t local_port = (mach_port_t)dm->ds_dkev->dk_kevent.ident;
- _dispatch_source_kevent_unregister(dm->_as_ds);
- if ((dm->dq_atomic_flags & DSF_STATE_MASK) == DSF_DELETED) {
- _dispatch_mach_msg_disconnected(dm, local_port, MACH_PORT_NULL);
- }
- } else {
- _dispatch_queue_atomic_flags_set_and_clear(dm->_as_dq, DSF_DELETED,
- DSF_ARMED | DSF_DEFERRED_DELETE);
- }
-}
-
-DISPATCH_NOINLINE
-static bool
-_dispatch_mach_reconnect_invoke(dispatch_mach_t dm, dispatch_object_t dou)
-{
- if (!_dispatch_mach_disconnect(dm)) return false;
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dr->dm_checkin = dou._dc->dc_data;
- dr->dm_send = (mach_port_t)dou._dc->dc_other;
- _dispatch_continuation_free(dou._dc);
- (void)os_atomic_dec2o(dr, dm_disconnect_cnt, relaxed);
- _dispatch_object_debug(dm, "%s", __func__);
- _dispatch_release(dm); // <rdar://problem/26266265>
- return true;
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_reconnect(dispatch_mach_t dm, mach_port_t send,
- dispatch_mach_msg_t checkin)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- (void)os_atomic_inc2o(dr, dm_disconnect_cnt, relaxed);
- if (MACH_PORT_VALID(send) && checkin) {
- dispatch_retain(checkin);
- checkin->dmsg_options = _dispatch_mach_checkin_options();
- dr->dm_checkin_port = _dispatch_mach_msg_get_remote_port(checkin);
- } else {
- checkin = NULL;
- dr->dm_checkin_port = MACH_PORT_NULL;
- }
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
- dc->dc_flags = DISPATCH_OBJ_CONSUME_BIT;
- // actually called manually in _dispatch_mach_send_drain
- dc->dc_func = (void*)_dispatch_mach_reconnect_invoke;
- dc->dc_ctxt = dc;
- dc->dc_data = checkin;
- dc->dc_other = (void*)(uintptr_t)send;
- dc->dc_voucher = DISPATCH_NO_VOUCHER;
- dc->dc_priority = DISPATCH_NO_PRIORITY;
- _dispatch_retain(dm); // <rdar://problem/26266265>
- return _dispatch_mach_send_push(dm, dc, 0);
-}
-
-DISPATCH_NOINLINE
-mach_port_t
-dispatch_mach_get_checkin_port(dispatch_mach_t dm)
-{
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- if (slowpath(dm->dq_atomic_flags & DSF_CANCELED)) {
- return MACH_PORT_DEAD;
- }
- return dr->dm_checkin_port;
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_connect_invoke(dispatch_mach_t dm)
-{
- dispatch_mach_refs_t dr = dm->ds_refs;
- _dispatch_client_callout4(dr->dm_handler_ctxt,
- DISPATCH_MACH_CONNECTED, NULL, 0, dr->dm_handler_func);
- dm->dm_connect_handler_called = 1;
-}
-
-DISPATCH_NOINLINE
-void
-_dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg,
- dispatch_invoke_flags_t flags)
-{
- dispatch_thread_frame_s dtf;
- dispatch_mach_refs_t dr;
- dispatch_mach_t dm;
- mach_error_t err;
- unsigned long reason = _dispatch_mach_msg_get_reason(dmsg, &err);
- _dispatch_thread_set_self_t adopt_flags = DISPATCH_PRIORITY_ENFORCE|
- DISPATCH_VOUCHER_CONSUME|DISPATCH_VOUCHER_REPLACE;
-
- // hide mach channel
- dm = (dispatch_mach_t)_dispatch_thread_frame_stash(&dtf);
- dr = dm->ds_refs;
- dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
- _dispatch_voucher_ktrace_dmsg_pop(dmsg);
- _dispatch_voucher_debug("mach-msg[%p] adopt", dmsg->dmsg_voucher, dmsg);
- (void)_dispatch_adopt_priority_and_set_voucher(dmsg->dmsg_priority,
- dmsg->dmsg_voucher, adopt_flags);
- dmsg->dmsg_voucher = NULL;
- dispatch_invoke_with_autoreleasepool(flags, {
- if (slowpath(!dm->dm_connect_handler_called)) {
- _dispatch_mach_connect_invoke(dm);
- }
- _dispatch_client_callout4(dr->dm_handler_ctxt, reason, dmsg, err,
- dr->dm_handler_func);
- });
- _dispatch_thread_frame_unstash(&dtf);
- _dispatch_introspection_queue_item_complete(dmsg);
- dispatch_release(dmsg);
-}
-
-DISPATCH_NOINLINE
-void
-_dispatch_mach_barrier_invoke(dispatch_continuation_t dc,
- dispatch_invoke_flags_t flags)
-{
- dispatch_thread_frame_s dtf;
- dispatch_mach_t dm = dc->dc_other;
- dispatch_mach_refs_t dr;
- uintptr_t dc_flags = (uintptr_t)dc->dc_data;
- unsigned long type = dc_type(dc);
-
- // hide mach channel from clients
- if (type == DISPATCH_CONTINUATION_TYPE(MACH_RECV_BARRIER)) {
- // on the send queue, the mach channel isn't the current queue
- // its target queue is the current one already
- _dispatch_thread_frame_stash(&dtf);
- }
- dr = dm->ds_refs;
- DISPATCH_COMPILER_CAN_ASSUME(dc_flags & DISPATCH_OBJ_CONSUME_BIT);
- _dispatch_continuation_pop_forwarded(dc, dm->dq_override_voucher, dc_flags,{
- dispatch_invoke_with_autoreleasepool(flags, {
- if (slowpath(!dm->dm_connect_handler_called)) {
- _dispatch_mach_connect_invoke(dm);
- }
- _dispatch_client_callout(dc->dc_ctxt, dc->dc_func);
- _dispatch_client_callout4(dr->dm_handler_ctxt,
- DISPATCH_MACH_BARRIER_COMPLETED, NULL, 0,
- dr->dm_handler_func);
- });
- });
- if (type == DISPATCH_CONTINUATION_TYPE(MACH_RECV_BARRIER)) {
- _dispatch_thread_frame_unstash(&dtf);
- }
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_send_barrier_f(dispatch_mach_t dm, void *context,
- dispatch_function_t func)
-{
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
- uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
- pthread_priority_t pp;
-
- _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags);
- dc->dc_data = (void *)dc->dc_flags;
- dc->dc_other = dm;
- dc->do_vtable = DC_VTABLE(MACH_SEND_BARRIER);
- _dispatch_trace_continuation_push(dm->_as_dq, dc);
- pp = _dispatch_continuation_get_override_priority(dm->_as_dq, dc);
- return _dispatch_mach_send_push(dm, dc, pp);
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_send_barrier(dispatch_mach_t dm, dispatch_block_t barrier)
-{
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
- uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
- pthread_priority_t pp;
-
- _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags);
- dc->dc_data = (void *)dc->dc_flags;
- dc->dc_other = dm;
- dc->do_vtable = DC_VTABLE(MACH_SEND_BARRIER);
- _dispatch_trace_continuation_push(dm->_as_dq, dc);
- pp = _dispatch_continuation_get_override_priority(dm->_as_dq, dc);
- return _dispatch_mach_send_push(dm, dc, pp);
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_receive_barrier_f(dispatch_mach_t dm, void *context,
- dispatch_function_t func)
-{
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
- uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
-
- _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags);
- dc->dc_data = (void *)dc->dc_flags;
- dc->dc_other = dm;
- dc->do_vtable = DC_VTABLE(MACH_RECV_BARRIER);
- return _dispatch_continuation_async(dm->_as_dq, dc);
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_receive_barrier(dispatch_mach_t dm, dispatch_block_t barrier)
-{
- dispatch_continuation_t dc = _dispatch_continuation_alloc();
- uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT;
-
- _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags);
- dc->dc_data = (void *)dc->dc_flags;
- dc->dc_other = dm;
- dc->do_vtable = DC_VTABLE(MACH_RECV_BARRIER);
- return _dispatch_continuation_async(dm->_as_dq, dc);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_mach_cancel_invoke(dispatch_mach_t dm, dispatch_invoke_flags_t flags)
-{
- dispatch_mach_refs_t dr = dm->ds_refs;
-
- dispatch_invoke_with_autoreleasepool(flags, {
- if (slowpath(!dm->dm_connect_handler_called)) {
- _dispatch_mach_connect_invoke(dm);
- }
- _dispatch_client_callout4(dr->dm_handler_ctxt,
- DISPATCH_MACH_CANCELED, NULL, 0, dr->dm_handler_func);
- });
- dm->dm_cancel_handler_called = 1;
- _dispatch_release(dm); // the retain is done at creation time
-}
-
-DISPATCH_NOINLINE
-void
-dispatch_mach_cancel(dispatch_mach_t dm)
-{
- dispatch_source_cancel(dm->_as_ds);
-}
-
-static void
-_dispatch_mach_install(dispatch_mach_t dm, pthread_priority_t pp)
-{
- uint32_t disconnect_cnt;
-
- if (dm->ds_dkev) {
- _dispatch_source_kevent_register(dm->_as_ds, pp);
- }
- if (dm->ds_is_direct_kevent) {
- pp &= (~_PTHREAD_PRIORITY_FLAGS_MASK |
- _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG |
- _PTHREAD_PRIORITY_OVERCOMMIT_FLAG);
- // _dispatch_mach_reply_kevent_register assumes this has been done
- // which is unlike regular sources or queues, the DEFAULTQUEUE flag
- // is used so that the priority of that channel doesn't act as a floor
- // QoS for incoming messages (26761457)
- dm->dq_priority = (dispatch_priority_t)pp;
- }
- dm->ds_is_installed = true;
- if (unlikely(!os_atomic_cmpxchgv2o(dm->dm_refs, dm_disconnect_cnt,
- DISPATCH_MACH_NEVER_INSTALLED, 0, &disconnect_cnt, release))) {
- DISPATCH_INTERNAL_CRASH(disconnect_cnt, "Channel already installed");
- }
-}
-
-void
-_dispatch_mach_finalize_activation(dispatch_mach_t dm)
-{
- if (dm->ds_is_direct_kevent && !dm->ds_is_installed) {
- dispatch_source_t ds = dm->_as_ds;
- pthread_priority_t pp = _dispatch_source_compute_kevent_priority(ds);
- if (pp) _dispatch_mach_install(dm, pp);
- }
-
- // call "super"
- _dispatch_queue_finalize_activation(dm->_as_dq);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_queue_t
-_dispatch_mach_invoke2(dispatch_object_t dou, dispatch_invoke_flags_t flags,
- uint64_t *owned, struct dispatch_object_s **dc_ptr DISPATCH_UNUSED)
-{
- dispatch_mach_t dm = dou._dm;
- dispatch_queue_t retq = NULL;
- dispatch_queue_t dq = _dispatch_queue_get_current();
-
- // This function performs all mach channel actions. Each action is
- // responsible for verifying that it takes place on the appropriate queue.
- // If the current queue is not the correct queue for this action, the
- // correct queue will be returned and the invoke will be re-driven on that
- // queue.
-
- // The order of tests here in invoke and in wakeup should be consistent.
-
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_queue_t dkq = &_dispatch_mgr_q;
-
- if (dm->ds_is_direct_kevent) {
- dkq = dm->do_targetq;
- }
-
- if (slowpath(!dm->ds_is_installed)) {
- // The channel needs to be installed on the kevent queue.
- if (dq != dkq) {
- return dkq;
- }
- _dispatch_mach_install(dm, _dispatch_get_defaultpriority());
- }
-
- if (_dispatch_queue_class_probe(dm)) {
- if (dq == dm->do_targetq) {
- retq = _dispatch_queue_serial_drain(dm->_as_dq, flags, owned, NULL);
- } else {
- retq = dm->do_targetq;
- }
- }
-
- dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
-
- if (dr->dm_tail) {
- bool requires_mgr = dr->dm_needs_mgr || (dr->dm_disconnect_cnt &&
- (dm->dm_dkev || !dm->ds_is_direct_kevent));
- if (!(dm->dm_dkev && DISPATCH_MACH_NOTIFICATION_ARMED(dm->dm_dkev)) ||
- (dqf & DSF_CANCELED) || dr->dm_disconnect_cnt) {
- // The channel has pending messages to send.
- if (unlikely(requires_mgr && dq != &_dispatch_mgr_q)) {
- return retq ? retq : &_dispatch_mgr_q;
- }
- dispatch_mach_send_invoke_flags_t send_flags = DM_SEND_INVOKE_NONE;
- if (dq != &_dispatch_mgr_q) {
- send_flags |= DM_SEND_INVOKE_CAN_RUN_BARRIER;
- }
- _dispatch_mach_send_invoke(dm, flags, send_flags);
- }
- } else if (dqf & DSF_CANCELED) {
- // The channel has been cancelled and needs to be uninstalled from the
- // manager queue. After uninstallation, the cancellation handler needs
- // to be delivered to the target queue.
- if ((dqf & DSF_STATE_MASK) == (DSF_ARMED | DSF_DEFERRED_DELETE)) {
- // waiting for the delivery of a deferred delete event
- return retq;
- }
- if ((dqf & DSF_STATE_MASK) != DSF_DELETED) {
- if (dq != &_dispatch_mgr_q) {
- return retq ? retq : &_dispatch_mgr_q;
- }
- _dispatch_mach_send_invoke(dm, flags, DM_SEND_INVOKE_CANCEL);
- dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
- if (unlikely((dqf & DSF_STATE_MASK) != DSF_DELETED)) {
- // waiting for the delivery of a deferred delete event
- // or deletion didn't happen because send_invoke couldn't
- // acquire the send lock
- return retq;
- }
- }
- if (!dm->dm_cancel_handler_called) {
- if (dq != dm->do_targetq) {
- return retq ? retq : dm->do_targetq;
- }
- _dispatch_mach_cancel_invoke(dm, flags);
- }
- }
-
- return retq;
-}
-
-DISPATCH_NOINLINE
-void
-_dispatch_mach_invoke(dispatch_mach_t dm, dispatch_invoke_flags_t flags)
-{
- _dispatch_queue_class_invoke(dm, flags, _dispatch_mach_invoke2);
-}
-
-void
-_dispatch_mach_wakeup(dispatch_mach_t dm, pthread_priority_t pp,
- dispatch_wakeup_flags_t flags)
-{
- // This function determines whether the mach channel needs to be invoked.
- // The order of tests here in probe and in invoke should be consistent.
-
- dispatch_mach_send_refs_t dr = dm->dm_refs;
- dispatch_queue_wakeup_target_t dkq = DISPATCH_QUEUE_WAKEUP_MGR;
- dispatch_queue_wakeup_target_t tq = DISPATCH_QUEUE_WAKEUP_NONE;
- dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
-
- if (dm->ds_is_direct_kevent) {
- dkq = DISPATCH_QUEUE_WAKEUP_TARGET;
- }
-
- if (!dm->ds_is_installed) {
- // The channel needs to be installed on the kevent queue.
- tq = dkq;
- goto done;
- }
-
- if (_dispatch_queue_class_probe(dm)) {
- tq = DISPATCH_QUEUE_WAKEUP_TARGET;
- goto done;
- }
-
- if (_dispatch_lock_is_locked(dr->dm_state_lock.dul_lock)) {
- // Sending and uninstallation below require the send lock, the channel
- // will be woken up when the lock is dropped <rdar://15132939&15203957>
- _dispatch_queue_reinstate_override_priority(dm, (dispatch_priority_t)pp);
- goto done;
- }
-
- if (dr->dm_tail) {
- bool requires_mgr = dr->dm_needs_mgr || (dr->dm_disconnect_cnt &&
- (dm->dm_dkev || !dm->ds_is_direct_kevent));
- if (!(dm->dm_dkev && DISPATCH_MACH_NOTIFICATION_ARMED(dm->dm_dkev)) ||
- (dqf & DSF_CANCELED) || dr->dm_disconnect_cnt) {
- if (unlikely(requires_mgr)) {
- tq = DISPATCH_QUEUE_WAKEUP_MGR;
- } else {
- tq = DISPATCH_QUEUE_WAKEUP_TARGET;
- }
- } else {
- // can happen when we can't send because the port is full
- // but we should not lose the override
- _dispatch_queue_reinstate_override_priority(dm,
- (dispatch_priority_t)pp);
- }
- } else if (dqf & DSF_CANCELED) {
- if ((dqf & DSF_STATE_MASK) == (DSF_ARMED | DSF_DEFERRED_DELETE)) {
- // waiting for the delivery of a deferred delete event
- } else if ((dqf & DSF_STATE_MASK) != DSF_DELETED) {
- // The channel needs to be uninstalled from the manager queue
- tq = DISPATCH_QUEUE_WAKEUP_MGR;
- } else if (!dm->dm_cancel_handler_called) {
- // the cancellation handler needs to be delivered to the target
- // queue.
- tq = DISPATCH_QUEUE_WAKEUP_TARGET;
- }
- }
-
-done:
- if (tq) {
- return _dispatch_queue_class_wakeup(dm->_as_dq, pp, flags, tq);
- } else if (pp) {
- return _dispatch_queue_class_override_drainer(dm->_as_dq, pp, flags);
- } else if (flags & DISPATCH_WAKEUP_CONSUME) {
- return _dispatch_release_tailcall(dm);
- }
-}
-
-#pragma mark -
-#pragma mark dispatch_mach_msg_t
-
-dispatch_mach_msg_t
-dispatch_mach_msg_create(mach_msg_header_t *msg, size_t size,
- dispatch_mach_msg_destructor_t destructor, mach_msg_header_t **msg_ptr)
-{
- if (slowpath(size < sizeof(mach_msg_header_t)) ||
- slowpath(destructor && !msg)) {
- DISPATCH_CLIENT_CRASH(size, "Empty message");
- }
- dispatch_mach_msg_t dmsg = _dispatch_alloc(DISPATCH_VTABLE(mach_msg),
- sizeof(struct dispatch_mach_msg_s) +
- (destructor ? 0 : size - sizeof(dmsg->dmsg_msg)));
- if (destructor) {
- dmsg->dmsg_msg = msg;
- } else if (msg) {
- memcpy(dmsg->dmsg_buf, msg, size);
- }
- dmsg->do_next = DISPATCH_OBJECT_LISTLESS;
- dmsg->do_targetq = _dispatch_get_root_queue(_DISPATCH_QOS_CLASS_DEFAULT,
- false);
- dmsg->dmsg_destructor = destructor;
- dmsg->dmsg_size = size;
- if (msg_ptr) {
- *msg_ptr = _dispatch_mach_msg_get_msg(dmsg);
- }
- return dmsg;
-}
-
-void
-_dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg)
-{
- if (dmsg->dmsg_voucher) {
- _voucher_release(dmsg->dmsg_voucher);
- dmsg->dmsg_voucher = NULL;
- }
- switch (dmsg->dmsg_destructor) {
- case DISPATCH_MACH_MSG_DESTRUCTOR_DEFAULT:
- break;
- case DISPATCH_MACH_MSG_DESTRUCTOR_FREE:
- free(dmsg->dmsg_msg);
- break;
- case DISPATCH_MACH_MSG_DESTRUCTOR_VM_DEALLOCATE: {
- mach_vm_size_t vm_size = dmsg->dmsg_size;
- mach_vm_address_t vm_addr = (uintptr_t)dmsg->dmsg_msg;
- (void)dispatch_assume_zero(mach_vm_deallocate(mach_task_self(),
- vm_addr, vm_size));
- break;
- }}
-}
-
-static inline mach_msg_header_t*
-_dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg)
-{
- return dmsg->dmsg_destructor ? dmsg->dmsg_msg :
- (mach_msg_header_t*)dmsg->dmsg_buf;
-}
-
-mach_msg_header_t*
-dispatch_mach_msg_get_msg(dispatch_mach_msg_t dmsg, size_t *size_ptr)
-{
- if (size_ptr) {
- *size_ptr = dmsg->dmsg_size;
- }
- return _dispatch_mach_msg_get_msg(dmsg);
-}
-
-size_t
-_dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf, size_t bufsiz)
-{
- size_t offset = 0;
- offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
- dx_kind(dmsg), dmsg);
- offset += dsnprintf(&buf[offset], bufsiz - offset, "xrefcnt = 0x%x, "
- "refcnt = 0x%x, ", dmsg->do_xref_cnt + 1, dmsg->do_ref_cnt + 1);
- offset += dsnprintf(&buf[offset], bufsiz - offset, "opts/err = 0x%x, "
- "msgh[%p] = { ", dmsg->dmsg_options, dmsg->dmsg_buf);
- mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg);
- if (hdr->msgh_id) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "id 0x%x, ",
- hdr->msgh_id);
- }
- if (hdr->msgh_size) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "size %u, ",
- hdr->msgh_size);
- }
- if (hdr->msgh_bits) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "bits <l %u, r %u",
- MACH_MSGH_BITS_LOCAL(hdr->msgh_bits),
- MACH_MSGH_BITS_REMOTE(hdr->msgh_bits));
- if (MACH_MSGH_BITS_OTHER(hdr->msgh_bits)) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, ", o 0x%x",
- MACH_MSGH_BITS_OTHER(hdr->msgh_bits));
- }
- offset += dsnprintf(&buf[offset], bufsiz - offset, ">, ");
- }
- if (hdr->msgh_local_port && hdr->msgh_remote_port) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "local 0x%x, "
- "remote 0x%x", hdr->msgh_local_port, hdr->msgh_remote_port);
- } else if (hdr->msgh_local_port) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "local 0x%x",
- hdr->msgh_local_port);
- } else if (hdr->msgh_remote_port) {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "remote 0x%x",
- hdr->msgh_remote_port);
- } else {
- offset += dsnprintf(&buf[offset], bufsiz - offset, "no ports");
- }
- offset += dsnprintf(&buf[offset], bufsiz - offset, " } }");
- return offset;
-}
-
-#pragma mark -
-#pragma mark dispatch_mig_server
-
-mach_msg_return_t
-dispatch_mig_server(dispatch_source_t ds, size_t maxmsgsz,
- dispatch_mig_callback_t callback)
-{
- mach_msg_options_t options = MACH_RCV_MSG | MACH_RCV_TIMEOUT
- | MACH_RCV_TRAILER_ELEMENTS(MACH_RCV_TRAILER_CTX)
- | MACH_RCV_TRAILER_TYPE(MACH_MSG_TRAILER_FORMAT_0) | MACH_RCV_VOUCHER;
- mach_msg_options_t tmp_options;
- mig_reply_error_t *bufTemp, *bufRequest, *bufReply;
- mach_msg_return_t kr = 0;
- uint64_t assertion_token = 0;
- unsigned int cnt = 1000; // do not stall out serial queues
- boolean_t demux_success;
- bool received = false;
- size_t rcv_size = maxmsgsz + MAX_TRAILER_SIZE;
-
- bufRequest = alloca(rcv_size);
- bufRequest->RetCode = 0;
- for (mach_vm_address_t p = mach_vm_trunc_page(bufRequest + vm_page_size);
- p < (mach_vm_address_t)bufRequest + rcv_size; p += vm_page_size) {
- *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
- }
-
- bufReply = alloca(rcv_size);
- bufReply->Head.msgh_size = 0;
- for (mach_vm_address_t p = mach_vm_trunc_page(bufReply + vm_page_size);
- p < (mach_vm_address_t)bufReply + rcv_size; p += vm_page_size) {
- *(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
- }
-
-#if DISPATCH_DEBUG
- options |= MACH_RCV_LARGE; // rdar://problem/8422992
-#endif
- tmp_options = options;
- // XXX FIXME -- change this to not starve out the target queue
- for (;;) {
- if (DISPATCH_QUEUE_IS_SUSPENDED(ds) || (--cnt == 0)) {
- options &= ~MACH_RCV_MSG;
- tmp_options &= ~MACH_RCV_MSG;
-
- if (!(tmp_options & MACH_SEND_MSG)) {
- goto out;
- }
- }
- kr = mach_msg(&bufReply->Head, tmp_options, bufReply->Head.msgh_size,
- (mach_msg_size_t)rcv_size, (mach_port_t)ds->ds_ident_hack, 0,0);
-
- tmp_options = options;
-
- if (slowpath(kr)) {
- switch (kr) {
- case MACH_SEND_INVALID_DEST:
- case MACH_SEND_TIMED_OUT:
- if (bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) {
- mach_msg_destroy(&bufReply->Head);
- }
- break;
- case MACH_RCV_TIMED_OUT:
- // Don't return an error if a message was sent this time or
- // a message was successfully received previously
- // rdar://problems/7363620&7791738
- if(bufReply->Head.msgh_remote_port || received) {
- kr = MACH_MSG_SUCCESS;
- }
- break;
- case MACH_RCV_INVALID_NAME:
- break;
-#if DISPATCH_DEBUG
- case MACH_RCV_TOO_LARGE:
- // receive messages that are too large and log their id and size
- // rdar://problem/8422992
- tmp_options &= ~MACH_RCV_LARGE;
- size_t large_size = bufReply->Head.msgh_size + MAX_TRAILER_SIZE;
- void *large_buf = malloc(large_size);
- if (large_buf) {
- rcv_size = large_size;
- bufReply = large_buf;
- }
- if (!mach_msg(&bufReply->Head, tmp_options, 0,
- (mach_msg_size_t)rcv_size,
- (mach_port_t)ds->ds_ident_hack, 0, 0)) {
- _dispatch_log("BUG in libdispatch client: "
- "dispatch_mig_server received message larger than "
- "requested size %zd: id = 0x%x, size = %d",
- maxmsgsz, bufReply->Head.msgh_id,
- bufReply->Head.msgh_size);
- }
- if (large_buf) {
- free(large_buf);
- }
- // fall through
-#endif
- default:
- _dispatch_bug_mach_client(
- "dispatch_mig_server: mach_msg() failed", kr);
- break;
- }
- goto out;
- }
-
- if (!(tmp_options & MACH_RCV_MSG)) {
- goto out;
- }
-
- if (assertion_token) {
-#if DISPATCH_USE_IMPORTANCE_ASSERTION
- int r = proc_importance_assertion_complete(assertion_token);
- (void)dispatch_assume_zero(r);
-#endif
- assertion_token = 0;
- }
- received = true;
-
- bufTemp = bufRequest;
- bufRequest = bufReply;
- bufReply = bufTemp;
-
-#if DISPATCH_USE_IMPORTANCE_ASSERTION
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wdeprecated-declarations"
- int r = proc_importance_assertion_begin_with_msg(&bufRequest->Head,
- NULL, &assertion_token);
- if (r && slowpath(r != EIO)) {
- (void)dispatch_assume_zero(r);
- }
-#pragma clang diagnostic pop
-#endif
- _voucher_replace(voucher_create_with_mach_msg(&bufRequest->Head));
- demux_success = callback(&bufRequest->Head, &bufReply->Head);
-
- if (!demux_success) {
- // destroy the request - but not the reply port
- bufRequest->Head.msgh_remote_port = 0;
- mach_msg_destroy(&bufRequest->Head);
- } else if (!(bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
- // if MACH_MSGH_BITS_COMPLEX is _not_ set, then bufReply->RetCode
- // is present
- if (slowpath(bufReply->RetCode)) {
- if (bufReply->RetCode == MIG_NO_REPLY) {
- continue;
- }
-
- // destroy the request - but not the reply port
- bufRequest->Head.msgh_remote_port = 0;
- mach_msg_destroy(&bufRequest->Head);
- }
- }
-
- if (bufReply->Head.msgh_remote_port) {
- tmp_options |= MACH_SEND_MSG;
- if (MACH_MSGH_BITS_REMOTE(bufReply->Head.msgh_bits) !=
- MACH_MSG_TYPE_MOVE_SEND_ONCE) {
- tmp_options |= MACH_SEND_TIMEOUT;
- }
- }
- }
-
-out:
- if (assertion_token) {
-#if DISPATCH_USE_IMPORTANCE_ASSERTION
- int r = proc_importance_assertion_complete(assertion_token);
- (void)dispatch_assume_zero(r);
-#endif
- }
-
- return kr;
-}
-
-#endif /* HAVE_MACH */
-
-#pragma mark -
-#pragma mark dispatch_source_debug
-
-DISPATCH_NOINLINE
-static const char *
-_evfiltstr(short filt)
-{
- switch (filt) {
-#define _evfilt2(f) case (f): return #f
- _evfilt2(EVFILT_READ);
- _evfilt2(EVFILT_WRITE);
- _evfilt2(EVFILT_AIO);
- _evfilt2(EVFILT_VNODE);
- _evfilt2(EVFILT_PROC);
- _evfilt2(EVFILT_SIGNAL);
- _evfilt2(EVFILT_TIMER);
-#if HAVE_MACH
- _evfilt2(EVFILT_MACHPORT);
- _evfilt2(DISPATCH_EVFILT_MACH_NOTIFICATION);
-#endif
- _evfilt2(EVFILT_FS);
- _evfilt2(EVFILT_USER);
-#ifdef EVFILT_SOCK
- _evfilt2(EVFILT_SOCK);
-#endif
-#ifdef EVFILT_MEMORYSTATUS
- _evfilt2(EVFILT_MEMORYSTATUS);
-#endif
-
- _evfilt2(DISPATCH_EVFILT_TIMER);
- _evfilt2(DISPATCH_EVFILT_CUSTOM_ADD);
- _evfilt2(DISPATCH_EVFILT_CUSTOM_OR);
- default:
- return "EVFILT_missing";
- }
-}
-
-#if DISPATCH_DEBUG
-static const char *
-_evflagstr2(uint16_t *flagsp)
-{
-#define _evflag2(f) \
- if ((*flagsp & (f)) == (f) && (f)) { \
- *flagsp &= ~(f); \
- return #f "|"; \
- }
- _evflag2(EV_ADD);
- _evflag2(EV_DELETE);
- _evflag2(EV_ENABLE);
- _evflag2(EV_DISABLE);
- _evflag2(EV_ONESHOT);
- _evflag2(EV_CLEAR);
- _evflag2(EV_RECEIPT);
- _evflag2(EV_DISPATCH);
- _evflag2(EV_UDATA_SPECIFIC);
-#ifdef EV_POLL
- _evflag2(EV_POLL);
-#endif
-#ifdef EV_OOBAND
- _evflag2(EV_OOBAND);
-#endif
- _evflag2(EV_ERROR);
- _evflag2(EV_EOF);
- _evflag2(EV_VANISHED);
- *flagsp = 0;
- return "EV_UNKNOWN ";
-}
-
-DISPATCH_NOINLINE
-static const char *
-_evflagstr(uint16_t flags, char *str, size_t strsize)
-{
- str[0] = 0;
- while (flags) {
- strlcat(str, _evflagstr2(&flags), strsize);
- }
- size_t sz = strlen(str);
- if (sz) str[sz-1] = 0;
- return str;
-}
-#endif
-
-static size_t
-_dispatch_source_debug_attr(dispatch_source_t ds, char* buf, size_t bufsiz)
-{
- dispatch_queue_t target = ds->do_targetq;
- return dsnprintf(buf, bufsiz, "target = %s[%p], ident = 0x%lx, "
- "mask = 0x%lx, pending_data = 0x%lx, registered = %d, "
- "armed = %d, deleted = %d%s, canceled = %d, ",
- target && target->dq_label ? target->dq_label : "", target,
- ds->ds_ident_hack, ds->ds_pending_data_mask, ds->ds_pending_data,
- ds->ds_is_installed, (bool)(ds->dq_atomic_flags & DSF_ARMED),
- (bool)(ds->dq_atomic_flags & DSF_DELETED),
- (ds->dq_atomic_flags & DSF_DEFERRED_DELETE) ? " (pending)" : "",
- (bool)(ds->dq_atomic_flags & DSF_CANCELED));
-}
-
-static size_t
-_dispatch_timer_debug_attr(dispatch_source_t ds, char* buf, size_t bufsiz)
-{
- dispatch_source_refs_t dr = ds->ds_refs;
- return dsnprintf(buf, bufsiz, "timer = { target = 0x%llx, deadline = 0x%llx"
- ", last_fire = 0x%llx, interval = 0x%llx, flags = 0x%lx }, ",
- (unsigned long long)ds_timer(dr).target,
- (unsigned long long)ds_timer(dr).deadline,
- (unsigned long long)ds_timer(dr).last_fire,
- (unsigned long long)ds_timer(dr).interval, ds_timer(dr).flags);
-}
-
-size_t
-_dispatch_source_debug(dispatch_source_t ds, char* buf, size_t bufsiz)
-{
- size_t offset = 0;
- offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
- dx_kind(ds), ds);
- offset += _dispatch_object_debug_attr(ds, &buf[offset], bufsiz - offset);
- offset += _dispatch_source_debug_attr(ds, &buf[offset], bufsiz - offset);
- if (ds->ds_is_timer) {
- offset += _dispatch_timer_debug_attr(ds, &buf[offset], bufsiz - offset);
- }
- const char *filter;
- if (!ds->ds_dkev) {
- filter = "????";
- } else if (ds->ds_is_custom_source) {
- filter = _evfiltstr((int16_t)(uintptr_t)ds->ds_dkev);
- } else {
- filter = _evfiltstr(ds->ds_dkev->dk_kevent.filter);
- }
- offset += dsnprintf(&buf[offset], bufsiz - offset, "kevent = %p%s, "
- "filter = %s }", ds->ds_dkev, ds->ds_is_direct_kevent ? " (direct)"
- : "", filter);
- return offset;
-}
-
-#if HAVE_MACH
-static size_t
-_dispatch_mach_debug_attr(dispatch_mach_t dm, char* buf, size_t bufsiz)
-{
- dispatch_queue_t target = dm->do_targetq;
- return dsnprintf(buf, bufsiz, "target = %s[%p], receive = 0x%x, "
- "send = 0x%x, send-possible = 0x%x%s, checkin = 0x%x%s, "
- "send state = %016llx, disconnected = %d, canceled = %d ",
- target && target->dq_label ? target->dq_label : "", target,
- dm->ds_dkev ?(mach_port_t)dm->ds_dkev->dk_kevent.ident:0,
- dm->dm_refs->dm_send,
- dm->dm_dkev ?(mach_port_t)dm->dm_dkev->dk_kevent.ident:0,
- dm->dm_dkev && DISPATCH_MACH_NOTIFICATION_ARMED(dm->dm_dkev) ?
- " (armed)" : "", dm->dm_refs->dm_checkin_port,
- dm->dm_refs->dm_checkin ? " (pending)" : "",
- dm->dm_refs->dm_state, dm->dm_refs->dm_disconnect_cnt,
- (bool)(dm->dq_atomic_flags & DSF_CANCELED));
-}
-
-size_t
-_dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz)
-{
- size_t offset = 0;
- offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
- dm->dq_label && !dm->dm_cancel_handler_called ? dm->dq_label :
- dx_kind(dm), dm);
- offset += _dispatch_object_debug_attr(dm, &buf[offset], bufsiz - offset);
- offset += _dispatch_mach_debug_attr(dm, &buf[offset], bufsiz - offset);
- offset += dsnprintf(&buf[offset], bufsiz - offset, "}");
- return offset;
-}
-#endif // HAVE_MACH
-
-#if DISPATCH_DEBUG
-DISPATCH_NOINLINE
-static void
-dispatch_kevent_debug(const char *verb, const _dispatch_kevent_qos_s *kev,
- int i, int n, const char *function, unsigned int line)
-{
- char flagstr[256];
- char i_n[31];
-
- if (n > 1) {
- snprintf(i_n, sizeof(i_n), "%d/%d ", i + 1, n);
- } else {
- i_n[0] = '\0';
- }
-#if DISPATCH_USE_KEVENT_QOS
- _dispatch_debug("%s kevent[%p] %s= { ident = 0x%llx, filter = %s, "
- "flags = %s (0x%x), fflags = 0x%x, data = 0x%llx, udata = 0x%llx, "
- "qos = 0x%x, ext[0] = 0x%llx, ext[1] = 0x%llx, ext[2] = 0x%llx, "
- "ext[3] = 0x%llx }: %s #%u", verb, kev, i_n, kev->ident,
- _evfiltstr(kev->filter), _evflagstr(kev->flags, flagstr,
- sizeof(flagstr)), kev->flags, kev->fflags, kev->data, kev->udata,
- kev->qos, kev->ext[0], kev->ext[1], kev->ext[2], kev->ext[3],
- function, line);
-#else
- _dispatch_debug("%s kevent[%p] %s= { ident = 0x%llx, filter = %s, "
- "flags = %s (0x%x), fflags = 0x%x, data = 0x%llx, udata = 0x%llx, "
- "ext[0] = 0x%llx, ext[1] = 0x%llx }: %s #%u", verb, kev, i_n,
- kev->ident, _evfiltstr(kev->filter), _evflagstr(kev->flags, flagstr,
- sizeof(flagstr)), kev->flags, kev->fflags, kev->data, kev->udata,
-#ifndef IGNORE_KEVENT64_EXT
- kev->ext[0], kev->ext[1],
-#else
- 0ull, 0ull,
-#endif
- function, line);
-#endif
-}
-
-#if HAVE_MACH
-
-#ifndef MACH_PORT_TYPE_SPREQUEST
-#define MACH_PORT_TYPE_SPREQUEST 0x40000000
-#endif
-
-DISPATCH_NOINLINE
-void
-dispatch_debug_machport(mach_port_t name, const char* str)
-{
- mach_port_type_t type;
- mach_msg_bits_t ns = 0, nr = 0, nso = 0, nd = 0;
- unsigned int dnreqs = 0, dnrsiz;
- kern_return_t kr = mach_port_type(mach_task_self(), name, &type);
- if (kr) {
- _dispatch_log("machport[0x%08x] = { error(0x%x) \"%s\" }: %s", name,
- kr, mach_error_string(kr), str);
- return;
- }
- if (type & MACH_PORT_TYPE_SEND) {
- (void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
- MACH_PORT_RIGHT_SEND, &ns));
- }
- if (type & MACH_PORT_TYPE_SEND_ONCE) {
- (void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
- MACH_PORT_RIGHT_SEND_ONCE, &nso));
- }
- if (type & MACH_PORT_TYPE_DEAD_NAME) {
- (void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
- MACH_PORT_RIGHT_DEAD_NAME, &nd));
- }
- if (type & (MACH_PORT_TYPE_RECEIVE|MACH_PORT_TYPE_SEND)) {
- kr = mach_port_dnrequest_info(mach_task_self(), name, &dnrsiz, &dnreqs);
- if (kr != KERN_INVALID_RIGHT) (void)dispatch_assume_zero(kr);
- }
- if (type & MACH_PORT_TYPE_RECEIVE) {
- mach_port_status_t status = { .mps_pset = 0, };
- mach_msg_type_number_t cnt = MACH_PORT_RECEIVE_STATUS_COUNT;
- (void)dispatch_assume_zero(mach_port_get_refs(mach_task_self(), name,
- MACH_PORT_RIGHT_RECEIVE, &nr));
- (void)dispatch_assume_zero(mach_port_get_attributes(mach_task_self(),
- name, MACH_PORT_RECEIVE_STATUS, (void*)&status, &cnt));
- _dispatch_log("machport[0x%08x] = { R(%03u) S(%03u) SO(%03u) D(%03u) "
- "dnreqs(%03u) spreq(%s) nsreq(%s) pdreq(%s) srights(%s) "
- "sorights(%03u) qlim(%03u) msgcount(%03u) mkscount(%03u) "
- "seqno(%03u) }: %s", name, nr, ns, nso, nd, dnreqs,
- type & MACH_PORT_TYPE_SPREQUEST ? "Y":"N",
- status.mps_nsrequest ? "Y":"N", status.mps_pdrequest ? "Y":"N",
- status.mps_srights ? "Y":"N", status.mps_sorights,
- status.mps_qlimit, status.mps_msgcount, status.mps_mscount,
- status.mps_seqno, str);
- } else if (type & (MACH_PORT_TYPE_SEND|MACH_PORT_TYPE_SEND_ONCE|
- MACH_PORT_TYPE_DEAD_NAME)) {
- _dispatch_log("machport[0x%08x] = { R(%03u) S(%03u) SO(%03u) D(%03u) "
- "dnreqs(%03u) spreq(%s) }: %s", name, nr, ns, nso, nd, dnreqs,
- type & MACH_PORT_TYPE_SPREQUEST ? "Y":"N", str);
- } else {
- _dispatch_log("machport[0x%08x] = { type(0x%08x) }: %s", name, type,
- str);
- }
-}
-
-#endif // HAVE_MACH
-
-#endif // DISPATCH_DEBUG
projects / apple / libdispatch.git / commitdiff
