+filt_machport_turnstile_prepare_lazily(
+ struct knote *kn,
+ mach_msg_type_name_t msgt_name,
+ ipc_port_t port)
+{
+ /* This is called from within filt_machportprocess */
+ assert((kn->kn_status & KN_SUPPRESSED) && (kn->kn_status & KN_LOCKED));
+
+ if (!filt_machport_kqueue_has_turnstile(kn)) {
+ return;
+ }
+
+ if (kn->kn_ext[3] == 0 || kn->kn_hook) {
+ return;
+ }
+
+ struct turnstile *ts = filt_ipc_kqueue_turnstile(kn);
+ if ((msgt_name == MACH_MSG_TYPE_PORT_SEND_ONCE && port->ip_specialreply) ||
+ (msgt_name == MACH_MSG_TYPE_PORT_RECEIVE)) {
+ struct turnstile *kn_ts = turnstile_alloc();
+ kn_ts = turnstile_prepare((uintptr_t)kn,
+ (struct turnstile **)&kn->kn_hook, kn_ts, TURNSTILE_KNOTE);
+ turnstile_update_inheritor(kn_ts, ts,
+ TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_TURNSTILE);
+ turnstile_cleanup();
+ }
+}
+
+static void
+filt_machport_turnstile_complete_port(struct knote *kn, ipc_port_t port,
+ ipc_mqueue_t mqueue)
+{
+ struct turnstile *ts = TURNSTILE_NULL;
+
+ ip_lock(port);
+ if (port->ip_specialreply) {
+ /*
+ * If the reply has been sent to the special reply port already,
+ * then the special reply port may already be reused to do something
+ * entirely different.
+ *
+ * However, the only reason for it to still point to this knote is
+ * that it's still waiting for a reply, so when this is the case,
+ * neuter the linkage.
+ */
+ if (port->ip_sync_link_state == PORT_SYNC_LINK_WORKLOOP_KNOTE &&
+ port->ip_sync_inheritor_knote == kn) {
+ ipc_port_adjust_special_reply_port_locked(port, NULL,
+ (IPC_PORT_ADJUST_SR_NONE | IPC_PORT_ADJUST_SR_ENABLE_EVENT), FALSE);
+ } else {
+ ip_unlock(port);
+ }
+ } else {
+ /*
+ * For receive rights, if their IMQ_KNOTE() is still this
+ * knote, then sever the link.
+ */
+ imq_lock(mqueue);
+ if (port->ip_sync_link_state == PORT_SYNC_LINK_WORKLOOP_KNOTE &&
+ mqueue->imq_inheritor_knote == kn) {
+ ipc_port_adjust_sync_link_state_locked(port, PORT_SYNC_LINK_ANY, NULL);
+ ts = port_send_turnstile(port);
+ }
+ if (ts) {
+ turnstile_reference(ts);
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL,
+ TURNSTILE_IMMEDIATE_UPDATE);
+ }
+ imq_unlock(mqueue);
+ ip_unlock(port);
+
+ if (ts) {
+ turnstile_update_inheritor_complete(ts,
+ TURNSTILE_INTERLOCK_NOT_HELD);
+ turnstile_deallocate(ts);
+ }
+ }
+
+ ip_release(port);
+}
+
+void
+filt_wldetach_sync_ipc(struct knote *kn)
+{
+ ipc_mqueue_t mqueue = kn->kn_mqueue;
+ filt_machport_turnstile_complete_port(kn, ip_from_mq(mqueue), mqueue);
+ kn->kn_mqueue = NULL;
+}
+
+/*
+ * Other half of filt_machport_turnstile_prepare_lazily()
+ *
+ * This is serialized by the knote state machine.
+ */
+static void
+filt_machport_turnstile_complete(struct knote *kn)
+{
+ if (kn->kn_ext[3]) {
+ ipc_port_t port = (ipc_port_t)kn->kn_ext[3];
+ filt_machport_turnstile_complete_port(kn, port, &port->ip_messages);
+ kn->kn_ext[3] = 0;
+ }
+
+ if (kn->kn_hook) {
+ struct turnstile *ts = kn->kn_hook;
+
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL,
+ TURNSTILE_IMMEDIATE_UPDATE);
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
+
+ turnstile_complete((uintptr_t)kn, (struct turnstile **)&kn->kn_hook, &ts, TURNSTILE_KNOTE);
+ turnstile_cleanup();
+
+ assert(ts);
+ turnstile_deallocate(ts);
+ }
+}
+
+static void
+filt_machport_link(ipc_mqueue_t mqueue, struct knote *kn)
+{
+ struct knote *hd = SLIST_FIRST(&mqueue->imq_klist);
+
+ if (hd && filt_machport_kqueue_has_turnstile(kn)) {
+ SLIST_INSERT_AFTER(hd, kn, kn_selnext);
+ } else {
+ SLIST_INSERT_HEAD(&mqueue->imq_klist, kn, kn_selnext);
+ }
+}
+
+static void
+filt_machport_unlink(ipc_mqueue_t mqueue, struct knote *kn)
+{
+ struct knote **knprev;
+
+ KNOTE_DETACH(&mqueue->imq_klist, kn);
+
+ /* make sure the first knote is a knote we can push on */
+ SLIST_FOREACH_PREVPTR(kn, knprev, &mqueue->imq_klist, kn_selnext) {
+ if (filt_machport_kqueue_has_turnstile(kn)) {
+ *knprev = SLIST_NEXT(kn, kn_selnext);
+ SLIST_INSERT_HEAD(&mqueue->imq_klist, kn, kn_selnext);
+ break;
+ }
+ }
+}
+
+int
+filt_wlattach_sync_ipc(struct knote *kn)
+{
+ mach_port_name_t name = (mach_port_name_t)kn->kn_id;
+ ipc_space_t space = current_space();
+ ipc_entry_t entry;
+ ipc_port_t port = IP_NULL;
+ int error = 0;
+
+ if (ipc_right_lookup_read(space, name, &entry) != KERN_SUCCESS) {
+ return ENOENT;
+ }
+
+ /* space is read-locked */
+
+ if (entry->ie_bits & MACH_PORT_TYPE_RECEIVE) {
+ port = ip_object_to_port(entry->ie_object);
+ if (port->ip_specialreply) {
+ error = ENOENT;
+ }
+ } else if (entry->ie_bits & MACH_PORT_TYPE_SEND_ONCE) {
+ port = ip_object_to_port(entry->ie_object);
+ if (!port->ip_specialreply) {
+ error = ENOENT;
+ }
+ } else {
+ error = ENOENT;
+ }
+ if (error) {
+ is_read_unlock(space);
+ return error;
+ }
+
+ ip_lock(port);
+ is_read_unlock(space);
+
+ if (port->ip_sync_link_state == PORT_SYNC_LINK_ANY) {
+ ip_unlock(port);
+ /*
+ * We cannot start a sync IPC inheritance chain, only further one
+ * Note: this can also happen if the inheritance chain broke
+ * because the original requestor died.
+ */
+ return ENOENT;
+ }
+
+ if (port->ip_specialreply) {
+ ipc_port_adjust_special_reply_port_locked(port, kn,
+ IPC_PORT_ADJUST_SR_LINK_WORKLOOP, FALSE);
+ } else {
+ ipc_port_adjust_port_locked(port, kn, FALSE);
+ }
+
+ /* make sure the port was stashed */
+ assert(kn->kn_mqueue == &port->ip_messages);
+
+ /* port has been unlocked by ipc_port_adjust_* */
+
+ return 0;
+}
+
+static int
+filt_machportattach(
+ struct knote *kn,
+ __unused struct kevent_qos_s *kev)
+{
+ mach_port_name_t name = (mach_port_name_t)kn->kn_id;
+ uint64_t wq_link_id = waitq_link_reserve(NULL);
+ ipc_space_t space = current_space();
+ ipc_kmsg_t first;
+ struct turnstile *send_turnstile = TURNSTILE_NULL;
+
+ int error;
+ int result = 0;
+ kern_return_t kr;
+ ipc_entry_t entry;
+ ipc_mqueue_t mqueue;
+
+ kn->kn_flags &= ~EV_EOF;
+ kn->kn_ext[3] = 0;
+
+ if (filt_machport_kqueue_has_turnstile(kn)) {
+ /*
+ * If the filter is likely to support sync IPC override,
+ * and it happens to be attaching to a workloop,
+ * make sure the workloop has an allocated turnstile.
+ */
+ kqueue_alloc_turnstile(knote_get_kq(kn));
+ }
+
+lookup_again:
+ kr = ipc_right_lookup_read(space, name, &entry);
+
+ if (kr != KERN_SUCCESS) {
+ error = ENOENT;
+ goto out;
+ }
+
+ /* space is read-locked and active */
+
+ if ((entry->ie_bits & MACH_PORT_TYPE_PORT_SET) &&
+ knote_link_waitqset_should_lazy_alloc(kn)) {
+ is_read_unlock(space);
+
+ /*
+ * We need to link the portset of the kn,
+ * to insure that the link is allocated before taking
+ * any spinlocks.
+ *
+ * Because we have to drop the space lock so that
+ * knote_link_waitqset_lazy_alloc() can allocate memory,
+ * we will need to redo the lookup.
+ */
+ knote_link_waitqset_lazy_alloc(kn);
+ goto lookup_again;
+ }
+
+ if (entry->ie_bits & MACH_PORT_TYPE_PORT_SET) {
+ ipc_pset_t pset;
+
+ pset = ips_object_to_pset(entry->ie_object);
+ mqueue = &pset->ips_messages;
+ ips_reference(pset);
+
+ imq_lock(mqueue);
+ kn->kn_mqueue = mqueue;
+
+ /*
+ * Bind the portset wait queue directly to knote/kqueue.
+ * This allows us to just use wait_queue foo to effect a wakeup,
+ * rather than having to call knote() from the Mach code on each
+ * message. We still attach the knote to the mqueue klist for
+ * NOTE_REVOKE purposes only.
+ */
+ error = knote_link_waitq(kn, &mqueue->imq_wait_queue, &wq_link_id);
+ if (!error) {
+ filt_machport_link(mqueue, kn);
+ imq_unlock(mqueue);
+ } else {
+ kn->kn_mqueue = IMQ_NULL;
+ imq_unlock(mqueue);
+ ips_release(pset);
+ }
+
+ is_read_unlock(space);
+
+ /*
+ * linked knotes are marked stay-active and therefore don't
+ * need an indication of their fired state to be returned
+ * from the attach operation.
+ */
+ } else if (entry->ie_bits & MACH_PORT_TYPE_RECEIVE) {
+ ipc_port_t port = ip_object_to_port(entry->ie_object);
+
+ if (port->ip_specialreply) {
+ /*
+ * Registering for kevents on special reply ports
+ * isn't supported for two reasons:
+ *
+ * 1. it really makes very little sense for a port that
+ * is supposed to be used synchronously
+ *
+ * 2. their mqueue's imq_klist field will be used to
+ * store the receive turnstile, so we can't possibly
+ * attach them anyway.
+ */
+ is_read_unlock(space);
+ error = ENOTSUP;
+ goto out;
+ }
+
+ mqueue = &port->ip_messages;
+ ip_reference(port);
+
+ /*
+ * attach knote to port and determine result
+ * If the filter requested direct message receipt,
+ * we may need to adjust the qos of the knote to
+ * reflect the requested and override qos of the
+ * first message in the queue.
+ */
+ ip_lock(port);
+ imq_lock(mqueue);
+
+ kn->kn_mqueue = mqueue;
+ if (port->ip_sync_link_state != PORT_SYNC_LINK_ANY) {
+ /*
+ * We're attaching a port that used to have an IMQ_KNOTE,
+ * clobber this state, we'll fixup its turnstile inheritor below.
+ */
+ ipc_port_adjust_sync_link_state_locked(port, PORT_SYNC_LINK_ANY, NULL);
+ }
+ filt_machport_link(mqueue, kn);
+
+ if ((first = ipc_kmsg_queue_first(&mqueue->imq_messages)) != IKM_NULL) {
+ result = FILTER_ACTIVE | filt_machport_adjust_qos(kn, first);
+ }
+
+ /*
+ * Update the port's turnstile inheritor
+ *
+ * Unlike filt_machportdetach(), we don't have to care about races for
+ * turnstile_workloop_pusher_info(): filt_machport_link() doesn't affect
+ * already pushing knotes, and if the current one becomes the new
+ * pusher, it'll only be visible when turnstile_workloop_pusher_info()
+ * returns.
+ */
+ send_turnstile = port_send_turnstile(port);
+ if (send_turnstile) {
+ turnstile_reference(send_turnstile);
+ ipc_port_send_update_inheritor(port, send_turnstile,
+ TURNSTILE_IMMEDIATE_UPDATE);
+
+ /*
+ * rdar://problem/48861190
+ *
+ * When a listener connection resumes a peer,
+ * updating the inheritor above has moved the push
+ * from the current thread to the workloop.
+ *
+ * However, we haven't told the workloop yet
+ * that it needs a thread request, and we risk
+ * to be preeempted as soon as we drop the space
+ * lock below.
+ *
+ * To avoid this disable preemption and let kevent
+ * reenable it after it takes the kqlock.
+ */
+ disable_preemption();
+ result |= FILTER_THREADREQ_NODEFEER;
+ }
+
+ imq_unlock(mqueue);
+ ip_unlock(port);
+
+ is_read_unlock(space);
+ if (send_turnstile) {
+ turnstile_update_inheritor_complete(send_turnstile,
+ TURNSTILE_INTERLOCK_NOT_HELD);
+ turnstile_deallocate_safe(send_turnstile);
+ }
+
+ error = 0;
+ } else {
+ is_read_unlock(space);
+ error = ENOTSUP;
+ }
+
+out:
+ waitq_link_release(wq_link_id);
+
+ /* bail out on errors */
+ if (error) {
+ knote_set_error(kn, error);
+ return 0;
+ }
+
+ return result;
+}
+
+/* Validate imq_to_object implementation "works" */
+_Static_assert(offsetof(struct ipc_pset, ips_messages) ==
+ offsetof(struct ipc_port, ip_messages),
+ "Make sure the mqueue aliases in both ports and psets");
+
+static void
+filt_machportdetach(
+ struct knote *kn)
+{
+ ipc_mqueue_t mqueue = kn->kn_mqueue;
+ ipc_object_t object = imq_to_object(mqueue);
+ struct turnstile *send_turnstile = TURNSTILE_NULL;
+
+ filt_machport_turnstile_complete(kn);
+
+ imq_lock(mqueue);
+ if ((kn->kn_status & KN_VANISHED) || (kn->kn_flags & EV_EOF)) {
+ /*
+ * ipc_mqueue_changed() already unhooked this knote from the mqueue,
+ */
+ } else {
+ ipc_port_t port = IP_NULL;
+
+ /*
+ * When the knote being detached is the first one in the list,
+ * then unlinking the knote *and* updating the turnstile inheritor
+ * need to happen atomically with respect to the callers of
+ * turnstile_workloop_pusher_info().
+ *
+ * The caller of turnstile_workloop_pusher_info() will use the kq req
+ * lock (and hence the kqlock), so we just need to hold the kqlock too.
+ */
+ if (io_otype(object) == IOT_PORT) {
+ port = ip_object_to_port(object);
+ assert(port->ip_sync_link_state == PORT_SYNC_LINK_ANY);
+ if (kn == SLIST_FIRST(&mqueue->imq_klist)) {
+ send_turnstile = port_send_turnstile(port);
+ }
+ }
+
+ filt_machport_unlink(mqueue, kn);
+
+ if (send_turnstile) {
+ turnstile_reference(send_turnstile);
+ ipc_port_send_update_inheritor(port, send_turnstile,
+ TURNSTILE_IMMEDIATE_UPDATE);
+ }
+ }
+
+ /* Clear the knote pointer once the knote has been removed from turnstile */
+ kn->kn_mqueue = IMQ_NULL;
+ imq_unlock(mqueue);
+
+ if (send_turnstile) {
+ turnstile_update_inheritor_complete(send_turnstile,
+ TURNSTILE_INTERLOCK_NOT_HELD);
+ turnstile_deallocate(send_turnstile);
+ }
+
+ if (io_otype(object) == IOT_PORT_SET) {
+ /*
+ * Unlink the portset wait queue from knote/kqueue.
+ * JMM - Does this need to be atomic under the mq lock?
+ */
+ (void)knote_unlink_waitq(kn, &mqueue->imq_wait_queue);
+ }
+ io_release(object);
+}
+
+/*
+ * filt_machportevent - deliver events into the mach port filter
+ *
+ * Mach port message arrival events are currently only posted via the
+ * kqueue filter routine for ports. Port sets are marked stay-active
+ * and the wait queue code will break any kqueue waiters out to go
+ * poll the stay-queued knotes again.
+ *
+ * If there is a message at the head of the queue,
+ * we indicate that the knote should go active. If
+ * the message is to be direct-received, we adjust the
+ * QoS of the knote according the requested and override
+ * QoS of that first message.
+ */
+static int
+filt_machportevent(struct knote *kn, long hint __assert_only)
projects / apple / xnu.git / blobdiff