+struct turnstile *
+ipc_port_get_inheritor(ipc_port_t port)
+{
+ ipc_mqueue_t mqueue = &port->ip_messages;
+ struct knote *kn;
+
+ assert(imq_held(mqueue));
+
+ if (!IMQ_KLIST_VALID(mqueue)) {
+ return IMQ_INHERITOR(mqueue);
+ }
+
+ SLIST_FOREACH(kn, &port->ip_messages.imq_klist, kn_selnext) {
+ if ((kn->kn_sfflags & MACH_RCV_MSG) && (kn->kn_status & KN_DISPATCH)) {
+ return filt_machport_kqueue_turnstile(kn);
+ }
+ }
+
+ return TURNSTILE_NULL;
+}
+
+/*
+ * Routine: ipc_port_send_turnstile_prepare
+ * Purpose:
+ * Get a reference on port's send turnstile, if
+ * port does not have a send turnstile then allocate one.
+ *
+ * Conditions:
+ * Nothing is locked.
+ */
+void
+ipc_port_send_turnstile_prepare(ipc_port_t port)
+{
+ struct turnstile *turnstile = TURNSTILE_NULL;
+ struct turnstile *inheritor = TURNSTILE_NULL;
+ struct turnstile *send_turnstile = TURNSTILE_NULL;
+
+retry_alloc:
+ imq_lock(&port->ip_messages);
+
+ if (port_send_turnstile(port) == NULL ||
+ port_send_turnstile(port)->ts_port_ref == 0) {
+
+ if (turnstile == TURNSTILE_NULL) {
+ imq_unlock(&port->ip_messages);
+ turnstile = turnstile_alloc();
+ goto retry_alloc;
+ }
+
+ send_turnstile = turnstile_prepare((uintptr_t)port,
+ port_send_turnstile_address(port),
+ turnstile, TURNSTILE_SYNC_IPC);
+ turnstile = TURNSTILE_NULL;
+
+ /*
+ * if port in transit, setup linkage for its turnstile,
+ * otherwise the link it to WL turnstile.
+ */
+ if (ip_active(port) &&
+ port->ip_receiver_name == MACH_PORT_NULL &&
+ port->ip_destination != IP_NULL) {
+ assert(port->ip_receiver_name == MACH_PORT_NULL);
+ assert(port->ip_destination != IP_NULL);
+
+ inheritor = port_send_turnstile(port->ip_destination);
+ } else {
+ inheritor = ipc_port_get_inheritor(port);
+ }
+ turnstile_update_inheritor(send_turnstile, inheritor,
+ TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE);
+ /* turnstile complete will be called in ipc_port_send_turnstile_complete */
+ }
+
+ /* Increment turnstile counter */
+ port_send_turnstile(port)->ts_port_ref++;
+ imq_unlock(&port->ip_messages);
+
+ if (send_turnstile) {
+ turnstile_update_inheritor_complete(send_turnstile,
+ TURNSTILE_INTERLOCK_NOT_HELD);
+ }
+ if (turnstile != TURNSTILE_NULL) {
+ turnstile_deallocate(turnstile);
+ }
+}
+
+
+/*
+ * Routine: ipc_port_send_turnstile_complete
+ * Purpose:
+ * Drop a ref on the port's send turnstile, if the
+ * ref becomes zero, deallocate the turnstile.
+ *
+ * Conditions:
+ * The space might be locked, use safe deallocate.
+ */
+void
+ipc_port_send_turnstile_complete(ipc_port_t port)
+{
+ struct turnstile *turnstile = TURNSTILE_NULL;
+
+ /* Drop turnstile count on dest port */
+ imq_lock(&port->ip_messages);
+
+ port_send_turnstile(port)->ts_port_ref--;
+ if (port_send_turnstile(port)->ts_port_ref == 0) {
+ turnstile_complete((uintptr_t)port, port_send_turnstile_address(port),
+ &turnstile);
+ assert(turnstile != TURNSTILE_NULL);
+ }
+ imq_unlock(&port->ip_messages);
+ turnstile_cleanup();
+
+ if (turnstile != TURNSTILE_NULL) {
+ turnstile_deallocate_safe(turnstile);
+ turnstile = TURNSTILE_NULL;
+ }
+}
+
+
+/*
+ * Routine: ipc_port_rcv_turnstile_waitq
+ * Purpose:
+ * Given the mqueue's waitq, find the port's
+ * rcv turnstile and return its waitq.
+ *
+ * Conditions:
+ * mqueue locked or thread waiting on turnstile is locked.
+ */
+struct waitq *
+ipc_port_rcv_turnstile_waitq(struct waitq *waitq)
+{
+ struct waitq *safeq;
+
+ ipc_mqueue_t mqueue = imq_from_waitq(waitq);
+ ipc_port_t port = ip_from_mq(mqueue);
+ struct turnstile *rcv_turnstile = ipc_port_rcv_turnstile(port);
+
+ /* Check if the port has a rcv turnstile */
+ if (rcv_turnstile != TURNSTILE_NULL) {
+ safeq = &rcv_turnstile->ts_waitq;
+ } else {
+ safeq = global_eventq(waitq);
+ }
+ return safeq;
+}
+
+
+/*
+ * Routine: ipc_port_rcv_turnstile
+ * Purpose:
+ * Get the port's receive turnstile
+ *
+ * Conditions:
+ * mqueue locked or thread waiting on turnstile is locked.
+ */
+struct turnstile *
+ipc_port_rcv_turnstile(ipc_port_t port)
+{
+ return turnstile_lookup_by_proprietor((uintptr_t)port);
+}
+
+
+/*
+ * Routine: ipc_port_link_special_reply_port
+ * Purpose:
+ * Link the special reply port with the destination port.
+ * Allocates turnstile to dest port.
+ *
+ * Conditions:
+ * Nothing is locked.
+ */
+void
+ipc_port_link_special_reply_port(
+ ipc_port_t special_reply_port,
+ ipc_port_t dest_port)
+{
+ boolean_t drop_turnstile_ref = FALSE;
+
+ /* Check if dest_port needs a turnstile */
+ ipc_port_send_turnstile_prepare(dest_port);
+
+ /* Lock the special reply port and establish the linkage */
+ ip_lock(special_reply_port);
+ imq_lock(&special_reply_port->ip_messages);
+
+ /* Check if we need to drop the acquired turnstile ref on dest port */
+ if (!special_reply_port->ip_specialreply ||
+ special_reply_port->ip_sync_link_state != PORT_SYNC_LINK_ANY ||
+ special_reply_port->ip_sync_inheritor_port != IPC_PORT_NULL) {
+ drop_turnstile_ref = TRUE;
+ } else {
+ /* take a reference on dest_port */
+ ip_reference(dest_port);
+ special_reply_port->ip_sync_inheritor_port = dest_port;
+ special_reply_port->ip_sync_link_state = PORT_SYNC_LINK_PORT;
+ }
+
+ imq_unlock(&special_reply_port->ip_messages);
+ ip_unlock(special_reply_port);
+
+ if (drop_turnstile_ref) {
+ ipc_port_send_turnstile_complete(dest_port);
+ }
+
+ return;
+}
+
+#if DEVELOPMENT || DEBUG
+inline void
+reset_ip_srp_bits(ipc_port_t special_reply_port)
+{
+ special_reply_port->ip_srp_lost_link = 0;
+ special_reply_port->ip_srp_msg_sent = 0;
+}
+
+inline void
+reset_ip_srp_msg_sent(ipc_port_t special_reply_port)
+{
+ if (special_reply_port->ip_specialreply == 1) {
+ special_reply_port->ip_srp_msg_sent = 0;
+ }
+}
+
+inline void
+set_ip_srp_msg_sent(ipc_port_t special_reply_port)
+{
+ if (special_reply_port->ip_specialreply == 1) {
+ special_reply_port->ip_srp_msg_sent = 1;
+ }
+}
+
+inline void
+set_ip_srp_lost_link(ipc_port_t special_reply_port)
+{
+ if (special_reply_port->ip_specialreply == 1 && special_reply_port->ip_srp_msg_sent == 0) {
+ special_reply_port->ip_srp_lost_link = 1;
+ }
+}
+
+#else /* DEVELOPMENT || DEBUG */
+inline void
+reset_ip_srp_bits(__unused ipc_port_t special_reply_port)
+{
+ return;
+}
+
+inline void
+reset_ip_srp_msg_sent(__unused ipc_port_t special_reply_port)
+{
+ return;
+}
+
+inline void
+set_ip_srp_msg_sent(__unused ipc_port_t special_reply_port)
+{
+ return;
+}
+
+inline void
+set_ip_srp_lost_link(__unused ipc_port_t special_reply_port)
+{
+ return;
+}
+#endif /* DEVELOPMENT || DEBUG */
+
+/*
+ * Routine: ipc_port_adjust_special_reply_port_locked
+ * Purpose:
+ * If the special port has a turnstile, update it's inheritor.
+ * Condition:
+ * Special reply port locked on entry.
+ * Special reply port unlocked on return.
+ * Returns:
+ * None.
+ */
+void
+ipc_port_adjust_special_reply_port_locked(
+ ipc_port_t special_reply_port,
+ struct knote *kn,
+ uint8_t flags,
+ boolean_t get_turnstile)
+{
+ ipc_port_t dest_port = IPC_PORT_NULL;
+ int sync_link_state = PORT_SYNC_LINK_NO_LINKAGE;
+ turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;
+ struct turnstile *dest_ts = TURNSTILE_NULL, *ts = TURNSTILE_NULL;
+
+ imq_lock(&special_reply_port->ip_messages);
+
+ if (flags & IPC_PORT_ADJUST_SR_RECEIVED_MSG) {
+ reset_ip_srp_msg_sent(special_reply_port);
+ }
+
+ /* Check if the special reply port is marked non-special */
+ if (special_reply_port->ip_specialreply == 0 ||
+ special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_ANY) {
+ if (get_turnstile) {
+ turnstile_complete((uintptr_t)special_reply_port,
+ port_rcv_turnstile_address(special_reply_port),
+ NULL);
+ }
+ imq_unlock(&special_reply_port->ip_messages);
+ ip_unlock(special_reply_port);
+ if (get_turnstile) {
+ turnstile_cleanup();
+ }
+ return;
+ }
+
+ /* Clear thread's special reply port and clear linkage */
+ if (flags & IPC_PORT_ADJUST_SR_CLEAR_SPECIAL_REPLY) {
+ /* This option should only be specified by a non blocking thread */
+ assert(get_turnstile == FALSE);
+ special_reply_port->ip_specialreply = 0;
+
+ reset_ip_srp_bits(special_reply_port);
+
+ /* Check if need to break linkage */
+ if (special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_NO_LINKAGE) {
+ imq_unlock(&special_reply_port->ip_messages);
+ ip_unlock(special_reply_port);
+ return;
+ }
+ } else if (flags & IPC_PORT_ADJUST_SR_LINK_WORKLOOP) {
+ if (special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_ANY ||
+ special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_PORT) {
+ if (ITH_KNOTE_VALID(kn, MACH_MSG_TYPE_PORT_SEND_ONCE)) {
+ inheritor = filt_machport_stash_port(kn, special_reply_port,
+ &sync_link_state);
+ }
+ }
+ } else if (flags & IPC_PORT_ADJUST_SR_ALLOW_SYNC_LINKAGE) {
+ sync_link_state = PORT_SYNC_LINK_ANY;
+ }
+
+ switch (special_reply_port->ip_sync_link_state) {
+ case PORT_SYNC_LINK_PORT:
+ dest_port = special_reply_port->ip_sync_inheritor_port;
+ special_reply_port->ip_sync_inheritor_port = IPC_PORT_NULL;
+ break;
+ case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+ special_reply_port->ip_sync_inheritor_knote = NULL;
+ break;
+ case PORT_SYNC_LINK_WORKLOOP_STASH:
+ dest_ts = special_reply_port->ip_sync_inheritor_ts;
+ special_reply_port->ip_sync_inheritor_ts = NULL;
+ break;
+ }
+
+ special_reply_port->ip_sync_link_state = sync_link_state;
+
+ switch (sync_link_state) {
+ case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+ special_reply_port->ip_sync_inheritor_knote = kn;
+ break;
+ case PORT_SYNC_LINK_WORKLOOP_STASH:
+ turnstile_reference(inheritor);
+ special_reply_port->ip_sync_inheritor_ts = inheritor;
+ break;
+ case PORT_SYNC_LINK_NO_LINKAGE:
+ if (flags & IPC_PORT_ADJUST_SR_ENABLE_EVENT) {
+ set_ip_srp_lost_link(special_reply_port);
+ }
+ break;
+ }
+
+ /* Get thread's turnstile donated to special reply port */
+ if (get_turnstile) {
+ turnstile_complete((uintptr_t)special_reply_port,
+ port_rcv_turnstile_address(special_reply_port),
+ NULL);
+ } else {
+ ts = ipc_port_rcv_turnstile(special_reply_port);
+ if (ts) {
+ turnstile_reference(ts);
+ turnstile_update_inheritor(ts, inheritor,
+ (TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE));
+ }
+ }
+
+ imq_unlock(&special_reply_port->ip_messages);
+ ip_unlock(special_reply_port);
+
+ if (get_turnstile) {
+ turnstile_cleanup();
+ } else if (ts) {
+ /* Call turnstile cleanup after dropping the interlock */
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+ turnstile_deallocate_safe(ts);
+ }
+
+ /* Release the ref on the dest port and it's turnstile */
+ if (dest_port) {
+ ipc_port_send_turnstile_complete(dest_port);
+ /* release the reference on the dest port */
+ ip_release(dest_port);
+ }
+
+ if (dest_ts) {
+ turnstile_deallocate_safe(dest_ts);
+ }
+}
+
+/*
+ * Routine: ipc_port_adjust_special_reply_port
+ * Purpose:
+ * If the special port has a turnstile, update it's inheritor.
+ * Condition:
+ * Nothing locked.
+ * Returns:
+ * None.
+ */
+void
+ipc_port_adjust_special_reply_port(
+ ipc_port_t special_reply_port,
+ uint8_t flags,
+ boolean_t get_turnstile)
+{
+ ip_lock(special_reply_port);
+ ipc_port_adjust_special_reply_port_locked(special_reply_port, NULL, flags, get_turnstile);
+ /* special_reply_port unlocked */
+}
+
+/*
+ * Routine: ipc_port_get_special_reply_port_inheritor
+ * Purpose:
+ * Returns the current inheritor of the special reply port
+ * Condition:
+ * mqueue is locked, port is a special reply port
+ * Returns:
+ * the current inheritor
+ */
+turnstile_inheritor_t
+ipc_port_get_special_reply_port_inheritor(
+ ipc_port_t port)
+{
+ assert(port->ip_specialreply);
+ imq_held(&port->ip_messages);
+
+ switch (port->ip_sync_link_state) {
+ case PORT_SYNC_LINK_PORT:
+ if (port->ip_sync_inheritor_port != NULL) {
+ return port_send_turnstile(port->ip_sync_inheritor_port);
+ }
+ break;
+ case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+ return filt_machport_stashed_special_reply_port_turnstile(port);
+ case PORT_SYNC_LINK_WORKLOOP_STASH:
+ return port->ip_sync_inheritor_ts;
+ }
+ return TURNSTILE_INHERITOR_NULL;
+}
+
projects / apple / xnu.git / blobdiff