- hdr->msgh_bits =
- MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
- 0);
- hdr->msgh_remote_port = dest_port;
-
- /* make sure we can queue to the destination */
-
- assert(dest_port->ip_receiver != ipc_space_kernel);
-
- /* optimized ipc_entry_lookup/ipc_mqueue_copyin */
-
- {
- register ipc_entry_t entry;
- register ipc_entry_bits_t bits;
-
- {
- register mach_port_index_t index;
- register mach_port_gen_t gen;
-
- index = MACH_PORT_INDEX(rcv_name);
- gen = MACH_PORT_GEN(rcv_name);
-
- if (index < size) {
- entry = &table[index];
- bits = entry->ie_bits;
- if (IE_BITS_GEN(bits) != gen ||
- (bits & IE_BITS_COLLISION)) {
- entry = IE_NULL;
- }
- } else {
- entry = IE_NULL;
- }
- if (entry == IE_NULL) {
- entry = ipc_entry_lookup(space, rcv_name);
- if (entry == IE_NULL) {
- HOT(c_mmot_cold_024++);
- goto abort_reply_rcv_copyin;
- }
- bits = entry->ie_bits;
- }
-
- }
-
- /* check type bits; looking for receive or set */
-#if 0
- /*
- * JMM - The check below for messages in the receive
- * mqueue is insufficient to work with port sets, since
- * the messages stay in the port queues. For now, don't
- * allow portsets (but receiving on portsets when sending
- * a message to a send-once right is actually a very
- * common case (so we should re-enable).
- */
- if (bits & MACH_PORT_TYPE_PORT_SET) {
- register ipc_pset_t rcv_pset;
-
- rcv_pset = (ipc_pset_t) entry->ie_object;
- assert(rcv_pset != IPS_NULL);
-
- ips_lock(rcv_pset);
- assert(ips_active(rcv_pset));
-
- rcv_object = (ipc_object_t) rcv_pset;
- rcv_mqueue = &rcv_pset->ips_messages;
- } else
-#endif /* 0 */
- if (bits & MACH_PORT_TYPE_RECEIVE) {
- register ipc_port_t rcv_port;
-
- rcv_port = (ipc_port_t) entry->ie_object;
- assert(rcv_port != IP_NULL);
-
- if (!ip_lock_try(rcv_port)) {
- HOT(c_mmot_cold_025++);
- goto abort_reply_rcv_copyin;
- }
- assert(ip_active(rcv_port));
-
- if (rcv_port->ip_pset_count != 0) {
- ip_unlock(rcv_port);
- HOT(c_mmot_cold_026++);
- goto abort_reply_rcv_copyin;
- }
-
- rcv_object = (ipc_object_t) rcv_port;
- rcv_mqueue = &rcv_port->ip_messages;
- } else {
- HOT(c_mmot_cold_027++);
- goto abort_reply_rcv_copyin;
- }
- }
-
- is_write_unlock(space);
- io_reference(rcv_object);
- io_unlock(rcv_object);
- HOT(c_mmot_hot_fSR_ok++);
- goto fast_send_receive;
-
- abort_reply_dest_copyin:
- is_write_unlock(space);
- HOT(c_mmot_cold_029++);
- goto slow_copyin;
-
- abort_reply_rcv_copyin:
- ip_unlock(dest_port);
- is_write_unlock(space);
- HOT(c_mmot_cold_030++);
- goto slow_send;
- }
-
- default:
- HOT(c_mmot_cold_031++);
- goto slow_copyin;
- }
- /*NOTREACHED*/
-
- fast_send_receive:
- /*
- * optimized ipc_mqueue_send/ipc_mqueue_receive
- *
- * Finished get/copyin of kmsg and copyin of rcv_name.
- * space is unlocked, dest_port is locked,
- * we can queue kmsg to dest_port,
- * rcv_mqueue is set, and rcv_object holds a ref
- * so the mqueue cannot go away.
- *
- * JMM - For now, rcv_object is just a port. Portsets
- * are disabled for the time being.
- */
-
- assert(ip_active(dest_port));
- assert(dest_port->ip_receiver != ipc_space_kernel);
-// assert(!imq_full(&dest_port->ip_messages) ||
-// (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) ==
-// MACH_MSG_TYPE_PORT_SEND_ONCE));
- assert((hdr->msgh_bits & MACH_MSGH_BITS_CIRCULAR) == 0);
-
- {
- register ipc_mqueue_t dest_mqueue;
- wait_queue_t waitq;
- thread_t receiver;
-#if THREAD_SWAPPER
- thread_act_t rcv_act;
-#endif
- spl_t s;
-
- s = splsched();
- dest_mqueue = &dest_port->ip_messages;
- waitq = &dest_mqueue->imq_wait_queue;
- imq_lock(dest_mqueue);
-
- wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
- /* queue still locked, thread locked - but still on q */
-
- if (receiver == THREAD_NULL) {
- abort_send_receive:
- imq_unlock(dest_mqueue);
- splx(s);
- ip_unlock(dest_port);
- ipc_object_release(rcv_object);
- HOT(c_mmot_cold_032++);
- goto slow_send;
- }
-
- assert(receiver->wait_queue == waitq);
- assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
-
- /*
- * See if it is still running on another processor (trying to
- * block itself). If so, fall off.
- *
- * JMM - We have an opportunity here. Since the thread is locked
- * and we find it runnable, it must still be trying to get into
- * thread_block on itself. We could just "hand him the message"
- * and let him go (thread_go_locked()) and then fall down into a
- * slow receive for ourselves. Only his RECEIVE_TOO_LARGE handling
- * runs afoul of that. Clean this up!
- */
- if ((receiver->state & (TH_RUN|TH_WAIT)) != TH_WAIT) {
- assert(NCPUS > 1);
- HOT(c_mmot_cold_033++);
- fall_off:
- thread_unlock(receiver);
- if (waitq != &dest_mqueue->imq_wait_queue)
- wait_queue_unlock(waitq);
- goto abort_send_receive;
- }
-
- /*
- * Check that the receiver can stay on the hot path.
- */
- if (send_size + REQUESTED_TRAILER_SIZE(receiver->ith_option) >
- receiver->ith_msize) {
- /*
- * The receiver can't accept the message.
- */
- HOT(c_mmot_bad_rcvr++);
- goto fall_off;
- }
-
-#if THREAD_SWAPPER
- /*
- * Receiver looks okay -- is it swapped in?
- */
- rcv_act = receiver->top_act;
- if (rcv_act->swap_state != TH_SW_IN &&
- rcv_act->swap_state != TH_SW_UNSWAPPABLE) {
- HOT(c_mmot_rcvr_swapped++);
- goto fall_off;
- }
-
- /*
- * Make sure receiver stays swapped in (if we can).
- */
- if (!act_lock_try(rcv_act)) { /* out of order! */
- HOT(c_mmot_rcvr_locked++);
- goto fall_off;
- }
-
- /*
- * Check for task swapping in progress affecting
- * receiver. Since rcv_act is attached to a shuttle,
- * its swap_state is covered by shuttle's thread_lock()
- * (sigh).
- */
- if ((rcv_act->swap_state != TH_SW_IN &&
- rcv_act->swap_state != TH_SW_UNSWAPPABLE) ||
- rcv_act->ast & AST_SWAPOUT) {
- act_unlock(rcv_act);
- HOT(c_mmot_rcvr_tswapped++);
- goto fall_off;
- }
-
- /*
- * We don't need to make receiver unswappable here -- holding
- * act_lock() of rcv_act is sufficient to prevent either thread
- * or task swapping from changing its state (see swapout_scan(),
- * task_swapout()). Don't release lock till receiver's state
- * is consistent. Its task may then be marked for swapout,
- * but that's life.
- */
- /*
- * NB: act_lock(rcv_act) still held
- */
-#endif /* THREAD_SWAPPER */
-
- /*
- * Before committing to the handoff, make sure that we are
- * really going to block (i.e. there are no messages already
- * queued for us. This violates lock ordering, so make sure
- * we don't deadlock. After the trylock succeeds below, we
- * may have up to 3 message queues locked:
- * - the dest port mqueue
- * - a portset mqueue (where waiting receiver was found)
- * - finally our own rcv_mqueue
- *
- * JMM - Need to make this check appropriate for portsets as
- * well before re-enabling them.
- */
- if (!imq_lock_try(rcv_mqueue)) {
- goto fall_off;
- }
- if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
- imq_unlock(rcv_mqueue);
- HOT(c_mmot_cold_033++);
- goto fall_off;
- }
-
- /* At this point we are committed to do the "handoff". */
- c_mach_msg_trap_switch_fast++;
-
- /*
- * JMM - Go ahead and pull the receiver from the runq. If the
- * runq wasn't the one for the mqueue, unlock it.
- */
- wait_queue_pull_thread_locked(waitq,
- receiver,
- (waitq != &dest_mqueue->imq_wait_queue));
-
- /*
- * Store the kmsg and seqno where the receiver can pick it up.
- */
- receiver->ith_state = MACH_MSG_SUCCESS;
- receiver->ith_kmsg = kmsg;
- receiver->ith_seqno = dest_mqueue->imq_seqno++;
-
- /*
- * Inline thread_go_locked
- *
- * JMM - Including hacked in version of setrun scheduler op
- * that doesn't try to put thread on a runq.
- */
- {
- receiver->state &= ~(TH_WAIT|TH_UNINT);
- hw_atomic_add(&receiver->processor_set->run_count, 1);
- receiver->state |= TH_RUN;
- receiver->wait_result = THREAD_AWAKENED;
-
- receiver->computation_metered = 0;
- receiver->reason = AST_NONE;
- }
-
- thread_unlock(receiver);
-#if THREAD_SWAPPER
- act_unlock(rcv_act);
-#endif /* THREAD_SWAPPER */
-
- imq_unlock(dest_mqueue);
- ip_unlock(dest_port);
- current_task()->messages_sent++;
-
-
- /*
- * Put self on receive port's queue.
- * Also save state that the sender of
- * our reply message needs to determine if it
- * can hand off directly back to us.
- */
- self->ith_msg = (rcv_msg) ? rcv_msg : msg;
- self->ith_object = rcv_object; /* still holds reference */
- self->ith_msize = rcv_size;
- self->ith_option = option;
- self->ith_scatter_list_size = scatter_list_size;
- self->ith_continuation = thread_syscall_return;
-
- waitq = &rcv_mqueue->imq_wait_queue;
- (void)wait_queue_assert_wait64_locked(waitq,
- IPC_MQUEUE_RECEIVE,
- THREAD_ABORTSAFE,
- TRUE); /* unlock? */
- /* rcv_mqueue is unlocked */
-
- /*
- * Switch directly to receiving thread, and block
- * this thread as though it had called ipc_mqueue_receive.
- */
- thread_run(self, ipc_mqueue_receive_continue, receiver);
- /* NOTREACHED */
- }
-
- fast_copyout:
- /*
- * Nothing locked and no references held, except
- * we have kmsg with msgh_seqno filled in. Must
- * still check against rcv_size and do
- * ipc_kmsg_copyout/ipc_kmsg_put.
- */
-
- reply_size = send_size + trailer->msgh_trailer_size;
- if (rcv_size < reply_size) {
- HOT(c_mmot_g_slow_copyout6++);
- goto slow_copyout;
- }
-
- /* optimized ipc_kmsg_copyout/ipc_kmsg_copyout_header */
-
- switch (hdr->msgh_bits) {
- case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
- MACH_MSG_TYPE_PORT_SEND_ONCE): {
- ipc_port_t reply_port =
- (ipc_port_t) hdr->msgh_local_port;
- mach_port_name_t dest_name, reply_name;
-
- /* receiving a request message */
-
- if (!IP_VALID(reply_port)) {
- HOT(c_mmot_g_slow_copyout5++);
- goto slow_copyout;
- }
-
- is_write_lock(space);
- assert(space->is_active);
-
- /*
- * To do an atomic copyout, need simultaneous
- * locks on both ports and the space. If
- * dest_port == reply_port, and simple locking is
- * enabled, then we will abort. Otherwise it's
- * OK to unlock twice.
- */
-
- ip_lock(dest_port);
- if (!ip_active(dest_port) ||
- !ip_lock_try(reply_port)) {
- HOT(c_mmot_cold_037++);
- goto abort_request_copyout;
- }
-
- if (!ip_active(reply_port)) {
- ip_unlock(reply_port);
- HOT(c_mmot_cold_038++);
- goto abort_request_copyout;
- }
-
- assert(reply_port->ip_sorights > 0);
- ip_unlock(reply_port);
-
- {
- register ipc_entry_t table;
- register ipc_entry_t entry;
- register mach_port_index_t index;
-
- /* optimized ipc_entry_get */
-
- table = space->is_table;
- index = table->ie_next;
-
- if (index == 0) {
- HOT(c_mmot_cold_039++);
- goto abort_request_copyout;
- }
-
- entry = &table[index];
- table->ie_next = entry->ie_next;
- entry->ie_request = 0;
-
- {
- register mach_port_gen_t gen;
-
- assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
- gen = IE_BITS_NEW_GEN(entry->ie_bits);
-
- reply_name = MACH_PORT_MAKE(index, gen);
-
- /* optimized ipc_right_copyout */
-
- entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
- }
-
- assert(MACH_PORT_VALID(reply_name));
- entry->ie_object = (ipc_object_t) reply_port;
- is_write_unlock(space);
- }
-
- /* optimized ipc_object_copyout_dest */
-
- assert(dest_port->ip_srights > 0);
- ip_release(dest_port);
-
- if (dest_port->ip_receiver == space)
- dest_name = dest_port->ip_receiver_name;
- else
- dest_name = MACH_PORT_NULL;
-
- if ((--dest_port->ip_srights == 0) &&
- (dest_port->ip_nsrequest != IP_NULL)) {
- ipc_port_t nsrequest;
- mach_port_mscount_t mscount;
-
- /* a rather rare case */
-
- nsrequest = dest_port->ip_nsrequest;
- mscount = dest_port->ip_mscount;
- dest_port->ip_nsrequest = IP_NULL;
- ip_unlock(dest_port);
- ipc_notify_no_senders(nsrequest, mscount);
- } else
- ip_unlock(dest_port);
-
- hdr->msgh_bits =
- MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE,
- MACH_MSG_TYPE_PORT_SEND);
- hdr->msgh_remote_port = (mach_port_t)reply_name;
- hdr->msgh_local_port = (mach_port_t)dest_name;
- HOT(c_mmot_hot_ok1++);
- goto fast_put;
-
- abort_request_copyout:
- ip_unlock(dest_port);
- is_write_unlock(space);
- HOT(c_mmot_g_slow_copyout4++);
- goto slow_copyout;
- }
-
- case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
- register mach_port_name_t dest_name;
-
- /* receiving a reply message */
-
- ip_lock(dest_port);
- if (!ip_active(dest_port)) {
- ip_unlock(dest_port);
- HOT(c_mmot_g_slow_copyout3++);
- goto slow_copyout;
- }
-
- /* optimized ipc_object_copyout_dest */
-
- assert(dest_port->ip_sorights > 0);
-
- if (dest_port->ip_receiver == space) {
- ip_release(dest_port);
- dest_port->ip_sorights--;
- dest_name = dest_port->ip_receiver_name;
- ip_unlock(dest_port);
- } else {
- ip_unlock(dest_port);
-
- ipc_notify_send_once(dest_port);
- dest_name = MACH_PORT_NULL;
- }
-
- hdr->msgh_bits = MACH_MSGH_BITS(0,
- MACH_MSG_TYPE_PORT_SEND_ONCE);
- hdr->msgh_remote_port = MACH_PORT_NULL;
- hdr->msgh_local_port = (ipc_port_t)dest_name;
- HOT(c_mmot_hot_ok2++);
- goto fast_put;
- }
-
- case MACH_MSGH_BITS_COMPLEX|
- MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND_ONCE, 0): {
- register mach_port_name_t dest_name;
-
- /* receiving a complex reply message */
-
- ip_lock(dest_port);
- if (!ip_active(dest_port)) {
- ip_unlock(dest_port);
- HOT(c_mmot_g_slow_copyout1++);
- goto slow_copyout;
- }
-
- /* optimized ipc_object_copyout_dest */
-
- assert(dest_port->ip_sorights > 0);
-
- if (dest_port->ip_receiver == space) {
- ip_release(dest_port);
- dest_port->ip_sorights--;
- dest_name = dest_port->ip_receiver_name;
- ip_unlock(dest_port);
- } else {
- ip_unlock(dest_port);
projects / apple / xnu.git / blobdiff