+ ipc_port_multiple_lock(); /* massive serialization */
+
+ /*
+ * Search for the end of the chain (a port not in transit),
+ * acquiring locks along the way.
+ */
+
+ for (;;) {
+ ip_lock(base);
+
+ if (!ip_active(base) ||
+ (base->ip_receiver_name != MACH_PORT_NULL) ||
+ (base->ip_destination == IP_NULL)) {
+ break;
+ }
+
+ base = base->ip_destination;
+ }
+
+ /* all ports in chain from dest to base, inclusive, are locked */
+
+ if (port == base) {
+ /* circularity detected! */
+
+ ipc_port_multiple_unlock();
+
+ /* port (== base) is in limbo */
+
+ require_ip_active(port);
+ assert(port->ip_receiver_name == MACH_PORT_NULL);
+ assert(port->ip_destination == IP_NULL);
+
+ base = dest;
+ while (base != IP_NULL) {
+ ipc_port_t next;
+
+ /* base is in transit or in limbo */
+
+ require_ip_active(base);
+ assert(base->ip_receiver_name == MACH_PORT_NULL);
+
+ next = base->ip_destination;
+ ip_unlock(base);
+ base = next;
+ }
+
+ if (imp_lock_held) {
+ ipc_importance_unlock();
+ }
+
+ ipc_port_send_turnstile_complete(dest);
+ return TRUE;
+ }
+
+ /*
+ * The guarantee: lock port while the entire chain is locked.
+ * Once port is locked, we can take a reference to dest,
+ * add port to the chain, and unlock everything.
+ */
+
+ ip_lock(port);
+ ipc_port_multiple_unlock();
+
+not_circular:
+ /* port is in limbo */
+ imq_lock(&port->ip_messages);
+
+ require_ip_active(port);
+ assert(port->ip_receiver_name == MACH_PORT_NULL);
+ assert(port->ip_destination == IP_NULL);
+
+ /* Port is being enqueued in a kmsg, remove the watchport boost in order to push on destination port */
+ watchport_elem = ipc_port_clear_watchport_elem_internal(port);
+
+ /* Check if the port is being enqueued as a part of sync bootstrap checkin */
+ if (dest->ip_specialreply && dest->ip_sync_bootstrap_checkin) {
+ port->ip_sync_bootstrap_checkin = 1;
+ }
+
+ ip_reference(dest);
+ port->ip_destination = dest;
+
+ /* must have been in limbo or still bound to a task */
+ assert(port->ip_tempowner != 0);
+
+ /*
+ * We delayed dropping assertions from a specific task.
+ * Cache that info now (we'll drop assertions and the
+ * task reference below).
+ */
+ release_imp_task = port->ip_imp_task;
+ if (IIT_NULL != release_imp_task) {
+ port->ip_imp_task = IIT_NULL;
+ }
+ assertcnt = port->ip_impcount;
+
+ /* take the port out of limbo w.r.t. assertions */
+ port->ip_tempowner = 0;
+
+ /*
+ * Setup linkage for source port if it has a send turnstile i.e. it has
+ * a thread waiting in send or has a port enqueued in it or has sync ipc
+ * push from a special reply port.
+ */
+ if (port_send_turnstile(port)) {
+ send_turnstile = turnstile_prepare((uintptr_t)port,
+ port_send_turnstile_address(port),
+ TURNSTILE_NULL, TURNSTILE_SYNC_IPC);
+
+ turnstile_update_inheritor(send_turnstile, port_send_turnstile(dest),
+ (TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE));
+
+ /* update complete and turnstile complete called after dropping all locks */
+ }
+ imq_unlock(&port->ip_messages);
+
+ /* now unlock chain */
+
+ ip_unlock(port);
+
+ for (;;) {
+ ipc_port_t next;
+ /* every port along chain track assertions behind it */
+ ipc_port_impcount_delta(dest, assertcnt, base);
+
+ if (dest == base) {
+ break;
+ }
+
+ /* port is in transit */
+
+ require_ip_active(dest);
+ assert(dest->ip_receiver_name == MACH_PORT_NULL);
+ assert(dest->ip_destination != IP_NULL);
+ assert(dest->ip_tempowner == 0);
+
+ next = dest->ip_destination;
+ ip_unlock(dest);
+ dest = next;
+ }
+
+ /* base is not in transit */
+ assert(!ip_active(base) ||
+ (base->ip_receiver_name != MACH_PORT_NULL) ||
+ (base->ip_destination == IP_NULL));
+
+ /*
+ * Find the task to boost (if any).
+ * We will boost "through" ports that don't know
+ * about inheritance to deliver receive rights that
+ * do.
+ */
+ if (ip_active(base) && (assertcnt > 0)) {
+ assert(imp_lock_held);
+ if (base->ip_tempowner != 0) {
+ if (IIT_NULL != base->ip_imp_task) {
+ /* specified tempowner task */
+ imp_task = base->ip_imp_task;
+ assert(ipc_importance_task_is_any_receiver_type(imp_task));
+ }
+ /* otherwise don't boost current task */
+ } else if (base->ip_receiver_name != MACH_PORT_NULL) {
+ ipc_space_t space = base->ip_receiver;
+
+ /* only spaces with boost-accepting tasks */
+ if (space->is_task != TASK_NULL &&
+ ipc_importance_task_is_any_receiver_type(space->is_task->task_imp_base)) {
+ imp_task = space->is_task->task_imp_base;
+ }
+ }
+
+ /* take reference before unlocking base */
+ if (imp_task != IIT_NULL) {
+ ipc_importance_task_reference(imp_task);
+ }
+ }
+
+ ip_unlock(base);
+
+ /* All locks dropped, call turnstile_update_inheritor_complete for source port's turnstile */
+ if (send_turnstile) {
+ turnstile_update_inheritor_complete(send_turnstile, TURNSTILE_INTERLOCK_NOT_HELD);
+
+ /* Take the mq lock to call turnstile complete */
+ imq_lock(&port->ip_messages);
+ turnstile_complete((uintptr_t)port, port_send_turnstile_address(port), NULL, TURNSTILE_SYNC_IPC);
+ send_turnstile = TURNSTILE_NULL;
+ imq_unlock(&port->ip_messages);
+ turnstile_cleanup();
+ }
+
+ /*
+ * Transfer assertions now that the ports are unlocked.
+ * Avoid extra overhead if transferring to/from the same task.
+ *
+ * NOTE: If a transfer is occurring, the new assertions will
+ * be added to imp_task BEFORE the importance lock is unlocked.
+ * This is critical - to avoid decrements coming from the kmsgs
+ * beating the increment to the task.
+ */
+ boolean_t transfer_assertions = (imp_task != release_imp_task);
+
+ if (imp_task != IIT_NULL) {
+ assert(imp_lock_held);
+ if (transfer_assertions) {
+ ipc_importance_task_hold_internal_assertion_locked(imp_task, assertcnt);
+ }
+ }
+
+ if (release_imp_task != IIT_NULL) {
+ assert(imp_lock_held);
+ if (transfer_assertions) {
+ ipc_importance_task_drop_internal_assertion_locked(release_imp_task, assertcnt);
+ }
+ }
+
+ if (imp_lock_held) {
+ ipc_importance_unlock();
+ }
+
+ if (imp_task != IIT_NULL) {
+ ipc_importance_task_release(imp_task);
+ }
+
+ if (release_imp_task != IIT_NULL) {
+ ipc_importance_task_release(release_imp_task);
+ }
+
+ if (watchport_elem) {
+ task_watchport_elem_deallocate(watchport_elem);
+ }
+
+ return FALSE;
+}
+
+/*
+ * Routine: ipc_importance_send
+ * Purpose:
+ * Post the importance voucher attribute [if sent] or a static
+ * importance boost depending upon options and conditions.
+ * Conditions:
+ * Destination port locked on entry and exit, may be dropped during the call.
+ * Returns:
+ * A boolean identifying if the port lock was tempoarily dropped.
+ */
+boolean_t
+ipc_importance_send(
+ ipc_kmsg_t kmsg,
+ mach_msg_option_t option)
+{
+ ipc_port_t port = kmsg->ikm_header->msgh_remote_port;
+ boolean_t port_lock_dropped = FALSE;
+ ipc_importance_elem_t elem;
+ task_t task;
+ ipc_importance_task_t task_imp;
+ kern_return_t kr;
+
+ assert(IP_VALID(port));
+
+ /* If no donation to be made, return quickly */
+ if ((port->ip_impdonation == 0) ||
+ (option & MACH_SEND_NOIMPORTANCE) != 0) {
+ return port_lock_dropped;
+ }
+
+ task = current_task();
+
+ /* If forced sending a static boost, go update the port */
+ if ((option & MACH_SEND_IMPORTANCE) != 0) {
+ /* acquire the importance lock while trying to hang on to port lock */
+ if (!ipc_importance_lock_try()) {
+ port_lock_dropped = TRUE;
+ ip_unlock(port);
+ ipc_importance_lock();
+ }
+ goto portupdate;
+ }
+
+ task_imp = task->task_imp_base;
+ assert(IIT_NULL != task_imp);
+
+ /* If the sender can never donate importance, nothing to do */
+ if (ipc_importance_task_is_never_donor(task_imp)) {
+ return port_lock_dropped;
+ }
+
+ elem = IIE_NULL;
+
+ /* If importance receiver and passing a voucher, look for importance in there */
+ if (IP_VALID(kmsg->ikm_voucher) &&
+ ipc_importance_task_is_marked_receiver(task_imp)) {
+ mach_voucher_attr_value_handle_t vals[MACH_VOUCHER_ATTR_VALUE_MAX_NESTED];
+ mach_voucher_attr_value_handle_array_size_t val_count;
+ ipc_voucher_t voucher;
+
+ assert(ip_kotype(kmsg->ikm_voucher) == IKOT_VOUCHER);
+ voucher = (ipc_voucher_t)kmsg->ikm_voucher->ip_kobject;
+
+ /* check to see if the voucher has an importance attribute */
+ val_count = MACH_VOUCHER_ATTR_VALUE_MAX_NESTED;
+ kr = mach_voucher_attr_control_get_values(ipc_importance_control, voucher,
+ vals, &val_count);
+ assert(KERN_SUCCESS == kr);
+
+ /*
+ * Only use importance associated with our task (either directly
+ * or through an inherit that donates to our task).
+ */
+ if (0 < val_count) {
+ ipc_importance_elem_t check_elem;
+
+ check_elem = (ipc_importance_elem_t)vals[0];
+ assert(IIE_NULL != check_elem);
+ if (IIE_TYPE_INHERIT == IIE_TYPE(check_elem)) {
+ ipc_importance_inherit_t inherit;
+ inherit = (ipc_importance_inherit_t) check_elem;
+ if (inherit->iii_to_task == task_imp) {
+ elem = check_elem;
+ }
+ } else if (check_elem == (ipc_importance_elem_t)task_imp) {
+ elem = check_elem;
+ }
+ }
+ }
+
+ /* If we haven't found an importance attribute to send yet, use the task's */
+ if (IIE_NULL == elem) {
+ elem = (ipc_importance_elem_t)task_imp;
+ }
+
+ /* take a reference for the message to hold */
+ ipc_importance_reference_internal(elem);
+
+ /* acquire the importance lock while trying to hang on to port lock */
+ if (!ipc_importance_lock_try()) {
+ port_lock_dropped = TRUE;
+ ip_unlock(port);
+ ipc_importance_lock();
+ }
+
+ /* link kmsg onto the donor element propagation chain */
projects / apple / xnu.git / blobdiff