X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/9bccf70c0258c7cac2dcb80011b2a964d884c552..d26ffc64f583ab2d29df48f13518685602bc8832:/osfmk/ipc/ipc_port.c diff --git a/osfmk/ipc/ipc_port.c b/osfmk/ipc/ipc_port.c index 9bb7ed781..871f98f49 100644 --- a/osfmk/ipc/ipc_port.c +++ b/osfmk/ipc/ipc_port.c @@ -1,23 +1,29 @@ /* - * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2007 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * The contents of this file constitute Original Code as defined in and - * are subject to the Apple Public Source License Version 1.1 (the - * "License"). You may not use this file except in compliance with the - * License. Please obtain a copy of the License at - * http://www.apple.com/publicsource and read it before using this file. + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. * - * This Original Code and all software distributed under the License are - * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the - * License for the specific language governing rights and limitations - * under the License. + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_FREE_COPYRIGHT@ @@ -47,6 +53,12 @@ * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ +/* + * NOTICE: This file was modified by McAfee Research in 2004 to introduce + * support for mandatory and extensible security protections. This notice + * is included in support of clause 2.2 (b) of the Apple Public License, + * Version 2.0. + */ /* */ /* @@ -57,18 +69,18 @@ * Functions to manipulate IPC ports. */ -#include -#include #include #include #include #include -#include #include #include #include -#include +#include +#include +#include +#include #include #include #include @@ -77,31 +89,41 @@ #include #include #include -#include #include +#include +#include -#if MACH_KDB -#include -#include -#include -#endif /* MACH_KDB */ +#include #include -decl_mutex_data(, ipc_port_multiple_lock_data) -decl_mutex_data(, ipc_port_timestamp_lock_data) +decl_lck_spin_data(, ipc_port_multiple_lock_data) ipc_port_timestamp_t ipc_port_timestamp_data; +int ipc_portbt; #if MACH_ASSERT void ipc_port_init_debug( - ipc_port_t port); + ipc_port_t port, + uintptr_t *callstack, + unsigned int callstack_max); + +void ipc_port_callstack_init_debug( + uintptr_t *callstack, + unsigned int callstack_max); + #endif /* MACH_ASSERT */ -#if MACH_KDB && ZONE_DEBUG -/* Forwards */ -void print_type_ports(unsigned, unsigned); -void print_ports(void); -#endif /* MACH_KDB && ZONE_DEBUG */ +void +ipc_port_release(ipc_port_t port) +{ + ip_release(port); +} + +void +ipc_port_reference(ipc_port_t port) +{ + ip_reference(port); +} /* * Routine: ipc_port_timestamp @@ -112,19 +134,13 @@ void print_ports(void); ipc_port_timestamp_t ipc_port_timestamp(void) { - ipc_port_timestamp_t timestamp; - - ipc_port_timestamp_lock(); - timestamp = ipc_port_timestamp_data++; - ipc_port_timestamp_unlock(); - - return timestamp; + return OSIncrementAtomic(&ipc_port_timestamp_data); } /* - * Routine: ipc_port_dnrequest + * Routine: ipc_port_request_alloc * Purpose: - * Try to allocate a dead-name request slot. + * Try to allocate a request slot. * If successful, returns the request index. * Otherwise returns zero. * Conditions: @@ -134,21 +150,41 @@ ipc_port_timestamp(void) * KERN_NO_SPACE No index allocated. */ +#if IMPORTANCE_INHERITANCE kern_return_t -ipc_port_dnrequest( +ipc_port_request_alloc( ipc_port_t port, mach_port_name_t name, ipc_port_t soright, + boolean_t send_possible, + boolean_t immediate, + ipc_port_request_index_t *indexp, + boolean_t *importantp) +#else +kern_return_t +ipc_port_request_alloc( + ipc_port_t port, + mach_port_name_t name, + ipc_port_t soright, + boolean_t send_possible, + boolean_t immediate, ipc_port_request_index_t *indexp) +#endif /* IMPORTANCE_INHERITANCE */ { ipc_port_request_t ipr, table; ipc_port_request_index_t index; + uintptr_t mask = 0; + +#if IMPORTANCE_INHERITANCE + *importantp = FALSE; +#endif /* IMPORTANCE_INHERITANCE */ assert(ip_active(port)); assert(name != MACH_PORT_NULL); assert(soright != IP_NULL); - table = port->ip_dnrequests; + table = port->ip_requests; + if (table == IPR_NULL) return KERN_NO_SPACE; @@ -161,16 +197,35 @@ ipc_port_dnrequest( table->ipr_next = ipr->ipr_next; ipr->ipr_name = name; - ipr->ipr_soright = soright; + + if (send_possible) { + mask |= IPR_SOR_SPREQ_MASK; + if (immediate) { + mask |= IPR_SOR_SPARM_MASK; + if (port->ip_sprequests == 0) { + port->ip_sprequests = 1; +#if IMPORTANCE_INHERITANCE + /* TODO: Live importance support in send-possible */ + if (port->ip_impdonation != 0 && + port->ip_spimportant == 0 && + (task_is_importance_donor(current_task()))) { + *importantp = TRUE; + } +#endif /* IMPORTANCE_INHERTANCE */ + } + } + } + ipr->ipr_soright = IPR_SOR_MAKE(soright, mask); *indexp = index; + return KERN_SUCCESS; } /* - * Routine: ipc_port_dngrow + * Routine: ipc_port_request_grow * Purpose: - * Grow a port's table of dead-name requests. + * Grow a port's table of requests. * Conditions: * The port must be locked and active. * Nothing else locked; will allocate memory. @@ -184,18 +239,18 @@ ipc_port_dnrequest( */ kern_return_t -ipc_port_dngrow( - ipc_port_t port, - int target_size) +ipc_port_request_grow( + ipc_port_t port, + ipc_table_elems_t target_size) { ipc_table_size_t its; ipc_port_request_t otable, ntable; assert(ip_active(port)); - otable = port->ip_dnrequests; + otable = port->ip_requests; if (otable == IPR_NULL) - its = &ipc_table_dnrequests[0]; + its = &ipc_table_requests[0]; else its = otable->ipr_size + 1; @@ -218,23 +273,21 @@ ipc_port_dngrow( ip_unlock(port); if ((its->its_size == 0) || - ((ntable = it_dnrequests_alloc(its)) == IPR_NULL)) { - ipc_port_release(port); + ((ntable = it_requests_alloc(its)) == IPR_NULL)) { + ip_release(port); return KERN_RESOURCE_SHORTAGE; } ip_lock(port); - ip_release(port); /* * Check that port is still active and that nobody else * has slipped in and grown the table on us. Note that - * just checking port->ip_dnrequests == otable isn't - * sufficient; must check ipr_size. + * just checking if the current table pointer == otable + * isn't sufficient; must check ipr_size. */ - if (ip_active(port) && - (port->ip_dnrequests == otable) && + if (ip_active(port) && (port->ip_requests == otable) && ((otable == IPR_NULL) || (otable->ipr_size+1 == its))) { ipc_table_size_t oits; ipc_table_elems_t osize, nsize; @@ -252,6 +305,7 @@ ipc_port_dngrow( (osize - 1) * sizeof(struct ipc_port_request)); } else { osize = 1; + oits = 0; free = 0; } @@ -270,55 +324,149 @@ ipc_port_dngrow( ntable->ipr_next = free; ntable->ipr_size = its; - port->ip_dnrequests = ntable; + port->ip_requests = ntable; ip_unlock(port); + ip_release(port); if (otable != IPR_NULL) { - it_dnrequests_free(oits, otable); + it_requests_free(oits, otable); } } else { - ip_check_unlock(port); - it_dnrequests_free(its, ntable); + ip_unlock(port); + ip_release(port); + it_requests_free(its, ntable); } return KERN_SUCCESS; } /* - * Routine: ipc_port_dncancel + * Routine: ipc_port_request_sparm + * Purpose: + * Arm delayed send-possible request. + * Conditions: + * The port must be locked and active. + * + * Returns TRUE if the request was armed + * (or armed with importance in that version). + */ + +boolean_t +ipc_port_request_sparm( + ipc_port_t port, + __assert_only mach_port_name_t name, + ipc_port_request_index_t index, + mach_msg_option_t option, + mach_msg_priority_t override) +{ + if (index != IE_REQ_NONE) { + ipc_port_request_t ipr, table; + + assert(ip_active(port)); + + table = port->ip_requests; + assert(table != IPR_NULL); + + ipr = &table[index]; + assert(ipr->ipr_name == name); + + /* Is there a valid destination? */ + if (IPR_SOR_SPREQ(ipr->ipr_soright)) { + ipr->ipr_soright = IPR_SOR_MAKE(ipr->ipr_soright, IPR_SOR_SPARM_MASK); + port->ip_sprequests = 1; + + if (option & MACH_SEND_OVERRIDE) { + /* apply override to message queue */ + ipc_mqueue_override_send(&port->ip_messages, override); + } + +#if IMPORTANCE_INHERITANCE + if (((option & MACH_SEND_NOIMPORTANCE) == 0) && + (port->ip_impdonation != 0) && + (port->ip_spimportant == 0) && + (((option & MACH_SEND_IMPORTANCE) != 0) || + (task_is_importance_donor(current_task())))) { + return TRUE; + } +#else + return TRUE; +#endif /* IMPORTANCE_INHERITANCE */ + } + } + return FALSE; +} + +/* + * Routine: ipc_port_request_type + * Purpose: + * Determine the type(s) of port requests enabled for a name. + * Conditions: + * The port must be locked or inactive (to avoid table growth). + * The index must not be IE_REQ_NONE and for the name in question. + */ +mach_port_type_t +ipc_port_request_type( + ipc_port_t port, + __assert_only mach_port_name_t name, + ipc_port_request_index_t index) +{ + ipc_port_request_t ipr, table; + mach_port_type_t type = 0; + + table = port->ip_requests; + assert (table != IPR_NULL); + + assert(index != IE_REQ_NONE); + ipr = &table[index]; + assert(ipr->ipr_name == name); + + if (IP_VALID(IPR_SOR_PORT(ipr->ipr_soright))) { + type |= MACH_PORT_TYPE_DNREQUEST; + + if (IPR_SOR_SPREQ(ipr->ipr_soright)) { + type |= MACH_PORT_TYPE_SPREQUEST; + + if (!IPR_SOR_SPARMED(ipr->ipr_soright)) { + type |= MACH_PORT_TYPE_SPREQUEST_DELAYED; + } + } + } + return type; +} + +/* + * Routine: ipc_port_request_cancel * Purpose: - * Cancel a dead-name request and return the send-once right. + * Cancel a dead-name/send-possible request and return the send-once right. * Conditions: - * The port must locked and active. + * The port must be locked and active. + * The index must not be IPR_REQ_NONE and must correspond with name. */ ipc_port_t -ipc_port_dncancel( +ipc_port_request_cancel( ipc_port_t port, - mach_port_name_t name, + __assert_only mach_port_name_t name, ipc_port_request_index_t index) { ipc_port_request_t ipr, table; - ipc_port_t dnrequest; + ipc_port_t request = IP_NULL; assert(ip_active(port)); - assert(name != MACH_PORT_NULL); - assert(index != 0); - - table = port->ip_dnrequests; + table = port->ip_requests; assert(table != IPR_NULL); + assert (index != IE_REQ_NONE); ipr = &table[index]; - dnrequest = ipr->ipr_soright; assert(ipr->ipr_name == name); + request = IPR_SOR_PORT(ipr->ipr_soright); /* return ipr to the free list inside the table */ - ipr->ipr_name = MACH_PORT_NULL; ipr->ipr_next = table->ipr_next; table->ipr_next = index; - return dnrequest; + return request; } /* @@ -394,38 +542,61 @@ ipc_port_nsrequest( /* * Routine: ipc_port_clear_receiver * Purpose: - * Prepares a receive right for transmission/destruction. + * Prepares a receive right for transmission/destruction, + * optionally performs mqueue destruction (with port lock held) + * * Conditions: * The port is locked and active. + * Returns: + * If should_destroy is TRUE, then the return value indicates + * whether the caller needs to reap kmsg structures that should + * be destroyed (by calling ipc_kmsg_reap_delayed) + * + * If should_destroy is FALSE, this always returns FALSE */ -void +boolean_t ipc_port_clear_receiver( - ipc_port_t port) + ipc_port_t port, + boolean_t should_destroy) { - spl_t s; - - assert(ip_active(port)); + ipc_mqueue_t mqueue = &port->ip_messages; + boolean_t reap_messages = FALSE; /* - * pull ourselves from any sets. + * Pull ourselves out of any sets to which we belong. + * We hold the port locked, so even though this acquires and releases + * the mqueue lock, we know we won't be added to any other sets. */ - if (port->ip_pset_count != 0) { + if (port->ip_in_pset != 0) { ipc_pset_remove_from_all(port); - assert(port->ip_pset_count == 0); + assert(port->ip_in_pset == 0); } /* * Send anyone waiting on the port's queue directly away. * Also clear the mscount and seqno. */ - s = splsched(); - imq_lock(&port->ip_messages); - ipc_mqueue_changed(&port->ip_messages); - ipc_port_set_mscount(port, 0); - port->ip_messages.imq_seqno = 0; + imq_lock(mqueue); + ipc_mqueue_changed(mqueue); + port->ip_mscount = 0; + mqueue->imq_seqno = 0; + port->ip_context = port->ip_guarded = port->ip_strict_guard = 0; + + if (should_destroy) { + /* + * Mark the mqueue invalid, preventing further send/receive + * operations from succeeding. It's important for this to be + * done under the same lock hold as the ipc_mqueue_changed + * call to avoid additional threads blocking on an mqueue + * that's being destroyed. + */ + reap_messages = ipc_mqueue_destroy_locked(mqueue); + } + imq_unlock(&port->ip_messages); - splx(s); + + return reap_messages; } /* @@ -452,16 +623,25 @@ ipc_port_init( port->ip_nsrequest = IP_NULL; port->ip_pdrequest = IP_NULL; - port->ip_dnrequests = IPR_NULL; + port->ip_requests = IPR_NULL; - port->ip_pset_count = 0; port->ip_premsg = IKM_NULL; + port->ip_context = 0; -#if MACH_ASSERT - ipc_port_init_debug(port); -#endif /* MACH_ASSERT */ + port->ip_sprequests = 0; + port->ip_spimportant = 0; + port->ip_impdonation = 0; + port->ip_tempowner = 0; - ipc_mqueue_init(&port->ip_messages, FALSE /* set */); + port->ip_guarded = 0; + port->ip_strict_guard = 0; + port->ip_impcount = 0; + + port->ip_specialreply = 0; + port->ip_link_sync_qos = 0; + + ipc_mqueue_init(&port->ip_messages, + FALSE /* !set */, NULL /* no reserved link */); } /* @@ -488,16 +668,27 @@ ipc_port_alloc( mach_port_name_t name; kern_return_t kr; +#if MACH_ASSERT + uintptr_t buf[IP_CALLSTACK_MAX]; + ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + kr = ipc_object_alloc(space, IOT_PORT, MACH_PORT_TYPE_RECEIVE, 0, &name, (ipc_object_t *) &port); if (kr != KERN_SUCCESS) return kr; - /* port is locked */ - + /* port and space are locked */ ipc_port_init(port, space, name); +#if MACH_ASSERT + ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + + /* unlock space after init */ + is_write_unlock(space); + *namep = name; *portp = port; @@ -527,6 +718,11 @@ ipc_port_alloc_name( ipc_port_t port; kern_return_t kr; +#if MACH_ASSERT + uintptr_t buf[IP_CALLSTACK_MAX]; + ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + kr = ipc_object_alloc_name(space, IOT_PORT, MACH_PORT_TYPE_RECEIVE, 0, name, (ipc_object_t *) &port); @@ -537,43 +733,126 @@ ipc_port_alloc_name( ipc_port_init(port, space, name); +#if MACH_ASSERT + ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + *portp = port; return KERN_SUCCESS; } /* - * Generate dead name notifications. Called from ipc_port_destroy. - * Port is unlocked but still has reference(s); - * dnrequests was taken from port while the port - * was locked but the port now has port->ip_dnrequests set to IPR_NULL. + * Routine: ipc_port_spnotify + * Purpose: + * Generate send-possible port notifications. + * Conditions: + * Nothing locked, reference held on port. */ void -ipc_port_dnnotify( - ipc_port_t port, - ipc_port_request_t dnrequests) +ipc_port_spnotify( + ipc_port_t port) { - ipc_table_size_t its = dnrequests->ipr_size; - ipc_table_elems_t size = its->its_size; - ipc_port_request_index_t index; + ipc_port_request_index_t index = 0; + ipc_table_elems_t size = 0; + + /* + * If the port has no send-possible request + * armed, don't bother to lock the port. + */ + if (port->ip_sprequests == 0) + return; + + ip_lock(port); + +#if IMPORTANCE_INHERITANCE + if (port->ip_spimportant != 0) { + port->ip_spimportant = 0; + if (ipc_port_importance_delta(port, IPID_OPTION_NORMAL, -1) == TRUE) { + ip_lock(port); + } + } +#endif /* IMPORTANCE_INHERITANCE */ + + if (port->ip_sprequests == 0) { + ip_unlock(port); + return; + } + port->ip_sprequests = 0; + +revalidate: + if (ip_active(port)) { + ipc_port_request_t requests; + + /* table may change each time port unlocked (reload) */ + requests = port->ip_requests; + assert(requests != IPR_NULL); + + /* + * no need to go beyond table size when first + * we entered - those are future notifications. + */ + if (size == 0) + size = requests->ipr_size->its_size; - for (index = 1; index < size; index++) { - ipc_port_request_t ipr = &dnrequests[index]; - mach_port_name_t name = ipr->ipr_name; - ipc_port_t soright; + /* no need to backtrack either */ + while (++index < size) { + ipc_port_request_t ipr = &requests[index]; + mach_port_name_t name = ipr->ipr_name; + ipc_port_t soright = IPR_SOR_PORT(ipr->ipr_soright); + boolean_t armed = IPR_SOR_SPARMED(ipr->ipr_soright); - if (name == MACH_PORT_NULL) - continue; + if (MACH_PORT_VALID(name) && armed && IP_VALID(soright)) { + /* claim send-once right - slot still inuse */ + ipr->ipr_soright = IP_NULL; + ip_unlock(port); - soright = ipr->ipr_soright; - assert(soright != IP_NULL); + ipc_notify_send_possible(soright, name); - ipc_notify_dead_name(soright, name); + ip_lock(port); + goto revalidate; + } + } } + ip_unlock(port); + return; +} - it_dnrequests_free(its, dnrequests); +/* + * Routine: ipc_port_dnnotify + * Purpose: + * Generate dead name notifications for + * all outstanding dead-name and send- + * possible requests. + * Conditions: + * Nothing locked. + * Port must be inactive. + * Reference held on port. + */ +void +ipc_port_dnnotify( + ipc_port_t port) +{ + ipc_port_request_t requests = port->ip_requests; + + assert(!ip_active(port)); + if (requests != IPR_NULL) { + ipc_table_size_t its = requests->ipr_size; + ipc_table_elems_t size = its->its_size; + ipc_port_request_index_t index; + for (index = 1; index < size; index++) { + ipc_port_request_t ipr = &requests[index]; + mach_port_name_t name = ipr->ipr_name; + ipc_port_t soright = IPR_SOR_PORT(ipr->ipr_soright); + + if (MACH_PORT_VALID(name) && IP_VALID(soright)) { + ipc_notify_dead_name(soright, name); + } + } + } } + /* * Routine: ipc_port_destroy * Purpose: @@ -588,25 +867,54 @@ ipc_port_dnnotify( */ void -ipc_port_destroy( - ipc_port_t port) +ipc_port_destroy(ipc_port_t port) { ipc_port_t pdrequest, nsrequest; ipc_mqueue_t mqueue; - ipc_kmsg_queue_t kmqueue; ipc_kmsg_t kmsg; - ipc_port_request_t dnrequests; + boolean_t special_reply = port->ip_specialreply; + +#if IMPORTANCE_INHERITANCE + ipc_importance_task_t release_imp_task = IIT_NULL; + thread_t self = current_thread(); + boolean_t top = (self->ith_assertions == 0); + natural_t assertcnt = 0; +#endif /* IMPORTANCE_INHERITANCE */ assert(ip_active(port)); /* port->ip_receiver_name is garbage */ /* port->ip_receiver/port->ip_destination is garbage */ - assert(port->ip_pset_count == 0); - assert(port->ip_mscount == 0); - - /* first check for a backup port */ + /* check for a backup port */ pdrequest = port->ip_pdrequest; + +#if IMPORTANCE_INHERITANCE + /* determine how many assertions to drop and from whom */ + if (port->ip_tempowner != 0) { + assert(top); + release_imp_task = port->ip_imp_task; + if (IIT_NULL != release_imp_task) { + port->ip_imp_task = IIT_NULL; + assertcnt = port->ip_impcount; + } + /* Otherwise, nothing to drop */ + } else { + assertcnt = port->ip_impcount; + if (pdrequest != IP_NULL) + /* mark in limbo for the journey */ + port->ip_tempowner = 1; + } + + if (top) + self->ith_assertions = assertcnt; +#endif /* IMPORTANCE_INHERITANCE */ + if (pdrequest != IP_NULL) { + /* clear receiver, don't destroy the port */ + (void)ipc_port_clear_receiver(port, FALSE); + assert(port->ip_in_pset == 0); + assert(port->ip_mscount == 0); + /* we assume the ref for pdrequest */ port->ip_pdrequest = IP_NULL; @@ -615,68 +923,110 @@ ipc_port_destroy( port->ip_destination = IP_NULL; ip_unlock(port); - if (!ipc_port_check_circularity(port, pdrequest)) { - /* consumes our refs for port and pdrequest */ - ipc_notify_port_destroyed(pdrequest, port); - return; - } else { - /* consume pdrequest and destroy port */ - ipc_port_release_sonce(pdrequest); + if (special_reply) { + ipc_port_unlink_special_reply_port(port, + IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE); } + /* consumes our refs for port and pdrequest */ + ipc_notify_port_destroyed(pdrequest, port); - ip_lock(port); - assert(ip_active(port)); - assert(port->ip_pset_count == 0); - assert(port->ip_mscount == 0); - assert(port->ip_pdrequest == IP_NULL); - assert(port->ip_receiver_name == MACH_PORT_NULL); - assert(port->ip_destination == IP_NULL); - - /* fall through and destroy the port */ + goto drop_assertions; } - /* once port is dead, we don't need to keep it locked */ - port->ip_object.io_bits &= ~IO_BITS_ACTIVE; port->ip_timestamp = ipc_port_timestamp(); + nsrequest = port->ip_nsrequest; - /* save for later */ - dnrequests = port->ip_dnrequests; - port->ip_dnrequests = IPR_NULL; + /* + * The mach_msg_* paths don't hold a port lock, they only hold a + * reference to the port object. If a thread raced us and is now + * blocked waiting for message reception on this mqueue (or waiting + * for ipc_mqueue_full), it will never be woken up. We call + * ipc_port_clear_receiver() here, _after_ the port has been marked + * inactive, to wakeup any threads which may be blocked and ensure + * that no other thread can get lost waiting for a wake up on a + * port/mqueue that's been destroyed. + */ + boolean_t reap_msgs = FALSE; + reap_msgs = ipc_port_clear_receiver(port, TRUE); /* marks mqueue inactive */ + assert(port->ip_in_pset == 0); + assert(port->ip_mscount == 0); /* * If the port has a preallocated message buffer and that buffer * is not inuse, free it. If it has an inuse one, then the kmsg * free will detect that we freed the association and it can free it * like a normal buffer. + * + * Once the port is marked inactive we don't need to keep it locked. */ if (IP_PREALLOC(port)) { + ipc_port_t inuse_port; + kmsg = port->ip_premsg; assert(kmsg != IKM_NULL); + inuse_port = ikm_prealloc_inuse_port(kmsg); IP_CLEAR_PREALLOC(port, kmsg); - if (!ikm_prealloc_inuse(kmsg)) + ip_unlock(port); + if (inuse_port != IP_NULL) { + assert(inuse_port == port); + } else { ipc_kmsg_free(kmsg); + } + } else { + ip_unlock(port); } - ip_unlock(port); - /* throw away no-senders request */ + /* unlink the kmsg from special reply port */ + if (special_reply) { + ipc_port_unlink_special_reply_port(port, + IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE); + } - nsrequest = port->ip_nsrequest; + /* throw away no-senders request */ if (nsrequest != IP_NULL) ipc_notify_send_once(nsrequest); /* consumes ref */ - /* destroy any queued messages */ + /* + * Reap any kmsg objects waiting to be destroyed. + * This must be done after we've released the port lock. + */ + if (reap_msgs) + ipc_kmsg_reap_delayed(); + mqueue = &port->ip_messages; - ipc_mqueue_destroy(mqueue); + + /* cleanup waitq related resources */ + ipc_mqueue_deinit(mqueue); /* generate dead-name notifications */ - if (dnrequests != IPR_NULL) { - ipc_port_dnnotify(port, dnrequests); - } + ipc_port_dnnotify(port); ipc_kobject_destroy(port); - ipc_port_release(port); /* consume caller's ref */ + ip_release(port); /* consume caller's ref */ + + drop_assertions: +#if IMPORTANCE_INHERITANCE + if (release_imp_task != IIT_NULL) { + if (assertcnt > 0) { + assert(top); + self->ith_assertions = 0; + assert(ipc_importance_task_is_any_receiver_type(release_imp_task)); + ipc_importance_task_drop_internal_assertion(release_imp_task, assertcnt); + } + ipc_importance_task_release(release_imp_task); + + } else if (assertcnt > 0) { + if (top) { + self->ith_assertions = 0; + release_imp_task = current_task()->task_imp_base; + if (ipc_importance_task_is_any_receiver_type(release_imp_task)) { + ipc_importance_task_drop_internal_assertion(release_imp_task, assertcnt); + } + } + } +#endif /* IMPORTANCE_INHERITANCE */ } /* @@ -691,6 +1041,7 @@ ipc_port_destroy( * That is, we want to set port->ip_destination == dest, * but guaranteeing that this doesn't create a circle * port->ip_destination->ip_destination->... == port + * * Conditions: * No ports locked. References held for "port" and "dest". */ @@ -700,7 +1051,14 @@ ipc_port_check_circularity( ipc_port_t port, ipc_port_t dest) { +#if IMPORTANCE_INHERITANCE + /* adjust importance counts at the same time */ + return ipc_importance_check_circularity(port, dest); +#else ipc_port_t base; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + boolean_t update_knote = FALSE; assert(port != IP_NULL); assert(dest != IP_NULL); @@ -713,7 +1071,6 @@ ipc_port_check_circularity( * First try a quick check that can run in parallel. * No circularity if dest is not in transit. */ - ip_lock(port); if (ip_lock_try(dest)) { if (!ip_active(dest) || @@ -783,7 +1140,8 @@ ipc_port_check_circularity( ip_lock(port); ipc_port_multiple_unlock(); - not_circular: +not_circular: + imq_lock(&base->ip_messages); /* port is in limbo */ @@ -794,80 +1152,796 @@ ipc_port_check_circularity( ip_reference(dest); port->ip_destination = dest; + /* Capture the sync qos count delta */ + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_delta_add[i] = port_sync_qos(port, i); + } + /* now unlock chain */ - while (port != base) { - ipc_port_t next; + ip_unlock(port); + + for (;;) { + /* every port along chain tracks override behind it */ + update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub); + if (dest == base) + break; /* port is in transit */ - assert(ip_active(port)); - assert(port->ip_receiver_name == MACH_PORT_NULL); - assert(port->ip_destination != IP_NULL); + assert(ip_active(dest)); + assert(dest->ip_receiver_name == MACH_PORT_NULL); + assert(dest->ip_destination != IP_NULL); - next = port->ip_destination; - ip_unlock(port); - port = next; + port = dest->ip_destination; + ip_unlock(dest); + dest = port; } /* base is not in transit */ - assert(!ip_active(base) || (base->ip_receiver_name != MACH_PORT_NULL) || (base->ip_destination == IP_NULL)); + + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); return FALSE; +#endif /* !IMPORTANCE_INHERITANCE */ } /* - * Routine: ipc_port_lookup_notify + * Routine: ipc_port_link_special_reply_port_with_qos * Purpose: - * Make a send-once notify port from a receive right. - * Returns IP_NULL if name doesn't denote a receive right. + * Link the special reply port with the destination port. + * Update the sync qos count of special reply port, + * destination port. + * * Conditions: - * The space must be locked (read or write) and active. - * Being the active space, we can rely on thread server_id - * context to give us the proper server level sub-order - * within the space. + * Nothing is locked. */ - -ipc_port_t -ipc_port_lookup_notify( - ipc_space_t space, - mach_port_name_t name) +kern_return_t +ipc_port_link_special_reply_port_with_qos( + ipc_port_t special_reply_port, + ipc_port_t dest_port, + int qos) { - ipc_port_t port; - ipc_entry_t entry; + ipc_port_t next, base; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + boolean_t update_knote = FALSE; + boolean_t multiple_lock = FALSE; + + ip_lock(dest_port); + + /* Check if dest is active */ + if (!ip_active(dest_port)) { + ip_unlock(dest_port); + return KERN_FAILURE; + } - assert(space->is_active); + if ((dest_port->ip_receiver_name == MACH_PORT_NULL) && + (dest_port->ip_destination != IP_NULL)) { + /* dest_port is in transit; need to take the serialize lock */ + ip_unlock(dest_port); + goto take_multiple_lock; + } - entry = ipc_entry_lookup(space, name); - if (entry == IE_NULL) - return IP_NULL; - if ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0) - return IP_NULL; + /* Check if the port is a special reply port */ + if (ip_lock_try(special_reply_port)) { + if (!special_reply_port->ip_specialreply || + !special_reply_port->ip_link_sync_qos || + (special_reply_port->ip_sync_qos_override_port != IP_NULL && + special_reply_port->ip_sync_qos_override_port != dest_port)) { + + boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos; + ip_unlock(special_reply_port); + ip_unlock(dest_port); + /* return KERN_SUCCESS when link_sync_qos is not set */ + if (!link_sync_qos) { + return KERN_SUCCESS; + } + return KERN_FAILURE; + } else { + goto both_ports_locked; + } + } - port = (ipc_port_t) entry->ie_object; - assert(port != IP_NULL); + ip_unlock(dest_port); - ip_lock(port); - assert(ip_active(port)); - assert(port->ip_receiver_name == name); - assert(port->ip_receiver == space); +take_multiple_lock: - ip_reference(port); - port->ip_sorights++; - ip_unlock(port); + ipc_port_multiple_lock(); /* massive serialization */ + multiple_lock = TRUE; - return port; -} + ip_lock(special_reply_port); -/* - * Routine: ipc_port_make_send_locked - * Purpose: - * Make a naked send right from a receive right. - * + /* Check if the special reply port is marked regular */ + if (!special_reply_port->ip_specialreply || + !special_reply_port->ip_link_sync_qos || + (special_reply_port->ip_sync_qos_override_port != IP_NULL && + special_reply_port->ip_sync_qos_override_port != dest_port)) { + + boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos; + ip_unlock(special_reply_port); + ipc_port_multiple_unlock(); + /* return KERN_SUCCESS when link_sync_qos is not set */ + if (!link_sync_qos) { + return KERN_SUCCESS; + } + return KERN_FAILURE; + } + + ip_lock(dest_port); + +both_ports_locked: + next = dest_port; + + /* Apply the qos to special reply port, capture the old qos */ + if (special_reply_port->ip_sync_qos_override_port != IP_NULL) { + /* Check if qos needs to be updated */ + if ((sync_qos_count_t)qos <= port_special_qos(special_reply_port)) { + imq_lock(&dest_port->ip_messages); + goto done_update; + } + sync_qos_delta_sub[port_special_qos(special_reply_port)]++; + } + + set_port_special_qos(special_reply_port, (sync_qos_count_t)qos); + sync_qos_delta_add[qos]++; + + /* Link the special reply port to dest port */ + if (special_reply_port->ip_sync_qos_override_port == IP_NULL) { + /* take a reference on dest_port */ + ip_reference(dest_port); + special_reply_port->ip_sync_qos_override_port = dest_port; + } + + /* Apply the sync qos delta to all in-transit ports */ + for (;;) { + boolean_t port_not_in_transit = FALSE; + if (!ip_active(next) || + (next->ip_receiver_name != MACH_PORT_NULL) || + (next->ip_destination == IP_NULL)) { + /* Get the mqueue lock for destination port to update knotes */ + imq_lock(&next->ip_messages); + port_not_in_transit = TRUE; + } + /* Apply the sync qos delta */ + update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub); + + if (port_not_in_transit) + break; + + next = next->ip_destination; + ip_lock(next); + } +done_update: + + if (multiple_lock) { + ipc_port_multiple_unlock(); + } + + ip_unlock(special_reply_port); + base = next; + next = dest_port; + + while (next != base) { + ipc_port_t prev = next; + next = next->ip_destination; + + ip_unlock(prev); + } + + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); + return KERN_SUCCESS; +} + +/* + * Routine: ipc_port_unlink_special_reply_port_locked + * Purpose: + * If the special port is linked to a port, adjust it's sync qos override and unlink the port. + * Condition: + * Special reply port locked on entry. + * Special reply port unlocked on return. + * Returns: + * None. + */ +void +ipc_port_unlink_special_reply_port_locked( + ipc_port_t special_reply_port, + struct knote *kn, + uint8_t flags) +{ + ipc_port_t dest_port; + sync_qos_count_t sync_qos; + sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0}; + sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0}; + + /* Return if called from copy out in pseudo receive */ + if (kn == ITH_KNOTE_PSEUDO) { + ip_unlock(special_reply_port); + return; + } + + /* check if special port has a port linked to it */ + if (special_reply_port->ip_specialreply == 0 || + special_reply_port->ip_sync_qos_override_port == IP_NULL) { + set_port_special_qos(special_reply_port, 0); + if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) { + special_reply_port->ip_specialreply = 0; + } + if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) { + special_reply_port->ip_link_sync_qos = 1; + } + ip_unlock(special_reply_port); + return; + } + + /* + * port->ip_sync_qos_override_port is not null and it is safe + * to access it since ip_specialreply is set. + */ + dest_port = special_reply_port->ip_sync_qos_override_port; + sync_qos_delta_sub[port_special_qos(special_reply_port)]++; + sync_qos = port_special_qos(special_reply_port); + + /* Clear qos delta for special reply port */ + set_port_special_qos(special_reply_port, 0); + special_reply_port->ip_sync_qos_override_port = IP_NULL; + if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) { + special_reply_port->ip_specialreply = 0; + } + + if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) { + special_reply_port->ip_link_sync_qos = 1; + } else { + special_reply_port->ip_link_sync_qos = 0; + } + + ip_unlock(special_reply_port); + + /* Add the sync qos on knote */ + if (ITH_KNOTE_VALID(kn)) { + knote_adjust_sync_qos(kn, sync_qos, TRUE); + } + + /* Adjust the sync qos of destination */ + ipc_port_adjust_sync_qos(dest_port, sync_qos_delta_add, sync_qos_delta_sub); + ip_release(dest_port); +} + +/* + * Routine: ipc_port_unlink_special_reply_port + * Purpose: + * If the special port is linked to a port, adjust it's sync qos override and unlink the port. + * Condition: + * Nothing locked. + * Returns: + * None. + */ +void +ipc_port_unlink_special_reply_port( + ipc_port_t special_reply_port, + uint8_t flags) +{ + ip_lock(special_reply_port); + ipc_port_unlink_special_reply_port_locked(special_reply_port, NULL, flags); + /* special_reply_port unlocked */ +} + +/* + * Routine: ipc_port_sync_qos_delta + * Purpose: + * Adjust the sync qos count associated with a port. + * + * For now, be defensive during deductions to make sure the + * sync_qos count for the port doesn't underflow zero. + * Returns: + * TRUE: if max sync qos of the port changes. + * FALSE: otherwise. + * Conditions: + * The port is referenced and locked. + * The mqueue is locked if port is not in-transit. + */ +boolean_t +ipc_port_sync_qos_delta( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub) +{ + sync_qos_count_t max_sync_qos_index; + + if (!ip_active(port)) { + return FALSE; + } + + max_sync_qos_index = ipc_port_get_max_sync_qos_index(port); + + for (int i = 0; i < THREAD_QOS_LAST; i++) { + sync_qos_count_t port_sync_qos_count = port_sync_qos(port, i); + /* Do not let the sync qos underflow */ + if (sync_qos_delta_sub[i] > port_sync_qos_count) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_UNDERFLOW), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_sub[i]); + + set_port_sync_qos(port, i, 0); + } else if (sync_qos_delta_sub[i] != 0) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_REMOVED), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_sub[i]); + + set_port_sync_qos(port, i, (port_sync_qos_count - sync_qos_delta_sub[i])); + } + + port_sync_qos_count = port_sync_qos(port, i); + /* Do not let the sync qos overflow */ + if (UCHAR_MAX - sync_qos_delta_add[i] < port_sync_qos_count) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_OVERFLOW), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_add[i]); + + set_port_sync_qos(port, i, UCHAR_MAX); + } else if (sync_qos_delta_add[i] != 0) { + KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_APPLIED), + i, VM_KERNEL_UNSLIDE_OR_PERM(port), + port_sync_qos_count, sync_qos_delta_add[i]); + + set_port_sync_qos(port, i, (port_sync_qos_count + sync_qos_delta_add[i])); + } + } + return (ipc_port_get_max_sync_qos_index(port) != max_sync_qos_index); +} + +/* + * Routine: ipc_port_get_max_sync_qos_index + * Purpose: + * Return the max sync qos of the port. + * + * Conditions: + */ +sync_qos_count_t +ipc_port_get_max_sync_qos_index( + ipc_port_t port) +{ + int i; + for (i = THREAD_QOS_LAST - 1; i >= 0; i--) { + if (port_sync_qos(port, i) != 0) { + return i; + } + } + return THREAD_QOS_UNSPECIFIED; +} + +/* + * Routine: ipc_port_adjust_sync_qos + * Purpose: + * Adjust sync qos of the port and it's destination + * port if the port is in transit. + * Conditions: + * Nothing locked. + * Returns: + * None. + */ +void +ipc_port_adjust_sync_qos( + ipc_port_t port, + sync_qos_count_t *sync_qos_delta_add, + sync_qos_count_t *sync_qos_delta_sub) +{ + boolean_t update_knote; + boolean_t multiple_lock = FALSE; + ipc_port_t dest, base, next; + + ip_lock(port); + + /* Check if the port is in transit */ + if (!ip_active(port) || + (port->ip_receiver_name != MACH_PORT_NULL) || + (port->ip_destination == IP_NULL)) { + /* lock the mqueue since port is not in-transit */ + imq_lock(&port->ip_messages); + update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub); + if (update_knote) { + KNOTE(&port->ip_messages.imq_klist, 0); + } + imq_unlock(&port->ip_messages); + ip_unlock(port); + return; + } + + dest = port->ip_destination; + assert(dest != IP_NULL); + + if (ip_lock_try(dest)) { + if (!ip_active(dest) || + (dest->ip_receiver_name != MACH_PORT_NULL) || + (dest->ip_destination == IP_NULL)) { + update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub); + ip_unlock(port); + + /* lock the mqueue since dest is not in-transit */ + imq_lock(&dest->ip_messages); + update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub); + if (update_knote) { + KNOTE(&dest->ip_messages.imq_klist, 0); + } + imq_unlock(&dest->ip_messages); + ip_unlock(dest); + return; + } + + /* dest is in transit; need to take the serialize lock */ + ip_unlock(dest); + } + + ip_unlock(port); + + ipc_port_multiple_lock(); /* massive serialization */ + multiple_lock = TRUE; + + ip_lock(port); + next = port; + + /* Apply the sync qos delta to all in-transit ports */ + for (;;) { + boolean_t port_not_in_transit = FALSE; + + if (!ip_active(next) || + (next->ip_receiver_name != MACH_PORT_NULL) || + (next->ip_destination == IP_NULL)) { + /* Get the mqueue lock for destination port to update knotes */ + imq_lock(&next->ip_messages); + port_not_in_transit = TRUE; + } + + /* Apply the sync qos delta */ + update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub); + + if (port_not_in_transit) + break; + + next = next->ip_destination; + ip_lock(next); + } + + if (multiple_lock) { + ipc_port_multiple_unlock(); + } + + base = next; + next = port; + + while (next != base) { + ipc_port_t prev = next; + next = next->ip_destination; + + ip_unlock(prev); + } + + if (update_knote) { + KNOTE(&base->ip_messages.imq_klist, 0); + } + imq_unlock(&base->ip_messages); + ip_unlock(base); +} + +/* + * Routine: ipc_port_impcount_delta + * Purpose: + * Adjust only the importance count associated with a port. + * If there are any adjustments to be made to receiver task, + * those are handled elsewhere. + * + * For now, be defensive during deductions to make sure the + * impcount for the port doesn't underflow zero. This will + * go away when the port boost addition is made atomic (see + * note in ipc_port_importance_delta()). + * Conditions: + * The port is referenced and locked. + * Nothing else is locked. + */ +mach_port_delta_t +ipc_port_impcount_delta( + ipc_port_t port, + mach_port_delta_t delta, + ipc_port_t __unused base) +{ + mach_port_delta_t absdelta; + + if (!ip_active(port)) { + return 0; + } + + /* adding/doing nothing is easy */ + if (delta >= 0) { + port->ip_impcount += delta; + return delta; + } + + absdelta = 0 - delta; + if (port->ip_impcount >= absdelta) { + port->ip_impcount -= absdelta; + return delta; + } + +#if (DEVELOPMENT || DEBUG) + if (port->ip_receiver_name != MACH_PORT_NULL) { + task_t target_task = port->ip_receiver->is_task; + ipc_importance_task_t target_imp = target_task->task_imp_base; + const char *target_procname; + int target_pid; + + if (target_imp != IIT_NULL) { + target_procname = target_imp->iit_procname; + target_pid = target_imp->iit_bsd_pid; + } else { + target_procname = "unknown"; + target_pid = -1; + } + printf("Over-release of importance assertions for port 0x%x receiver pid %d (%s), " + "dropping %d assertion(s) but port only has %d remaining.\n", + port->ip_receiver_name, + target_pid, target_procname, + absdelta, port->ip_impcount); + + } else if (base != IP_NULL) { + task_t target_task = base->ip_receiver->is_task; + ipc_importance_task_t target_imp = target_task->task_imp_base; + const char *target_procname; + int target_pid; + + if (target_imp != IIT_NULL) { + target_procname = target_imp->iit_procname; + target_pid = target_imp->iit_bsd_pid; + } else { + target_procname = "unknown"; + target_pid = -1; + } + printf("Over-release of importance assertions for port 0x%lx " + "enqueued on port 0x%x with receiver pid %d (%s), " + "dropping %d assertion(s) but port only has %d remaining.\n", + (unsigned long)VM_KERNEL_UNSLIDE_OR_PERM((uintptr_t)port), + base->ip_receiver_name, + target_pid, target_procname, + absdelta, port->ip_impcount); + } +#endif + + delta = 0 - port->ip_impcount; + port->ip_impcount = 0; + return delta; +} + +/* + * Routine: ipc_port_importance_delta_internal + * Purpose: + * Adjust the importance count through the given port. + * If the port is in transit, apply the delta throughout + * the chain. Determine if the there is a task at the + * base of the chain that wants/needs to be adjusted, + * and if so, apply the delta. + * Conditions: + * The port is referenced and locked on entry. + * Importance may be locked. + * Nothing else is locked. + * The lock may be dropped on exit. + * Returns TRUE if lock was dropped. + */ +#if IMPORTANCE_INHERITANCE + +boolean_t +ipc_port_importance_delta_internal( + ipc_port_t port, + natural_t options, + mach_port_delta_t *deltap, + ipc_importance_task_t *imp_task) +{ + ipc_port_t next, base; + boolean_t dropped = FALSE; + + *imp_task = IIT_NULL; + + if (*deltap == 0) + return FALSE; + + assert(options == IPID_OPTION_NORMAL || options == IPID_OPTION_SENDPOSSIBLE); + + base = port; + + /* if port is in transit, have to search for end of chain */ + if (ip_active(port) && + port->ip_destination != IP_NULL && + port->ip_receiver_name == MACH_PORT_NULL) { + + dropped = TRUE; + + ip_unlock(port); + ipc_port_multiple_lock(); /* massive serialization */ + ip_lock(base); + + while(ip_active(base) && + base->ip_destination != IP_NULL && + base->ip_receiver_name == MACH_PORT_NULL) { + + base = base->ip_destination; + ip_lock(base); + } + ipc_port_multiple_unlock(); + } + + /* + * If the port lock is dropped b/c the port is in transit, there is a + * race window where another thread can drain messages and/or fire a + * send possible notification before we get here. + * + * We solve this race by checking to see if our caller armed the send + * possible notification, whether or not it's been fired yet, and + * whether or not we've already set the port's ip_spimportant bit. If + * we don't need a send-possible boost, then we'll just apply a + * harmless 0-boost to the port. + */ + if (options & IPID_OPTION_SENDPOSSIBLE) { + assert(*deltap == 1); + if (port->ip_sprequests && port->ip_spimportant == 0) + port->ip_spimportant = 1; + else + *deltap = 0; + } + + /* unlock down to the base, adjusting boost(s) at each level */ + for (;;) { + *deltap = ipc_port_impcount_delta(port, *deltap, base); + + if (port == base) { + break; + } + + /* port is in transit */ + assert(port->ip_tempowner == 0); + next = port->ip_destination; + ip_unlock(port); + port = next; + } + + /* find the task (if any) to boost according to the base */ + if (ip_active(base)) { + if (base->ip_tempowner != 0) { + if (IIT_NULL != base->ip_imp_task) + *imp_task = base->ip_imp_task; + /* otherwise don't boost */ + + } else if (base->ip_receiver_name != MACH_PORT_NULL) { + ipc_space_t space = base->ip_receiver; + + /* only spaces with boost-accepting tasks */ + if (space->is_task != TASK_NULL && + ipc_importance_task_is_any_receiver_type(space->is_task->task_imp_base)) { + *imp_task = space->is_task->task_imp_base; + } + } + } + + /* + * Only the base is locked. If we have to hold or drop task + * importance assertions, we'll have to drop that lock as well. + */ + if (*imp_task != IIT_NULL) { + /* take a reference before unlocking base */ + ipc_importance_task_reference(*imp_task); + } + + if (dropped == TRUE) { + ip_unlock(base); + } + + return dropped; +} +#endif /* IMPORTANCE_INHERITANCE */ + +/* + * Routine: ipc_port_importance_delta + * Purpose: + * Adjust the importance count through the given port. + * If the port is in transit, apply the delta throughout + * the chain. + * + * If there is a task at the base of the chain that wants/needs + * to be adjusted, apply the delta. + * Conditions: + * The port is referenced and locked on entry. + * Nothing else is locked. + * The lock may be dropped on exit. + * Returns TRUE if lock was dropped. + */ +#if IMPORTANCE_INHERITANCE + +boolean_t +ipc_port_importance_delta( + ipc_port_t port, + natural_t options, + mach_port_delta_t delta) +{ + ipc_importance_task_t imp_task = IIT_NULL; + boolean_t dropped; + + dropped = ipc_port_importance_delta_internal(port, options, &delta, &imp_task); + + if (IIT_NULL == imp_task || delta == 0) + return dropped; + + if (!dropped) + ip_unlock(port); + + assert(ipc_importance_task_is_any_receiver_type(imp_task)); + + if (delta > 0) + ipc_importance_task_hold_internal_assertion(imp_task, delta); + else + ipc_importance_task_drop_internal_assertion(imp_task, -delta); + + ipc_importance_task_release(imp_task); + return TRUE; +} +#endif /* IMPORTANCE_INHERITANCE */ + +/* + * Routine: ipc_port_lookup_notify + * Purpose: + * Make a send-once notify port from a receive right. + * Returns IP_NULL if name doesn't denote a receive right. + * Conditions: + * The space must be locked (read or write) and active. + * Being the active space, we can rely on thread server_id + * context to give us the proper server level sub-order + * within the space. + */ + +ipc_port_t +ipc_port_lookup_notify( + ipc_space_t space, + mach_port_name_t name) +{ + ipc_port_t port; + ipc_entry_t entry; + + assert(is_active(space)); + + entry = ipc_entry_lookup(space, name); + if (entry == IE_NULL) + return IP_NULL; + if ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0) + return IP_NULL; + + __IGNORE_WCASTALIGN(port = (ipc_port_t) entry->ie_object); + assert(port != IP_NULL); + + ip_lock(port); + assert(ip_active(port)); + assert(port->ip_receiver_name == name); + assert(port->ip_receiver == space); + + ip_reference(port); + port->ip_sorights++; + ip_unlock(port); + + return port; +} + +/* + * Routine: ipc_port_make_send_locked + * Purpose: + * Make a naked send right from a receive right. + * * Conditions: * port locked and active. */ @@ -879,7 +1953,6 @@ ipc_port_make_send_locked( port->ip_mscount++; port->ip_srights++; ip_reference(port); - ip_unlock(port); return port; } @@ -974,7 +2047,7 @@ ipc_port_copyout_send( name = MACH_PORT_NULL; } } else - name = (mach_port_name_t) sright; + name = CAST_MACH_PORT_TO_NAME(sright); return name; } @@ -982,7 +2055,7 @@ ipc_port_copyout_send( /* * Routine: ipc_port_release_send * Purpose: - * Release a (valid) naked send right. + * Release a naked send right. * Consumes a ref for the port. * Conditions: * Nothing locked. @@ -995,32 +2068,54 @@ ipc_port_release_send( ipc_port_t nsrequest = IP_NULL; mach_port_mscount_t mscount; - assert(IP_VALID(port)); + if (!IP_VALID(port)) + return; ip_lock(port); - ip_release(port); + + assert(port->ip_srights > 0); + if (port->ip_srights == 0) { + panic("Over-release of port %p send right!", port); + } + + port->ip_srights--; if (!ip_active(port)) { - ip_check_unlock(port); + ip_unlock(port); + ip_release(port); return; } - assert(port->ip_srights > 0); - - if (--port->ip_srights == 0 && + if (port->ip_srights == 0 && port->ip_nsrequest != IP_NULL) { nsrequest = port->ip_nsrequest; port->ip_nsrequest = IP_NULL; mscount = port->ip_mscount; ip_unlock(port); + ip_release(port); ipc_notify_no_senders(nsrequest, mscount); - /* - * Check that there are no other locks taken, because - * [norma_]ipc_notify_no_senders routines may block. - */ - check_simple_locks(); - } else + } else { ip_unlock(port); + ip_release(port); + } +} + +/* + * Routine: ipc_port_make_sonce_locked + * Purpose: + * Make a naked send-once right from a receive right. + * Conditions: + * The port is locked and active. + */ + +ipc_port_t +ipc_port_make_sonce_locked( + ipc_port_t port) +{ + assert(ip_active(port)); + port->ip_sorights++; + ip_reference(port); + return port; } /* @@ -1028,22 +2123,25 @@ ipc_port_release_send( * Purpose: * Make a naked send-once right from a receive right. * Conditions: - * The port is not locked but it is active. + * The port is not locked. */ ipc_port_t ipc_port_make_sonce( ipc_port_t port) { - assert(IP_VALID(port)); + if (!IP_VALID(port)) + return port; ip_lock(port); - assert(ip_active(port)); - port->ip_sorights++; - ip_reference(port); + if (ip_active(port)) { + port->ip_sorights++; + ip_reference(port); + ip_unlock(port); + return port; + } ip_unlock(port); - - return port; + return IP_DEAD; } /* @@ -1064,22 +2162,22 @@ void ipc_port_release_sonce( ipc_port_t port) { - assert(IP_VALID(port)); + if (!IP_VALID(port)) + return; + + ipc_port_unlink_special_reply_port(port, IPC_PORT_UNLINK_SR_NONE); ip_lock(port); assert(port->ip_sorights > 0); + if (port->ip_sorights == 0) { + panic("Over-release of port %p send-once right!", port); + } port->ip_sorights--; - ip_release(port); - - if (!ip_active(port)) { - ip_check_unlock(port); - return; - } - ip_unlock(port); + ip_release(port); } /* @@ -1097,7 +2195,8 @@ ipc_port_release_receive( { ipc_port_t dest; - assert(IP_VALID(port)); + if (!IP_VALID(port)) + return; ip_lock(port); assert(ip_active(port)); @@ -1107,7 +2206,7 @@ ipc_port_release_receive( ipc_port_destroy(port); /* consumes ref, unlocks */ if (dest != IP_NULL) - ipc_port_release(dest); + ip_release(dest); } /* @@ -1126,10 +2225,15 @@ ipc_port_alloc_special( { ipc_port_t port; - port = (ipc_port_t) io_alloc(IOT_PORT); + __IGNORE_WCASTALIGN(port = (ipc_port_t) io_alloc(IOT_PORT)); if (port == IP_NULL) return IP_NULL; +#if MACH_ASSERT + uintptr_t buf[IP_CALLSTACK_MAX]; + ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + bzero((char *)port, sizeof(*port)); io_lock_init(&port->ip_object); port->ip_references = 1; @@ -1137,6 +2241,10 @@ ipc_port_alloc_special( ipc_port_init(port, space, 1); +#if MACH_ASSERT + ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX); +#endif /* MACH_ASSERT */ + return port; } @@ -1151,12 +2259,12 @@ ipc_port_alloc_special( void ipc_port_dealloc_special( - ipc_port_t port, - ipc_space_t space) + ipc_port_t port, + __assert_only ipc_space_t space) { ip_lock(port); assert(ip_active(port)); - assert(port->ip_receiver_name != MACH_PORT_NULL); +// assert(port->ip_receiver_name != MACH_PORT_NULL); assert(port->ip_receiver == space); /* @@ -1174,15 +2282,161 @@ ipc_port_dealloc_special( ipc_port_destroy(port); } +/* + * Routine: ipc_port_finalize + * Purpose: + * Called on last reference deallocate to + * free any remaining data associated with the + * port. + * Conditions: + * Nothing locked. + */ +void +ipc_port_finalize( + ipc_port_t port) +{ + ipc_port_request_t requests = port->ip_requests; + + assert(!ip_active(port)); + if (requests != IPR_NULL) { + ipc_table_size_t its = requests->ipr_size; + it_requests_free(its, requests); + port->ip_requests = IPR_NULL; + } + + ipc_mqueue_deinit(&port->ip_messages); + +#if MACH_ASSERT + ipc_port_track_dealloc(port); +#endif /* MACH_ASSERT */ +} + +/* + * Routine: kdp_mqueue_send_find_owner + * Purpose: + * Discover the owner of the ipc_mqueue that contains the input + * waitq object. The thread blocked on the waitq should be + * waiting for an IPC_MQUEUE_FULL event. + * Conditions: + * The 'waitinfo->wait_type' value should already be set to + * kThreadWaitPortSend. + * Note: + * If we find out that the containing port is actually in + * transit, we reset the wait_type field to reflect this. + */ +void +kdp_mqueue_send_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo) +{ + assert(waitinfo->wait_type == kThreadWaitPortSend); + assert(event == IPC_MQUEUE_FULL); + + ipc_mqueue_t mqueue = imq_from_waitq(waitq); + ipc_port_t port = ip_from_mq(mqueue); /* we are blocking on send */ + assert(kdp_is_in_zone(port, "ipc ports")); + + waitinfo->owner = 0; + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(port); + if (ip_lock_held_kdp(port)) { + /* + * someone has the port locked: it may be in an + * inconsistent state: bail + */ + waitinfo->owner = STACKSHOT_WAITOWNER_PORT_LOCKED; + return; + } + + if (ip_active(port)) { + if (port->ip_tempowner) { + if (port->ip_imp_task != IIT_NULL && port->ip_imp_task->iit_task != NULL) { + /* port is held by a tempowner */ + waitinfo->owner = pid_from_task(port->ip_imp_task->iit_task); + } else { + waitinfo->owner = STACKSHOT_WAITOWNER_INTRANSIT; + } + } else if (port->ip_receiver_name) { + /* port in a space */ + if (port->ip_receiver == ipc_space_kernel) { + /* + * The kernel pid is 0, make this + * distinguishable from no-owner and + * inconsistent port state. + */ + waitinfo->owner = STACKSHOT_WAITOWNER_KERNEL; + } else { + waitinfo->owner = pid_from_task(port->ip_receiver->is_task); + } + } else if (port->ip_destination != IP_NULL) { + /* port in transit */ + waitinfo->wait_type = kThreadWaitPortSendInTransit; + waitinfo->owner = VM_KERNEL_UNSLIDE_OR_PERM(port->ip_destination); + } + } +} + +/* + * Routine: kdp_mqueue_recv_find_owner + * Purpose: + * Discover the "owner" of the ipc_mqueue that contains the input + * waitq object. The thread blocked on the waitq is trying to + * receive on the mqueue. + * Conditions: + * The 'waitinfo->wait_type' value should already be set to + * kThreadWaitPortReceive. + * Note: + * If we find that we are actualy waiting on a port set, we reset + * the wait_type field to reflect this. + */ +void +kdp_mqueue_recv_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo) +{ + assert(waitinfo->wait_type == kThreadWaitPortReceive); + assert(event == IPC_MQUEUE_RECEIVE); + + ipc_mqueue_t mqueue = imq_from_waitq(waitq); + waitinfo->owner = 0; + if (imq_is_set(mqueue)) { /* we are waiting on a port set */ + ipc_pset_t set = ips_from_mq(mqueue); + assert(kdp_is_in_zone(set, "ipc port sets")); + + /* Reset wait type to specify waiting on port set receive */ + waitinfo->wait_type = kThreadWaitPortSetReceive; + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(set); + if (ips_lock_held_kdp(set)) { + waitinfo->owner = STACKSHOT_WAITOWNER_PSET_LOCKED; + } + /* There is no specific owner "at the other end" of a port set, so leave unset. */ + } else { + ipc_port_t port = ip_from_mq(mqueue); + assert(kdp_is_in_zone(port, "ipc ports")); + + waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(port); + if (ip_lock_held_kdp(port)) { + waitinfo->owner = STACKSHOT_WAITOWNER_PORT_LOCKED; + return; + } + + if (ip_active(port)) { + if (port->ip_receiver_name != MACH_PORT_NULL) { + waitinfo->owner = port->ip_receiver_name; + } else { + waitinfo->owner = STACKSHOT_WAITOWNER_INTRANSIT; + } + } + } +} #if MACH_ASSERT +#include + /* * Keep a list of all allocated ports. * Allocation is intercepted via ipc_port_init; * deallocation is intercepted via io_free. */ +#if 0 queue_head_t port_alloc_queue; -decl_mutex_data(,port_alloc_queue_lock) +lck_spin_t port_alloc_queue_lock; +#endif unsigned long port_count = 0; unsigned long port_count_warning = 20000; @@ -1205,10 +2459,18 @@ int db_port_walk( void ipc_port_debug_init(void) { +#if 0 queue_init(&port_alloc_queue); - mutex_init(&port_alloc_queue_lock, ETAP_IPC_PORT_ALLOCQ); + lck_spin_init(&port_alloc_queue_lock, &ipc_lck_grp, &ipc_lck_attr); +#endif + + if (!PE_parse_boot_argn("ipc_portbt", &ipc_portbt, sizeof (ipc_portbt))) + ipc_portbt = 0; } +#ifdef MACH_BSD +extern int proc_pid(struct proc*); +#endif /* MACH_BSD */ /* * Initialize all of the debugging state in a port. @@ -1216,713 +2478,85 @@ ipc_port_debug_init(void) */ void ipc_port_init_debug( - ipc_port_t port) + ipc_port_t port, + uintptr_t *callstack, + unsigned int callstack_max) { unsigned int i; - port->ip_thread = (unsigned long) current_thread(); + port->ip_thread = current_thread(); port->ip_timetrack = port_timestamp++; - for (i = 0; i < IP_CALLSTACK_MAX; ++i) - port->ip_callstack[i] = 0; + for (i = 0; i < callstack_max; ++i) + port->ip_callstack[i] = callstack[i]; for (i = 0; i < IP_NSPARES; ++i) - port->ip_spares[i] = 0; - - /* - * Machine-dependent routine to fill in an - * array with up to IP_CALLSTACK_MAX levels - * of return pc information. - */ - machine_callstack(&port->ip_callstack[0], IP_CALLSTACK_MAX); + port->ip_spares[i] = 0; + +#ifdef MACH_BSD + task_t task = current_task(); + if (task != TASK_NULL) { + struct proc* proc = (struct proc*) get_bsdtask_info(task); + if (proc) + port->ip_spares[0] = proc_pid(proc); + } +#endif /* MACH_BSD */ #if 0 - mutex_lock(&port_alloc_queue_lock); + lck_spin_lock(&port_alloc_queue_lock); ++port_count; if (port_count_warning > 0 && port_count >= port_count_warning) assert(port_count < port_count_warning); queue_enter(&port_alloc_queue, port, ipc_port_t, ip_port_links); - mutex_unlock(&port_alloc_queue_lock); -#endif -} - - -/* - * Remove a port from the queue of allocated ports. - * This routine should be invoked JUST prior to - * deallocating the actual memory occupied by the port. - */ -void -ipc_port_track_dealloc( - ipc_port_t port) -{ -#if 0 - mutex_lock(&port_alloc_queue_lock); - assert(port_count > 0); - --port_count; - queue_remove(&port_alloc_queue, port, ipc_port_t, ip_port_links); - mutex_unlock(&port_alloc_queue_lock); + lck_spin_unlock(&port_alloc_queue_lock); #endif } -#endif /* MACH_ASSERT */ - - -#if MACH_KDB - -#include -#include - -#define printf kdbprintf -extern int db_indent; - -int -db_port_queue_print( - ipc_port_t port); - -/* - * ipc_entry_print - pretty-print an ipc_entry - */ -static void ipc_entry_print(struct ipc_entry *, char *); /* forward */ - -static void ipc_entry_print(struct ipc_entry *iep, char *tag) -{ - ipc_entry_bits_t bits = iep->ie_bits; - - iprintf("%s @", tag); - printf(" 0x%x, bits=%x object=%x\n", iep, bits, iep->ie_object); - db_indent += 2; - iprintf("urefs=%x ", IE_BITS_UREFS(bits)); - printf("type=%x gen=%x\n", IE_BITS_TYPE(bits), IE_BITS_GEN(bits)); - db_indent -= 2; -} - /* - * Routine: ipc_port_print + * Routine: ipc_port_callstack_init_debug * Purpose: - * Pretty-print a port for kdb. - */ -int ipc_port_print_long = 0; /* set for more detail */ - -void -ipc_port_print( - ipc_port_t port, - boolean_t have_addr, - db_expr_t count, - char *modif) -{ - extern int db_indent; - db_addr_t task; - int task_id; - int nmsgs; - int verbose = 0; -#if MACH_ASSERT - int i, needs_db_indent, items_printed; -#endif /* MACH_ASSERT */ - - if (db_option(modif, 'l') || db_option(modif, 'v')) - ++verbose; - - printf("port 0x%x\n", port); - - db_indent += 2; - - ipc_object_print(&port->ip_object); - - if (ipc_port_print_long) { - printf("\n"); - } - - if (!ip_active(port)) { - iprintf("timestamp=0x%x", port->ip_timestamp); - } else if (port->ip_receiver_name == MACH_PORT_NULL) { - iprintf("destination=0x%x (", port->ip_destination); - if (port->ip_destination != MACH_PORT_NULL && - (task = db_task_from_space(port->ip_destination-> - ip_receiver, &task_id))) - printf("task%d at 0x%x", task_id, task); - else - printf("unknown"); - printf(")"); - } else { - iprintf("receiver=0x%x (", port->ip_receiver); - if (port->ip_receiver == ipc_space_kernel) - printf("kernel"); - else if (port->ip_receiver == ipc_space_reply) - printf("reply"); - else if (port->ip_receiver == default_pager_space) - printf("default_pager"); - else if (task = db_task_from_space(port->ip_receiver, &task_id)) - printf("task%d at 0x%x", task_id, task); - else - printf("unknown"); - printf(")"); - } - printf(", receiver_name=0x%x\n", port->ip_receiver_name); - - iprintf("mscount=%d", port->ip_mscount); - printf(", srights=%d", port->ip_srights); - printf(", sorights=%d\n", port->ip_sorights); - - iprintf("nsrequest=0x%x", port->ip_nsrequest); - printf(", pdrequest=0x%x", port->ip_pdrequest); - printf(", dnrequests=0x%x\n", port->ip_dnrequests); - - iprintf("pset_count=0x%x", port->ip_pset_count); - printf(", seqno=%d", port->ip_messages.imq_seqno); - printf(", msgcount=%d", port->ip_messages.imq_msgcount); - printf(", qlimit=%d\n", port->ip_messages.imq_qlimit); - - iprintf("kmsgs=0x%x", port->ip_messages.imq_messages.ikmq_base); - printf(", rcvrs queue=0x%x", port->ip_messages.imq_wait_queue); - printf(", kobj=0x%x\n", port->ip_kobject); - - iprintf("premsg=0x%x", port->ip_premsg); - -#if MACH_ASSERT - /* don't bother printing callstack or queue links */ - iprintf("ip_thread=0x%x, ip_timetrack=0x%x\n", - port->ip_thread, port->ip_timetrack); - items_printed = 0; - needs_db_indent = 1; - for (i = 0; i < IP_NSPARES; ++i) { - if (port->ip_spares[i] != 0) { - if (needs_db_indent) { - iprintf(""); - needs_db_indent = 0; - } - printf("%sip_spares[%d] = %d", - items_printed ? ", " : "", i, - port->ip_spares[i]); - if (++items_printed >= 4) { - needs_db_indent = 1; - printf("\n"); - items_printed = 0; - } - } - } -#endif /* MACH_ASSERT */ - - if (verbose) { - iprintf("kmsg queue contents:\n"); - db_indent += 2; - nmsgs = db_port_queue_print(port); - db_indent -= 2; - iprintf("...total kmsgs: %d\n", nmsgs); - } - - db_indent -=2; -} - -ipc_port_t -ipc_name_to_data( - task_t task, - mach_port_name_t name) -{ - ipc_space_t space; - ipc_entry_t entry; - - if (task == TASK_NULL) { - db_printf("port_name_to_data: task is null\n"); - return (0); - } - if ((space = task->itk_space) == 0) { - db_printf("port_name_to_data: task->itk_space is null\n"); - return (0); - } - if (!space->is_active) { - db_printf("port_name_to_data: task->itk_space not active\n"); - return (0); - } - if ((entry = ipc_entry_lookup(space, name)) == 0) { - db_printf("port_name_to_data: lookup yields zero\n"); - return (0); - } - return ((ipc_port_t)entry->ie_object); -} - -#if ZONE_DEBUG -void -print_type_ports(type, dead) - unsigned type; - unsigned dead; -{ - ipc_port_t port; - int n; - - n = 0; - for (port = (ipc_port_t)first_element(ipc_object_zones[IOT_PORT]); - port; - port = (ipc_port_t)next_element(ipc_object_zones[IOT_PORT], - (vm_offset_t)port)) - if (ip_kotype(port) == type && - (!dead || !ip_active(port))) { - if (++n % 5) - printf("0x%x\t", port); - else - printf("0x%x\n", port); - } - if (n % 5) - printf("\n"); -} - -void -print_ports(void) -{ - ipc_port_t port; - int total_port_count; - int space_null_count; - int space_kernel_count; - int space_reply_count; - int space_pager_count; - int space_other_count; - - struct { - int total_count; - int dead_count; - } port_types[IKOT_MAX_TYPE]; - - total_port_count = 0; - - bzero((char *)&port_types[0], sizeof(port_types)); - space_null_count = 0; - space_kernel_count = 0; - space_reply_count = 0; - space_pager_count = 0; - space_other_count = 0; - - for (port = (ipc_port_t)first_element(ipc_object_zones[IOT_PORT]); - port; - port = (ipc_port_t)next_element(ipc_object_zones[IOT_PORT], - (vm_offset_t)port)) { - total_port_count++; - if (ip_kotype(port) >= IKOT_MAX_TYPE) { - port_types[IKOT_UNKNOWN].total_count++; - if (!io_active(&port->ip_object)) - port_types[IKOT_UNKNOWN].dead_count++; - } else { - port_types[ip_kotype(port)].total_count++; - if (!io_active(&port->ip_object)) - port_types[ip_kotype(port)].dead_count++; - } - - if (!port->ip_receiver) - space_null_count++; - else if (port->ip_receiver == ipc_space_kernel) - space_kernel_count++; - else if (port->ip_receiver == ipc_space_reply) - space_reply_count++; - else if (port->ip_receiver == default_pager_space) - space_pager_count++; - else - space_other_count++; - } - printf("\n%7d total ports\n\n", total_port_count); - -#define PRINT_ONE_PORT_TYPE(name) \ - printf("%7d %s", port_types[IKOT_##name].total_count, # name); \ - if (port_types[IKOT_##name].dead_count) \ - printf(" (%d dead ports)", port_types[IKOT_##name].dead_count);\ - printf("\n"); - - PRINT_ONE_PORT_TYPE(NONE); - PRINT_ONE_PORT_TYPE(THREAD); - PRINT_ONE_PORT_TYPE(TASK); - PRINT_ONE_PORT_TYPE(HOST); - PRINT_ONE_PORT_TYPE(HOST_PRIV); - PRINT_ONE_PORT_TYPE(PROCESSOR); - PRINT_ONE_PORT_TYPE(PSET); - PRINT_ONE_PORT_TYPE(PSET_NAME); - PRINT_ONE_PORT_TYPE(PAGING_REQUEST); - PRINT_ONE_PORT_TYPE(MEMORY_OBJECT); - PRINT_ONE_PORT_TYPE(MIG); - PRINT_ONE_PORT_TYPE(XMM_PAGER); - PRINT_ONE_PORT_TYPE(XMM_KERNEL); - PRINT_ONE_PORT_TYPE(XMM_REPLY); - PRINT_ONE_PORT_TYPE(CLOCK); - PRINT_ONE_PORT_TYPE(CLOCK_CTRL); - PRINT_ONE_PORT_TYPE(MASTER_DEVICE); - PRINT_ONE_PORT_TYPE(UNKNOWN); - printf("\nipc_space:\n\n"); - printf("NULL KERNEL REPLY PAGER OTHER\n"); - printf("%d %d %d %d %d\n", - space_null_count, - space_kernel_count, - space_reply_count, - space_pager_count, - space_other_count - ); -} - -#endif /* ZONE_DEBUG */ - - -/* - * Print out all the kmsgs in a queue. Aggregate kmsgs with - * identical message ids into a single entry. Count up the - * amount of inline and out-of-line data consumed by each - * and every kmsg. - * - */ - -#define KMSG_MATCH_FIELD(kmsg) ((unsigned int) kmsg->ikm_header.msgh_id) -#define DKQP_LONG(kmsg) FALSE -char *dkqp_long_format = "(%3d) <%10d> 0x%x %10d %10d\n"; -char *dkqp_format = "(%3d) <%10d> 0x%x %10d %10d\n"; - -int -db_kmsg_queue_print( - ipc_kmsg_t kmsg); -int -db_kmsg_queue_print( - ipc_kmsg_t kmsg) -{ - ipc_kmsg_t ikmsg, first_kmsg; - register int icount; - mach_msg_id_t cur_id; - unsigned int inline_total, ool_total; - int nmsgs; - - iprintf("Count msgh_id kmsg addr inline bytes ool bytes\n"); - inline_total = ool_total = (vm_size_t) 0; - cur_id = KMSG_MATCH_FIELD(kmsg); - for (icount = 0, nmsgs = 0, first_kmsg = ikmsg = kmsg; - kmsg != IKM_NULL && (kmsg != first_kmsg || nmsgs == 0); - kmsg = kmsg->ikm_next) { - ++nmsgs; - if (!(KMSG_MATCH_FIELD(kmsg) == cur_id)) { - iprintf(DKQP_LONG(kmsg) ? dkqp_long_format:dkqp_format, - icount, cur_id, ikmsg, inline_total,ool_total); - cur_id = KMSG_MATCH_FIELD(kmsg); - icount = 1; - ikmsg = kmsg; - inline_total = ool_total = 0; - } else { - icount++; - } - if (DKQP_LONG(kmsg)) - inline_total += kmsg->ikm_size; - else - inline_total += kmsg->ikm_header.msgh_size; - } - iprintf(DKQP_LONG(kmsg) ? dkqp_long_format : dkqp_format, - icount, cur_id, ikmsg, inline_total, ool_total); - return nmsgs; -} - - -/* - * Process all of the messages on a port - prints out the - * number of occurences of each message type, and the first - * kmsg with a particular msgh_id. - */ -int -db_port_queue_print( - ipc_port_t port) -{ - ipc_kmsg_t kmsg; - - if (ipc_kmsg_queue_empty(&port->ip_messages.imq_messages)) - return 0; - kmsg = ipc_kmsg_queue_first(&port->ip_messages.imq_messages); - return db_kmsg_queue_print(kmsg); -} - - -#if MACH_ASSERT -#include -#include - -#define FUNC_NULL ((void (*)) 0) -#define MAX_REFS 5 /* bins for tracking ref counts */ - -/* - * Translate port's cache of call stack pointers - * into symbolic names. + * Calls the machine-dependent routine to + * fill in an array with up to IP_CALLSTACK_MAX + * levels of return pc information + * Conditions: + * May block (via copyin) */ void -db_port_stack_trace( - ipc_port_t port) +ipc_port_callstack_init_debug( + uintptr_t *callstack, + unsigned int callstack_max) { unsigned int i; - for (i = 0; i < IP_CALLSTACK_MAX; ++i) { - iprintf("[%d] 0x%x\t", i, port->ip_callstack[i]); - if (port->ip_callstack[i] != 0 && - DB_VALID_KERN_ADDR(port->ip_callstack[i])) - db_printsym(port->ip_callstack[i], DB_STGY_PROC); - printf("\n"); - } -} - - -typedef struct port_item { - unsigned long item; - unsigned long count; -} port_item; - - -#define ITEM_MAX 400 -typedef struct port_track { - char *name; - unsigned long max; - unsigned long warning; - port_item items[ITEM_MAX]; -} port_track; - -port_track port_callers; /* match against calling addresses */ -port_track port_threads; /* match against allocating threads */ -port_track port_spaces; /* match against ipc spaces */ - -void port_track_init( - port_track *trackp, - char *name); -void port_item_add( - port_track *trackp, - unsigned long item); -void port_track_sort( - port_track *trackp); -void port_track_print( - port_track *trackp, - void (*func)(port_item *)); -void port_callers_print( - port_item *p); - -void -port_track_init( - port_track *trackp, - char *name) -{ - port_item *i; - - trackp->max = trackp->warning = 0; - trackp->name = name; - for (i = trackp->items; i < trackp->items + ITEM_MAX; ++i) - i->item = i->count = 0; -} - - -void -port_item_add( - port_track *trackp, - unsigned long item) -{ - port_item *limit, *i; + /* guarantee the callstack is initialized */ + for (i=0; i < callstack_max; i++) + callstack[i] = 0; - limit = trackp->items + trackp->max; - for (i = trackp->items; i < limit; ++i) - if (i->item == item) { - i->count++; - return; - } - if (trackp->max >= ITEM_MAX) { - if (trackp->warning++ == 0) - iprintf("%s: no room\n", trackp->name); - return; - } - i->item = item; - i->count = 1; - trackp->max++; + if (ipc_portbt) + machine_callstack(callstack, callstack_max); } - /* - * Simple (and slow) bubble sort. + * Remove a port from the queue of allocated ports. + * This routine should be invoked JUST prior to + * deallocating the actual memory occupied by the port. */ +#if 1 void -port_track_sort( - port_track *trackp) -{ - port_item *limit, *p; - port_item temp; - boolean_t unsorted; - - limit = trackp->items + trackp->max - 1; - do { - unsorted = FALSE; - for (p = trackp->items; p < limit - 1; ++p) { - if (p->count < (p+1)->count) { - temp = *p; - *p = *(p+1); - *(p+1) = temp; - unsorted = TRUE; - } - } - } while (unsorted == TRUE); -} - - -void -port_track_print( - port_track *trackp, - void (*func)(port_item *)) -{ - port_item *limit, *p; - - limit = trackp->items + trackp->max; - iprintf("%s:\n", trackp->name); - for (p = trackp->items; p < limit; ++p) { - if (func != FUNC_NULL) - (*func)(p); - else - iprintf("0x%x\t%8d\n", p->item, p->count); - } -} - - -void -port_callers_print( - port_item *p) +ipc_port_track_dealloc( + __unused ipc_port_t port) { - iprintf("0x%x\t%8d\t", p->item, p->count); - db_printsym(p->item, DB_STGY_PROC); - printf("\n"); } - - -/* - * Show all ports with a given reference count. - */ +#else void -db_ref( - int refs) -{ - db_port_walk(1, 1, 1, refs); -} - - -/* - * Examine all currently allocated ports. - * Options: - * verbose display suspicious ports - * display print out each port encountered - * ref_search restrict examination to ports with - * a specified reference count - * ref_target reference count for ref_search - */ -int -db_port_walk( - unsigned int verbose, - unsigned int display, - unsigned int ref_search, - unsigned int ref_target) +ipc_port_track_dealloc( + ipc_port_t port) { - ipc_port_t port; - unsigned int ref_overflow, refs, i, ref_inactive_overflow; - unsigned int no_receiver, no_match; - unsigned int ref_counts[MAX_REFS]; - unsigned int inactive[MAX_REFS]; - unsigned int ipc_ports = 0; - unsigned int proxies = 0, principals = 0; - - iprintf("Allocated port count is %d\n", port_count); - no_receiver = no_match = ref_overflow = 0; - ref_inactive_overflow = 0; - for (i = 0; i < MAX_REFS; ++i) { - ref_counts[i] = 0; - inactive[i] = 0; - } - port_track_init(&port_callers, "port callers"); - port_track_init(&port_threads, "port threads"); - port_track_init(&port_spaces, "port spaces"); - if (ref_search) - iprintf("Walking ports of ref_count=%d.\n", ref_target); - else - iprintf("Walking all ports.\n"); - - queue_iterate(&port_alloc_queue, port, ipc_port_t, ip_port_links) { - char *port_type; - - port_type = " IPC port"; - if (ip_active(port)) - ipc_ports++; - - refs = port->ip_references; - if (ref_search && refs != ref_target) - continue; - - if (refs >= MAX_REFS) { - if (ip_active(port)) - ++ref_overflow; - else - ++ref_inactive_overflow; - } else { - if (refs == 0 && verbose) - iprintf("%s 0x%x has ref count of zero!\n", - port_type, port); - if (ip_active(port)) - ref_counts[refs]++; - else - inactive[refs]++; - } - port_item_add(&port_threads, (unsigned long) port->ip_thread); - for (i = 0; i < IP_CALLSTACK_MAX; ++i) { - if (port->ip_callstack[i] != 0 && - DB_VALID_KERN_ADDR(port->ip_callstack[i])) - port_item_add(&port_callers, - port->ip_callstack[i]); - } - if (!ip_active(port)) { - if (verbose) - iprintf("%s 0x%x, inactive, refcnt %d\n", - port_type, port, refs); - continue; - } - - if (port->ip_receiver_name == MACH_PORT_NULL) { - iprintf("%s 0x%x, no receiver, refcnt %d\n", - port, refs); - ++no_receiver; - continue; - } - if (port->ip_receiver == ipc_space_kernel || - port->ip_receiver == ipc_space_reply || - ipc_entry_lookup(port->ip_receiver, - port->ip_receiver_name) - != IE_NULL) { - port_item_add(&port_spaces, - (unsigned long)port->ip_receiver); - if (display) { - iprintf( "%s 0x%x time 0x%x ref_cnt %d\n", - port_type, port, - port->ip_timetrack, refs); - } - continue; - } - iprintf("%s 0x%x, rcvr 0x%x, name 0x%x, ref %d, no match\n", - port_type, port, port->ip_receiver, - port->ip_receiver_name, refs); - ++no_match; - } - iprintf("Active port type summary:\n"); - iprintf("\tlocal IPC %6d\n", ipc_ports); - iprintf("summary:\tcallers %d threads %d spaces %d\n", - port_callers.max, port_threads.max, port_spaces.max); - - iprintf("\tref_counts:\n"); - for (i = 0; i < MAX_REFS; ++i) - iprintf("\t ref_counts[%d] = %d\n", i, ref_counts[i]); - - iprintf("\t%d ports w/o receivers, %d w/o matches\n", - no_receiver, no_match); - - iprintf("\tinactives:"); - if ( ref_inactive_overflow || inactive[0] || inactive[1] || - inactive[2] || inactive[3] || inactive[4] ) - printf(" [0]=%d [1]=%d [2]=%d [3]=%d [4]=%d [5+]=%d\n", - inactive[0], inactive[1], inactive[2], - inactive[3], inactive[4], ref_inactive_overflow); - else - printf(" No inactive ports.\n"); - - port_track_sort(&port_spaces); - port_track_print(&port_spaces, FUNC_NULL); - port_track_sort(&port_threads); - port_track_print(&port_threads, FUNC_NULL); - port_track_sort(&port_callers); - port_track_print(&port_callers, port_callers_print); - return 0; + lck_spin_lock(&port_alloc_queue_lock); + assert(port_count > 0); + --port_count; + queue_remove(&port_alloc_queue, port, ipc_port_t, ip_port_links); + lck_spin_unlock(&port_alloc_queue_lock); } +#endif #endif /* MACH_ASSERT */ - -#endif /* MACH_KDB */