xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / ipc / ipc_port.c

diff --git a/osfmk/ipc/ipc_port.c b/osfmk/ipc/ipc_port.c
index 871f98f49f5685e75ae612d709a3b05515b37059..8ba9fcf0a05b7c0039a369e78db5c92c52a29840 100644 (file)
--- a/osfmk/ipc/ipc_port.c
+++ b/osfmk/ipc/ipc_port.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2019 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -69,7 +69,6 @@
  *     Functions to manipulate IPC ports.
  */
 
  *     Functions to manipulate IPC ports.
  */
 

-#include <zone_debug.h>

 #include <mach_assert.h>
 
 #include <mach/port.h>
 #include <mach_assert.h>
 
 #include <mach/port.h>


@@ -91,27 +90,39 @@
 #include <ipc/ipc_notify.h>
 #include <ipc/ipc_table.h>
 #include <ipc/ipc_importance.h>
 #include <ipc/ipc_notify.h>
 #include <ipc/ipc_table.h>
 #include <ipc/ipc_importance.h>

-#include <machine/machlimits.h>
+#include <machine/limits.h>
+#include <kern/turnstile.h>
+#include <kern/machine.h>

 
 #include <security/mac_mach_internal.h>
 
 #include <string.h>
 
 
 #include <security/mac_mach_internal.h>
 
 #include <string.h>
 

-decl_lck_spin_data(,   ipc_port_multiple_lock_data)
-ipc_port_timestamp_t   ipc_port_timestamp_data;
-int ipc_portbt;
+static TUNABLE(bool, prioritize_launch, "prioritize_launch", true);
+TUNABLE_WRITEABLE(int, ipc_portbt, "ipc_portbt", false);
+
+LCK_SPIN_DECLARE_ATTR(ipc_port_multiple_lock_data, &ipc_lck_grp, &ipc_lck_attr);
+ipc_port_timestamp_t ipc_port_timestamp_data;
+
+#if     MACH_ASSERT
+void    ipc_port_init_debug(
+       ipc_port_t      port,
+       uintptr_t       *callstack,
+       unsigned int    callstack_max);
+
+void    ipc_port_callstack_init_debug(
+       uintptr_t       *callstack,
+       unsigned int    callstack_max);
+
+#endif  /* MACH_ASSERT */

 
 

-#if    MACH_ASSERT
-void   ipc_port_init_debug(
-               ipc_port_t      port,
-               uintptr_t       *callstack,
-               unsigned int    callstack_max);
+static void
+ipc_port_send_turnstile_recompute_push_locked(
+       ipc_port_t port);

 
 

-void   ipc_port_callstack_init_debug(
-               uintptr_t       *callstack,
-               unsigned int    callstack_max);
-       
-#endif /* MACH_ASSERT */
+static thread_t
+ipc_port_get_watchport_inheritor(
+       ipc_port_t port);

 
 void
 ipc_port_release(ipc_port_t port)
 
 void
 ipc_port_release(ipc_port_t port)


@@ -153,22 +164,22 @@ ipc_port_timestamp(void)
 #if IMPORTANCE_INHERITANCE
 kern_return_t
 ipc_port_request_alloc(
 #if IMPORTANCE_INHERITANCE
 kern_return_t
 ipc_port_request_alloc(

-       ipc_port_t                      port,
-       mach_port_name_t                name,
-       ipc_port_t                      soright,
-       boolean_t                       send_possible,
-       boolean_t                       immediate,
-       ipc_port_request_index_t        *indexp,
-       boolean_t                       *importantp)
+       ipc_port_t                      port,
+       mach_port_name_t                name,
+       ipc_port_t                      soright,
+       boolean_t                       send_possible,
+       boolean_t                       immediate,
+       ipc_port_request_index_t        *indexp,
+       boolean_t                       *importantp)

 #else
 kern_return_t
 ipc_port_request_alloc(
 #else
 kern_return_t
 ipc_port_request_alloc(

-       ipc_port_t                      port,
-       mach_port_name_t                name,
-       ipc_port_t                      soright,
-       boolean_t                       send_possible,
-       boolean_t                       immediate,
-       ipc_port_request_index_t        *indexp)
+       ipc_port_t                      port,
+       mach_port_name_t                name,
+       ipc_port_t                      soright,
+       boolean_t                       send_possible,
+       boolean_t                       immediate,
+       ipc_port_request_index_t        *indexp)

 #endif /* IMPORTANCE_INHERITANCE */
 {
        ipc_port_request_t ipr, table;
 #endif /* IMPORTANCE_INHERITANCE */
 {
        ipc_port_request_t ipr, table;


@@ -179,25 +190,27 @@ ipc_port_request_alloc(
        *importantp = FALSE;
 #endif /* IMPORTANCE_INHERITANCE */
 
        *importantp = FALSE;
 #endif /* IMPORTANCE_INHERITANCE */
 

-       assert(ip_active(port));
+       require_ip_active(port);

        assert(name != MACH_PORT_NULL);
        assert(soright != IP_NULL);
 
        table = port->ip_requests;
 
        assert(name != MACH_PORT_NULL);
        assert(soright != IP_NULL);
 
        table = port->ip_requests;
 

-       if (table == IPR_NULL)
+       if (table == IPR_NULL) {

                return KERN_NO_SPACE;
                return KERN_NO_SPACE;

+       }

 
        index = table->ipr_next;
 
        index = table->ipr_next;

-       if (index == 0)
+       if (index == 0) {

                return KERN_NO_SPACE;
                return KERN_NO_SPACE;

+       }

 
        ipr = &table[index];
        assert(ipr->ipr_name == MACH_PORT_NULL);
 
        table->ipr_next = ipr->ipr_next;
        ipr->ipr_name = name;
 
        ipr = &table[index];
        assert(ipr->ipr_name == MACH_PORT_NULL);
 
        table->ipr_next = ipr->ipr_next;
        ipr->ipr_name = name;

-       
+

        if (send_possible) {
                mask |= IPR_SOR_SPREQ_MASK;
                if (immediate) {
        if (send_possible) {
                mask |= IPR_SOR_SPREQ_MASK;
                if (immediate) {


@@ -240,26 +253,26 @@ ipc_port_request_alloc(
 
 kern_return_t
 ipc_port_request_grow(
 
 kern_return_t
 ipc_port_request_grow(

-       ipc_port_t              port,
-       ipc_table_elems_t       target_size)
+       ipc_port_t              port,
+       ipc_table_elems_t       target_size)

 {
        ipc_table_size_t its;
        ipc_port_request_t otable, ntable;
 {
        ipc_table_size_t its;
        ipc_port_request_t otable, ntable;

-
-       assert(ip_active(port));
+       require_ip_active(port);

 
        otable = port->ip_requests;
 
        otable = port->ip_requests;

-       if (otable == IPR_NULL)
+       if (otable == IPR_NULL) {

                its = &ipc_table_requests[0];
                its = &ipc_table_requests[0];

-       else
+       } else {

                its = otable->ipr_size + 1;
                its = otable->ipr_size + 1;

+       }

 
        if (target_size != ITS_SIZE_NONE) {
                if ((otable != IPR_NULL) &&
                    (target_size <= otable->ipr_size->its_size)) {
                        ip_unlock(port);
                        return KERN_SUCCESS;
 
        if (target_size != ITS_SIZE_NONE) {
                if ((otable != IPR_NULL) &&
                    (target_size <= otable->ipr_size->its_size)) {
                        ip_unlock(port);
                        return KERN_SUCCESS;

-               }
+               }

                while ((its->its_size) && (its->its_size < target_size)) {
                        its++;
                }
                while ((its->its_size) && (its->its_size < target_size)) {
                        its++;
                }


@@ -288,7 +301,7 @@ ipc_port_request_grow(
         */
 
        if (ip_active(port) && (port->ip_requests == otable) &&
         */
 
        if (ip_active(port) && (port->ip_requests == otable) &&

-           ((otable == IPR_NULL) || (otable->ipr_size+1 == its))) {
+           ((otable == IPR_NULL) || (otable->ipr_size + 1 == its))) {

                ipc_table_size_t oits;
                ipc_table_elems_t osize, nsize;
                ipc_port_request_index_t free, i;
                ipc_table_size_t oits;
                ipc_table_elems_t osize, nsize;
                ipc_port_request_index_t free, i;


@@ -301,8 +314,8 @@ ipc_port_request_grow(
                        free = otable->ipr_next;
 
                        (void) memcpy((void *)(ntable + 1),
                        free = otable->ipr_next;
 
                        (void) memcpy((void *)(ntable + 1),

-                             (const void *)(otable + 1),
-                             (osize - 1) * sizeof(struct ipc_port_request));
+                           (const void *)(otable + 1),
+                           (osize - 1) * sizeof(struct ipc_port_request));

                } else {
                        osize = 1;
                        oits = 0;
                } else {
                        osize = 1;
                        oits = 0;


@@ -330,7 +343,7 @@ ipc_port_request_grow(
 
                if (otable != IPR_NULL) {
                        it_requests_free(oits, otable);
 
                if (otable != IPR_NULL) {
                        it_requests_free(oits, otable);

-               }
+               }

        } else {
                ip_unlock(port);
                ip_release(port);
        } else {
                ip_unlock(port);
                ip_release(port);


@@ -339,7 +352,7 @@ ipc_port_request_grow(
 
        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }

- 
+

 /*
  *     Routine:        ipc_port_request_sparm
  *     Purpose:
 /*
  *     Routine:        ipc_port_request_sparm
  *     Purpose:


@@ -353,17 +366,17 @@ ipc_port_request_grow(
 
 boolean_t
 ipc_port_request_sparm(
 
 boolean_t
 ipc_port_request_sparm(

-       ipc_port_t                      port,
-       __assert_only mach_port_name_t  name,
-       ipc_port_request_index_t        index,
-       mach_msg_option_t       option,
-       mach_msg_priority_t override)
+       ipc_port_t                      port,
+       __assert_only mach_port_name_t  name,
+       ipc_port_request_index_t        index,
+       mach_msg_option_t               option,
+       mach_msg_priority_t             priority)

 {
        if (index != IE_REQ_NONE) {
                ipc_port_request_t ipr, table;
 
 {
        if (index != IE_REQ_NONE) {
                ipc_port_request_t ipr, table;
 

-               assert(ip_active(port));
-       
+               require_ip_active(port);
+

                table = port->ip_requests;
                assert(table != IPR_NULL);
 
                table = port->ip_requests;
                assert(table != IPR_NULL);
 


@@ -377,7 +390,15 @@ ipc_port_request_sparm(
 
                        if (option & MACH_SEND_OVERRIDE) {
                                /* apply override to message queue */
 
                        if (option & MACH_SEND_OVERRIDE) {
                                /* apply override to message queue */

-                               ipc_mqueue_override_send(&port->ip_messages, override);
+                               mach_msg_qos_t qos_ovr;
+                               if (mach_msg_priority_is_pthread_priority(priority)) {
+                                       qos_ovr = _pthread_priority_thread_qos(priority);
+                               } else {
+                                       qos_ovr = mach_msg_priority_overide_qos(priority);
+                               }
+                               if (qos_ovr) {
+                                       ipc_mqueue_override_send(&port->ip_messages, qos_ovr);
+                               }

                        }
 
 #if IMPORTANCE_INHERITANCE
                        }
 
 #if IMPORTANCE_INHERITANCE


@@ -385,13 +406,13 @@ ipc_port_request_sparm(
                            (port->ip_impdonation != 0) &&
                            (port->ip_spimportant == 0) &&
                            (((option & MACH_SEND_IMPORTANCE) != 0) ||
                            (port->ip_impdonation != 0) &&
                            (port->ip_spimportant == 0) &&
                            (((option & MACH_SEND_IMPORTANCE) != 0) ||

-                            (task_is_importance_donor(current_task())))) {
+                           (task_is_importance_donor(current_task())))) {

                                return TRUE;
                        }
 #else
                        return TRUE;
 #endif /* IMPORTANCE_INHERITANCE */
                                return TRUE;
                        }
 #else
                        return TRUE;
 #endif /* IMPORTANCE_INHERITANCE */

-               } 
+               }

        }
        return FALSE;
 }
        }
        return FALSE;
 }


@@ -406,15 +427,15 @@ ipc_port_request_sparm(
  */
 mach_port_type_t
 ipc_port_request_type(
  */
 mach_port_type_t
 ipc_port_request_type(

-       ipc_port_t                      port,
-       __assert_only mach_port_name_t  name,
-       ipc_port_request_index_t        index)
+       ipc_port_t                      port,
+       __assert_only mach_port_name_t  name,
+       ipc_port_request_index_t        index)

 {
        ipc_port_request_t ipr, table;
        mach_port_type_t type = 0;
 
        table = port->ip_requests;
 {
        ipc_port_request_t ipr, table;
        mach_port_type_t type = 0;
 
        table = port->ip_requests;

-       assert (table != IPR_NULL);
+       assert(table != IPR_NULL);

 
        assert(index != IE_REQ_NONE);
        ipr = &table[index];
 
        assert(index != IE_REQ_NONE);
        ipr = &table[index];


@@ -445,18 +466,18 @@ ipc_port_request_type(
 
 ipc_port_t
 ipc_port_request_cancel(
 
 ipc_port_t
 ipc_port_request_cancel(

-       ipc_port_t                      port,
-       __assert_only mach_port_name_t  name,
-       ipc_port_request_index_t        index)
+       ipc_port_t                      port,
+       __assert_only mach_port_name_t  name,
+       ipc_port_request_index_t        index)

 {
        ipc_port_request_t ipr, table;
        ipc_port_t request = IP_NULL;
 
 {
        ipc_port_request_t ipr, table;
        ipc_port_t request = IP_NULL;
 

-       assert(ip_active(port));
+       require_ip_active(port);

        table = port->ip_requests;
        assert(table != IPR_NULL);
 
        table = port->ip_requests;
        assert(table != IPR_NULL);
 

-       assert (index != IE_REQ_NONE);
+       assert(index != IE_REQ_NONE);

        ipr = &table[index];
        assert(ipr->ipr_name == name);
        request = IPR_SOR_PORT(ipr->ipr_soright);
        ipr = &table[index];
        assert(ipr->ipr_name == name);
        request = IPR_SOR_PORT(ipr->ipr_soright);


@@ -483,13 +504,12 @@ ipc_port_request_cancel(
 
 void
 ipc_port_pdrequest(
 
 void
 ipc_port_pdrequest(

-       ipc_port_t      port,
-       ipc_port_t      notify,
-       ipc_port_t      *previousp)
+       ipc_port_t      port,
+       ipc_port_t      notify,
+       ipc_port_t      *previousp)

 {
        ipc_port_t previous;
 {
        ipc_port_t previous;

-
-       assert(ip_active(port));
+       require_ip_active(port);

 
        previous = port->ip_pdrequest;
        port->ip_pdrequest = notify;
 
        previous = port->ip_pdrequest;
        port->ip_pdrequest = notify;


@@ -512,15 +532,14 @@ ipc_port_pdrequest(
 
 void
 ipc_port_nsrequest(
 
 void
 ipc_port_nsrequest(

-       ipc_port_t              port,
-       mach_port_mscount_t     sync,
-       ipc_port_t              notify,
-       ipc_port_t              *previousp)
+       ipc_port_t              port,
+       mach_port_mscount_t     sync,
+       ipc_port_t              notify,
+       ipc_port_t              *previousp)

 {
        ipc_port_t previous;
        mach_port_mscount_t mscount;
 {
        ipc_port_t previous;
        mach_port_mscount_t mscount;

-
-       assert(ip_active(port));
+       require_ip_active(port);

 
        previous = port->ip_nsrequest;
        mscount = port->ip_mscount;
 
        previous = port->ip_nsrequest;
        mscount = port->ip_mscount;


@@ -552,16 +571,16 @@ ipc_port_nsrequest(
  *             whether the caller needs to reap kmsg structures that should
  *             be destroyed (by calling ipc_kmsg_reap_delayed)
  *
  *             whether the caller needs to reap kmsg structures that should
  *             be destroyed (by calling ipc_kmsg_reap_delayed)
  *

- *             If should_destroy is FALSE, this always returns FALSE
+ *              If should_destroy is FALSE, this always returns FALSE

  */
 
 boolean_t
 ipc_port_clear_receiver(
  */
 
 boolean_t
 ipc_port_clear_receiver(

-       ipc_port_t      port,
-       boolean_t       should_destroy)
+       ipc_port_t      port,
+       boolean_t       should_destroy)

 {
 {

-       ipc_mqueue_t    mqueue = &port->ip_messages;
-       boolean_t       reap_messages = FALSE;
+       ipc_mqueue_t    mqueue = &port->ip_messages;
+       boolean_t       reap_messages = FALSE;

 
        /*
         * Pull ourselves out of any sets to which we belong.
 
        /*
         * Pull ourselves out of any sets to which we belong.


@@ -575,23 +594,41 @@ ipc_port_clear_receiver(
 
        /*
         * Send anyone waiting on the port's queue directly away.
 
        /*
         * Send anyone waiting on the port's queue directly away.

-        * Also clear the mscount and seqno.
+        * Also clear the mscount, seqno, guard bits

         */
        imq_lock(mqueue);
         */
        imq_lock(mqueue);

-       ipc_mqueue_changed(mqueue);
+       if (port->ip_receiver_name) {
+               ipc_mqueue_changed(port->ip_receiver, mqueue);
+       } else {
+               ipc_mqueue_changed(NULL, mqueue);
+       }

        port->ip_mscount = 0;
        mqueue->imq_seqno = 0;
        port->ip_context = port->ip_guarded = port->ip_strict_guard = 0;
        port->ip_mscount = 0;
        mqueue->imq_seqno = 0;
        port->ip_context = port->ip_guarded = port->ip_strict_guard = 0;

+       /*
+        * clear the immovable bit so the port can move back to anyone listening
+        * for the port destroy notification
+        */
+       port->ip_immovable_receive = 0;

 
        if (should_destroy) {
                /*
 
        if (should_destroy) {
                /*

-                * Mark the mqueue invalid, preventing further send/receive
+                * Mark the port and mqueue invalid, preventing further send/receive

                 * operations from succeeding. It's important for this to be
                 * done under the same lock hold as the ipc_mqueue_changed
                 * call to avoid additional threads blocking on an mqueue
                 * that's being destroyed.
                 * operations from succeeding. It's important for this to be
                 * done under the same lock hold as the ipc_mqueue_changed
                 * call to avoid additional threads blocking on an mqueue
                 * that's being destroyed.

+                *
+                * The port active bit needs to be guarded under mqueue lock for
+                * turnstiles

                 */
                 */

+               port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
+               port->ip_timestamp = ipc_port_timestamp();

                reap_messages = ipc_mqueue_destroy_locked(mqueue);
                reap_messages = ipc_mqueue_destroy_locked(mqueue);

+       } else {
+               /* make port be in limbo */
+               port->ip_receiver_name = MACH_PORT_NULL;
+               port->ip_destination = IP_NULL;

        }
 
        imq_unlock(&port->ip_messages);
        }
 
        imq_unlock(&port->ip_messages);


@@ -608,9 +645,10 @@ ipc_port_clear_receiver(
 
 void
 ipc_port_init(
 
 void
 ipc_port_init(

-       ipc_port_t              port,
-       ipc_space_t             space,
-       mach_port_name_t        name)
+       ipc_port_t              port,
+       ipc_space_t             space,
+       ipc_port_init_flags_t   flags,
+       mach_port_name_t        name)

 {
        /* port->ip_kobject doesn't have to be initialized */
 
 {
        /* port->ip_kobject doesn't have to be initialized */
 


@@ -620,6 +658,10 @@ ipc_port_init(
        port->ip_mscount = 0;
        port->ip_srights = 0;
        port->ip_sorights = 0;
        port->ip_mscount = 0;
        port->ip_srights = 0;
        port->ip_sorights = 0;

+       if (flags & IPC_PORT_INIT_MAKE_SEND_RIGHT) {
+               port->ip_srights = 1;
+               port->ip_mscount = 1;
+       }

 
        port->ip_nsrequest = IP_NULL;
        port->ip_pdrequest = IP_NULL;
 
        port->ip_nsrequest = IP_NULL;
        port->ip_pdrequest = IP_NULL;


@@ -627,6 +669,7 @@ ipc_port_init(
 
        port->ip_premsg = IKM_NULL;
        port->ip_context = 0;
 
        port->ip_premsg = IKM_NULL;
        port->ip_context = 0;

+       port->ip_reply_context = 0;

 
        port->ip_sprequests  = 0;
        port->ip_spimportant = 0;
 
        port->ip_sprequests  = 0;
        port->ip_spimportant = 0;


@@ -635,13 +678,29 @@ ipc_port_init(
 
        port->ip_guarded      = 0;
        port->ip_strict_guard = 0;
 
        port->ip_guarded      = 0;
        port->ip_strict_guard = 0;

+       port->ip_immovable_receive = 0;
+       port->ip_no_grant    = 0;
+       port->ip_immovable_send = 0;

        port->ip_impcount    = 0;
 
        port->ip_impcount    = 0;
 

-       port->ip_specialreply = 0;
-       port->ip_link_sync_qos = 0;
+       if (flags & IPC_PORT_INIT_FILTER_MESSAGE) {
+               port->ip_object.io_bits |= IP_BIT_FILTER_MSG;
+       }
+
+       port->ip_tg_block_tracking = (flags & IPC_PORT_INIT_TG_BLOCK_TRACKING) != 0;
+       port->ip_specialreply = (flags & IPC_PORT_INIT_SPECIAL_REPLY) != 0;
+       port->ip_sync_link_state = PORT_SYNC_LINK_ANY;
+       port->ip_sync_bootstrap_checkin = 0;
+
+       ipc_special_reply_port_bits_reset(port);

 
 

-       ipc_mqueue_init(&port->ip_messages,
-                       FALSE /* !set */, NULL /* no reserved link */);
+       port->ip_send_turnstile = TURNSTILE_NULL;
+
+       ipc_mqueue_kind_t kind = IPC_MQUEUE_KIND_NONE;
+       if (flags & IPC_PORT_INIT_MESSAGE_QUEUE) {
+               kind = IPC_MQUEUE_KIND_PORT;
+       }
+       ipc_mqueue_init(&port->ip_messages, kind);

 }
 
 /*
 }
 
 /*


@@ -660,27 +719,34 @@ ipc_port_init(
 
 kern_return_t
 ipc_port_alloc(
 
 kern_return_t
 ipc_port_alloc(

-       ipc_space_t             space,
-       mach_port_name_t        *namep,
-       ipc_port_t              *portp)
+       ipc_space_t             space,
+       ipc_port_init_flags_t   flags,
+       mach_port_name_t        *namep,
+       ipc_port_t              *portp)

 {
        ipc_port_t port;
        mach_port_name_t name;
        kern_return_t kr;
 {
        ipc_port_t port;
        mach_port_name_t name;
        kern_return_t kr;

+       mach_port_type_t type = MACH_PORT_TYPE_RECEIVE;
+       mach_port_urefs_t urefs = 0;

 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);
 #endif /* MACH_ASSERT */
 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);
 #endif /* MACH_ASSERT */

-           
-       kr = ipc_object_alloc(space, IOT_PORT,
-                             MACH_PORT_TYPE_RECEIVE, 0,
-                             &name, (ipc_object_t *) &port);
-       if (kr != KERN_SUCCESS)
+
+       if (flags & IPC_PORT_INIT_MAKE_SEND_RIGHT) {
+               type |= MACH_PORT_TYPE_SEND;
+               urefs = 1;
+       }
+       kr = ipc_object_alloc(space, IOT_PORT, type, urefs,
+           &name, (ipc_object_t *) &port);
+       if (kr != KERN_SUCCESS) {

                return kr;
                return kr;

+       }

 
        /* port and space are locked */
 
        /* port and space are locked */

-       ipc_port_init(port, space, name);
+       ipc_port_init(port, space, flags, name);

 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);
 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);


@@ -711,31 +777,38 @@ ipc_port_alloc(
 
 kern_return_t
 ipc_port_alloc_name(
 
 kern_return_t
 ipc_port_alloc_name(

-       ipc_space_t             space,
-       mach_port_name_t        name,
-       ipc_port_t              *portp)
+       ipc_space_t             space,
+       ipc_port_init_flags_t   flags,
+       mach_port_name_t        name,
+       ipc_port_t              *portp)

 {
        ipc_port_t port;
        kern_return_t kr;
 {
        ipc_port_t port;
        kern_return_t kr;

+       mach_port_type_t type = MACH_PORT_TYPE_RECEIVE;
+       mach_port_urefs_t urefs = 0;

 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);
 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);

-#endif /* MACH_ASSERT */       
+#endif /* MACH_ASSERT */

 
 

-       kr = ipc_object_alloc_name(space, IOT_PORT,
-                                  MACH_PORT_TYPE_RECEIVE, 0,
-                                  name, (ipc_object_t *) &port);
-       if (kr != KERN_SUCCESS)
+       if (flags & IPC_PORT_INIT_MAKE_SEND_RIGHT) {
+               type |= MACH_PORT_TYPE_SEND;
+               urefs = 1;
+       }
+       kr = ipc_object_alloc_name(space, IOT_PORT, type, urefs,
+           name, (ipc_object_t *) &port);
+       if (kr != KERN_SUCCESS) {

                return kr;
                return kr;

+       }

 
        /* port is locked */
 
 
        /* port is locked */
 

-       ipc_port_init(port, space, name);
+       ipc_port_init(port, space, flags, name);

 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);
 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);

-#endif  /* MACH_ASSERT */      
+#endif  /* MACH_ASSERT */

 
        *portp = port;
 
 
        *portp = port;
 


@@ -743,7 +816,7 @@ ipc_port_alloc_name(
 }
 
 /*
 }
 
 /*

- *     Routine:        ipc_port_spnotify
+ *      Routine:       ipc_port_spnotify

  *     Purpose:
  *             Generate send-possible port notifications.
  *     Conditions:
  *     Purpose:
  *             Generate send-possible port notifications.
  *     Conditions:


@@ -751,7 +824,7 @@ ipc_port_alloc_name(
  */
 void
 ipc_port_spnotify(
  */
 void
 ipc_port_spnotify(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        ipc_port_request_index_t index = 0;
        ipc_table_elems_t size = 0;
 {
        ipc_port_request_index_t index = 0;
        ipc_table_elems_t size = 0;


@@ -760,11 +833,12 @@ ipc_port_spnotify(
         * If the port has no send-possible request
         * armed, don't bother to lock the port.
         */
         * If the port has no send-possible request
         * armed, don't bother to lock the port.
         */

-       if (port->ip_sprequests == 0)
+       if (port->ip_sprequests == 0) {

                return;
                return;

+       }

 
        ip_lock(port);
 
        ip_lock(port);

-       
+

 #if IMPORTANCE_INHERITANCE
        if (port->ip_spimportant != 0) {
                port->ip_spimportant = 0;
 #if IMPORTANCE_INHERITANCE
        if (port->ip_spimportant != 0) {
                port->ip_spimportant = 0;


@@ -792,8 +866,9 @@ revalidate:
                 * no need to go beyond table size when first
                 * we entered - those are future notifications.
                 */
                 * no need to go beyond table size when first
                 * we entered - those are future notifications.
                 */

-               if (size == 0)
+               if (size == 0) {

                        size = requests->ipr_size->its_size;
                        size = requests->ipr_size->its_size;

+               }

 
                /* no need to backtrack either */
                while (++index < size) {
 
                /* no need to backtrack either */
                while (++index < size) {


@@ -819,7 +894,7 @@ revalidate:
 }
 
 /*
 }
 
 /*

- *     Routine:        ipc_port_dnnotify
+ *      Routine:       ipc_port_dnnotify

  *     Purpose:
  *             Generate dead name notifications for
  *             all outstanding dead-name and send-
  *     Purpose:
  *             Generate dead name notifications for
  *             all outstanding dead-name and send-


@@ -831,7 +906,7 @@ revalidate:
  */
 void
 ipc_port_dnnotify(
  */
 void
 ipc_port_dnnotify(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        ipc_port_request_t requests = port->ip_requests;
 
 {
        ipc_port_request_t requests = port->ip_requests;
 


@@ -873,6 +948,7 @@ ipc_port_destroy(ipc_port_t port)
        ipc_mqueue_t mqueue;
        ipc_kmsg_t kmsg;
        boolean_t special_reply = port->ip_specialreply;
        ipc_mqueue_t mqueue;
        ipc_kmsg_t kmsg;
        boolean_t special_reply = port->ip_specialreply;

+       struct task_watchport_elem *watchport_elem = NULL;

 
 #if IMPORTANCE_INHERITANCE
        ipc_importance_task_t release_imp_task = IIT_NULL;
 
 #if IMPORTANCE_INHERITANCE
        ipc_importance_task_t release_imp_task = IIT_NULL;


@@ -881,10 +957,13 @@ ipc_port_destroy(ipc_port_t port)
        natural_t assertcnt = 0;
 #endif /* IMPORTANCE_INHERITANCE */
 
        natural_t assertcnt = 0;
 #endif /* IMPORTANCE_INHERITANCE */
 

-       assert(ip_active(port));
+       require_ip_active(port);

        /* port->ip_receiver_name is garbage */
        /* port->ip_receiver/port->ip_destination is garbage */
 
        /* port->ip_receiver_name is garbage */
        /* port->ip_receiver/port->ip_destination is garbage */
 

+       /* clear any reply-port context */
+       port->ip_reply_context = 0;
+

        /* check for a backup port */
        pdrequest = port->ip_pdrequest;
 
        /* check for a backup port */
        pdrequest = port->ip_pdrequest;
 


@@ -900,13 +979,15 @@ ipc_port_destroy(ipc_port_t port)
                /* Otherwise, nothing to drop */
        } else {
                assertcnt = port->ip_impcount;
                /* Otherwise, nothing to drop */
        } else {
                assertcnt = port->ip_impcount;

-               if (pdrequest != IP_NULL)
+               if (pdrequest != IP_NULL) {

                        /* mark in limbo for the journey */
                        port->ip_tempowner = 1;
                        /* mark in limbo for the journey */
                        port->ip_tempowner = 1;

+               }

        }
 
        }
 

-       if (top)
+       if (top) {

                self->ith_assertions = assertcnt;
                self->ith_assertions = assertcnt;

+       }

 #endif /* IMPORTANCE_INHERITANCE */
 
        if (pdrequest != IP_NULL) {
 #endif /* IMPORTANCE_INHERITANCE */
 
        if (pdrequest != IP_NULL) {


@@ -918,14 +999,19 @@ ipc_port_destroy(ipc_port_t port)
                /* we assume the ref for pdrequest */
                port->ip_pdrequest = IP_NULL;
 
                /* we assume the ref for pdrequest */
                port->ip_pdrequest = IP_NULL;
 

-               /* make port be in limbo */
-               port->ip_receiver_name = MACH_PORT_NULL;
-               port->ip_destination = IP_NULL;
-               ip_unlock(port);
+               imq_lock(&port->ip_messages);
+               watchport_elem = ipc_port_clear_watchport_elem_internal(port);
+               ipc_port_send_turnstile_recompute_push_locked(port);
+               /* mqueue and port unlocked */

 
                if (special_reply) {
 
                if (special_reply) {

-                       ipc_port_unlink_special_reply_port(port,
-                               IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE);
+                       ipc_port_adjust_special_reply_port(port,
+                           IPC_PORT_ADJUST_SR_ALLOW_SYNC_LINKAGE);
+               }
+
+               if (watchport_elem) {
+                       task_watchport_elem_deallocate(watchport_elem);
+                       watchport_elem = NULL;

                }
                /* consumes our refs for port and pdrequest */
                ipc_notify_port_destroyed(pdrequest, port);
                }
                /* consumes our refs for port and pdrequest */
                ipc_notify_port_destroyed(pdrequest, port);


@@ -933,10 +1019,6 @@ ipc_port_destroy(ipc_port_t port)
                goto drop_assertions;
        }
 
                goto drop_assertions;
        }
 

-       port->ip_object.io_bits &= ~IO_BITS_ACTIVE;
-       port->ip_timestamp = ipc_port_timestamp();
-       nsrequest = port->ip_nsrequest;
-

        /*
         * The mach_msg_* paths don't hold a port lock, they only hold a
         * reference to the port object. If a thread raced us and is now
        /*
         * The mach_msg_* paths don't hold a port lock, they only hold a
         * reference to the port object. If a thread raced us and is now


@@ -948,10 +1030,15 @@ ipc_port_destroy(ipc_port_t port)
         * port/mqueue that's been destroyed.
         */
        boolean_t reap_msgs = FALSE;
         * port/mqueue that's been destroyed.
         */
        boolean_t reap_msgs = FALSE;

-       reap_msgs = ipc_port_clear_receiver(port, TRUE); /* marks mqueue inactive */
+       reap_msgs = ipc_port_clear_receiver(port, TRUE); /* marks port and mqueue inactive */

        assert(port->ip_in_pset == 0);
        assert(port->ip_mscount == 0);
 
        assert(port->ip_in_pset == 0);
        assert(port->ip_mscount == 0);
 

+       imq_lock(&port->ip_messages);
+       watchport_elem = ipc_port_clear_watchport_elem_internal(port);
+       imq_unlock(&port->ip_messages);
+       nsrequest = port->ip_nsrequest;
+

        /*
         * If the port has a preallocated message buffer and that buffer
         * is not inuse, free it.  If it has an inuse one, then the kmsg
        /*
         * If the port has a preallocated message buffer and that buffer
         * is not inuse, free it.  If it has an inuse one, then the kmsg


@@ -966,33 +1053,46 @@ ipc_port_destroy(ipc_port_t port)
                kmsg = port->ip_premsg;
                assert(kmsg != IKM_NULL);
                inuse_port = ikm_prealloc_inuse_port(kmsg);
                kmsg = port->ip_premsg;
                assert(kmsg != IKM_NULL);
                inuse_port = ikm_prealloc_inuse_port(kmsg);

-               IP_CLEAR_PREALLOC(port, kmsg);
-               ip_unlock(port);
+               ipc_kmsg_clear_prealloc(kmsg, port);
+
+               imq_lock(&port->ip_messages);
+               ipc_port_send_turnstile_recompute_push_locked(port);
+               /* mqueue and port unlocked */
+

                if (inuse_port != IP_NULL) {
                        assert(inuse_port == port);
                } else {
                        ipc_kmsg_free(kmsg);
                }
        } else {
                if (inuse_port != IP_NULL) {
                        assert(inuse_port == port);
                } else {
                        ipc_kmsg_free(kmsg);
                }
        } else {

-               ip_unlock(port);
+               imq_lock(&port->ip_messages);
+               ipc_port_send_turnstile_recompute_push_locked(port);
+               /* mqueue and port unlocked */
+       }
+
+       /* Deallocate the watchport element */
+       if (watchport_elem) {
+               task_watchport_elem_deallocate(watchport_elem);
+               watchport_elem = NULL;

        }
 
        /* unlink the kmsg from special reply port */
        if (special_reply) {
        }
 
        /* unlink the kmsg from special reply port */
        if (special_reply) {

-               ipc_port_unlink_special_reply_port(port,
-                       IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE);
+               ipc_port_adjust_special_reply_port(port,
+                   IPC_PORT_ADJUST_SR_ALLOW_SYNC_LINKAGE);

        }
 
        /* throw away no-senders request */
        }
 
        /* throw away no-senders request */

-       if (nsrequest != IP_NULL)
+       if (nsrequest != IP_NULL) {

                ipc_notify_send_once(nsrequest); /* consumes ref */
                ipc_notify_send_once(nsrequest); /* consumes ref */

-
+       }

        /*
         * Reap any kmsg objects waiting to be destroyed.
         * This must be done after we've released the port lock.
         */
        /*
         * Reap any kmsg objects waiting to be destroyed.
         * This must be done after we've released the port lock.
         */

-       if (reap_msgs)
+       if (reap_msgs) {

                ipc_kmsg_reap_delayed();
                ipc_kmsg_reap_delayed();

+       }

 
        mqueue = &port->ip_messages;
 
 
        mqueue = &port->ip_messages;
 


@@ -1006,7 +1106,7 @@ ipc_port_destroy(ipc_port_t port)
 
        ip_release(port); /* consume caller's ref */
 
 
        ip_release(port); /* consume caller's ref */
 

- drop_assertions:
+drop_assertions:

 #if IMPORTANCE_INHERITANCE
        if (release_imp_task != IIT_NULL) {
                if (assertcnt > 0) {
 #if IMPORTANCE_INHERITANCE
        if (release_imp_task != IIT_NULL) {
                if (assertcnt > 0) {


@@ -1016,7 +1116,6 @@ ipc_port_destroy(ipc_port_t port)
                        ipc_importance_task_drop_internal_assertion(release_imp_task, assertcnt);
                }
                ipc_importance_task_release(release_imp_task);
                        ipc_importance_task_drop_internal_assertion(release_imp_task, assertcnt);
                }
                ipc_importance_task_release(release_imp_task);

-

        } else if (assertcnt > 0) {
                if (top) {
                        self->ith_assertions = 0;
        } else if (assertcnt > 0) {
                if (top) {
                        self->ith_assertions = 0;


@@ -1048,25 +1147,27 @@ ipc_port_destroy(ipc_port_t port)
 
 boolean_t
 ipc_port_check_circularity(
 
 boolean_t
 ipc_port_check_circularity(

-       ipc_port_t      port,
-       ipc_port_t      dest)
+       ipc_port_t      port,
+       ipc_port_t      dest)

 {
 #if IMPORTANCE_INHERITANCE
        /* adjust importance counts at the same time */
        return ipc_importance_check_circularity(port, dest);
 #else
        ipc_port_t base;
 {
 #if IMPORTANCE_INHERITANCE
        /* adjust importance counts at the same time */
        return ipc_importance_check_circularity(port, dest);
 #else
        ipc_port_t base;

-       sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0};
-       sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0};
-       boolean_t update_knote = FALSE;
+       struct task_watchport_elem *watchport_elem = NULL;

 
        assert(port != IP_NULL);
        assert(dest != IP_NULL);
 
 
        assert(port != IP_NULL);
        assert(dest != IP_NULL);
 

-       if (port == dest)
+       if (port == dest) {

                return TRUE;
                return TRUE;

+       }

        base = dest;
 
        base = dest;
 

+       /* Check if destination needs a turnstile */
+       ipc_port_send_turnstile_prepare(dest);
+

        /*
         *      First try a quick check that can run in parallel.
         *      No circularity if dest is not in transit.
        /*
         *      First try a quick check that can run in parallel.
         *      No circularity if dest is not in transit.


@@ -1075,8 +1176,9 @@ ipc_port_check_circularity(
        if (ip_lock_try(dest)) {
                if (!ip_active(dest) ||
                    (dest->ip_receiver_name != MACH_PORT_NULL) ||
        if (ip_lock_try(dest)) {
                if (!ip_active(dest) ||
                    (dest->ip_receiver_name != MACH_PORT_NULL) ||

-                   (dest->ip_destination == IP_NULL))
+                   (dest->ip_destination == IP_NULL)) {

                        goto not_circular;
                        goto not_circular;

+               }

 
                /* dest is in transit; further checking necessary */
 
 
                /* dest is in transit; further checking necessary */
 


@@ -1096,8 +1198,9 @@ ipc_port_check_circularity(
 
                if (!ip_active(base) ||
                    (base->ip_receiver_name != MACH_PORT_NULL) ||
 
                if (!ip_active(base) ||
                    (base->ip_receiver_name != MACH_PORT_NULL) ||

-                   (base->ip_destination == IP_NULL))
+                   (base->ip_destination == IP_NULL)) {

                        break;
                        break;

+               }

 
                base = base->ip_destination;
        }
 
                base = base->ip_destination;
        }


@@ -1110,24 +1213,24 @@ ipc_port_check_circularity(
                ipc_port_multiple_unlock();
 
                /* port (== base) is in limbo */
                ipc_port_multiple_unlock();
 
                /* port (== base) is in limbo */

-
-               assert(ip_active(port));
+               require_ip_active(port);

                assert(port->ip_receiver_name == MACH_PORT_NULL);
                assert(port->ip_destination == IP_NULL);
 
                assert(port->ip_receiver_name == MACH_PORT_NULL);
                assert(port->ip_destination == IP_NULL);
 

-               while (dest != IP_NULL) {
+               base = dest;
+               while (base != IP_NULL) {

                        ipc_port_t next;
 
                        /* dest is in transit or in limbo */
                        ipc_port_t next;
 
                        /* dest is in transit or in limbo */

+                       require_ip_active(base);
+                       assert(base->ip_receiver_name == MACH_PORT_NULL);

 
 

-                       assert(ip_active(dest));
-                       assert(dest->ip_receiver_name == MACH_PORT_NULL);
-
-                       next = dest->ip_destination;
-                       ip_unlock(dest);
-                       dest = next;
+                       next = base->ip_destination;
+                       ip_unlock(base);
+                       base = next;

                }
 
                }
 

+               ipc_port_send_turnstile_complete(dest);

                return TRUE;
        }
 
                return TRUE;
        }
 


@@ -1141,498 +1244,1031 @@ ipc_port_check_circularity(
        ipc_port_multiple_unlock();
 
 not_circular:
        ipc_port_multiple_unlock();
 
 not_circular:

-       imq_lock(&base->ip_messages);
+       imq_lock(&port->ip_messages);

 
        /* port is in limbo */
 
        /* port is in limbo */

-
-       assert(ip_active(port));
+       require_ip_active(port);

        assert(port->ip_receiver_name == MACH_PORT_NULL);
        assert(port->ip_destination == IP_NULL);
 
        assert(port->ip_receiver_name == MACH_PORT_NULL);
        assert(port->ip_destination == IP_NULL);
 

+       /* Clear the watchport boost */
+       watchport_elem = ipc_port_clear_watchport_elem_internal(port);
+
+       /* Check if the port is being enqueued as a part of sync bootstrap checkin */
+       if (dest->ip_specialreply && dest->ip_sync_bootstrap_checkin) {
+               port->ip_sync_bootstrap_checkin = 1;
+       }
+

        ip_reference(dest);
        port->ip_destination = dest;
 
        ip_reference(dest);
        port->ip_destination = dest;
 

-       /* Capture the sync qos count delta */
-       for (int i = 0; i < THREAD_QOS_LAST; i++) {
-               sync_qos_delta_add[i] = port_sync_qos(port, i);
+       /* Setup linkage for source port if it has sync ipc push */
+       struct turnstile *send_turnstile = TURNSTILE_NULL;
+       if (port_send_turnstile(port)) {
+               send_turnstile = turnstile_prepare((uintptr_t)port,
+                   port_send_turnstile_address(port),
+                   TURNSTILE_NULL, TURNSTILE_SYNC_IPC);
+
+               /*
+                * What ipc_port_adjust_port_locked would do,
+                * but we need to also drop even more locks before
+                * calling turnstile_update_inheritor_complete().
+                */
+               ipc_port_adjust_sync_link_state_locked(port, PORT_SYNC_LINK_ANY, NULL);
+
+               turnstile_update_inheritor(send_turnstile, port_send_turnstile(dest),
+                   (TURNSTILE_INHERITOR_TURNSTILE | TURNSTILE_IMMEDIATE_UPDATE));
+
+               /* update complete and turnstile complete called after dropping all locks */

        }
        }

+       imq_unlock(&port->ip_messages);

 
        /* now unlock chain */
 
        ip_unlock(port);
 
        for (;;) {
 
        /* now unlock chain */
 
        ip_unlock(port);
 
        for (;;) {

-               /* every port along chain tracks override behind it */
-               update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub);
-               if (dest == base)
+               ipc_port_t next;
+
+               if (dest == base) {

                        break;
                        break;

+               }

 
                /* port is in transit */
 
                /* port is in transit */

-
-               assert(ip_active(dest));
+               require_ip_active(dest);

                assert(dest->ip_receiver_name == MACH_PORT_NULL);
                assert(dest->ip_destination != IP_NULL);
 
                assert(dest->ip_receiver_name == MACH_PORT_NULL);
                assert(dest->ip_destination != IP_NULL);
 

-               port = dest->ip_destination;
+               next = dest->ip_destination;

                ip_unlock(dest);
                ip_unlock(dest);

-               dest = port;
+               dest = next;

        }
 
        /* base is not in transit */
        assert(!ip_active(base) ||
        }
 
        /* base is not in transit */
        assert(!ip_active(base) ||

-              (base->ip_receiver_name != MACH_PORT_NULL) ||
-              (base->ip_destination == IP_NULL));
+           (base->ip_receiver_name != MACH_PORT_NULL) ||
+           (base->ip_destination == IP_NULL));
+
+       ip_unlock(base);

 
 

-       if (update_knote) {
-               KNOTE(&base->ip_messages.imq_klist, 0);
+       /* All locks dropped, call turnstile_update_inheritor_complete for source port's turnstile */
+       if (send_turnstile) {
+               turnstile_update_inheritor_complete(send_turnstile, TURNSTILE_INTERLOCK_NOT_HELD);
+
+               /* Take the mq lock to call turnstile complete */
+               imq_lock(&port->ip_messages);
+               turnstile_complete((uintptr_t)port, port_send_turnstile_address(port), NULL, TURNSTILE_SYNC_IPC);
+               send_turnstile = TURNSTILE_NULL;
+               imq_unlock(&port->ip_messages);
+               turnstile_cleanup();

        }
        }

-       imq_unlock(&base->ip_messages);

 
 

-       ip_unlock(base);
+       if (watchport_elem) {
+               task_watchport_elem_deallocate(watchport_elem);
+       }

 
        return FALSE;
 #endif /* !IMPORTANCE_INHERITANCE */
 }
 
 /*
 
        return FALSE;
 #endif /* !IMPORTANCE_INHERITANCE */
 }
 
 /*

- *     Routine:        ipc_port_link_special_reply_port_with_qos
+ *     Routine:        ipc_port_watchport_elem

  *     Purpose:
  *     Purpose:

- *             Link the special reply port with the destination port.
- *             Update the sync qos count of special reply port,
- *             destination port.
+ *             Get the port's watchport elem field

  *
  *     Conditions:
  *
  *     Conditions:

- *             Nothing is locked.
+ *             mqueue locked

  */
  */

-kern_return_t
-ipc_port_link_special_reply_port_with_qos(
-       ipc_port_t special_reply_port,
-       ipc_port_t dest_port,
-       int qos)
+static struct task_watchport_elem *
+ipc_port_watchport_elem(ipc_port_t port)

 {
 {

-       ipc_port_t next, base;
-       sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0};
-       sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0};
-       boolean_t update_knote = FALSE;
-       boolean_t multiple_lock = FALSE;
-
-       ip_lock(dest_port);
-
-       /* Check if dest is active */
-       if (!ip_active(dest_port)) {
-               ip_unlock(dest_port);
-               return KERN_FAILURE;
-       }
-
-       if ((dest_port->ip_receiver_name == MACH_PORT_NULL) &&
-           (dest_port->ip_destination != IP_NULL)) {
-               /* dest_port is in transit; need to take the serialize lock */
-               ip_unlock(dest_port);
-               goto take_multiple_lock;
-       }
+       return port->ip_messages.imq_wait_queue.waitq_tspriv;
+}

 
 

-       /* Check if the port is a special reply port */
-       if (ip_lock_try(special_reply_port)) {
-               if (!special_reply_port->ip_specialreply ||
-                   !special_reply_port->ip_link_sync_qos ||
-                   (special_reply_port->ip_sync_qos_override_port != IP_NULL &&
-                    special_reply_port->ip_sync_qos_override_port != dest_port)) {
+/*
+ *     Routine:        ipc_port_update_watchport_elem
+ *     Purpose:
+ *             Set the port's watchport elem field
+ *
+ *     Conditions:
+ *             mqueue locked
+ */
+static inline struct task_watchport_elem *
+ipc_port_update_watchport_elem(ipc_port_t port, struct task_watchport_elem *we)
+{
+       assert(!port->ip_specialreply);
+       struct task_watchport_elem *old_we = ipc_port_watchport_elem(port);
+       port->ip_messages.imq_wait_queue.waitq_tspriv = we;
+       return old_we;
+}

 
 

-                       boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos;
-                       ip_unlock(special_reply_port);
-                       ip_unlock(dest_port);
-                       /* return KERN_SUCCESS when link_sync_qos is not set */
-                       if (!link_sync_qos) {
-                               return KERN_SUCCESS;
-                       }
-                       return KERN_FAILURE;
-               } else {
-                       goto both_ports_locked;
-               }
-       }
+/*
+ *     Routine:        ipc_special_reply_stash_pid_locked
+ *     Purpose:
+ *             Set the pid of process that copied out send once right to special reply port.
+ *
+ *     Conditions:
+ *             port locked
+ */
+static inline void
+ipc_special_reply_stash_pid_locked(ipc_port_t port, int pid)
+{
+       assert(port->ip_specialreply);
+       port->ip_messages.imq_wait_queue.waitq_priv_pid = pid;
+       return;
+}

 
 

-       ip_unlock(dest_port);
+/*
+ *     Routine:        ipc_special_reply_get_pid_locked
+ *     Purpose:
+ *             Get the pid of process that copied out send once right to special reply port.
+ *
+ *     Conditions:
+ *             port locked
+ */
+int
+ipc_special_reply_get_pid_locked(ipc_port_t port)
+{
+       assert(port->ip_specialreply);
+       return port->ip_messages.imq_wait_queue.waitq_priv_pid;
+}

 
 

-take_multiple_lock:
+/*
+ * Update the recv turnstile inheritor for a port.
+ *
+ * Sync IPC through the port receive turnstile only happens for the special
+ * reply port case. It has three sub-cases:
+ *
+ * 1. a send-once right is in transit, and pushes on the send turnstile of its
+ *    destination mqueue.
+ *
+ * 2. a send-once right has been stashed on a knote it was copied out "through",
+ *    as the first such copied out port.
+ *
+ * 3. a send-once right has been stashed on a knote it was copied out "through",
+ *    as the second or more copied out port.
+ */
+void
+ipc_port_recv_update_inheritor(
+       ipc_port_t port,
+       struct turnstile *rcv_turnstile,
+       turnstile_update_flags_t flags)
+{
+       struct turnstile *inheritor = TURNSTILE_NULL;
+       struct knote *kn;

 
 

-       ipc_port_multiple_lock(); /* massive serialization */
-       multiple_lock = TRUE;
+       if (ip_active(port) && port->ip_specialreply) {
+               imq_held(&port->ip_messages);

 
 

-       ip_lock(special_reply_port);
+               switch (port->ip_sync_link_state) {
+               case PORT_SYNC_LINK_PORT:
+                       if (port->ip_sync_inheritor_port != NULL) {
+                               inheritor = port_send_turnstile(port->ip_sync_inheritor_port);
+                       }
+                       break;

 
 

-       /* Check if the special reply port is marked regular */
-       if (!special_reply_port->ip_specialreply ||
-           !special_reply_port->ip_link_sync_qos ||
-           (special_reply_port->ip_sync_qos_override_port != IP_NULL &&
-            special_reply_port->ip_sync_qos_override_port != dest_port)) {
+               case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+                       kn = port->ip_sync_inheritor_knote;
+                       inheritor = filt_ipc_kqueue_turnstile(kn);
+                       break;

 
 

-               boolean_t link_sync_qos = special_reply_port->ip_link_sync_qos;
-               ip_unlock(special_reply_port);
-               ipc_port_multiple_unlock();
-               /* return KERN_SUCCESS when link_sync_qos is not set */
-               if (!link_sync_qos) {
-                       return KERN_SUCCESS;
+               case PORT_SYNC_LINK_WORKLOOP_STASH:
+                       inheritor = port->ip_sync_inheritor_ts;
+                       break;

                }
                }

-               return KERN_FAILURE;

        }
 
        }
 

-       ip_lock(dest_port);
+       turnstile_update_inheritor(rcv_turnstile, inheritor,
+           flags | TURNSTILE_INHERITOR_TURNSTILE);
+}
+
+/*
+ * Update the send turnstile inheritor for a port.
+ *
+ * Sync IPC through the port send turnstile has 7 possible reasons to be linked:
+ *
+ * 1. a special reply port is part of sync ipc for bootstrap checkin and needs
+ *    to push on thread doing the sync ipc.
+ *
+ * 2. a receive right is in transit, and pushes on the send turnstile of its
+ *    destination mqueue.
+ *
+ * 3. port was passed as an exec watchport and port is pushing on main thread
+ *    of the task.
+ *
+ * 4. a receive right has been stashed on a knote it was copied out "through",
+ *    as the first such copied out port (same as PORT_SYNC_LINK_WORKLOOP_KNOTE
+ *    for the special reply port)
+ *
+ * 5. a receive right has been stashed on a knote it was copied out "through",
+ *    as the second or more copied out port (same as
+ *    PORT_SYNC_LINK_WORKLOOP_STASH for the special reply port)
+ *
+ * 6. a receive right has been copied out as a part of sync bootstrap checkin
+ *    and needs to push on thread doing the sync bootstrap checkin.
+ *
+ * 7. the receive right is monitored by a knote, and pushes on any that is
+ *    registered on a workloop. filt_machport makes sure that if such a knote
+ *    exists, it is kept as the first item in the knote list, so we never need
+ *    to walk.
+ */
+void
+ipc_port_send_update_inheritor(
+       ipc_port_t port,
+       struct turnstile *send_turnstile,
+       turnstile_update_flags_t flags)
+{
+       ipc_mqueue_t mqueue = &port->ip_messages;
+       turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;
+       struct knote *kn;
+       turnstile_update_flags_t inheritor_flags = TURNSTILE_INHERITOR_TURNSTILE;

 
 

-both_ports_locked:
-       next = dest_port;
+       assert(imq_held(mqueue));

 
 

-       /* Apply the qos to special reply port, capture the old qos */
-       if (special_reply_port->ip_sync_qos_override_port != IP_NULL) {
-               /* Check if qos needs to be updated */
-               if ((sync_qos_count_t)qos <= port_special_qos(special_reply_port)) {
-                       imq_lock(&dest_port->ip_messages);
-                       goto done_update;
+       if (!ip_active(port)) {
+               /* this port is no longer active, it should not push anywhere */
+       } else if (port->ip_specialreply) {
+               /* Case 1. */
+               if (port->ip_sync_bootstrap_checkin && prioritize_launch) {
+                       inheritor = port->ip_messages.imq_srp_owner_thread;
+                       inheritor_flags = TURNSTILE_INHERITOR_THREAD;
+               }
+       } else if (port->ip_receiver_name == MACH_PORT_NULL &&
+           port->ip_destination != NULL) {
+               /* Case 2. */
+               inheritor = port_send_turnstile(port->ip_destination);
+       } else if (ipc_port_watchport_elem(port) != NULL) {
+               /* Case 3. */
+               if (prioritize_launch) {
+                       assert(port->ip_sync_link_state == PORT_SYNC_LINK_ANY);
+                       inheritor = ipc_port_get_watchport_inheritor(port);
+                       inheritor_flags = TURNSTILE_INHERITOR_THREAD;
+               }
+       } else if (port->ip_sync_link_state == PORT_SYNC_LINK_WORKLOOP_KNOTE) {
+               /* Case 4. */
+               inheritor = filt_ipc_kqueue_turnstile(mqueue->imq_inheritor_knote);
+       } else if (port->ip_sync_link_state == PORT_SYNC_LINK_WORKLOOP_STASH) {
+               /* Case 5. */
+               inheritor = mqueue->imq_inheritor_turnstile;
+       } else if (port->ip_sync_link_state == PORT_SYNC_LINK_RCV_THREAD) {
+               /* Case 6. */
+               if (prioritize_launch) {
+                       inheritor = port->ip_messages.imq_inheritor_thread_ref;
+                       inheritor_flags = TURNSTILE_INHERITOR_THREAD;
+               }
+       } else if ((kn = SLIST_FIRST(&mqueue->imq_klist))) {
+               /* Case 7. Push on a workloop that is interested */
+               if (filt_machport_kqueue_has_turnstile(kn)) {
+                       assert(port->ip_sync_link_state == PORT_SYNC_LINK_ANY);
+                       inheritor = filt_ipc_kqueue_turnstile(kn);

                }
                }

-               sync_qos_delta_sub[port_special_qos(special_reply_port)]++;

        }
 
        }
 

-       set_port_special_qos(special_reply_port, (sync_qos_count_t)qos);
-       sync_qos_delta_add[qos]++;
-
-       /* Link the special reply port to dest port */
-       if (special_reply_port->ip_sync_qos_override_port == IP_NULL) {
-               /* take a reference on dest_port */
-               ip_reference(dest_port);
-               special_reply_port->ip_sync_qos_override_port = dest_port;
-       }
+       turnstile_update_inheritor(send_turnstile, inheritor,
+           flags | inheritor_flags);
+}

 
 

-       /* Apply the sync qos delta to all in-transit ports */
-       for (;;) {
-               boolean_t port_not_in_transit = FALSE;
-               if (!ip_active(next) ||
-                   (next->ip_receiver_name != MACH_PORT_NULL) ||
-                   (next->ip_destination == IP_NULL)) {
-                       /* Get the mqueue lock for destination port to update knotes */
-                       imq_lock(&next->ip_messages);
-                       port_not_in_transit = TRUE;
+/*
+ *     Routine:        ipc_port_send_turnstile_prepare
+ *     Purpose:
+ *             Get a reference on port's send turnstile, if
+ *             port does not have a send turnstile then allocate one.
+ *
+ *     Conditions:
+ *             Nothing is locked.
+ */
+void
+ipc_port_send_turnstile_prepare(ipc_port_t port)
+{
+       struct turnstile *turnstile = TURNSTILE_NULL;
+       struct turnstile *send_turnstile = TURNSTILE_NULL;
+
+retry_alloc:
+       imq_lock(&port->ip_messages);
+
+       if (port_send_turnstile(port) == NULL ||
+           port_send_turnstile(port)->ts_port_ref == 0) {
+               if (turnstile == TURNSTILE_NULL) {
+                       imq_unlock(&port->ip_messages);
+                       turnstile = turnstile_alloc();
+                       goto retry_alloc;

                }
                }

-               /* Apply the sync qos delta */
-               update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub);

 
 

-               if (port_not_in_transit)
-                       break;
+               send_turnstile = turnstile_prepare((uintptr_t)port,
+                   port_send_turnstile_address(port),
+                   turnstile, TURNSTILE_SYNC_IPC);
+               turnstile = TURNSTILE_NULL;

 
 

-               next = next->ip_destination;
-               ip_lock(next);
-       }
-done_update:
+               ipc_port_send_update_inheritor(port, send_turnstile,
+                   TURNSTILE_IMMEDIATE_UPDATE);

 
 

-       if (multiple_lock) {
-               ipc_port_multiple_unlock();
+               /* turnstile complete will be called in ipc_port_send_turnstile_complete */

        }
 
        }
 

-       ip_unlock(special_reply_port);
-       base = next;
-       next = dest_port;
-
-       while (next != base) {
-               ipc_port_t prev = next;
-               next = next->ip_destination;
+       /* Increment turnstile counter */
+       port_send_turnstile(port)->ts_port_ref++;
+       imq_unlock(&port->ip_messages);

 
 

-               ip_unlock(prev);
+       if (send_turnstile) {
+               turnstile_update_inheritor_complete(send_turnstile,
+                   TURNSTILE_INTERLOCK_NOT_HELD);

        }
        }

-
-       if (update_knote) {
-               KNOTE(&base->ip_messages.imq_klist, 0);
+       if (turnstile != TURNSTILE_NULL) {
+               turnstile_deallocate(turnstile);

        }
        }

-       imq_unlock(&base->ip_messages);
-       ip_unlock(base);
-       return KERN_SUCCESS;

 }
 
 }
 

+

 /*
 /*

- *     Routine:        ipc_port_unlink_special_reply_port_locked
+ *     Routine:        ipc_port_send_turnstile_complete

  *     Purpose:
  *     Purpose:

- *             If the special port is linked to a port, adjust it's sync qos override and unlink the port.
- *     Condition:
- *             Special reply port locked on entry.
- *             Special reply port unlocked on return.
- *     Returns:
- *             None.
+ *             Drop a ref on the port's send turnstile, if the
+ *             ref becomes zero, deallocate the turnstile.
+ *
+ *     Conditions:
+ *             The space might be locked, use safe deallocate.

  */
 void
  */
 void

-ipc_port_unlink_special_reply_port_locked(
-       ipc_port_t special_reply_port,
-       struct knote *kn,
-       uint8_t flags)
+ipc_port_send_turnstile_complete(ipc_port_t port)

 {
 {

-       ipc_port_t dest_port;
-       sync_qos_count_t sync_qos;
-       sync_qos_count_t sync_qos_delta_add[THREAD_QOS_LAST] = {0};
-       sync_qos_count_t sync_qos_delta_sub[THREAD_QOS_LAST] = {0};
+       struct turnstile *turnstile = TURNSTILE_NULL;

 
 

-       /* Return if called from copy out in pseudo receive */
-       if (kn == ITH_KNOTE_PSEUDO) {
-               ip_unlock(special_reply_port);
-               return;
-       }
-
-       /* check if special port has a port linked to it */
-       if (special_reply_port->ip_specialreply == 0 ||
-           special_reply_port->ip_sync_qos_override_port == IP_NULL) {
-               set_port_special_qos(special_reply_port, 0);
-               if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) {
-                       special_reply_port->ip_specialreply = 0;
-               }
-               if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) {
-                       special_reply_port->ip_link_sync_qos = 1;
-               }
-               ip_unlock(special_reply_port);
-               return;
-       }
-
-       /*
-        * port->ip_sync_qos_override_port is not null and it is safe
-        * to access it since ip_specialreply is set.
-        */
-       dest_port = special_reply_port->ip_sync_qos_override_port;
-       sync_qos_delta_sub[port_special_qos(special_reply_port)]++;
-       sync_qos = port_special_qos(special_reply_port);
+       /* Drop turnstile count on dest port */
+       imq_lock(&port->ip_messages);

 
 

-       /* Clear qos delta for special reply port */
-       set_port_special_qos(special_reply_port, 0);
-       special_reply_port->ip_sync_qos_override_port = IP_NULL;
-       if (flags & IPC_PORT_UNLINK_SR_CLEAR_SPECIAL_REPLY) {
-               special_reply_port->ip_specialreply = 0;
+       port_send_turnstile(port)->ts_port_ref--;
+       if (port_send_turnstile(port)->ts_port_ref == 0) {
+               turnstile_complete((uintptr_t)port, port_send_turnstile_address(port),
+                   &turnstile, TURNSTILE_SYNC_IPC);
+               assert(turnstile != TURNSTILE_NULL);

        }
        }

+       imq_unlock(&port->ip_messages);
+       turnstile_cleanup();

 
 

-       if (flags & IPC_PORT_UNLINK_SR_ALLOW_SYNC_QOS_LINKAGE) {
-               special_reply_port->ip_link_sync_qos = 1;
-       } else {
-               special_reply_port->ip_link_sync_qos = 0;
-       }
-
-       ip_unlock(special_reply_port);
-
-       /* Add the sync qos on knote */
-       if (ITH_KNOTE_VALID(kn)) {
-               knote_adjust_sync_qos(kn, sync_qos, TRUE);
+       if (turnstile != TURNSTILE_NULL) {
+               turnstile_deallocate_safe(turnstile);
+               turnstile = TURNSTILE_NULL;

        }
        }

-
-       /* Adjust the sync qos of destination */
-       ipc_port_adjust_sync_qos(dest_port, sync_qos_delta_add, sync_qos_delta_sub);
-       ip_release(dest_port);

 }
 
 /*
 }
 
 /*

- *     Routine:        ipc_port_unlink_special_reply_port
+ *     Routine:        ipc_port_rcv_turnstile

  *     Purpose:
  *     Purpose:

- *             If the special port is linked to a port, adjust it's sync qos override and unlink the port.
- *     Condition:
- *             Nothing locked.
- *     Returns:
- *             None.
+ *             Get the port's receive turnstile
+ *
+ *     Conditions:
+ *             mqueue locked or thread waiting on turnstile is locked.

  */
  */

-void
-ipc_port_unlink_special_reply_port(
-       ipc_port_t special_reply_port,
-       uint8_t flags)
+static struct turnstile *
+ipc_port_rcv_turnstile(ipc_port_t port)

 {
 {

-       ip_lock(special_reply_port);
-       ipc_port_unlink_special_reply_port_locked(special_reply_port, NULL, flags);
-       /* special_reply_port unlocked */
+       return *port_rcv_turnstile_address(port);

 }
 
 }
 

+

 /*
 /*

- *     Routine:        ipc_port_sync_qos_delta
+ *     Routine:        ipc_port_link_special_reply_port

  *     Purpose:
  *     Purpose:

- *             Adjust the sync qos count associated with a port.
+ *             Link the special reply port with the destination port.
+ *              Allocates turnstile to dest port.

  *
  *

- *             For now, be defensive during deductions to make sure the
- *             sync_qos count for the port doesn't underflow zero.
- *     Returns:
- *             TRUE: if max sync qos of the port changes.
- *             FALSE: otherwise.

  *     Conditions:
  *     Conditions:

- *             The port is referenced and locked.
- *             The mqueue is locked if port is not in-transit.
+ *             Nothing is locked.

  */
  */

-boolean_t
-ipc_port_sync_qos_delta(
-       ipc_port_t        port,
-       sync_qos_count_t *sync_qos_delta_add,
-       sync_qos_count_t *sync_qos_delta_sub)
+void
+ipc_port_link_special_reply_port(
+       ipc_port_t special_reply_port,
+       ipc_port_t dest_port,
+       boolean_t sync_bootstrap_checkin)

 {
 {

-       sync_qos_count_t max_sync_qos_index;
+       boolean_t drop_turnstile_ref = FALSE;
+       boolean_t special_reply = FALSE;

 
 

-       if (!ip_active(port)) {
-               return FALSE;
+       /* Check if dest_port needs a turnstile */
+       ipc_port_send_turnstile_prepare(dest_port);
+
+       /* Lock the special reply port and establish the linkage */
+       ip_lock(special_reply_port);
+       imq_lock(&special_reply_port->ip_messages);
+
+       special_reply = special_reply_port->ip_specialreply;
+
+       if (sync_bootstrap_checkin && special_reply) {
+               special_reply_port->ip_sync_bootstrap_checkin = 1;

        }
 
        }
 

-       max_sync_qos_index = ipc_port_get_max_sync_qos_index(port);
+       /* Check if we need to drop the acquired turnstile ref on dest port */
+       if (!special_reply ||
+           special_reply_port->ip_sync_link_state != PORT_SYNC_LINK_ANY ||
+           special_reply_port->ip_sync_inheritor_port != IPC_PORT_NULL) {
+               drop_turnstile_ref = TRUE;
+       } else {
+               /* take a reference on dest_port */
+               ip_reference(dest_port);
+               special_reply_port->ip_sync_inheritor_port = dest_port;
+               special_reply_port->ip_sync_link_state = PORT_SYNC_LINK_PORT;
+       }

 
 

-       for (int i = 0; i < THREAD_QOS_LAST; i++) {
-               sync_qos_count_t port_sync_qos_count = port_sync_qos(port, i);
-               /* Do not let the sync qos underflow */
-               if (sync_qos_delta_sub[i] > port_sync_qos_count) {
-                       KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_UNDERFLOW),
-                             i, VM_KERNEL_UNSLIDE_OR_PERM(port),
-                             port_sync_qos_count, sync_qos_delta_sub[i]);
+       imq_unlock(&special_reply_port->ip_messages);
+       ip_unlock(special_reply_port);
+
+       if (special_reply) {
+               /*
+                * For special reply ports, if the destination port is
+                * marked with the thread group blocked tracking flag,
+                * callout to the performance controller.
+                */
+               ipc_port_thread_group_blocked(dest_port);
+       }

 
 

-                       set_port_sync_qos(port, i, 0);
-               } else if (sync_qos_delta_sub[i] != 0) {
-                       KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_REMOVED),
-                             i, VM_KERNEL_UNSLIDE_OR_PERM(port),
-                             port_sync_qos_count, sync_qos_delta_sub[i]);
+       if (drop_turnstile_ref) {
+               ipc_port_send_turnstile_complete(dest_port);
+       }

 
 

-                       set_port_sync_qos(port, i, (port_sync_qos_count - sync_qos_delta_sub[i]));
-               }
+       return;
+}

 
 

-               port_sync_qos_count = port_sync_qos(port, i);
-               /* Do not let the sync qos overflow */
-               if (UCHAR_MAX - sync_qos_delta_add[i] < port_sync_qos_count) {
-                       KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_OVERFLOW),
-                             i, VM_KERNEL_UNSLIDE_OR_PERM(port),
-                             port_sync_qos_count, sync_qos_delta_add[i]);
+/*
+ *     Routine:        ipc_port_thread_group_blocked
+ *     Purpose:
+ *             Call thread_group_blocked callout if the port
+ *             has ip_tg_block_tracking bit set and the thread
+ *             has not made this callout already.
+ *
+ *     Conditions:
+ *             Nothing is locked.
+ */
+void
+ipc_port_thread_group_blocked(ipc_port_t port __unused)
+{
+#if CONFIG_THREAD_GROUPS
+       bool port_tg_block_tracking = false;
+       thread_t self = current_thread();

 
 

-                       set_port_sync_qos(port, i, UCHAR_MAX);
-               } else if (sync_qos_delta_add[i] != 0) {
-                       KDBG_FILTERED(IMPORTANCE_CODE(IMP_SYNC_IPC_QOS, IMP_SYNC_IPC_QOS_APPLIED),
-                             i, VM_KERNEL_UNSLIDE_OR_PERM(port),
-                             port_sync_qos_count, sync_qos_delta_add[i]);
+       if (self->thread_group == NULL ||
+           (self->options & TH_OPT_IPC_TG_BLOCKED)) {
+               return;
+       }

 
 

-                       set_port_sync_qos(port, i, (port_sync_qos_count + sync_qos_delta_add[i]));
-               }
+       port_tg_block_tracking = port->ip_tg_block_tracking;
+       if (!port_tg_block_tracking) {
+               return;

        }
        }

-       return (ipc_port_get_max_sync_qos_index(port) != max_sync_qos_index);
+
+       machine_thread_group_blocked(self->thread_group, NULL,
+           PERFCONTROL_CALLOUT_BLOCKING_TG_RENDER_SERVER, self);
+
+       self->options |= TH_OPT_IPC_TG_BLOCKED;
+#endif

 }
 
 /*
 }
 
 /*

- *     Routine:        ipc_port_get_max_sync_qos_index
+ *     Routine:        ipc_port_thread_group_unblocked

  *     Purpose:
  *     Purpose:

- *             Return the max sync qos of the port.
+ *             Call thread_group_unblocked callout if the
+ *             thread had previously made a thread_group_blocked
+ *             callout before (indicated by TH_OPT_IPC_TG_BLOCKED
+ *             flag on the thread).

  *
  *     Conditions:
  *
  *     Conditions:

+ *             Nothing is locked.
+ */
+void
+ipc_port_thread_group_unblocked(void)
+{
+#if CONFIG_THREAD_GROUPS
+       thread_t self = current_thread();
+
+       if (!(self->options & TH_OPT_IPC_TG_BLOCKED)) {
+               return;
+       }
+
+       machine_thread_group_unblocked(self->thread_group, NULL,
+           PERFCONTROL_CALLOUT_BLOCKING_TG_RENDER_SERVER, self);
+
+       self->options &= ~TH_OPT_IPC_TG_BLOCKED;
+#endif
+}
+
+#if DEVELOPMENT || DEBUG
+inline void
+ipc_special_reply_port_bits_reset(ipc_port_t special_reply_port)
+{
+       special_reply_port->ip_srp_lost_link = 0;
+       special_reply_port->ip_srp_msg_sent = 0;
+}
+
+static inline void
+ipc_special_reply_port_msg_sent_reset(ipc_port_t special_reply_port)
+{
+       if (special_reply_port->ip_specialreply == 1) {
+               special_reply_port->ip_srp_msg_sent = 0;
+       }
+}
+
+inline void
+ipc_special_reply_port_msg_sent(ipc_port_t special_reply_port)
+{
+       if (special_reply_port->ip_specialreply == 1) {
+               special_reply_port->ip_srp_msg_sent = 1;
+       }
+}
+
+static inline void
+ipc_special_reply_port_lost_link(ipc_port_t special_reply_port)
+{
+       if (special_reply_port->ip_specialreply == 1 && special_reply_port->ip_srp_msg_sent == 0) {
+               special_reply_port->ip_srp_lost_link = 1;
+       }
+}
+
+#else /* DEVELOPMENT || DEBUG */
+inline void
+ipc_special_reply_port_bits_reset(__unused ipc_port_t special_reply_port)
+{
+       return;
+}
+
+static inline void
+ipc_special_reply_port_msg_sent_reset(__unused ipc_port_t special_reply_port)
+{
+       return;
+}
+
+inline void
+ipc_special_reply_port_msg_sent(__unused ipc_port_t special_reply_port)
+{
+       return;
+}
+
+static inline void
+ipc_special_reply_port_lost_link(__unused ipc_port_t special_reply_port)
+{
+       return;
+}
+#endif /* DEVELOPMENT || DEBUG */
+
+/*
+ *     Routine:        ipc_port_adjust_special_reply_port_locked
+ *     Purpose:
+ *             If the special port has a turnstile, update its inheritor.
+ *     Condition:
+ *             Special reply port locked on entry.
+ *             Special reply port unlocked on return.
+ *             The passed in port is a special reply port.
+ *     Returns:
+ *             None.

  */
  */

-sync_qos_count_t
-ipc_port_get_max_sync_qos_index(
-       ipc_port_t      port)
+void
+ipc_port_adjust_special_reply_port_locked(
+       ipc_port_t special_reply_port,
+       struct knote *kn,
+       uint8_t flags,
+       boolean_t get_turnstile)

 {
 {

-       int i;
-       for (i = THREAD_QOS_LAST - 1; i >= 0; i--) {
-               if (port_sync_qos(port, i) != 0) {
-                       return i;
+       ipc_port_t dest_port = IPC_PORT_NULL;
+       int sync_link_state = PORT_SYNC_LINK_NO_LINKAGE;
+       turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;
+       struct turnstile *ts = TURNSTILE_NULL;
+
+       ip_lock_held(special_reply_port); // ip_sync_link_state is touched
+       imq_lock(&special_reply_port->ip_messages);
+
+       if (!special_reply_port->ip_specialreply) {
+               // only mach_msg_receive_results_complete() calls this with any port
+               assert(get_turnstile);
+               goto not_special;
+       }
+
+       if (flags & IPC_PORT_ADJUST_SR_RECEIVED_MSG) {
+               ipc_special_reply_port_msg_sent_reset(special_reply_port);
+       }
+
+       if (flags & IPC_PORT_ADJUST_UNLINK_THREAD) {
+               special_reply_port->ip_messages.imq_srp_owner_thread = NULL;
+       }
+
+       if (flags & IPC_PORT_ADJUST_RESET_BOOSTRAP_CHECKIN) {
+               special_reply_port->ip_sync_bootstrap_checkin = 0;
+       }
+
+       /* Check if the special reply port is marked non-special */
+       if (special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_ANY) {
+not_special:
+               if (get_turnstile) {
+                       turnstile_complete((uintptr_t)special_reply_port,
+                           port_rcv_turnstile_address(special_reply_port), NULL, TURNSTILE_SYNC_IPC);
+               }
+               imq_unlock(&special_reply_port->ip_messages);
+               ip_unlock(special_reply_port);
+               if (get_turnstile) {
+                       turnstile_cleanup();
+               }
+               return;
+       }
+
+       if (flags & IPC_PORT_ADJUST_SR_LINK_WORKLOOP) {
+               if (ITH_KNOTE_VALID(kn, MACH_MSG_TYPE_PORT_SEND_ONCE)) {
+                       inheritor = filt_machport_stash_port(kn, special_reply_port,
+                           &sync_link_state);
+               }
+       } else if (flags & IPC_PORT_ADJUST_SR_ALLOW_SYNC_LINKAGE) {
+               sync_link_state = PORT_SYNC_LINK_ANY;
+       }
+
+       /* Check if need to break linkage */
+       if (!get_turnstile && sync_link_state == PORT_SYNC_LINK_NO_LINKAGE &&
+           special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_NO_LINKAGE) {
+               imq_unlock(&special_reply_port->ip_messages);
+               ip_unlock(special_reply_port);
+               return;
+       }
+
+       switch (special_reply_port->ip_sync_link_state) {
+       case PORT_SYNC_LINK_PORT:
+               dest_port = special_reply_port->ip_sync_inheritor_port;
+               special_reply_port->ip_sync_inheritor_port = IPC_PORT_NULL;
+               break;
+       case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+               special_reply_port->ip_sync_inheritor_knote = NULL;
+               break;
+       case PORT_SYNC_LINK_WORKLOOP_STASH:
+               special_reply_port->ip_sync_inheritor_ts = NULL;
+               break;
+       }
+
+       /*
+        * Stash (or unstash) the server's PID in the ip_sorights field of the
+        * special reply port, so that stackshot can later retrieve who the client
+        * is blocked on.
+        */
+       if (special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_PORT &&
+           sync_link_state == PORT_SYNC_LINK_NO_LINKAGE) {
+               ipc_special_reply_stash_pid_locked(special_reply_port, pid_from_task(current_task()));
+       } else if (special_reply_port->ip_sync_link_state == PORT_SYNC_LINK_NO_LINKAGE &&
+           sync_link_state == PORT_SYNC_LINK_ANY) {
+               /* If we are resetting the special reply port, remove the stashed pid. */
+               ipc_special_reply_stash_pid_locked(special_reply_port, 0);
+       }
+
+       special_reply_port->ip_sync_link_state = sync_link_state;
+
+       switch (sync_link_state) {
+       case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+               special_reply_port->ip_sync_inheritor_knote = kn;
+               break;
+       case PORT_SYNC_LINK_WORKLOOP_STASH:
+               special_reply_port->ip_sync_inheritor_ts = inheritor;
+               break;
+       case PORT_SYNC_LINK_NO_LINKAGE:
+               if (flags & IPC_PORT_ADJUST_SR_ENABLE_EVENT) {
+                       ipc_special_reply_port_lost_link(special_reply_port);

                }
                }

+               break;
+       }
+
+       /* Get thread's turnstile donated to special reply port */
+       if (get_turnstile) {
+               turnstile_complete((uintptr_t)special_reply_port,
+                   port_rcv_turnstile_address(special_reply_port), NULL, TURNSTILE_SYNC_IPC);
+       } else {
+               ts = ipc_port_rcv_turnstile(special_reply_port);
+               if (ts) {
+                       turnstile_reference(ts);
+                       ipc_port_recv_update_inheritor(special_reply_port, ts,
+                           TURNSTILE_IMMEDIATE_UPDATE);
+               }
+       }
+
+       imq_unlock(&special_reply_port->ip_messages);
+       ip_unlock(special_reply_port);
+
+       if (get_turnstile) {
+               turnstile_cleanup();
+       } else if (ts) {
+               /* Call turnstile cleanup after dropping the interlock */
+               turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+               turnstile_deallocate_safe(ts);
+       }
+
+       /* Release the ref on the dest port and its turnstile */
+       if (dest_port) {
+               ipc_port_send_turnstile_complete(dest_port);
+               /* release the reference on the dest port */
+               ip_release(dest_port);

        }
        }

-       return THREAD_QOS_UNSPECIFIED;

 }
 
 /*
 }
 
 /*

- *     Routine:        ipc_port_adjust_sync_qos
+ *     Routine:        ipc_port_adjust_special_reply_port

  *     Purpose:
  *     Purpose:

- *             Adjust sync qos of the port and it's destination
- *             port if the port is in transit.
- *     Conditions:
+ *             If the special port has a turnstile, update its inheritor.
+ *     Condition:

  *             Nothing locked.
  *     Returns:
  *             None.
  */
 void
  *             Nothing locked.
  *     Returns:
  *             None.
  */
 void

-ipc_port_adjust_sync_qos(
+ipc_port_adjust_special_reply_port(
+       ipc_port_t port,
+       uint8_t flags)
+{
+       if (port->ip_specialreply) {
+               ip_lock(port);
+               ipc_port_adjust_special_reply_port_locked(port, NULL, flags, FALSE);
+       }
+}
+
+/*
+ *     Routine:        ipc_port_adjust_sync_link_state_locked
+ *     Purpose:
+ *             Update the sync link state of the port and the
+ *             turnstile inheritor.
+ *     Condition:
+ *             Port and mqueue locked on entry.
+ *             Port and mqueue locked on return.
+ *     Returns:
+ *              None.
+ */
+void
+ipc_port_adjust_sync_link_state_locked(
+       ipc_port_t port,
+       int sync_link_state,
+       turnstile_inheritor_t inheritor)
+{
+       switch (port->ip_sync_link_state) {
+       case PORT_SYNC_LINK_RCV_THREAD:
+               /* deallocate the thread reference for the inheritor */
+               thread_deallocate_safe(port->ip_messages.imq_inheritor_thread_ref);
+               OS_FALLTHROUGH;
+       default:
+               klist_init(&port->ip_messages.imq_klist);
+       }
+
+       switch (sync_link_state) {
+       case PORT_SYNC_LINK_WORKLOOP_KNOTE:
+               port->ip_messages.imq_inheritor_knote = inheritor;
+               break;
+       case PORT_SYNC_LINK_WORKLOOP_STASH:
+               port->ip_messages.imq_inheritor_turnstile = inheritor;
+               break;
+       case PORT_SYNC_LINK_RCV_THREAD:
+               /* The thread could exit without clearing port state, take a thread ref */
+               thread_reference((thread_t)inheritor);
+               port->ip_messages.imq_inheritor_thread_ref = inheritor;
+               break;
+       default:
+               klist_init(&port->ip_messages.imq_klist);
+               sync_link_state = PORT_SYNC_LINK_ANY;
+       }
+
+       port->ip_sync_link_state = sync_link_state;
+}
+
+
+/*
+ *     Routine:        ipc_port_adjust_port_locked
+ *     Purpose:
+ *             If the port has a turnstile, update its inheritor.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port unlocked on return.
+ *     Returns:
+ *             None.
+ */
+void
+ipc_port_adjust_port_locked(

        ipc_port_t port,
        ipc_port_t port,

-       sync_qos_count_t *sync_qos_delta_add,
-       sync_qos_count_t *sync_qos_delta_sub)
+       struct knote *kn,
+       boolean_t sync_bootstrap_checkin)

 {
 {

-       boolean_t update_knote;
-       boolean_t multiple_lock = FALSE;
-       ipc_port_t dest, base, next;
+       int sync_link_state = PORT_SYNC_LINK_ANY;
+       turnstile_inheritor_t inheritor = TURNSTILE_INHERITOR_NULL;

 
 

-       ip_lock(port);
+       ip_lock_held(port); // ip_sync_link_state is touched
+       imq_held(&port->ip_messages);

 
 

-       /* Check if the port is in transit */
-       if (!ip_active(port) ||
-           (port->ip_receiver_name != MACH_PORT_NULL) ||
-           (port->ip_destination == IP_NULL)) {
-               /* lock the mqueue since port is not in-transit */
-               imq_lock(&port->ip_messages);
-               update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub);
-               if (update_knote) {
-                       KNOTE(&port->ip_messages.imq_klist, 0);
+       assert(!port->ip_specialreply);
+
+       if (kn) {
+               inheritor = filt_machport_stash_port(kn, port, &sync_link_state);
+               if (sync_link_state == PORT_SYNC_LINK_WORKLOOP_KNOTE) {
+                       inheritor = kn;

                }
                }

-               imq_unlock(&port->ip_messages);
+       } else if (sync_bootstrap_checkin) {
+               inheritor = current_thread();
+               sync_link_state = PORT_SYNC_LINK_RCV_THREAD;
+       }
+
+       ipc_port_adjust_sync_link_state_locked(port, sync_link_state, inheritor);
+       port->ip_sync_bootstrap_checkin = 0;
+
+       ipc_port_send_turnstile_recompute_push_locked(port);
+       /* port and mqueue unlocked */
+}
+
+/*
+ *     Routine:        ipc_port_clear_sync_rcv_thread_boost_locked
+ *     Purpose:
+ *             If the port is pushing on rcv thread, clear it.
+ *     Condition:
+ *             Port locked on entry
+ *             mqueue is not locked.
+ *             Port unlocked on return.
+ *     Returns:
+ *             None.
+ */
+void
+ipc_port_clear_sync_rcv_thread_boost_locked(
+       ipc_port_t port)
+{
+       ip_lock_held(port); // ip_sync_link_state is touched
+
+       if (port->ip_sync_link_state != PORT_SYNC_LINK_RCV_THREAD) {

                ip_unlock(port);
                return;
        }
 
                ip_unlock(port);
                return;
        }
 

-       dest = port->ip_destination;
-       assert(dest != IP_NULL);
+       imq_lock(&port->ip_messages);
+       ipc_port_adjust_sync_link_state_locked(port, PORT_SYNC_LINK_ANY, NULL);

 
 

-       if (ip_lock_try(dest)) {
-               if (!ip_active(dest) ||
-                   (dest->ip_receiver_name != MACH_PORT_NULL) ||
-                   (dest->ip_destination == IP_NULL)) {
-                       update_knote = ipc_port_sync_qos_delta(port, sync_qos_delta_add, sync_qos_delta_sub);
-                       ip_unlock(port);
+       ipc_port_send_turnstile_recompute_push_locked(port);
+       /* port and mqueue unlocked */
+}

 
 

-                       /* lock the mqueue since dest is not in-transit */
-                       imq_lock(&dest->ip_messages);
-                       update_knote = ipc_port_sync_qos_delta(dest, sync_qos_delta_add, sync_qos_delta_sub);
-                       if (update_knote) {
-                               KNOTE(&dest->ip_messages.imq_klist, 0);
-                       }
-                       imq_unlock(&dest->ip_messages);
-                       ip_unlock(dest);
-                       return;
-               }
+/*
+ *     Routine:        ipc_port_add_watchport_elem_locked
+ *     Purpose:
+ *             Transfer the turnstile boost of watchport to task calling exec.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port unlocked on return.
+ *     Returns:
+ *             KERN_SUCESS on success.
+ *             KERN_FAILURE otherwise.
+ */
+kern_return_t
+ipc_port_add_watchport_elem_locked(
+       ipc_port_t                 port,
+       struct task_watchport_elem *watchport_elem,
+       struct task_watchport_elem **old_elem)
+{
+       ip_lock_held(port);
+       imq_held(&port->ip_messages);

 
 

-               /* dest is in transit; need to take the serialize lock */
-               ip_unlock(dest);
+       /* Watchport boost only works for non-special active ports mapped in an ipc space */
+       if (!ip_active(port) || port->ip_specialreply ||
+           port->ip_receiver_name == MACH_PORT_NULL) {
+               imq_unlock(&port->ip_messages);
+               ip_unlock(port);
+               return KERN_FAILURE;

        }
 
        }
 

-       ip_unlock(port);
+       if (port->ip_sync_link_state != PORT_SYNC_LINK_ANY) {
+               /* Sever the linkage if the port was pushing on knote */
+               ipc_port_adjust_sync_link_state_locked(port, PORT_SYNC_LINK_ANY, NULL);
+       }

 
 

-       ipc_port_multiple_lock(); /* massive serialization */
-       multiple_lock = TRUE;
+       *old_elem = ipc_port_update_watchport_elem(port, watchport_elem);

 
 

-       ip_lock(port);
-       next = port;
+       ipc_port_send_turnstile_recompute_push_locked(port);
+       /* port and mqueue unlocked */
+       return KERN_SUCCESS;
+}

 
 

-       /* Apply the sync qos delta to all in-transit ports */
-       for (;;) {
-               boolean_t port_not_in_transit = FALSE;
-
-               if (!ip_active(next) ||
-                   (next->ip_receiver_name != MACH_PORT_NULL) ||
-                   (next->ip_destination == IP_NULL)) {
-                       /* Get the mqueue lock for destination port to update knotes */
-                       imq_lock(&next->ip_messages);
-                       port_not_in_transit = TRUE;
-               }
+/*
+ *     Routine:        ipc_port_clear_watchport_elem_internal_conditional_locked
+ *     Purpose:
+ *             Remove the turnstile boost of watchport and recompute the push.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port unlocked on return.
+ *     Returns:
+ *             KERN_SUCESS on success.
+ *             KERN_FAILURE otherwise.
+ */
+kern_return_t
+ipc_port_clear_watchport_elem_internal_conditional_locked(
+       ipc_port_t                 port,
+       struct task_watchport_elem *watchport_elem)
+{
+       ip_lock_held(port);
+       imq_held(&port->ip_messages);

 
 

-               /* Apply the sync qos delta */
-               update_knote = ipc_port_sync_qos_delta(next, sync_qos_delta_add, sync_qos_delta_sub);
+       if (ipc_port_watchport_elem(port) != watchport_elem) {
+               imq_unlock(&port->ip_messages);
+               ip_unlock(port);
+               return KERN_FAILURE;
+       }

 
 

-               if (port_not_in_transit)
-                       break;
+       ipc_port_clear_watchport_elem_internal(port);
+       ipc_port_send_turnstile_recompute_push_locked(port);
+       /* port and mqueue unlocked */
+       return KERN_SUCCESS;
+}

 
 

-               next = next->ip_destination;
-               ip_lock(next);
-       }
+/*
+ *     Routine:        ipc_port_replace_watchport_elem_conditional_locked
+ *     Purpose:
+ *             Replace the turnstile boost of watchport and recompute the push.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port unlocked on return.
+ *     Returns:
+ *             KERN_SUCESS on success.
+ *             KERN_FAILURE otherwise.
+ */
+kern_return_t
+ipc_port_replace_watchport_elem_conditional_locked(
+       ipc_port_t                 port,
+       struct task_watchport_elem *old_watchport_elem,
+       struct task_watchport_elem *new_watchport_elem)
+{
+       ip_lock_held(port);
+       imq_held(&port->ip_messages);

 
 

-       if (multiple_lock) {
-               ipc_port_multiple_unlock();
+       if (ipc_port_watchport_elem(port) != old_watchport_elem) {
+               imq_unlock(&port->ip_messages);
+               ip_unlock(port);
+               return KERN_FAILURE;

        }
 
        }
 

-       base = next;
-       next = port;
+       ipc_port_update_watchport_elem(port, new_watchport_elem);
+       ipc_port_send_turnstile_recompute_push_locked(port);
+       /* port and mqueue unlocked */
+       return KERN_SUCCESS;
+}

 
 

-       while (next != base) {
-               ipc_port_t prev = next;
-               next = next->ip_destination;
+/*
+ *     Routine:        ipc_port_clear_watchport_elem_internal
+ *     Purpose:
+ *             Remove the turnstile boost of watchport.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port locked on return.
+ *     Returns:
+ *             Old task_watchport_elem returned.
+ */
+struct task_watchport_elem *
+ipc_port_clear_watchport_elem_internal(
+       ipc_port_t                 port)
+{
+       ip_lock_held(port);
+       imq_held(&port->ip_messages);

 
 

-               ip_unlock(prev);
+       if (port->ip_specialreply) {
+               return NULL;
+       }
+
+       return ipc_port_update_watchport_elem(port, NULL);
+}
+
+/*
+ *     Routine:        ipc_port_send_turnstile_recompute_push_locked
+ *     Purpose:
+ *             Update send turnstile inheritor of port and recompute the push.
+ *     Condition:
+ *             Port locked on entry.
+ *             Port unlocked on return.
+ *     Returns:
+ *             None.
+ */
+static void
+ipc_port_send_turnstile_recompute_push_locked(
+       ipc_port_t port)
+{
+       struct turnstile *send_turnstile = port_send_turnstile(port);
+       if (send_turnstile) {
+               turnstile_reference(send_turnstile);
+               ipc_port_send_update_inheritor(port, send_turnstile,
+                   TURNSTILE_IMMEDIATE_UPDATE);

        }
        }

+       imq_unlock(&port->ip_messages);
+       ip_unlock(port);

 
 

-       if (update_knote) {
-               KNOTE(&base->ip_messages.imq_klist, 0);
+       if (send_turnstile) {
+               turnstile_update_inheritor_complete(send_turnstile,
+                   TURNSTILE_INTERLOCK_NOT_HELD);
+               turnstile_deallocate_safe(send_turnstile);

        }
        }

-       imq_unlock(&base->ip_messages);
-       ip_unlock(base);
+}
+
+/*
+ *     Routine:        ipc_port_get_watchport_inheritor
+ *     Purpose:
+ *             Returns inheritor for watchport.
+ *
+ *     Conditions:
+ *             mqueue locked.
+ *     Returns:
+ *             watchport inheritor.
+ */
+static thread_t
+ipc_port_get_watchport_inheritor(
+       ipc_port_t port)
+{
+       imq_held(&port->ip_messages);
+       return ipc_port_watchport_elem(port)->twe_task->watchports->tw_thread;

 }
 
 /*
 }
 
 /*


@@ -1656,7 +2292,7 @@ ipc_port_impcount_delta(
        mach_port_delta_t delta,
        ipc_port_t        __unused base)
 {
        mach_port_delta_t delta,
        ipc_port_t        __unused base)
 {

-       mach_port_delta_t absdelta; 
+       mach_port_delta_t absdelta;

 
        if (!ip_active(port)) {
                return 0;
 
        if (!ip_active(port)) {
                return 0;


@@ -1668,7 +2304,7 @@ ipc_port_impcount_delta(
                return delta;
        }
 
                return delta;
        }
 

-       absdelta = 0 - delta;           
+       absdelta = 0 - delta;

        if (port->ip_impcount >= absdelta) {
                port->ip_impcount -= absdelta;
                return delta;
        if (port->ip_impcount >= absdelta) {
                port->ip_impcount -= absdelta;
                return delta;


@@ -1689,11 +2325,10 @@ ipc_port_impcount_delta(
                        target_pid = -1;
                }
                printf("Over-release of importance assertions for port 0x%x receiver pid %d (%s), "
                        target_pid = -1;
                }
                printf("Over-release of importance assertions for port 0x%x receiver pid %d (%s), "

-                      "dropping %d assertion(s) but port only has %d remaining.\n",
-                      port->ip_receiver_name, 
-                      target_pid, target_procname,
-                      absdelta, port->ip_impcount);
-
+                   "dropping %d assertion(s) but port only has %d remaining.\n",
+                   port->ip_receiver_name,
+                   target_pid, target_procname,
+                   absdelta, port->ip_impcount);

        } else if (base != IP_NULL) {
                task_t target_task = base->ip_receiver->is_task;
                ipc_importance_task_t target_imp = target_task->task_imp_base;
        } else if (base != IP_NULL) {
                task_t target_task = base->ip_receiver->is_task;
                ipc_importance_task_t target_imp = target_task->task_imp_base;


@@ -1708,12 +2343,12 @@ ipc_port_impcount_delta(
                        target_pid = -1;
                }
                printf("Over-release of importance assertions for port 0x%lx "
                        target_pid = -1;
                }
                printf("Over-release of importance assertions for port 0x%lx "

-                      "enqueued on port 0x%x with receiver pid %d (%s), "
-                      "dropping %d assertion(s) but port only has %d remaining.\n",
-                      (unsigned long)VM_KERNEL_UNSLIDE_OR_PERM((uintptr_t)port),
-                      base->ip_receiver_name,
-                      target_pid, target_procname,
-                      absdelta, port->ip_impcount);
+                   "enqueued on port 0x%x with receiver pid %d (%s), "
+                   "dropping %d assertion(s) but port only has %d remaining.\n",
+                   (unsigned long)VM_KERNEL_UNSLIDE_OR_PERM((uintptr_t)port),
+                   base->ip_receiver_name,
+                   target_pid, target_procname,
+                   absdelta, port->ip_impcount);

        }
 #endif
 
        }
 #endif
 


@@ -1741,18 +2376,19 @@ ipc_port_impcount_delta(
 
 boolean_t
 ipc_port_importance_delta_internal(
 
 boolean_t
 ipc_port_importance_delta_internal(

-       ipc_port_t              port,
-       natural_t               options,
-       mach_port_delta_t       *deltap,
-       ipc_importance_task_t   *imp_task)
+       ipc_port_t              port,
+       natural_t               options,
+       mach_port_delta_t       *deltap,
+       ipc_importance_task_t   *imp_task)

 {
        ipc_port_t next, base;
        boolean_t dropped = FALSE;
 
        *imp_task = IIT_NULL;
 
 {
        ipc_port_t next, base;
        boolean_t dropped = FALSE;
 
        *imp_task = IIT_NULL;
 

-       if (*deltap == 0)
+       if (*deltap == 0) {

                return FALSE;
                return FALSE;

+       }

 
        assert(options == IPID_OPTION_NORMAL || options == IPID_OPTION_SENDPOSSIBLE);
 
 
        assert(options == IPID_OPTION_NORMAL || options == IPID_OPTION_SENDPOSSIBLE);
 


@@ -1762,17 +2398,15 @@ ipc_port_importance_delta_internal(
        if (ip_active(port) &&
            port->ip_destination != IP_NULL &&
            port->ip_receiver_name == MACH_PORT_NULL) {
        if (ip_active(port) &&
            port->ip_destination != IP_NULL &&
            port->ip_receiver_name == MACH_PORT_NULL) {

-

                dropped = TRUE;
 
                ip_unlock(port);
                ipc_port_multiple_lock(); /* massive serialization */
                ip_lock(base);
 
                dropped = TRUE;
 
                ip_unlock(port);
                ipc_port_multiple_lock(); /* massive serialization */
                ip_lock(base);
 

-               while(ip_active(base) &&
-                     base->ip_destination != IP_NULL &&
-                     base->ip_receiver_name == MACH_PORT_NULL) {
-
+               while (ip_active(base) &&
+                   base->ip_destination != IP_NULL &&
+                   base->ip_receiver_name == MACH_PORT_NULL) {

                        base = base->ip_destination;
                        ip_lock(base);
                }
                        base = base->ip_destination;
                        ip_lock(base);
                }


@@ -1792,10 +2426,11 @@ ipc_port_importance_delta_internal(
         */
        if (options & IPID_OPTION_SENDPOSSIBLE) {
                assert(*deltap == 1);
         */
        if (options & IPID_OPTION_SENDPOSSIBLE) {
                assert(*deltap == 1);

-               if (port->ip_sprequests && port->ip_spimportant == 0)
+               if (port->ip_sprequests && port->ip_spimportant == 0) {

                        port->ip_spimportant = 1;
                        port->ip_spimportant = 1;

-               else
+               } else {

                        *deltap = 0;
                        *deltap = 0;

+               }

        }
 
        /* unlock down to the base, adjusting boost(s) at each level */
        }
 
        /* unlock down to the base, adjusting boost(s) at each level */


@@ -1816,10 +2451,10 @@ ipc_port_importance_delta_internal(
        /* find the task (if any) to boost according to the base */
        if (ip_active(base)) {
                if (base->ip_tempowner != 0) {
        /* find the task (if any) to boost according to the base */
        if (ip_active(base)) {
                if (base->ip_tempowner != 0) {

-                       if (IIT_NULL != base->ip_imp_task)
+                       if (IIT_NULL != base->ip_imp_task) {

                                *imp_task = base->ip_imp_task;
                                *imp_task = base->ip_imp_task;

+                       }

                        /* otherwise don't boost */
                        /* otherwise don't boost */

-

                } else if (base->ip_receiver_name != MACH_PORT_NULL) {
                        ipc_space_t space = base->ip_receiver;
 
                } else if (base->ip_receiver_name != MACH_PORT_NULL) {
                        ipc_space_t space = base->ip_receiver;
 


@@ -1867,76 +2502,36 @@ ipc_port_importance_delta_internal(
 
 boolean_t
 ipc_port_importance_delta(
 
 boolean_t
 ipc_port_importance_delta(

-       ipc_port_t              port,
-       natural_t               options,
-       mach_port_delta_t       delta)
+       ipc_port_t              port,
+       natural_t               options,
+       mach_port_delta_t       delta)

 {
        ipc_importance_task_t imp_task = IIT_NULL;
        boolean_t dropped;
 
        dropped = ipc_port_importance_delta_internal(port, options, &delta, &imp_task);
 
 {
        ipc_importance_task_t imp_task = IIT_NULL;
        boolean_t dropped;
 
        dropped = ipc_port_importance_delta_internal(port, options, &delta, &imp_task);
 

-       if (IIT_NULL == imp_task || delta == 0)
+       if (IIT_NULL == imp_task || delta == 0) {

                return dropped;
                return dropped;

+       }

 
 

-       if (!dropped)
+       if (!dropped) {

                ip_unlock(port);
                ip_unlock(port);

+       }

 
        assert(ipc_importance_task_is_any_receiver_type(imp_task));
 
 
        assert(ipc_importance_task_is_any_receiver_type(imp_task));
 

-       if (delta > 0)
+       if (delta > 0) {

                ipc_importance_task_hold_internal_assertion(imp_task, delta);
                ipc_importance_task_hold_internal_assertion(imp_task, delta);

-       else
+       } else {

                ipc_importance_task_drop_internal_assertion(imp_task, -delta);
                ipc_importance_task_drop_internal_assertion(imp_task, -delta);

+       }

 
        ipc_importance_task_release(imp_task);
        return TRUE;
 }
 #endif /* IMPORTANCE_INHERITANCE */
 
 
        ipc_importance_task_release(imp_task);
        return TRUE;
 }
 #endif /* IMPORTANCE_INHERITANCE */
 

-/*
- *     Routine:        ipc_port_lookup_notify
- *     Purpose:
- *             Make a send-once notify port from a receive right.
- *             Returns IP_NULL if name doesn't denote a receive right.
- *     Conditions:
- *             The space must be locked (read or write) and active.
- *             Being the active space, we can rely on thread server_id
- *             context to give us the proper server level sub-order
- *             within the space.
- */
-
-ipc_port_t
-ipc_port_lookup_notify(
-       ipc_space_t             space,
-       mach_port_name_t        name)
-{
-       ipc_port_t port;
-       ipc_entry_t entry;
-
-       assert(is_active(space));
-
-       entry = ipc_entry_lookup(space, name);
-       if (entry == IE_NULL)
-               return IP_NULL;
-       if ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0)
-               return IP_NULL;
-
-       __IGNORE_WCASTALIGN(port = (ipc_port_t) entry->ie_object);
-       assert(port != IP_NULL);
-
-       ip_lock(port);
-       assert(ip_active(port));
-       assert(port->ip_receiver_name == name);
-       assert(port->ip_receiver == space);
-
-       ip_reference(port);
-       port->ip_sorights++;
-       ip_unlock(port);
-
-       return port;
-}
-

 /*
  *     Routine:        ipc_port_make_send_locked
  *     Purpose:
 /*
  *     Routine:        ipc_port_make_send_locked
  *     Purpose:


@@ -1947,9 +2542,9 @@ ipc_port_lookup_notify(
  */
 ipc_port_t
 ipc_port_make_send_locked(
  */
 ipc_port_t
 ipc_port_make_send_locked(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
 {

-       assert(ip_active(port));
+       require_ip_active(port);

        port->ip_mscount++;
        port->ip_srights++;
        ip_reference(port);
        port->ip_mscount++;
        port->ip_srights++;
        ip_reference(port);


@@ -1964,17 +2559,15 @@ ipc_port_make_send_locked(
 
 ipc_port_t
 ipc_port_make_send(
 
 ipc_port_t
 ipc_port_make_send(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
 {

-       
-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return port;
                return port;

+       }

 
        ip_lock(port);
        if (ip_active(port)) {
 
        ip_lock(port);
        if (ip_active(port)) {

-               port->ip_mscount++;
-               port->ip_srights++;
-               ip_reference(port);
+               ipc_port_make_send_locked(port);

                ip_unlock(port);
                return port;
        }
                ip_unlock(port);
                return port;
        }


@@ -1982,6 +2575,22 @@ ipc_port_make_send(
        return IP_DEAD;
 }
 
        return IP_DEAD;
 }
 

+/*
+ *     Routine:        ipc_port_copy_send_locked
+ *     Purpose:
+ *             Make a naked send right from another naked send right.
+ *     Conditions:
+ *             port locked and active.
+ */
+void
+ipc_port_copy_send_locked(
+       ipc_port_t      port)
+{
+       assert(port->ip_srights > 0);
+       port->ip_srights++;
+       ip_reference(port);
+}
+

 /*
  *     Routine:        ipc_port_copy_send
  *     Purpose:
 /*
  *     Routine:        ipc_port_copy_send
  *     Purpose:


@@ -1996,22 +2605,21 @@ ipc_port_make_send(
 
 ipc_port_t
 ipc_port_copy_send(
 
 ipc_port_t
 ipc_port_copy_send(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        ipc_port_t sright;
 
 {
        ipc_port_t sright;
 

-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return port;
                return port;

+       }

 
        ip_lock(port);
        if (ip_active(port)) {
 
        ip_lock(port);
        if (ip_active(port)) {

-               assert(port->ip_srights > 0);
-
-               ip_reference(port);
-               port->ip_srights++;
+               ipc_port_copy_send_locked(port);

                sright = port;
                sright = port;

-       } else
+       } else {

                sright = IP_DEAD;
                sright = IP_DEAD;

+       }

        ip_unlock(port);
 
        return sright;
        ip_unlock(port);
 
        return sright;


@@ -2028,26 +2636,28 @@ ipc_port_copy_send(
 
 mach_port_name_t
 ipc_port_copyout_send(
 
 mach_port_name_t
 ipc_port_copyout_send(

-       ipc_port_t      sright,
-       ipc_space_t     space)
+       ipc_port_t      sright,
+       ipc_space_t     space)

 {
        mach_port_name_t name;
 
        if (IP_VALID(sright)) {
                kern_return_t kr;
 
 {
        mach_port_name_t name;
 
        if (IP_VALID(sright)) {
                kern_return_t kr;
 

-               kr = ipc_object_copyout(space, (ipc_object_t) sright,
-                                       MACH_MSG_TYPE_PORT_SEND, TRUE, &name);
+               kr = ipc_object_copyout(space, ip_to_object(sright),
+                   MACH_MSG_TYPE_PORT_SEND, NULL, NULL, &name);

                if (kr != KERN_SUCCESS) {
                        ipc_port_release_send(sright);
 
                if (kr != KERN_SUCCESS) {
                        ipc_port_release_send(sright);
 

-                       if (kr == KERN_INVALID_CAPABILITY)
+                       if (kr == KERN_INVALID_CAPABILITY) {

                                name = MACH_PORT_DEAD;
                                name = MACH_PORT_DEAD;

-                       else
+                       } else {

                                name = MACH_PORT_NULL;
                                name = MACH_PORT_NULL;

+                       }

                }
                }

-       } else
+       } else {

                name = CAST_MACH_PORT_TO_NAME(sright);
                name = CAST_MACH_PORT_TO_NAME(sright);

+       }

 
        return name;
 }
 
        return name;
 }


@@ -2063,13 +2673,14 @@ ipc_port_copyout_send(
 
 void
 ipc_port_release_send(
 
 void
 ipc_port_release_send(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        ipc_port_t nsrequest = IP_NULL;
        mach_port_mscount_t mscount;
 
 {
        ipc_port_t nsrequest = IP_NULL;
        mach_port_mscount_t mscount;
 

-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return;
                return;

+       }

 
        ip_lock(port);
 
 
        ip_lock(port);
 


@@ -2110,9 +2721,9 @@ ipc_port_release_send(
 
 ipc_port_t
 ipc_port_make_sonce_locked(
 
 ipc_port_t
 ipc_port_make_sonce_locked(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
 {

-       assert(ip_active(port));
+       require_ip_active(port);

        port->ip_sorights++;
        ip_reference(port);
        return port;
        port->ip_sorights++;
        ip_reference(port);
        return port;


@@ -2128,15 +2739,15 @@ ipc_port_make_sonce_locked(
 
 ipc_port_t
 ipc_port_make_sonce(
 
 ipc_port_t
 ipc_port_make_sonce(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
 {

-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return port;
                return port;

+       }

 
        ip_lock(port);
        if (ip_active(port)) {
 
        ip_lock(port);
        if (ip_active(port)) {

-               port->ip_sorights++;
-               ip_reference(port);
+               ipc_port_make_sonce_locked(port);

                ip_unlock(port);
                return port;
        }
                ip_unlock(port);
                return port;
        }


@@ -2160,12 +2771,13 @@ ipc_port_make_sonce(
 
 void
 ipc_port_release_sonce(
 
 void
 ipc_port_release_sonce(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
 {

-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return;
                return;

+       }

 
 

-       ipc_port_unlink_special_reply_port(port, IPC_PORT_UNLINK_SR_NONE);
+       ipc_port_adjust_special_reply_port(port, IPC_PORT_ADJUST_RESET_BOOSTRAP_CHECKIN);

 
        ip_lock(port);
 
 
        ip_lock(port);
 


@@ -2191,22 +2803,25 @@ ipc_port_release_sonce(
 
 void
 ipc_port_release_receive(
 
 void
 ipc_port_release_receive(

-       ipc_port_t      port)
+       ipc_port_t      port)

 {
        ipc_port_t dest;
 
 {
        ipc_port_t dest;
 

-       if (!IP_VALID(port))
+       if (!IP_VALID(port)) {

                return;
                return;

+       }

 
        ip_lock(port);
 
        ip_lock(port);

-       assert(ip_active(port));
+       require_ip_active(port);

        assert(port->ip_receiver_name == MACH_PORT_NULL);
        dest = port->ip_destination;
 
        ipc_port_destroy(port); /* consumes ref, unlocks */
 
        assert(port->ip_receiver_name == MACH_PORT_NULL);
        dest = port->ip_destination;
 
        ipc_port_destroy(port); /* consumes ref, unlocks */
 

-       if (dest != IP_NULL)
+       if (dest != IP_NULL) {
+               ipc_port_send_turnstile_complete(dest);

                ip_release(dest);
                ip_release(dest);

+       }

 }
 
 /*
 }
 
 /*


@@ -2221,29 +2836,31 @@ ipc_port_release_receive(
 
 ipc_port_t
 ipc_port_alloc_special(
 
 ipc_port_t
 ipc_port_alloc_special(

-       ipc_space_t     space)
+       ipc_space_t             space,
+       ipc_port_init_flags_t   flags)

 {
        ipc_port_t port;
 
 {
        ipc_port_t port;
 

-       __IGNORE_WCASTALIGN(port = (ipc_port_t) io_alloc(IOT_PORT));
-       if (port == IP_NULL)
+       port = ip_object_to_port(io_alloc(IOT_PORT));
+       if (port == IP_NULL) {

                return IP_NULL;
                return IP_NULL;

+       }

 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);
 
 #if     MACH_ASSERT
        uintptr_t buf[IP_CALLSTACK_MAX];
        ipc_port_callstack_init_debug(&buf[0], IP_CALLSTACK_MAX);

-#endif /* MACH_ASSERT */       
+#endif /* MACH_ASSERT */

 
        bzero((char *)port, sizeof(*port));
 
        bzero((char *)port, sizeof(*port));

-       io_lock_init(&port->ip_object);
+       io_lock_init(ip_to_object(port));

        port->ip_references = 1;
        port->ip_object.io_bits = io_makebits(TRUE, IOT_PORT, 0);
 
        port->ip_references = 1;
        port->ip_object.io_bits = io_makebits(TRUE, IOT_PORT, 0);
 

-       ipc_port_init(port, space, 1);
+       ipc_port_init(port, space, flags, 1);

 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);
 
 #if     MACH_ASSERT
        ipc_port_init_debug(port, &buf[0], IP_CALLSTACK_MAX);

-#endif  /* MACH_ASSERT */              
+#endif  /* MACH_ASSERT */

 
        return port;
 }
 
        return port;
 }


@@ -2259,11 +2876,11 @@ ipc_port_alloc_special(
 
 void
 ipc_port_dealloc_special(
 
 void
 ipc_port_dealloc_special(

-       ipc_port_t                      port,
-       __assert_only ipc_space_t       space)
+       ipc_port_t                      port,
+       __assert_only ipc_space_t       space)

 {
        ip_lock(port);
 {
        ip_lock(port);

-       assert(ip_active(port));
+       require_ip_active(port);

 //     assert(port->ip_receiver_name != MACH_PORT_NULL);
        assert(port->ip_receiver == space);
 
 //     assert(port->ip_receiver_name != MACH_PORT_NULL);
        assert(port->ip_receiver == space);
 


@@ -2272,11 +2889,13 @@ ipc_port_dealloc_special(
         *      the ipc_space_kernel check in ipc_mqueue_send.
         */
 
         *      the ipc_space_kernel check in ipc_mqueue_send.
         */
 

+       imq_lock(&port->ip_messages);

        port->ip_receiver_name = MACH_PORT_NULL;
        port->ip_receiver = IS_NULL;
        port->ip_receiver_name = MACH_PORT_NULL;
        port->ip_receiver = IS_NULL;

+       imq_unlock(&port->ip_messages);

 
        /* relevant part of ipc_port_clear_receiver */
 
        /* relevant part of ipc_port_clear_receiver */

-       ipc_port_set_mscount(port, 0);
+       port->ip_mscount = 0;

        port->ip_messages.imq_seqno = 0;
 
        ipc_port_destroy(port);
        port->ip_messages.imq_seqno = 0;
 
        ipc_port_destroy(port);


@@ -2293,11 +2912,19 @@ ipc_port_dealloc_special(
  */
 void
 ipc_port_finalize(
  */
 void
 ipc_port_finalize(

-       ipc_port_t              port)
+       ipc_port_t              port)

 {
        ipc_port_request_t requests = port->ip_requests;
 
 {
        ipc_port_request_t requests = port->ip_requests;
 

-       assert(!ip_active(port));
+       assert(port_send_turnstile(port) == TURNSTILE_NULL);
+       if (imq_is_turnstile_proxy(&port->ip_messages)) {
+               assert(ipc_port_rcv_turnstile(port) == TURNSTILE_NULL);
+       }
+
+       if (ip_active(port)) {
+               panic("Trying to free an active port. port %p", port);
+       }
+

        if (requests != IPR_NULL) {
                ipc_table_size_t its = requests->ipr_size;
                it_requests_free(its, requests);
        if (requests != IPR_NULL) {
                ipc_table_size_t its = requests->ipr_size;
                it_requests_free(its, requests);


@@ -2305,10 +2932,10 @@ ipc_port_finalize(
        }
 
        ipc_mqueue_deinit(&port->ip_messages);
        }
 
        ipc_mqueue_deinit(&port->ip_messages);

-       
-#if    MACH_ASSERT
+
+#if     MACH_ASSERT

        ipc_port_track_dealloc(port);
        ipc_port_track_dealloc(port);

-#endif /* MACH_ASSERT */
+#endif  /* MACH_ASSERT */

 }
 
 /*
 }
 
 /*


@@ -2327,12 +2954,15 @@ ipc_port_finalize(
 void
 kdp_mqueue_send_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo)
 {
 void
 kdp_mqueue_send_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo)
 {

+       struct turnstile *turnstile;

        assert(waitinfo->wait_type == kThreadWaitPortSend);
        assert(event == IPC_MQUEUE_FULL);
        assert(waitinfo->wait_type == kThreadWaitPortSend);
        assert(event == IPC_MQUEUE_FULL);

+       assert(waitq_is_turnstile_queue(waitq));

 
 

-       ipc_mqueue_t mqueue = imq_from_waitq(waitq);
-       ipc_port_t port     = ip_from_mq(mqueue); /* we are blocking on send */
-       assert(kdp_is_in_zone(port, "ipc ports"));
+       turnstile = waitq_to_turnstile(waitq);
+       ipc_port_t port = (ipc_port_t)turnstile->ts_proprietor; /* we are blocking on send */
+
+       zone_id_require(ZONE_ID_IPC_PORT, sizeof(struct ipc_port), port);

 
        waitinfo->owner = 0;
        waitinfo->context  = VM_KERNEL_UNSLIDE_OR_PERM(port);
 
        waitinfo->owner = 0;
        waitinfo->context  = VM_KERNEL_UNSLIDE_OR_PERM(port);


@@ -2396,7 +3026,8 @@ kdp_mqueue_recv_find_owner(struct waitq * waitq, __assert_only event64_t event,
        waitinfo->owner     = 0;
        if (imq_is_set(mqueue)) { /* we are waiting on a port set */
                ipc_pset_t set = ips_from_mq(mqueue);
        waitinfo->owner     = 0;
        if (imq_is_set(mqueue)) { /* we are waiting on a port set */
                ipc_pset_t set = ips_from_mq(mqueue);

-               assert(kdp_is_in_zone(set, "ipc port sets"));
+
+               zone_id_require(ZONE_ID_IPC_PORT_SET, sizeof(struct ipc_pset), set);

 
                /* Reset wait type to specify waiting on port set receive */
                waitinfo->wait_type = kThreadWaitPortSetReceive;
 
                /* Reset wait type to specify waiting on port set receive */
                waitinfo->wait_type = kThreadWaitPortSetReceive;


@@ -2407,7 +3038,8 @@ kdp_mqueue_recv_find_owner(struct waitq * waitq, __assert_only event64_t event,
                /* There is no specific owner "at the other end" of a port set, so leave unset. */
        } else {
                ipc_port_t port   = ip_from_mq(mqueue);
                /* There is no specific owner "at the other end" of a port set, so leave unset. */
        } else {
                ipc_port_t port   = ip_from_mq(mqueue);

-               assert(kdp_is_in_zone(port, "ipc ports"));
+
+               zone_id_require(ZONE_ID_IPC_PORT, sizeof(struct ipc_port), port);

 
                waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(port);
                if (ip_lock_held_kdp(port)) {
 
                waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(port);
                if (ip_lock_held_kdp(port)) {


@@ -2425,7 +3057,7 @@ kdp_mqueue_recv_find_owner(struct waitq * waitq, __assert_only event64_t event,
        }
 }
 
        }
 }
 

-#if    MACH_ASSERT
+#if     MACH_ASSERT

 #include <kern/machine.h>
 
 /*
 #include <kern/machine.h>
 
 /*


@@ -2434,39 +3066,23 @@ kdp_mqueue_recv_find_owner(struct waitq * waitq, __assert_only event64_t event,
  *     deallocation is intercepted via io_free.
  */
 #if 0
  *     deallocation is intercepted via io_free.
  */
 #if 0

-queue_head_t   port_alloc_queue;
-lck_spin_t     port_alloc_queue_lock;
+queue_head_t    port_alloc_queue = QUEUE_HEAD_INITIALIZER(port_alloc_queue);
+LCK_SPIN_DECLARE(port_alloc_queue_lock, &ipc_lck_grp, &ipc_lck_attr);

 #endif
 
 #endif
 

-unsigned long  port_count = 0;
-unsigned long  port_count_warning = 20000;
-unsigned long  port_timestamp = 0;
-
-void           db_port_stack_trace(
-                       ipc_port_t      port);
-void           db_ref(
-                       int             refs);
-int            db_port_walk(
-                       unsigned int    verbose,
-                       unsigned int    display,
-                       unsigned int    ref_search,
-                       unsigned int    ref_target);
-
-/*
- *     Initialize global state needed for run-time
- *     port debugging.
- */
-void
-ipc_port_debug_init(void)
-{
-#if 0
-       queue_init(&port_alloc_queue);
-       lck_spin_init(&port_alloc_queue_lock, &ipc_lck_grp, &ipc_lck_attr);
-#endif
+unsigned long   port_count = 0;
+unsigned long   port_count_warning = 20000;
+unsigned long   port_timestamp = 0;

 
 

-       if (!PE_parse_boot_argn("ipc_portbt", &ipc_portbt, sizeof (ipc_portbt)))
-               ipc_portbt = 0;
-}
+void            db_port_stack_trace(
+       ipc_port_t      port);
+void            db_ref(
+       int             refs);
+int             db_port_walk(
+       unsigned int    verbose,
+       unsigned int    display,
+       unsigned int    ref_search,
+       unsigned int    ref_target);

 
 #ifdef MACH_BSD
 extern int proc_pid(struct proc*);
 
 #ifdef MACH_BSD
 extern int proc_pid(struct proc*);


@@ -2478,33 +3094,37 @@ extern int proc_pid(struct proc*);
  */
 void
 ipc_port_init_debug(
  */
 void
 ipc_port_init_debug(

-       ipc_port_t      port,
-       uintptr_t       *callstack,
-       unsigned int    callstack_max)
+       ipc_port_t      port,
+       uintptr_t       *callstack,
+       unsigned int    callstack_max)

 {
 {

-       unsigned int    i;
+       unsigned int    i;

 
        port->ip_thread = current_thread();
        port->ip_timetrack = port_timestamp++;
 
        port->ip_thread = current_thread();
        port->ip_timetrack = port_timestamp++;

-       for (i = 0; i < callstack_max; ++i)
-               port->ip_callstack[i] = callstack[i];   
-       for (i = 0; i < IP_NSPARES; ++i)
-               port->ip_spares[i] = 0; 
+       for (i = 0; i < callstack_max; ++i) {
+               port->ip_callstack[i] = callstack[i];
+       }
+       for (i = 0; i < IP_NSPARES; ++i) {
+               port->ip_spares[i] = 0;
+       }

 
 #ifdef MACH_BSD
        task_t task = current_task();
        if (task != TASK_NULL) {
                struct proc* proc = (struct proc*) get_bsdtask_info(task);
 
 #ifdef MACH_BSD
        task_t task = current_task();
        if (task != TASK_NULL) {
                struct proc* proc = (struct proc*) get_bsdtask_info(task);

-               if (proc)
+               if (proc) {

                        port->ip_spares[0] = proc_pid(proc);
                        port->ip_spares[0] = proc_pid(proc);

+               }

        }
 #endif /* MACH_BSD */
 
 #if 0
        lck_spin_lock(&port_alloc_queue_lock);
        ++port_count;
        }
 #endif /* MACH_BSD */
 
 #if 0
        lck_spin_lock(&port_alloc_queue_lock);
        ++port_count;

-       if (port_count_warning > 0 && port_count >= port_count_warning)
+       if (port_count_warning > 0 && port_count >= port_count_warning) {

                assert(port_count < port_count_warning);
                assert(port_count < port_count_warning);

+       }

        queue_enter(&port_alloc_queue, port, ipc_port_t, ip_port_links);
        lck_spin_unlock(&port_alloc_queue_lock);
 #endif
        queue_enter(&port_alloc_queue, port, ipc_port_t, ip_port_links);
        lck_spin_unlock(&port_alloc_queue_lock);
 #endif


@@ -2521,17 +3141,19 @@ ipc_port_init_debug(
  */
 void
 ipc_port_callstack_init_debug(
  */
 void
 ipc_port_callstack_init_debug(

-       uintptr_t       *callstack,
-       unsigned int    callstack_max)
+       uintptr_t       *callstack,
+       unsigned int    callstack_max)

 {
 {

-       unsigned int    i;
+       unsigned int    i;

 
        /* guarantee the callstack is initialized */
 
        /* guarantee the callstack is initialized */

-       for (i=0; i < callstack_max; i++)
-               callstack[i] = 0;       
+       for (i = 0; i < callstack_max; i++) {
+               callstack[i] = 0;
+       }

 
 

-       if (ipc_portbt)
+       if (ipc_portbt) {

                machine_callstack(callstack, callstack_max);
                machine_callstack(callstack, callstack_max);

+       }

 }
 
 /*
 }
 
 /*


@@ -2542,13 +3164,13 @@ ipc_port_callstack_init_debug(
 #if 1
 void
 ipc_port_track_dealloc(
 #if 1
 void
 ipc_port_track_dealloc(

-       __unused ipc_port_t     port)
+       __unused ipc_port_t     port)

 {
 }
 #else
 void
 ipc_port_track_dealloc(
 {
 }
 #else
 void
 ipc_port_track_dealloc(

-       ipc_port_t              port)
+       ipc_port_t              port)

 {
        lck_spin_lock(&port_alloc_queue_lock);
        assert(port_count > 0);
 {
        lck_spin_lock(&port_alloc_queue_lock);
        assert(port_count > 0);


@@ -2559,4 +3181,4 @@ ipc_port_track_dealloc(
 #endif
 
 
 #endif
 
 

-#endif /* MACH_ASSERT */
+#endif  /* MACH_ASSERT */