xnu-1504.3.12.tar.gz

[apple/xnu.git] / bsd / net / kpi_protocol.c

diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c
index f1611a11ec90da6859239857af3b28a14b8db19f..a48cd249abf001780215f9bd5ebe0998c32b23a8 100644 (file)
--- a/bsd/net/kpi_protocol.c
+++ b/bsd/net/kpi_protocol.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *

- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.

  * 
  * 

- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.

  * 
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 
 #include "kpi_protocol.h"
  */
 
 #include "kpi_protocol.h"


@@ -32,36 +38,50 @@
 #include <net/dlil.h>
 #include <libkern/OSAtomic.h>
 
 #include <net/dlil.h>
 #include <libkern/OSAtomic.h>
 

-void proto_kpi_init(void);

 void proto_input_run(void);
 
 void proto_input_run(void);
 

-typedef int (*attach_t)(struct ifnet *ifp, u_long protocol_family);
-typedef int (*detach_t)(struct ifnet *ifp, u_long protocol_family);
+typedef int (*attach_t)(struct ifnet *ifp, uint32_t protocol_family);
+typedef int (*detach_t)(struct ifnet *ifp, uint32_t protocol_family);

 
 

-/****************************************************************************/
-/* WARNING: Big assumption made here - there can be only one input thread      */

 struct proto_input_entry {
        struct proto_input_entry                *next;
        int                                                             detach;
        struct domain                                   *domain;
 struct proto_input_entry {
        struct proto_input_entry                *next;
        int                                                             detach;
        struct domain                                   *domain;

+       int                                                             hash;
+       int                                                             chain;

        
        protocol_family_t                               protocol;
        proto_input_handler                             input;
        proto_input_detached_handler    detached;
        
        
        protocol_family_t                               protocol;
        proto_input_handler                             input;
        proto_input_detached_handler    detached;
        

-       mbuf_t                                                  first_packet;
-       mbuf_t                                                  last_packet;
+       mbuf_t                                                  inject_first;
+       mbuf_t                                                  inject_last;
+       
+       struct proto_input_entry                *input_next;
+       mbuf_t                                                  input_first;
+       mbuf_t                                                  input_last;
+};
+
+
+struct proto_family_str {
+       TAILQ_ENTRY(proto_family_str)   proto_fam_next;
+       protocol_family_t                               proto_family;
+       ifnet_family_t                                  if_family;
+       proto_plumb_handler                             attach_proto;
+       proto_unplumb_handler                   detach_proto;

 };
 
 #define PROTO_HASH_SLOTS       5
 
 };
 
 #define PROTO_HASH_SLOTS       5
 

-static struct proto_input_entry        *proto_hash[PROTO_HASH_SLOTS];
-static struct proto_input_entry *proto_input_add_list;
-static lck_mtx_t                               *proto_input_lock = 0;
-__private_extern__ u_int32_t   inject_buckets = 0;
+static struct proto_input_entry                        *proto_hash[PROTO_HASH_SLOTS];
+static int                                                             proto_total_waiting = 0;
+static struct proto_input_entry                *proto_input_add_list = NULL;
+static lck_mtx_t                                               *proto_family_mutex = 0;
+static TAILQ_HEAD(, proto_family_str)  proto_family_head =
+                       TAILQ_HEAD_INITIALIZER(proto_family_head);

 
 

-extern thread_t        dlil_input_thread_ptr;
-extern int             dlil_input_thread_wakeup;
+extern lck_mtx_t       *domain_proto_mtx;
+extern struct dlil_threading_info *dlil_lo_thread_ptr;

 
 static int
 proto_hash_value(
 
 static int
 proto_hash_value(


@@ -89,24 +109,26 @@ proto_kpi_init(void)
        
        /* Allocate a mtx lock */
        grp_attrib = lck_grp_attr_alloc_init();
        
        /* Allocate a mtx lock */
        grp_attrib = lck_grp_attr_alloc_init();

-       lck_grp_attr_setdefault(grp_attrib);

        lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib);
        lck_grp_attr_free(grp_attrib);
        lck_attrib = lck_attr_alloc_init();
        lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib);
        lck_grp_attr_free(grp_attrib);
        lck_attrib = lck_attr_alloc_init();

-       lck_attr_setdefault(lck_attrib);
-       proto_input_lock = lck_mtx_alloc_init(lck_group, lck_attrib);
+       proto_family_mutex = lck_mtx_alloc_init(lck_group, lck_attrib);

        lck_grp_free(lck_group);
        lck_attr_free(lck_attrib);
        lck_grp_free(lck_group);
        lck_attr_free(lck_attrib);

+       
+       bzero(proto_hash, sizeof(proto_hash));

 }
 
 __private_extern__ errno_t
 proto_register_input(
        protocol_family_t protocol,
        proto_input_handler input,
 }
 
 __private_extern__ errno_t
 proto_register_input(
        protocol_family_t protocol,
        proto_input_handler input,

-       proto_input_detached_handler detached)
+       proto_input_detached_handler detached,
+       int     chains)

 {
        
        struct proto_input_entry *entry;
 {
        
        struct proto_input_entry *entry;

+       struct dlil_threading_info *thread = dlil_lo_thread_ptr;

        
        entry = _MALLOC(sizeof(*entry), M_IFADDR, M_WAITOK);
        
        
        entry = _MALLOC(sizeof(*entry), M_IFADDR, M_WAITOK);
        


@@ -117,25 +139,29 @@ proto_register_input(
        entry->protocol = protocol;
        entry->input = input;
        entry->detached = detached;
        entry->protocol = protocol;
        entry->input = input;
        entry->detached = detached;

+       entry->hash = proto_hash_value(protocol);
+       entry->chain = chains;

        
        {
                struct domain *dp = domains;
        
        {
                struct domain *dp = domains;

-               extern lck_mtx_t *domain_proto_mtx;

 
                lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
                lck_mtx_lock(domain_proto_mtx);
 
                lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
                lck_mtx_lock(domain_proto_mtx);

-               while (dp && dp->dom_family != protocol)
+               while (dp && (protocol_family_t)dp->dom_family != protocol)

                        dp = dp->dom_next;
                entry->domain = dp;
                lck_mtx_unlock(domain_proto_mtx);       
        }
 
        
                        dp = dp->dom_next;
                entry->domain = dp;
                lck_mtx_unlock(domain_proto_mtx);       
        }
 
        

-       do {
-               entry->next = proto_input_add_list;
-       } while(!OSCompareAndSwap((UInt32)entry->next, (UInt32)entry, (UInt32*)&proto_input_add_list));
+       lck_mtx_lock(thread->input_lck);
+       entry->next = proto_input_add_list;
+       proto_input_add_list = entry;

        
        

-       wakeup((caddr_t)&dlil_input_thread_wakeup);
+       thread->input_waiting |= DLIL_PROTO_REGISTER;
+       if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0)
+               wakeup((caddr_t)&thread->input_waiting);
+       lck_mtx_unlock(thread->input_lck);

        
        return 0;
 }
        
        return 0;
 }


@@ -185,71 +211,68 @@ proto_delayed_attach(
        }
 }
 
        }
 }
 

-static void
-proto_delayed_inject(
-       struct proto_input_entry *entry)
-{
-       mbuf_t  packet_list;
-       mbuf_t  packet;
-       int             locked = 0;
-       
-       lck_mtx_lock(proto_input_lock);
-       packet_list = entry->first_packet;
-       entry->first_packet = entry->last_packet = 0;
-       lck_mtx_unlock(proto_input_lock);
-       
-       if (packet_list == NULL)
-               return;
-       
-       if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
-               lck_mtx_lock(entry->domain->dom_mtx);
-               locked = 1;
-       }
-       
-       for (packet = packet_list; packet; packet = packet_list) {
-               packet_list = mbuf_nextpkt(packet);
-               mbuf_setnextpkt(packet, NULL);
-               entry->input(entry->protocol, packet);
-       }
-       
-       if (locked) {
-               lck_mtx_unlock(entry->domain->dom_mtx);
-       }
-}
-
-/* This function must be called from a single dlil input thread */

 __private_extern__ void
 proto_input_run(void)
 {
        struct proto_input_entry        *entry;
 __private_extern__ void
 proto_input_run(void)
 {
        struct proto_input_entry        *entry;

-       u_int32_t                                       inject;
-       int                                                     i;
-       
-       if (current_thread() != dlil_input_thread_ptr)
-               panic("proto_input_run called from a thread other than dlil_input_thread!\n");
+       struct dlil_threading_info *thread = dlil_lo_thread_ptr;
+       mbuf_t packet_list;
+       int i, locked = 0;
+
+       lck_mtx_assert(thread->input_lck,  LCK_MTX_ASSERT_NOTOWNED);

 
 

-       do {
+       if ((thread->input_waiting & DLIL_PROTO_REGISTER) != 0) {
+               lck_mtx_lock(thread->input_lck);

                entry = proto_input_add_list;
                entry = proto_input_add_list;

-       } while (entry && !OSCompareAndSwap((UInt32)entry, 0, (UInt32*)&proto_input_add_list));
-       
-       if (entry)
+               proto_input_add_list = NULL;
+               thread->input_waiting &= ~DLIL_PROTO_REGISTER;
+               lck_mtx_unlock(thread->input_lck);

                proto_delayed_attach(entry);
                proto_delayed_attach(entry);

-       
-       do {
-               inject = inject_buckets;
-       } while (inject && !OSCompareAndSwap(inject, 0, (UInt32*)&inject_buckets));
-       
-       if (inject) {
-               for (i = 0; i < PROTO_HASH_SLOTS; i++) {
-                       if ((inject & (1L << i)) != 0) {
-                               for (entry = proto_hash[i]; entry; entry = entry->next) {
-                                       if (entry->first_packet) {
-                                               proto_delayed_inject(entry);
+       }
+       /*
+         Move everything from the lock protected list to the thread
+         specific list.
+        */
+       for (i = 0; proto_total_waiting != 0 && i < PROTO_HASH_SLOTS; i++) {
+               for (entry = proto_hash[i]; entry && proto_total_waiting;
+                        entry = entry->next) {
+                       if (entry->inject_first) {
+                               lck_mtx_lock(thread->input_lck);
+                               thread->input_waiting &= ~DLIL_PROTO_WAITING;
+
+                               packet_list = entry->inject_first;
+
+                               entry->inject_first = NULL;
+                               entry->inject_last = NULL;
+                               proto_total_waiting--;
+
+                               lck_mtx_unlock(thread->input_lck);
+
+                               if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
+                                       lck_mtx_lock(entry->domain->dom_mtx);
+                                       locked = 1;
+                               }
+               
+                               if (entry->chain) {
+                                       entry->input(entry->protocol, packet_list);
+                               }
+                               else {
+                                       mbuf_t  packet;
+                               
+                                       for (packet = packet_list; packet; packet = packet_list) {
+                                               packet_list = mbuf_nextpkt(packet);
+                                               mbuf_setnextpkt(packet, NULL);
+                                               entry->input(entry->protocol, packet);

                                        }
                                }
                                        }
                                }

-                       }
+                               if (locked) {
+                                       locked = 0;
+                                       lck_mtx_unlock(entry->domain->dom_mtx);
+                               }       

                }
        }
                }
        }

+       }
+

 }
 
 errno_t
 }
 
 errno_t


@@ -257,53 +280,37 @@ proto_input(
        protocol_family_t       protocol,
        mbuf_t                          packet_list)
 {
        protocol_family_t       protocol,
        mbuf_t                          packet_list)
 {

-       struct proto_input_entry *entry;
-       
-       if (current_thread() != dlil_input_thread_ptr)
-               panic("proto_input called from a thread other than dlil_input_thread!\n");
-       
-       for (entry = proto_hash[proto_hash_value(protocol)]; entry; entry = entry->next) {
+       struct proto_input_entry        *entry;
+       errno_t                         locked =0, result = 0;
+
+       for (entry = proto_hash[proto_hash_value(protocol)]; entry;
+                entry = entry->next) {

                if (entry->protocol == protocol)
                        break;
        }
 
                if (entry->protocol == protocol)
                        break;
        }
 

-       if (entry) {
+       if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
+               lck_mtx_lock(entry->domain->dom_mtx);
+               locked = 1;
+       }
+       
+       if (entry->chain) {
+               entry->input(entry->protocol, packet_list);
+       }
+       else {

                mbuf_t  packet;
                mbuf_t  packet;

-#if DIRECT_PROTO_INPUT
-               // See <rdar://problem/3687868> for why this is disabled
-               // We need to release the dlil lock before taking the protocol lock
+               

                for (packet = packet_list; packet; packet = packet_list) {
                        packet_list = mbuf_nextpkt(packet);
                        mbuf_setnextpkt(packet, NULL);
                        entry->input(entry->protocol, packet);
                }
                for (packet = packet_list; packet; packet = packet_list) {
                        packet_list = mbuf_nextpkt(packet);
                        mbuf_setnextpkt(packet, NULL);
                        entry->input(entry->protocol, packet);
                }

-#else
-               mbuf_t  last_packet;
-               int             hash_slot = proto_hash_value(protocol);
-               
-               for (last_packet = packet_list; mbuf_nextpkt(last_packet);
-                        last_packet = mbuf_nextpkt(last_packet))
-                       /* find the last packet */;
-               
-               lck_mtx_lock(proto_input_lock);
-               if (entry->first_packet == NULL) {
-                       entry->first_packet = packet_list;
-               }
-               else {
-                       mbuf_setnextpkt(entry->last_packet, packet_list);
-               }
-               entry->last_packet = last_packet;
-               lck_mtx_unlock(proto_input_lock);
-       
-               OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets);
-#endif
-       }
-       else
-       {
-               return ENOENT;

        }
        
        }
        

-       return 0;
+       if (locked) {
+               lck_mtx_unlock(entry->domain->dom_mtx);
+       }       
+       return result;

 }
 
 errno_t
 }
 
 errno_t


@@ -312,8 +319,9 @@ proto_inject(
        mbuf_t                          packet_list)
 {
        struct proto_input_entry        *entry;
        mbuf_t                          packet_list)
 {
        struct proto_input_entry        *entry;

-       mbuf_t                                          last_packet;
-       int                                                     hash_slot = proto_hash_value(protocol);
+       mbuf_t                          last_packet;
+       int                             hash_slot = proto_hash_value(protocol);
+       struct dlil_threading_info      *thread = dlil_lo_thread_ptr;

        
        for (last_packet = packet_list; mbuf_nextpkt(last_packet);
                 last_packet = mbuf_nextpkt(last_packet))
        
        for (last_packet = packet_list; mbuf_nextpkt(last_packet);
                 last_packet = mbuf_nextpkt(last_packet))


@@ -325,19 +333,20 @@ proto_inject(
        }
        
        if (entry) {
        }
        
        if (entry) {

-               lck_mtx_lock(proto_input_lock);
-               if (entry->first_packet == NULL) {
-                       entry->first_packet = packet_list;
+               lck_mtx_lock(thread->input_lck);
+               if (entry->inject_first == NULL) {
+                       proto_total_waiting++;
+                       thread->input_waiting |= DLIL_PROTO_WAITING;
+                       entry->inject_first = packet_list;

                }
                else {
                }
                else {

-                       mbuf_setnextpkt(entry->last_packet, packet_list);
+                       mbuf_setnextpkt(entry->inject_last, packet_list);

                }
                }

-               entry->last_packet = last_packet;
-               lck_mtx_unlock(proto_input_lock);
-       
-               OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets);
-               
-               wakeup((caddr_t)&dlil_input_thread_wakeup);
+               entry->inject_last = last_packet;
+               if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0) {
+                       wakeup((caddr_t)&thread->input_waiting);
+               }
+               lck_mtx_unlock(thread->input_lck);

        }
        else
        {
        }
        else
        {


@@ -347,20 +356,120 @@ proto_inject(
        return 0;
 }
 
        return 0;
 }
 

+static struct proto_family_str*
+proto_plumber_find(
+       protocol_family_t       proto_family,
+       ifnet_family_t          if_family)
+{
+       struct proto_family_str  *mod = NULL;
+
+       TAILQ_FOREACH(mod, &proto_family_head, proto_fam_next) {
+               if ((mod->proto_family == (proto_family & 0xffff)) 
+                       && (mod->if_family == (if_family & 0xffff))) 
+                       break;
+               }
+
+       return mod;
+}
+

 errno_t
 proto_register_plumber(
 errno_t
 proto_register_plumber(

-       protocol_family_t proto_fam,
-       ifnet_family_t if_fam,
-       proto_plumb_handler plumb,
-       proto_unplumb_handler unplumb)
+       protocol_family_t               protocol_family,
+       ifnet_family_t                  interface_family, 
+       proto_plumb_handler             attach,
+       proto_unplumb_handler   detach)

 {
 {

-       return dlil_reg_proto_module(proto_fam, if_fam, (attach_t)plumb, (detach_t)unplumb);
+       struct proto_family_str *proto_family;
+
+       if (attach == NULL) return EINVAL;
+
+       lck_mtx_lock(proto_family_mutex);
+       
+       TAILQ_FOREACH(proto_family, &proto_family_head, proto_fam_next) {
+               if (proto_family->proto_family == protocol_family &&
+                       proto_family->if_family == interface_family) {
+                       lck_mtx_unlock(proto_family_mutex);
+                       return EEXIST;
+               }
+       }
+
+       proto_family = (struct proto_family_str *) _MALLOC(sizeof(struct proto_family_str), M_IFADDR, M_WAITOK);
+       if (!proto_family) {
+               lck_mtx_unlock(proto_family_mutex);
+               return ENOMEM;
+       }
+
+       bzero(proto_family, sizeof(struct proto_family_str));
+       proto_family->proto_family      = protocol_family;
+       proto_family->if_family         = interface_family & 0xffff;
+       proto_family->attach_proto      = attach;
+       proto_family->detach_proto      = detach;
+
+       TAILQ_INSERT_TAIL(&proto_family_head, proto_family, proto_fam_next);
+       lck_mtx_unlock(proto_family_mutex);
+       return 0;

 }
 
 void
 proto_unregister_plumber(
 }
 
 void
 proto_unregister_plumber(

-       protocol_family_t proto_fam,
-       ifnet_family_t if_fam)
+       protocol_family_t       protocol_family,
+       ifnet_family_t          interface_family)

 {
 {

-       (void)dlil_dereg_proto_module(proto_fam, if_fam);
+       struct proto_family_str  *proto_family;
+
+       lck_mtx_lock(proto_family_mutex);
+
+       proto_family = proto_plumber_find(protocol_family, interface_family);
+       if (proto_family == 0) {
+               lck_mtx_unlock(proto_family_mutex);
+               return;
+       }
+
+       TAILQ_REMOVE(&proto_family_head, proto_family, proto_fam_next);
+       FREE(proto_family, M_IFADDR);
+       
+       lck_mtx_unlock(proto_family_mutex);
+       return;
+}
+
+__private_extern__ errno_t
+proto_plumb(
+       protocol_family_t       protocol_family,
+       ifnet_t                         ifp)
+{
+       struct proto_family_str  *proto_family;
+       int ret = 0;
+
+       lck_mtx_lock(proto_family_mutex);
+       proto_family = proto_plumber_find(protocol_family, ifp->if_family);
+       if (proto_family == 0) {
+               lck_mtx_unlock(proto_family_mutex);
+               return ENXIO;
+       }
+
+       ret = proto_family->attach_proto(ifp, protocol_family);
+
+       lck_mtx_unlock(proto_family_mutex);
+       return ret;
+}
+
+
+__private_extern__ errno_t
+proto_unplumb(
+       protocol_family_t       protocol_family,
+       ifnet_t                         ifp)
+{
+       struct proto_family_str  *proto_family;
+       int ret = 0;
+
+       lck_mtx_lock(proto_family_mutex);
+
+       proto_family = proto_plumber_find(protocol_family, ifp->if_family);
+       if (proto_family && proto_family->detach_proto)
+               proto_family->detach_proto(ifp, protocol_family);
+       else
+               ret = ifnet_detach_protocol(ifp, protocol_family);
+    
+       lck_mtx_unlock(proto_family_mutex);
+       return ret;

 }
 }