X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8f6c56a50524aa785f7e596d52dddfb331e18961..743345f9a4b36f7e2f9ba37691e70c50baecb56e:/bsd/net/kpi_protocol.c?ds=sidebyside diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c index 5dda02574..f35b2b10b 100644 --- a/bsd/net/kpi_protocol.c +++ b/bsd/net/kpi_protocol.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ @@ -38,335 +38,403 @@ #include #include -void proto_kpi_init(void); void proto_input_run(void); -typedef int (*attach_t)(struct ifnet *ifp, u_long protocol_family); -typedef int (*detach_t)(struct ifnet *ifp, u_long protocol_family); +typedef int (*attach_t)(struct ifnet *ifp, uint32_t protocol_family); +typedef int (*detach_t)(struct ifnet *ifp, uint32_t protocol_family); -/****************************************************************************/ -/* WARNING: Big assumption made here - there can be only one input thread */ struct proto_input_entry { - struct proto_input_entry *next; - int detach; - struct domain *domain; - - protocol_family_t protocol; - proto_input_handler input; + struct proto_input_entry *next; + int detach; + struct domain *domain; + int hash; + int chain; + + protocol_family_t protocol; + proto_input_handler input; proto_input_detached_handler detached; - - mbuf_t first_packet; - mbuf_t last_packet; -}; -#define PROTO_HASH_SLOTS 5 + mbuf_t inject_first; + mbuf_t inject_last; + + struct proto_input_entry *input_next; + mbuf_t input_first; + mbuf_t input_last; +}; -static struct proto_input_entry *proto_hash[PROTO_HASH_SLOTS]; -static struct proto_input_entry *proto_input_add_list; -static lck_mtx_t *proto_input_lock = 0; -__private_extern__ u_int32_t inject_buckets = 0; -extern thread_t dlil_input_thread_ptr; -extern int dlil_input_thread_wakeup; +struct proto_family_str { + TAILQ_ENTRY(proto_family_str) proto_fam_next; + protocol_family_t proto_family; + ifnet_family_t if_family; + proto_plumb_handler attach_proto; + proto_unplumb_handler detach_proto; +}; -static int -proto_hash_value( - protocol_family_t protocol) -{ - switch(protocol) { - case PF_INET: - return 0; - case PF_INET6: - return 1; - case PF_APPLETALK: - return 2; - case PF_VLAN: - return 3; - } - return 4; -} +static struct proto_input_entry *proto_hash[PROTO_HASH_SLOTS]; +static int proto_total_waiting = 0; +static struct proto_input_entry *proto_input_add_list = NULL; +decl_lck_mtx_data(static, proto_family_mutex_data); +static lck_mtx_t *proto_family_mutex = &proto_family_mutex_data; +static TAILQ_HEAD(, proto_family_str) proto_family_head = + TAILQ_HEAD_INITIALIZER(proto_family_head); __private_extern__ void proto_kpi_init(void) { - lck_grp_attr_t *grp_attrib = 0; - lck_attr_t *lck_attrib = 0; - lck_grp_t *lck_group = 0; - + lck_grp_attr_t *grp_attrib = NULL; + lck_attr_t *lck_attrib = NULL; + lck_grp_t *lck_group = NULL; + /* Allocate a mtx lock */ grp_attrib = lck_grp_attr_alloc_init(); - lck_grp_attr_setdefault(grp_attrib); lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib); lck_grp_attr_free(grp_attrib); lck_attrib = lck_attr_alloc_init(); - lck_attr_setdefault(lck_attrib); - proto_input_lock = lck_mtx_alloc_init(lck_group, lck_attrib); + lck_mtx_init(proto_family_mutex, lck_group, lck_attrib); lck_grp_free(lck_group); lck_attr_free(lck_attrib); + + bzero(proto_hash, sizeof (proto_hash)); } __private_extern__ errno_t -proto_register_input( - protocol_family_t protocol, - proto_input_handler input, - proto_input_detached_handler detached) +proto_register_input(protocol_family_t protocol, proto_input_handler input, + proto_input_detached_handler detached, int chains) { - struct proto_input_entry *entry; - - entry = _MALLOC(sizeof(*entry), M_IFADDR, M_WAITOK); - + struct dlil_threading_info *inp = dlil_main_input_thread; + struct domain *dp; + domain_guard_t guard; + + entry = _MALLOC(sizeof (*entry), M_IFADDR, M_WAITOK | M_ZERO); if (entry == NULL) - return ENOMEM; - - bzero(entry, sizeof(*entry)); + return (ENOMEM); + entry->protocol = protocol; entry->input = input; entry->detached = detached; - - { - struct domain *dp = domains; - extern lck_mtx_t *domain_proto_mtx; - - lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); - lck_mtx_lock(domain_proto_mtx); - while (dp && dp->dom_family != protocol) - dp = dp->dom_next; - entry->domain = dp; - lck_mtx_unlock(domain_proto_mtx); + entry->hash = proto_hash_value(protocol); + entry->chain = chains; + + guard = domain_guard_deploy(); + TAILQ_FOREACH(dp, &domains, dom_entry) { + if (dp->dom_family == (int)protocol) + break; } + domain_guard_release(guard); + if (dp == NULL) + return (EINVAL); - - do { - entry->next = proto_input_add_list; - } while(!OSCompareAndSwap((UInt32)entry->next, (UInt32)entry, (UInt32*)&proto_input_add_list)); - - wakeup((caddr_t)&dlil_input_thread_wakeup); - - return 0; -} + entry->domain = dp; + + lck_mtx_lock(&inp->input_lck); + entry->next = proto_input_add_list; + proto_input_add_list = entry; + + inp->input_waiting |= DLIL_PROTO_REGISTER; + if ((inp->input_waiting & DLIL_INPUT_RUNNING) == 0) + wakeup((caddr_t)&inp->input_waiting); + lck_mtx_unlock(&inp->input_lck); + return (0); +} __private_extern__ void -proto_unregister_input( - protocol_family_t protocol) +proto_unregister_input(protocol_family_t protocol) { struct proto_input_entry *entry = NULL; - - for (entry = proto_hash[proto_hash_value(protocol)]; entry; entry = entry->next) + + for (entry = proto_hash[proto_hash_value(protocol)]; entry != NULL; + entry = entry->next) { if (entry->protocol == protocol) break; - - if (entry) + } + + if (entry != NULL) entry->detach = 1; } - static void -proto_delayed_attach( - struct proto_input_entry *entry) +proto_delayed_attach(struct proto_input_entry *entry) { struct proto_input_entry *next_entry; - for (next_entry = entry->next; entry; entry = next_entry) { + + for (next_entry = entry->next; entry != NULL; entry = next_entry) { struct proto_input_entry *exist; int hash_slot; - + hash_slot = proto_hash_value(entry->protocol); next_entry = entry->next; - - for (exist = proto_hash[hash_slot]; exist; exist = exist->next) + + for (exist = proto_hash[hash_slot]; exist != NULL; + exist = exist->next) { if (exist->protocol == entry->protocol) break; - + } + /* If the entry already exists, call detached and dispose */ - if (exist) { + if (exist != NULL) { if (entry->detached) entry->detached(entry->protocol); FREE(entry, M_IFADDR); - } - else { + } else { entry->next = proto_hash[hash_slot]; proto_hash[hash_slot] = entry; } } } -static void -proto_delayed_inject( - struct proto_input_entry *entry) -{ - mbuf_t packet_list; - mbuf_t packet; - int locked = 0; - - lck_mtx_lock(proto_input_lock); - packet_list = entry->first_packet; - entry->first_packet = entry->last_packet = 0; - lck_mtx_unlock(proto_input_lock); - - if (packet_list == NULL) - return; - - if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) { - lck_mtx_lock(entry->domain->dom_mtx); - locked = 1; - } - - for (packet = packet_list; packet; packet = packet_list) { - packet_list = mbuf_nextpkt(packet); - mbuf_setnextpkt(packet, NULL); - entry->input(entry->protocol, packet); - } - - if (locked) { - lck_mtx_unlock(entry->domain->dom_mtx); - } -} - -/* This function must be called from a single dlil input thread */ __private_extern__ void proto_input_run(void) { - struct proto_input_entry *entry; - u_int32_t inject; - int i; - - if (current_thread() != dlil_input_thread_ptr) - panic("proto_input_run called from a thread other than dlil_input_thread!\n"); - - do { + struct proto_input_entry *entry; + struct dlil_threading_info *inp = dlil_main_input_thread; + mbuf_t packet_list; + int i, locked = 0; + + lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_NOTOWNED); + + if (inp->input_waiting & DLIL_PROTO_REGISTER) { + lck_mtx_lock_spin(&inp->input_lck); entry = proto_input_add_list; - } while (entry && !OSCompareAndSwap((UInt32)entry, 0, (UInt32*)&proto_input_add_list)); - - if (entry) + proto_input_add_list = NULL; + inp->input_waiting &= ~DLIL_PROTO_REGISTER; + lck_mtx_unlock(&inp->input_lck); proto_delayed_attach(entry); - - do { - inject = inject_buckets; - } while (inject && !OSCompareAndSwap(inject, 0, (UInt32*)&inject_buckets)); - - if (inject) { - for (i = 0; i < PROTO_HASH_SLOTS; i++) { - if ((inject & (1L << i)) != 0) { - for (entry = proto_hash[i]; entry; entry = entry->next) { - if (entry->first_packet) { - proto_delayed_inject(entry); + } + + /* + * Move everything from the lock protected list to the thread + * specific list. + */ + for (i = 0; proto_total_waiting != 0 && i < PROTO_HASH_SLOTS; i++) { + for (entry = proto_hash[i]; + entry != NULL && proto_total_waiting; entry = entry->next) { + if (entry->inject_first != NULL) { + lck_mtx_lock_spin(&inp->input_lck); + inp->input_waiting &= ~DLIL_PROTO_WAITING; + + packet_list = entry->inject_first; + + entry->inject_first = NULL; + entry->inject_last = NULL; + proto_total_waiting--; + + lck_mtx_unlock(&inp->input_lck); + + if (entry->domain != NULL && !(entry->domain-> + dom_flags & DOM_REENTRANT)) { + lck_mtx_lock(entry->domain->dom_mtx); + locked = 1; + } + + if (entry->chain) { + entry->input(entry->protocol, + packet_list); + } else { + mbuf_t packet; + + for (packet = packet_list; + packet != NULL; + packet = packet_list) { + packet_list = + mbuf_nextpkt(packet); + mbuf_setnextpkt(packet, NULL); + entry->input(entry->protocol, + packet); } } + if (locked) { + locked = 0; + lck_mtx_unlock(entry->domain->dom_mtx); + } } } } } errno_t -proto_input( - protocol_family_t protocol, - mbuf_t packet_list) +proto_input(protocol_family_t protocol, mbuf_t packet_list) { struct proto_input_entry *entry; - - if (current_thread() != dlil_input_thread_ptr) - panic("proto_input called from a thread other than dlil_input_thread!\n"); - - for (entry = proto_hash[proto_hash_value(protocol)]; entry; entry = entry->next) { + errno_t locked = 0, result = 0; + + for (entry = proto_hash[proto_hash_value(protocol)]; entry != NULL; + entry = entry->next) { if (entry->protocol == protocol) break; } - if (entry) { + if (entry->domain && !(entry->domain->dom_flags & DOM_REENTRANT)) { + lck_mtx_lock(entry->domain->dom_mtx); + locked = 1; + } + + if (entry->chain) { + entry->input(entry->protocol, packet_list); + } else { mbuf_t packet; -#if DIRECT_PROTO_INPUT - // See for why this is disabled - // We need to release the dlil lock before taking the protocol lock - for (packet = packet_list; packet; packet = packet_list) { + + for (packet = packet_list; packet != NULL; + packet = packet_list) { packet_list = mbuf_nextpkt(packet); mbuf_setnextpkt(packet, NULL); entry->input(entry->protocol, packet); } -#else - mbuf_t last_packet; - int hash_slot = proto_hash_value(protocol); - - for (last_packet = packet_list; mbuf_nextpkt(last_packet); - last_packet = mbuf_nextpkt(last_packet)) - /* find the last packet */; - - lck_mtx_lock(proto_input_lock); - if (entry->first_packet == NULL) { - entry->first_packet = packet_list; - } - else { - mbuf_setnextpkt(entry->last_packet, packet_list); - } - entry->last_packet = last_packet; - lck_mtx_unlock(proto_input_lock); - - OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets); -#endif } - else - { - return ENOENT; + + if (locked) { + lck_mtx_unlock(entry->domain->dom_mtx); } - - return 0; + return (result); } errno_t -proto_inject( - protocol_family_t protocol, - mbuf_t packet_list) +proto_inject(protocol_family_t protocol, mbuf_t packet_list) { - struct proto_input_entry *entry; - mbuf_t last_packet; - int hash_slot = proto_hash_value(protocol); - - for (last_packet = packet_list; mbuf_nextpkt(last_packet); - last_packet = mbuf_nextpkt(last_packet)) + struct proto_input_entry *entry; + mbuf_t last_packet; + int hash_slot = proto_hash_value(protocol); + struct dlil_threading_info *inp = dlil_main_input_thread; + + for (last_packet = packet_list; mbuf_nextpkt(last_packet) != NULL; + last_packet = mbuf_nextpkt(last_packet)) /* find the last packet */; - - for (entry = proto_hash[hash_slot]; entry; entry = entry->next) { + + for (entry = proto_hash[hash_slot]; entry != NULL; + entry = entry->next) { if (entry->protocol == protocol) break; } - - if (entry) { - lck_mtx_lock(proto_input_lock); - if (entry->first_packet == NULL) { - entry->first_packet = packet_list; + + if (entry != NULL) { + lck_mtx_lock(&inp->input_lck); + if (entry->inject_first == NULL) { + proto_total_waiting++; + inp->input_waiting |= DLIL_PROTO_WAITING; + entry->inject_first = packet_list; + } else { + mbuf_setnextpkt(entry->inject_last, packet_list); } - else { - mbuf_setnextpkt(entry->last_packet, packet_list); + entry->inject_last = last_packet; + if ((inp->input_waiting & DLIL_INPUT_RUNNING) == 0) { + wakeup((caddr_t)&inp->input_waiting); } - entry->last_packet = last_packet; - lck_mtx_unlock(proto_input_lock); - - OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets); - - wakeup((caddr_t)&dlil_input_thread_wakeup); + lck_mtx_unlock(&inp->input_lck); + } else { + return (ENOENT); } - else - { - return ENOENT; + + return (0); +} + +static struct proto_family_str * +proto_plumber_find(protocol_family_t proto_family, ifnet_family_t if_family) +{ + struct proto_family_str *mod = NULL; + + TAILQ_FOREACH(mod, &proto_family_head, proto_fam_next) { + if ((mod->proto_family == (proto_family & 0xffff)) && + (mod->if_family == (if_family & 0xffff))) + break; } - return 0; + return (mod); } errno_t -proto_register_plumber( - protocol_family_t proto_fam, - ifnet_family_t if_fam, - proto_plumb_handler plumb, - proto_unplumb_handler unplumb) +proto_register_plumber(protocol_family_t protocol_family, + ifnet_family_t interface_family, proto_plumb_handler attach, + proto_unplumb_handler detach) { - return dlil_reg_proto_module(proto_fam, if_fam, (attach_t)plumb, (detach_t)unplumb); + struct proto_family_str *proto_family; + + if (attach == NULL) + return (EINVAL); + + lck_mtx_lock(proto_family_mutex); + + TAILQ_FOREACH(proto_family, &proto_family_head, proto_fam_next) { + if (proto_family->proto_family == protocol_family && + proto_family->if_family == interface_family) { + lck_mtx_unlock(proto_family_mutex); + return (EEXIST); + } + } + + proto_family = (struct proto_family_str *) + _MALLOC(sizeof (struct proto_family_str), M_IFADDR, + M_WAITOK | M_ZERO); + if (!proto_family) { + lck_mtx_unlock(proto_family_mutex); + return (ENOMEM); + } + + proto_family->proto_family = protocol_family; + proto_family->if_family = interface_family & 0xffff; + proto_family->attach_proto = attach; + proto_family->detach_proto = detach; + + TAILQ_INSERT_TAIL(&proto_family_head, proto_family, proto_fam_next); + lck_mtx_unlock(proto_family_mutex); + return (0); } void -proto_unregister_plumber( - protocol_family_t proto_fam, - ifnet_family_t if_fam) +proto_unregister_plumber(protocol_family_t protocol_family, + ifnet_family_t interface_family) +{ + struct proto_family_str *proto_family; + + lck_mtx_lock(proto_family_mutex); + + proto_family = proto_plumber_find(protocol_family, interface_family); + if (proto_family == NULL) { + lck_mtx_unlock(proto_family_mutex); + return; + } + + TAILQ_REMOVE(&proto_family_head, proto_family, proto_fam_next); + FREE(proto_family, M_IFADDR); + + lck_mtx_unlock(proto_family_mutex); +} + +__private_extern__ errno_t +proto_plumb(protocol_family_t protocol_family, ifnet_t ifp) +{ + struct proto_family_str *proto_family; + int ret = 0; + + lck_mtx_lock(proto_family_mutex); + proto_family = proto_plumber_find(protocol_family, ifp->if_family); + if (proto_family == NULL) { + lck_mtx_unlock(proto_family_mutex); + return (ENXIO); + } + + ret = proto_family->attach_proto(ifp, protocol_family); + + lck_mtx_unlock(proto_family_mutex); + return (ret); +} + + +__private_extern__ errno_t +proto_unplumb(protocol_family_t protocol_family, ifnet_t ifp) { - (void)dlil_dereg_proto_module(proto_fam, if_fam); + struct proto_family_str *proto_family; + int ret = 0; + + lck_mtx_lock(proto_family_mutex); + + proto_family = proto_plumber_find(protocol_family, ifp->if_family); + if (proto_family != NULL && proto_family->detach_proto) + proto_family->detach_proto(ifp, protocol_family); + else + ret = ifnet_detach_protocol(ifp, protocol_family); + + lck_mtx_unlock(proto_family_mutex); + return (ret); }