X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/21362eb3e66fd2c787aee132bce100a44d71a99c..e2fac8b15b12a7979f72090454d850e612fc5b13:/bsd/net/kpi_protocol.c diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c index 5dda02574..9b63ec840 100644 --- a/bsd/net/kpi_protocol.c +++ b/bsd/net/kpi_protocol.c @@ -38,36 +38,50 @@ #include #include -void proto_kpi_init(void); void proto_input_run(void); typedef int (*attach_t)(struct ifnet *ifp, u_long protocol_family); typedef int (*detach_t)(struct ifnet *ifp, u_long protocol_family); -/****************************************************************************/ -/* WARNING: Big assumption made here - there can be only one input thread */ struct proto_input_entry { struct proto_input_entry *next; int detach; struct domain *domain; + int hash; + int chain; protocol_family_t protocol; proto_input_handler input; proto_input_detached_handler detached; - mbuf_t first_packet; - mbuf_t last_packet; + mbuf_t inject_first; + mbuf_t inject_last; + + struct proto_input_entry *input_next; + mbuf_t input_first; + mbuf_t input_last; +}; + + +struct proto_family_str { + TAILQ_ENTRY(proto_family_str) proto_fam_next; + protocol_family_t proto_family; + ifnet_family_t if_family; + proto_plumb_handler attach_proto; + proto_unplumb_handler detach_proto; }; #define PROTO_HASH_SLOTS 5 -static struct proto_input_entry *proto_hash[PROTO_HASH_SLOTS]; -static struct proto_input_entry *proto_input_add_list; -static lck_mtx_t *proto_input_lock = 0; -__private_extern__ u_int32_t inject_buckets = 0; +static struct proto_input_entry *proto_hash[PROTO_HASH_SLOTS]; +static int proto_total_waiting = 0; +static struct proto_input_entry *proto_input_add_list = NULL; +static lck_mtx_t *proto_family_mutex = 0; +static TAILQ_HEAD(, proto_family_str) proto_family_head = + TAILQ_HEAD_INITIALIZER(proto_family_head); -extern thread_t dlil_input_thread_ptr; -extern int dlil_input_thread_wakeup; +extern lck_mtx_t *domain_proto_mtx; +extern struct dlil_threading_info *dlil_lo_thread_ptr; static int proto_hash_value( @@ -95,24 +109,26 @@ proto_kpi_init(void) /* Allocate a mtx lock */ grp_attrib = lck_grp_attr_alloc_init(); - lck_grp_attr_setdefault(grp_attrib); lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib); lck_grp_attr_free(grp_attrib); lck_attrib = lck_attr_alloc_init(); - lck_attr_setdefault(lck_attrib); - proto_input_lock = lck_mtx_alloc_init(lck_group, lck_attrib); + proto_family_mutex = lck_mtx_alloc_init(lck_group, lck_attrib); lck_grp_free(lck_group); lck_attr_free(lck_attrib); + + bzero(proto_hash, sizeof(proto_hash)); } __private_extern__ errno_t proto_register_input( protocol_family_t protocol, proto_input_handler input, - proto_input_detached_handler detached) + proto_input_detached_handler detached, + int chains) { struct proto_input_entry *entry; + struct dlil_threading_info *thread = dlil_lo_thread_ptr; entry = _MALLOC(sizeof(*entry), M_IFADDR, M_WAITOK); @@ -123,25 +139,29 @@ proto_register_input( entry->protocol = protocol; entry->input = input; entry->detached = detached; + entry->hash = proto_hash_value(protocol); + entry->chain = chains; { struct domain *dp = domains; - extern lck_mtx_t *domain_proto_mtx; lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED); lck_mtx_lock(domain_proto_mtx); - while (dp && dp->dom_family != protocol) + while (dp && (protocol_family_t)dp->dom_family != protocol) dp = dp->dom_next; entry->domain = dp; lck_mtx_unlock(domain_proto_mtx); } - do { - entry->next = proto_input_add_list; - } while(!OSCompareAndSwap((UInt32)entry->next, (UInt32)entry, (UInt32*)&proto_input_add_list)); + lck_mtx_lock(thread->input_lck); + entry->next = proto_input_add_list; + proto_input_add_list = entry; - wakeup((caddr_t)&dlil_input_thread_wakeup); + thread->input_waiting |= DLIL_PROTO_REGISTER; + if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0) + wakeup((caddr_t)&thread->input_waiting); + lck_mtx_unlock(thread->input_lck); return 0; } @@ -191,71 +211,68 @@ proto_delayed_attach( } } -static void -proto_delayed_inject( - struct proto_input_entry *entry) -{ - mbuf_t packet_list; - mbuf_t packet; - int locked = 0; - - lck_mtx_lock(proto_input_lock); - packet_list = entry->first_packet; - entry->first_packet = entry->last_packet = 0; - lck_mtx_unlock(proto_input_lock); - - if (packet_list == NULL) - return; - - if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) { - lck_mtx_lock(entry->domain->dom_mtx); - locked = 1; - } - - for (packet = packet_list; packet; packet = packet_list) { - packet_list = mbuf_nextpkt(packet); - mbuf_setnextpkt(packet, NULL); - entry->input(entry->protocol, packet); - } - - if (locked) { - lck_mtx_unlock(entry->domain->dom_mtx); - } -} - -/* This function must be called from a single dlil input thread */ __private_extern__ void proto_input_run(void) { struct proto_input_entry *entry; - u_int32_t inject; - int i; - - if (current_thread() != dlil_input_thread_ptr) - panic("proto_input_run called from a thread other than dlil_input_thread!\n"); + struct dlil_threading_info *thread = dlil_lo_thread_ptr; + mbuf_t packet_list; + int i, locked = 0; - do { + lck_mtx_assert(thread->input_lck, LCK_MTX_ASSERT_NOTOWNED); + + if ((thread->input_waiting & DLIL_PROTO_REGISTER) != 0) { + lck_mtx_lock(thread->input_lck); entry = proto_input_add_list; - } while (entry && !OSCompareAndSwap((UInt32)entry, 0, (UInt32*)&proto_input_add_list)); - - if (entry) + proto_input_add_list = NULL; + thread->input_waiting &= ~DLIL_PROTO_REGISTER; + lck_mtx_unlock(thread->input_lck); proto_delayed_attach(entry); - - do { - inject = inject_buckets; - } while (inject && !OSCompareAndSwap(inject, 0, (UInt32*)&inject_buckets)); - - if (inject) { - for (i = 0; i < PROTO_HASH_SLOTS; i++) { - if ((inject & (1L << i)) != 0) { - for (entry = proto_hash[i]; entry; entry = entry->next) { - if (entry->first_packet) { - proto_delayed_inject(entry); + } + /* + Move everything from the lock protected list to the thread + specific list. + */ + for (i = 0; proto_total_waiting != 0 && i < PROTO_HASH_SLOTS; i++) { + for (entry = proto_hash[i]; entry && proto_total_waiting; + entry = entry->next) { + if (entry->inject_first) { + lck_mtx_lock(thread->input_lck); + thread->input_waiting &= ~DLIL_PROTO_WAITING; + + packet_list = entry->inject_first; + + entry->inject_first = NULL; + entry->inject_last = NULL; + proto_total_waiting--; + + lck_mtx_unlock(thread->input_lck); + + if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) { + lck_mtx_lock(entry->domain->dom_mtx); + locked = 1; + } + + if (entry->chain) { + entry->input(entry->protocol, packet_list); + } + else { + mbuf_t packet; + + for (packet = packet_list; packet; packet = packet_list) { + packet_list = mbuf_nextpkt(packet); + mbuf_setnextpkt(packet, NULL); + entry->input(entry->protocol, packet); } } - } + if (locked) { + locked = 0; + lck_mtx_unlock(entry->domain->dom_mtx); + } } } + } + } errno_t @@ -263,53 +280,37 @@ proto_input( protocol_family_t protocol, mbuf_t packet_list) { - struct proto_input_entry *entry; - - if (current_thread() != dlil_input_thread_ptr) - panic("proto_input called from a thread other than dlil_input_thread!\n"); - - for (entry = proto_hash[proto_hash_value(protocol)]; entry; entry = entry->next) { + struct proto_input_entry *entry; + errno_t locked =0, result = 0; + + for (entry = proto_hash[proto_hash_value(protocol)]; entry; + entry = entry->next) { if (entry->protocol == protocol) break; } - if (entry) { + if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) { + lck_mtx_lock(entry->domain->dom_mtx); + locked = 1; + } + + if (entry->chain) { + entry->input(entry->protocol, packet_list); + } + else { mbuf_t packet; -#if DIRECT_PROTO_INPUT - // See for why this is disabled - // We need to release the dlil lock before taking the protocol lock + for (packet = packet_list; packet; packet = packet_list) { packet_list = mbuf_nextpkt(packet); mbuf_setnextpkt(packet, NULL); entry->input(entry->protocol, packet); } -#else - mbuf_t last_packet; - int hash_slot = proto_hash_value(protocol); - - for (last_packet = packet_list; mbuf_nextpkt(last_packet); - last_packet = mbuf_nextpkt(last_packet)) - /* find the last packet */; - - lck_mtx_lock(proto_input_lock); - if (entry->first_packet == NULL) { - entry->first_packet = packet_list; - } - else { - mbuf_setnextpkt(entry->last_packet, packet_list); - } - entry->last_packet = last_packet; - lck_mtx_unlock(proto_input_lock); - - OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets); -#endif - } - else - { - return ENOENT; } - return 0; + if (locked) { + lck_mtx_unlock(entry->domain->dom_mtx); + } + return result; } errno_t @@ -318,8 +319,9 @@ proto_inject( mbuf_t packet_list) { struct proto_input_entry *entry; - mbuf_t last_packet; - int hash_slot = proto_hash_value(protocol); + mbuf_t last_packet; + int hash_slot = proto_hash_value(protocol); + struct dlil_threading_info *thread = dlil_lo_thread_ptr; for (last_packet = packet_list; mbuf_nextpkt(last_packet); last_packet = mbuf_nextpkt(last_packet)) @@ -331,19 +333,20 @@ proto_inject( } if (entry) { - lck_mtx_lock(proto_input_lock); - if (entry->first_packet == NULL) { - entry->first_packet = packet_list; + lck_mtx_lock(thread->input_lck); + if (entry->inject_first == NULL) { + proto_total_waiting++; + thread->input_waiting |= DLIL_PROTO_WAITING; + entry->inject_first = packet_list; } else { - mbuf_setnextpkt(entry->last_packet, packet_list); + mbuf_setnextpkt(entry->inject_last, packet_list); } - entry->last_packet = last_packet; - lck_mtx_unlock(proto_input_lock); - - OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets); - - wakeup((caddr_t)&dlil_input_thread_wakeup); + entry->inject_last = last_packet; + if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0) { + wakeup((caddr_t)&thread->input_waiting); + } + lck_mtx_unlock(thread->input_lck); } else { @@ -353,20 +356,120 @@ proto_inject( return 0; } +static struct proto_family_str* +proto_plumber_find( + protocol_family_t proto_family, + ifnet_family_t if_family) +{ + struct proto_family_str *mod = NULL; + + TAILQ_FOREACH(mod, &proto_family_head, proto_fam_next) { + if ((mod->proto_family == (proto_family & 0xffff)) + && (mod->if_family == (if_family & 0xffff))) + break; + } + + return mod; +} + errno_t proto_register_plumber( - protocol_family_t proto_fam, - ifnet_family_t if_fam, - proto_plumb_handler plumb, - proto_unplumb_handler unplumb) + protocol_family_t protocol_family, + ifnet_family_t interface_family, + proto_plumb_handler attach, + proto_unplumb_handler detach) { - return dlil_reg_proto_module(proto_fam, if_fam, (attach_t)plumb, (detach_t)unplumb); + struct proto_family_str *proto_family; + + if (attach == NULL) return EINVAL; + + lck_mtx_lock(proto_family_mutex); + + TAILQ_FOREACH(proto_family, &proto_family_head, proto_fam_next) { + if (proto_family->proto_family == protocol_family && + proto_family->if_family == interface_family) { + lck_mtx_unlock(proto_family_mutex); + return EEXIST; + } + } + + proto_family = (struct proto_family_str *) _MALLOC(sizeof(struct proto_family_str), M_IFADDR, M_WAITOK); + if (!proto_family) { + lck_mtx_unlock(proto_family_mutex); + return ENOMEM; + } + + bzero(proto_family, sizeof(struct proto_family_str)); + proto_family->proto_family = protocol_family; + proto_family->if_family = interface_family & 0xffff; + proto_family->attach_proto = attach; + proto_family->detach_proto = detach; + + TAILQ_INSERT_TAIL(&proto_family_head, proto_family, proto_fam_next); + lck_mtx_unlock(proto_family_mutex); + return 0; } void proto_unregister_plumber( - protocol_family_t proto_fam, - ifnet_family_t if_fam) + protocol_family_t protocol_family, + ifnet_family_t interface_family) { - (void)dlil_dereg_proto_module(proto_fam, if_fam); + struct proto_family_str *proto_family; + + lck_mtx_lock(proto_family_mutex); + + proto_family = proto_plumber_find(protocol_family, interface_family); + if (proto_family == 0) { + lck_mtx_unlock(proto_family_mutex); + return; + } + + TAILQ_REMOVE(&proto_family_head, proto_family, proto_fam_next); + FREE(proto_family, M_IFADDR); + + lck_mtx_unlock(proto_family_mutex); + return; +} + +__private_extern__ errno_t +proto_plumb( + protocol_family_t protocol_family, + ifnet_t ifp) +{ + struct proto_family_str *proto_family; + int ret = 0; + + lck_mtx_lock(proto_family_mutex); + proto_family = proto_plumber_find(protocol_family, ifp->if_family); + if (proto_family == 0) { + lck_mtx_unlock(proto_family_mutex); + return ENXIO; + } + + ret = proto_family->attach_proto(ifp, protocol_family); + + lck_mtx_unlock(proto_family_mutex); + return ret; +} + + +__private_extern__ errno_t +proto_unplumb( + protocol_family_t protocol_family, + ifnet_t ifp) +{ + struct proto_family_str *proto_family; + int ret = 0; + + lck_mtx_lock(proto_family_mutex); + + proto_family = proto_plumber_find(protocol_family, ifp->if_family); + if (proto_family && proto_family->detach_proto) + proto_family->detach_proto(ifp, protocol_family); + else + ret = ifnet_detach_protocol(ifp, protocol_family); + + lck_mtx_unlock(proto_family_mutex); + return ret; }