/*
- * Copyright (c) 1999-2015 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2018 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <net/kpi_protocol.h>
#include <net/if_types.h>
+#include <net/if_ipsec.h>
#include <net/if_llreach.h>
+#include <net/if_utun.h>
#include <net/kpi_interfacefilter.h>
#include <net/classq/classq.h>
#include <net/classq/classq_sfb.h>
#include <net/flowhash.h>
#include <net/ntstat.h>
+#include <net/if_llatbl.h>
+#include <net/net_api_stats.h>
+#include <net/if_ports_used.h>
#if INET
#include <netinet/in_var.h>
#include <netinet/udp_var.h>
#include <netinet/if_ether.h>
#include <netinet/in_pcb.h>
+#include <netinet/in_tclass.h>
#endif /* INET */
#if INET6
#if PF
#include <net/pfvar.h>
#endif /* PF */
-#if PF_ALTQ
-#include <net/altq/altq.h>
-#endif /* PF_ALTQ */
#include <net/pktsched/pktsched.h>
-#define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0)
-#define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2)
-#define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
-#define DBG_FNC_DLIL_OUTPUT DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
-#define DBG_FNC_DLIL_IFOUT DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
+#if NECP
+#include <net/necp.h>
+#endif /* NECP */
-#define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
-#define MAX_LINKADDR 4 /* LONGWORDS */
-#define M_NKE M_IFADDR
+
+#define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0)
+#define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2)
+#define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
+#define DBG_FNC_DLIL_OUTPUT DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
+#define DBG_FNC_DLIL_IFOUT DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
+
+#define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
+#define MAX_LINKADDR 4 /* LONGWORDS */
+#define M_NKE M_IFADDR
#if 1
-#define DLIL_PRINTF printf
+#define DLIL_PRINTF printf
#else
-#define DLIL_PRINTF kprintf
+#define DLIL_PRINTF kprintf
#endif
#define IF_DATA_REQUIRE_ALIGNED_64(f) \
SLIST_HEAD(proto_hash_entry, if_proto);
-#define DLIL_SDLMAXLEN 64
#define DLIL_SDLDATALEN \
(DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0]))
#define DLIF_PROTO_ZONE_MAX (DLIF_ZONE_MAX*2) /* maximum elements in zone */
#define DLIF_PROTO_ZONE_NAME "ifnet_proto" /* zone name */
-static unsigned int dlif_tcpstat_size; /* size of tcpstat_local to allocate */
-static unsigned int dlif_tcpstat_bufsize; /* size of dlif_tcpstat_size + headroom */
+static unsigned int dlif_tcpstat_size; /* size of tcpstat_local to allocate */
+static unsigned int dlif_tcpstat_bufsize; /* size of dlif_tcpstat_size + headroom */
static struct zone *dlif_tcpstat_zone; /* zone for tcpstat_local */
#define DLIF_TCPSTAT_ZONE_MAX 1 /* maximum elements in zone */
#define DLIF_TCPSTAT_ZONE_NAME "ifnet_tcpstat" /* zone name */
-static unsigned int dlif_udpstat_size; /* size of udpstat_local to allocate */
+static unsigned int dlif_udpstat_size; /* size of udpstat_local to allocate */
static unsigned int dlif_udpstat_bufsize; /* size of dlif_udpstat_size + headroom */
static struct zone *dlif_udpstat_zone; /* zone for udpstat_local */
#define DLIF_UDPSTAT_ZONE_MAX 1 /* maximum elements in zone */
#define DLIF_UDPSTAT_ZONE_NAME "ifnet_udpstat" /* zone name */
-/*
- * Updating this variable should be done by first acquiring the global
- * radix node head (rnh_lock), in tandem with settting/clearing the
- * PR_AGGDRAIN for routedomain.
- */
-u_int32_t ifnet_aggressive_drainers;
static u_int32_t net_rtref;
static struct dlil_main_threading_info dlil_main_input_thread_info;
__private_extern__ struct dlil_threading_info *dlil_main_input_thread =
(struct dlil_threading_info *)&dlil_main_input_thread_info;
-static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg);
+static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg, bool update_generation);
static int dlil_detach_filter_internal(interface_filter_t filter, int detached);
static void dlil_if_trace(struct dlil_ifnet *, int);
static void if_proto_ref(struct if_proto *);
static void if_proto_free(struct if_proto *);
static struct if_proto *find_attached_proto(struct ifnet *, u_int32_t);
-static int dlil_ifp_proto_count(struct ifnet *);
+static u_int32_t dlil_ifp_protolist(struct ifnet *ifp, protocol_family_t *list,
+ u_int32_t list_count);
static void if_flt_monitor_busy(struct ifnet *);
static void if_flt_monitor_unbusy(struct ifnet *);
static void if_flt_monitor_enter(struct ifnet *);
const struct sockaddr_dl *, const struct sockaddr *,
const struct sockaddr_dl *, const struct sockaddr *);
-static errno_t ifp_if_output(struct ifnet *, struct mbuf *);
-static void ifp_if_start(struct ifnet *);
+static errno_t ifp_if_input(struct ifnet *ifp, struct mbuf *m_head,
+ struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
+ boolean_t poll, struct thread *tp);
static void ifp_if_input_poll(struct ifnet *, u_int32_t, u_int32_t,
struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *);
static errno_t ifp_if_ctl(struct ifnet *, ifnet_ctl_cmd_t, u_int32_t, void *);
const struct ifnet_demux_desc *, u_int32_t);
static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t);
static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *);
+#if CONFIG_EMBEDDED
+static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
+ const struct sockaddr *, const char *, const char *,
+ u_int32_t *, u_int32_t *);
+#else
static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
const struct sockaddr *, const char *, const char *);
+#endif /* CONFIG_EMBEDDED */
static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **,
const struct sockaddr *, const char *, const char *,
u_int32_t *, u_int32_t *);
static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *,
const struct ifnet_stat_increment_param *, boolean_t, boolean_t);
-#if DEBUG
+#if DEBUG || DEVELOPMENT
static void dlil_verify_sum16(void);
-#endif /* DEBUG */
+#endif /* DEBUG || DEVELOPMENT */
static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t,
protocol_family_t);
static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *,
static void ifnet_start_thread_fn(void *, wait_result_t);
static void ifnet_poll_thread_fn(void *, wait_result_t);
static void ifnet_poll(struct ifnet *);
+static errno_t ifnet_enqueue_common(struct ifnet *, void *,
+ classq_pkt_type_t, boolean_t, boolean_t *);
static void ifp_src_route_copyout(struct ifnet *, struct route *);
static void ifp_src_route_copyin(struct ifnet *, struct route *);
static int sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS;
static int sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS;
static int sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS;
-static int sysctl_get_ports_used SYSCTL_HANDLER_ARGS;
struct chain_len_stats tx_chain_len_stats;
static int sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS;
+#if TEST_INPUT_THREAD_TERMINATION
+static int sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS;
+#endif /* TEST_INPUT_THREAD_TERMINATION */
+
/* The following are protected by dlil_ifnet_lock */
static TAILQ_HEAD(, ifnet) ifnet_detaching_head;
static u_int32_t ifnet_detaching_cnt;
#define IFNET_FC_ZONE_NAME "ifnet_fc_zone"
#define IFNET_FC_ZONE_MAX 32
-extern void bpfdetach(struct ifnet*);
+extern void bpfdetach(struct ifnet *);
extern void proto_input_run(void);
-extern uint32_t udp_count_opportunistic(unsigned int ifindex,
+extern uint32_t udp_count_opportunistic(unsigned int ifindex,
u_int32_t flags);
-extern uint32_t tcp_count_opportunistic(unsigned int ifindex,
+extern uint32_t tcp_count_opportunistic(unsigned int ifindex,
u_int32_t flags);
__private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
#if CONFIG_MACF
+#ifdef CONFIG_EMBEDDED
+int dlil_lladdr_ckreq = 1;
+#else
int dlil_lladdr_ckreq = 0;
#endif
+#endif
#if DEBUG
int dlil_verbose = 1;
SYSCTL_DECL(_net_link_generic_system);
-#if CONFIG_MACF
-SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_lladdr_ckreq,
- CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_lladdr_ckreq, 0,
- "Require MACF system info check to expose link-layer address");
-#endif
-
SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_verbose,
CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_verbose, 0, "Log DLIL error messages");
sysctl_sndq_maxlen, "I", "Default transmit queue max length");
#define IF_RCVQ_MINLEN 32
-#define IF_RCVQ_MAXLEN 256
+#define IF_RCVQ_MAXLEN 256
u_int32_t if_rcvq_maxlen = IF_RCVQ_MAXLEN;
SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rcvq_maxlen,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rcvq_maxlen, IFQ_MAXLEN,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0,
sysctl_rxpoll, "I", "enable opportunistic input polling");
-u_int32_t if_bw_smoothing_val = 3;
-SYSCTL_UINT(_net_link_generic_system, OID_AUTO, if_bw_smoothing_val,
- CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_smoothing_val, 0, "");
-
-u_int32_t if_bw_measure_size = 10;
-SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_bw_measure_size,
- CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_measure_size, 0, "");
+#if TEST_INPUT_THREAD_TERMINATION
+static u_int32_t if_input_thread_termination_spin = 0;
+SYSCTL_PROC(_net_link_generic_system, OID_AUTO, input_thread_termination_spin,
+ CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
+ &if_input_thread_termination_spin, 0,
+ sysctl_input_thread_termination_spin,
+ "I", "input thread termination spin limit");
+#endif /* TEST_INPUT_THREAD_TERMINATION */
static u_int32_t cur_dlil_input_threads = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads,
- CTLFLAG_RD | CTLFLAG_LOCKED, &cur_dlil_input_threads , 0,
+ CTLFLAG_RD | CTLFLAG_LOCKED, &cur_dlil_input_threads, 0,
"Current number of DLIL input threads");
#if IFNET_INPUT_SANITY_CHK
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_sanity_check,
- CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_input_sanity_check , 0,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_input_sanity_check, 0,
"Turn on sanity checking in DLIL input");
#endif /* IFNET_INPUT_SANITY_CHK */
uint32_t tx_chain_len_count = 0;
SYSCTL_UINT(_net_link_generic_system, OID_AUTO, tx_chain_len_count,
- CTLFLAG_RW | CTLFLAG_LOCKED, &tx_chain_len_count, 0,
- "");
+ CTLFLAG_RW | CTLFLAG_LOCKED, &tx_chain_len_count, 0, "");
+
+static uint32_t threshold_notify = 1; /* enable/disable */
+SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_notify,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_notify, 0, "");
+
+static uint32_t threshold_interval = 2; /* in seconds */
+SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_interval,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_interval, 0, "");
+
+#if (DEVELOPMENT || DEBUG)
+static int sysctl_get_kao_frames SYSCTL_HANDLER_ARGS;
+SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_kao_frames,
+ CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_kao_frames, "");
+#endif /* DEVELOPMENT || DEBUG */
+
+struct net_api_stats net_api_stats;
+SYSCTL_STRUCT(_net, OID_AUTO, api_stats, CTLFLAG_RD|CTLFLAG_LOCKED,
+ &net_api_stats, net_api_stats, "");
-SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_ports_used,
- CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_ports_used, "");
unsigned int net_rxpoll = 1;
unsigned int net_affinity = 1;
static lck_grp_attr_t *dlil_grp_attributes = NULL;
static lck_attr_t *dlil_lck_attributes = NULL;
+/* DLIL data threshold thread call */
+static void dlil_dt_tcall_fn(thread_call_param_t, thread_call_param_t);
+
+static void dlil_mit_tcall_fn(thread_call_param_t, thread_call_param_t);
+
+uint32_t dlil_rcv_mit_pkts_min = 5;
+uint32_t dlil_rcv_mit_pkts_max = 64;
+uint32_t dlil_rcv_mit_interval = (500 * 1000);
+
+#if (DEVELOPMENT || DEBUG)
+SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_min,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_min, 0, "");
+SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_max,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_max, 0, "");
+SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_interval,
+ CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_interval, 0, "");
+#endif /* DEVELOPMENT || DEBUG */
+
#define DLIL_INPUT_CHECK(m, ifp) { \
struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \
* the hash bucket index and the protocol family defined
* here; future changes must be applied there as well.
*/
- switch(protocol_family) {
+ switch (protocol_family) {
case PF_INET:
return (0);
case PF_INET6:
*/
ifnet_lock_shared(ifp);
ev_pr_data.proto_family = proto_family;
- ev_pr_data.proto_remaining_count = dlil_ifp_proto_count(ifp);
+ ev_pr_data.proto_remaining_count = dlil_ifp_protolist(ifp, NULL, 0);
ifnet_lock_done(ifp);
dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_DETACHED,
(struct net_event_data *)&ev_pr_data,
- sizeof(struct kev_dl_proto_data));
+ sizeof (struct kev_dl_proto_data));
+
+ if (ev_pr_data.proto_remaining_count == 0) {
+ /*
+ * The protocol count has gone to zero, mark the interface down.
+ * This used to be done by configd.KernelEventMonitor, but that
+ * is inherently prone to races (rdar://problem/30810208).
+ */
+ (void) ifnet_set_flags(ifp, 0, IFF_UP);
+ (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
+ dlil_post_sifflags_msg(ifp);
+ }
zfree(dlif_proto_zone, proto);
}
__private_extern__ void
ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what)
{
+#if !MACH_ASSERT
+#pragma unused(ifp)
+#endif
unsigned int type = 0;
int ass = 1;
/* NOTREACHED */
}
if (ass)
- lck_rw_assert(&ifp->if_lock, type);
+ LCK_RW_ASSERT(&ifp->if_lock, type);
}
__private_extern__ void
lck_rw_done(&ifnet_head_lock);
}
+__private_extern__ void
+ifnet_head_assert_exclusive(void)
+{
+ LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE);
+}
+
/*
- * Caller must already be holding ifnet lock.
+ * dlil_ifp_protolist
+ * - get the list of protocols attached to the interface, or just the number
+ * of attached protocols
+ * - if the number returned is greater than 'list_count', truncation occurred
+ *
+ * Note:
+ * - caller must already be holding ifnet lock.
*/
-static int
-dlil_ifp_proto_count(struct ifnet * ifp)
+static u_int32_t
+dlil_ifp_protolist(struct ifnet *ifp, protocol_family_t *list,
+ u_int32_t list_count)
{
- int i, count = 0;
+ u_int32_t count = 0;
+ int i;
ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
for (i = 0; i < PROTO_HASH_SLOTS; i++) {
struct if_proto *proto;
SLIST_FOREACH(proto, &ifp->if_proto_hash[i], next_hash) {
+ if (list != NULL && count < list_count) {
+ list[count] = proto->protocol_family;
+ }
count++;
}
}
return (count);
}
+__private_extern__ u_int32_t
+if_get_protolist(struct ifnet * ifp, u_int32_t *protolist, u_int32_t count)
+{
+ ifnet_lock_shared(ifp);
+ count = dlil_ifp_protolist(ifp, protolist, count);
+ ifnet_lock_done(ifp);
+ return (count);
+}
+
+__private_extern__ void
+if_free_protolist(u_int32_t *list)
+{
+ _FREE(list, M_TEMP);
+}
+
__private_extern__ void
dlil_post_msg(struct ifnet *ifp, u_int32_t event_subclass,
u_int32_t event_code, struct net_event_data *event_data,
if (event_data == NULL) {
event_data = &ev_data;
- event_data_len = sizeof(struct net_event_data);
+ event_data_len = sizeof (struct net_event_data);
}
strlcpy(&event_data->if_name[0], ifp->if_name, IFNAMSIZ);
event_data->if_family = ifp->if_family;
- event_data->if_unit = (u_int32_t) ifp->if_unit;
+ event_data->if_unit = (u_int32_t)ifp->if_unit;
ev_msg.dv[0].data_length = event_data_len;
ev_msg.dv[0].data_ptr = event_data;
ev_msg.dv[1].data_length = 0;
- dlil_event_internal(ifp, &ev_msg);
+ /* Don't update interface generation for quality and RRC state changess */
+ bool update_generation = (event_subclass != KEV_DL_SUBCLASS ||
+ (event_code != KEV_DL_LINK_QUALITY_METRIC_CHANGED &&
+ event_code != KEV_DL_RRC_STATE_CHANGED));
+
+ dlil_event_internal(ifp, &ev_msg, update_generation);
}
__private_extern__ int
if (ifp->if_ipv4_stat == NULL) {
MALLOC(ifp->if_ipv4_stat, struct if_tcp_ecn_stat *,
- sizeof(struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
+ sizeof (struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
if (ifp->if_ipv4_stat == NULL) {
ret = ENOMEM;
goto end;
if (ifp->if_ipv6_stat == NULL) {
MALLOC(ifp->if_ipv6_stat, struct if_tcp_ecn_stat *,
- sizeof(struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
+ sizeof (struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
if (ifp->if_ipv6_stat == NULL) {
ret = ENOMEM;
goto end;
limit = (u_int32_t)-1;
}
- _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit);
+ _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF);
if (inp == dlil_main_input_thread) {
struct dlil_main_threading_info *inpm =
(struct dlil_main_threading_info *)inp;
- _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit);
+ _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF);
}
error = kernel_thread_start(func, inp, &inp->input_thr);
* Randomize to reduce the probability
* of affinity tag namespace collision.
*/
- read_random(&tag, sizeof (tag));
+ read_frandom(&tag, sizeof (tag));
if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) {
thread_reference(tp);
inp->tag = tag;
return (error);
}
-static void
-dlil_terminate_input_thread(struct dlil_threading_info *inp)
+#if TEST_INPUT_THREAD_TERMINATION
+static int
+sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS
{
- struct ifnet *ifp;
+#pragma unused(arg1, arg2)
+ uint32_t i;
+ int err;
- VERIFY(current_thread() == inp->input_thr);
- VERIFY(inp != dlil_main_input_thread);
+ i = if_input_thread_termination_spin;
- OSAddAtomic(-1, &cur_dlil_input_threads);
+ err = sysctl_handle_int(oidp, &i, 0, req);
+ if (err != 0 || req->newptr == USER_ADDR_NULL)
+ return (err);
+
+ if (net_rxpoll == 0)
+ return (ENXIO);
+
+ if_input_thread_termination_spin = i;
+ return (err);
+}
+#endif /* TEST_INPUT_THREAD_TERMINATION */
+static void
+dlil_clean_threading_info(struct dlil_threading_info *inp)
+{
lck_mtx_destroy(&inp->input_lck, inp->lck_grp);
lck_grp_free(inp->lck_grp);
inp->input_waiting = 0;
inp->wtot = 0;
bzero(inp->input_name, sizeof (inp->input_name));
- ifp = inp->ifp;
inp->ifp = NULL;
VERIFY(qhead(&inp->rcvq_pkts) == NULL && qempty(&inp->rcvq_pkts));
qlimit(&inp->rcvq_pkts) = 0;
#if IFNET_INPUT_SANITY_CHK
inp->input_mbuf_cnt = 0;
#endif /* IFNET_INPUT_SANITY_CHK */
+}
- if (dlil_verbose) {
- printf("%s: input thread terminated\n",
- if_name(ifp));
+static void
+dlil_terminate_input_thread(struct dlil_threading_info *inp)
+{
+ struct ifnet *ifp = inp->ifp;
+
+ VERIFY(current_thread() == inp->input_thr);
+ VERIFY(inp != dlil_main_input_thread);
+
+ OSAddAtomic(-1, &cur_dlil_input_threads);
+
+#if TEST_INPUT_THREAD_TERMINATION
+ { /* do something useless that won't get optimized away */
+ uint32_t v = 1;
+ for (uint32_t i = 0;
+ i < if_input_thread_termination_spin;
+ i++) {
+ v = (i + 1) * v;
+ }
+ printf("the value is %d\n", v);
}
+#endif /* TEST_INPUT_THREAD_TERMINATION */
+
+ lck_mtx_lock_spin(&inp->input_lck);
+ VERIFY((inp->input_waiting & DLIL_INPUT_TERMINATE) != 0);
+ inp->input_waiting |= DLIL_INPUT_TERMINATE_COMPLETE;
+ wakeup_one((caddr_t)&inp->input_waiting);
+ lck_mtx_unlock(&inp->input_lck);
/* for the extra refcnt from kernel_thread_start() */
thread_deallocate(current_thread());
+ if (dlil_verbose) {
+ printf("%s: input thread terminated\n",
+ if_name(ifp));
+ }
+
/* this is the end */
thread_terminate(current_thread());
/* NOTREACHED */
* The following fields must be 64-bit aligned for atomic operations.
*/
IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
- IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors)
+ IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors);
IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
IF_DATA_REQUIRE_ALIGNED_64(ifi_fbytes);
IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
- IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors)
+ IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors);
IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
_CASSERT(IF_HWASSIST_CSUM_UDPIPV6 == IFNET_CSUM_UDPIPV6);
_CASSERT(IF_HWASSIST_CSUM_FRAGMENT_IPV6 == IFNET_IPV6_FRAGMENT);
_CASSERT(IF_HWASSIST_CSUM_PARTIAL == IFNET_CSUM_PARTIAL);
+ _CASSERT(IF_HWASSIST_CSUM_ZERO_INVERT == IFNET_CSUM_ZERO_INVERT);
_CASSERT(IF_HWASSIST_VLAN_TAGGING == IFNET_VLAN_TAGGING);
_CASSERT(IF_HWASSIST_VLAN_MTU == IFNET_VLAN_MTU);
_CASSERT(IF_HWASSIST_TSO_V4 == IFNET_TSO_IPV4);
_CASSERT(CSUM_UDPIPV6 == IF_HWASSIST_CSUM_UDPIPV6);
_CASSERT(CSUM_FRAGMENT_IPV6 == IF_HWASSIST_CSUM_FRAGMENT_IPV6);
_CASSERT(CSUM_PARTIAL == IF_HWASSIST_CSUM_PARTIAL);
+ _CASSERT(CSUM_ZERO_INVERT == IF_HWASSIST_CSUM_ZERO_INVERT);
_CASSERT(CSUM_VLAN_TAG_VALID == IF_HWASSIST_VLAN_TAGGING);
/*
_CASSERT(IFRTYPE_SUBFAMILY_WIFI == IFNET_SUBFAMILY_WIFI);
_CASSERT(IFRTYPE_SUBFAMILY_THUNDERBOLT == IFNET_SUBFAMILY_THUNDERBOLT);
_CASSERT(IFRTYPE_SUBFAMILY_RESERVED == IFNET_SUBFAMILY_RESERVED);
+ _CASSERT(IFRTYPE_SUBFAMILY_INTCOPROC == IFNET_SUBFAMILY_INTCOPROC);
_CASSERT(DLIL_MODIDLEN == IFNET_MODIDLEN);
_CASSERT(DLIL_MODARGLEN == IFNET_MODARGLEN);
zone_change(dlif_udpstat_zone, Z_CALLERACCT, FALSE);
ifnet_llreach_init();
+ eventhandler_lists_ctxt_init(&ifnet_evhdlr_ctxt);
TAILQ_INIT(&dlil_ifnet_head);
TAILQ_INIT(&ifnet_head);
TAILQ_INIT(&ifnet_detaching_head);
+ TAILQ_INIT(&ifnet_ordered_head);
/* Setup the lock groups we will use */
dlil_grp_attributes = lck_grp_attr_alloc_init();
/* Initialize the pktap virtual interface */
pktap_init();
-#if DEBUG
+ /* Initialize the service class to dscp map */
+ net_qos_map_init();
+
+ /* Initialize the interface port list */
+ if_ports_used_init();
+
+#if DEBUG || DEVELOPMENT
/* Run self-tests */
dlil_verify_sum16();
-#endif /* DEBUG */
+#endif /* DEBUG || DEVELOPMENT */
+
+ /* Initialize link layer table */
+ lltable_glbl_init();
/*
* Create and start up the main DLIL input thread and the interface
/* NOTREACHED */
}
thread_deallocate(thread);
+
}
static void
if_flt_monitor_busy(struct ifnet *ifp)
{
- lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
++ifp->if_flt_busy;
VERIFY(ifp->if_flt_busy != 0);
static void
if_flt_monitor_enter(struct ifnet *ifp)
{
- lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
while (ifp->if_flt_busy) {
++ifp->if_flt_waiters;
static void
if_flt_monitor_leave(struct ifnet *ifp)
{
- lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
VERIFY(ifp->if_flt_busy != 0);
--ifp->if_flt_busy;
filter->filt_cookie = if_filter->iff_cookie;
filter->filt_name = if_filter->iff_name;
filter->filt_protocol = if_filter->iff_protocol;
- filter->filt_input = if_filter->iff_input;
- filter->filt_output = if_filter->iff_output;
- filter->filt_event = if_filter->iff_event;
- filter->filt_ioctl = if_filter->iff_ioctl;
+ /*
+ * Do not install filter callbacks for internal coproc interface
+ */
+ if (!IFNET_IS_INTCOPROC(ifp)) {
+ filter->filt_input = if_filter->iff_input;
+ filter->filt_output = if_filter->iff_output;
+ filter->filt_event = if_filter->iff_event;
+ filter->filt_ioctl = if_filter->iff_ioctl;
+ }
filter->filt_detached = if_filter->iff_detached;
lck_mtx_lock(&ifp->if_flt_lock);
if_flt_monitor_enter(ifp);
- lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
TAILQ_INSERT_TAIL(&ifp->if_flt_head, filter, filt_next);
if_flt_monitor_leave(ifp);
OSAddAtomic(1, &dlil_filter_disable_tso_count);
routegenid_update();
}
+ OSIncrementAtomic64(&net_api_stats.nas_iflt_attach_count);
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_total);
+ if ((filter->filt_flags & DLIL_IFF_INTERNAL)) {
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_os_total);
+ }
if (dlil_verbose) {
printf("%s: %s filter attached\n", if_name(ifp),
if_filter->iff_name);
lck_mtx_lock(&ifp->if_flt_lock);
if_flt_monitor_enter(ifp);
- lck_mtx_assert(&ifp->if_flt_lock,
+ LCK_MTX_ASSERT(&ifp->if_flt_lock,
LCK_MTX_ASSERT_OWNED);
/* Remove the filter from the list */
if (filter->filt_detached)
filter->filt_detached(filter->filt_cookie, filter->filt_ifp);
- /* Free the filter */
- zfree(dlif_filt_zone, filter);
-
/*
* Decrease filter count and route_generation ID to let TCP
* know it should reevalute doing TSO or not
OSAddAtomic(-1, &dlil_filter_disable_tso_count);
routegenid_update();
}
+
+ VERIFY(OSDecrementAtomic64(&net_api_stats.nas_iflt_attach_count) > 0);
+
+ /* Free the filter */
+ zfree(dlif_filt_zone, filter);
+ filter = NULL;
done:
- if (retval != 0) {
+ if (retval != 0 && filter != NULL) {
DLIL_PRINTF("failed to detach %s filter (err=%d)\n",
filter->filt_name, retval);
}
+
return (retval);
}
* c) protocol registrations
* d) packet injections
*/
+__attribute__((noreturn))
static void
dlil_main_input_thread_func(void *v, wait_result_t w)
{
/* Packets for non-dedicated interfaces other than lo0 */
m_cnt = qlen(&inp->rcvq_pkts);
- m = _getq_all(&inp->rcvq_pkts);
+ m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
/* Packets exclusive to lo0 */
m_cnt_loop = qlen(&inpm->lo_rcvq_pkts);
- m_loop = _getq_all(&inpm->lo_rcvq_pkts);
+ m_loop = _getq_all(&inpm->lo_rcvq_pkts, NULL, NULL, NULL);
inp->wtot = 0;
lck_mtx_unlock(&inp->input_lck);
/*
- * NOTE warning %%% attention !!!!
- * We should think about putting some thread starvation
- * safeguards if we deal with long chains of packets.
- */
+ * NOTE warning %%% attention !!!!
+ * We should think about putting some thread starvation
+ * safeguards if we deal with long chains of packets.
+ */
if (m_loop != NULL)
dlil_input_packet_list_extended(lo_ifp, m_loop,
m_cnt_loop, inp->mode);
dlil_input_thread_func(void *v, wait_result_t w)
{
#pragma unused(w)
+ char thread_name[MAXTHREADNAMESIZE];
struct dlil_threading_info *inp = v;
struct ifnet *ifp = inp->ifp;
+ /* Construct the name for this thread, and then apply it. */
+ bzero(thread_name, sizeof(thread_name));
+ snprintf(thread_name, sizeof(thread_name), "dlil_input_%s", ifp->if_xname);
+ thread_set_thread_name(inp->input_thr, thread_name);
+
VERIFY(inp != dlil_main_input_thread);
VERIFY(ifp != NULL);
VERIFY(!(ifp->if_eflags & IFEF_RXPOLL) || !net_rxpoll);
/* Packets for this interface */
m_cnt = qlen(&inp->rcvq_pkts);
- m = _getq_all(&inp->rcvq_pkts);
+ m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
lck_mtx_unlock(&inp->input_lck);
inp->wtot = 0;
- dlil_input_stats_sync(ifp, inp);
+ dlil_input_stats_sync(ifp, inp);
lck_mtx_unlock(&inp->input_lck);
/*
- * NOTE warning %%% attention !!!!
- * We should think about putting some thread starvation
- * safeguards if we deal with long chains of packets.
- */
+ * NOTE warning %%% attention !!!!
+ * We should think about putting some thread starvation
+ * safeguards if we deal with long chains of packets.
+ */
if (m != NULL)
dlil_input_packet_list_extended(NULL, m,
m_cnt, inp->mode);
if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
/* Free up pending packets */
+ lck_mtx_convert_spin(&inp->input_lck);
_flushq(&inp->rcvq_pkts);
+ if (inp->input_mit_tcall != NULL) {
+ if (thread_call_isactive(inp->input_mit_tcall))
+ thread_call_cancel(inp->input_mit_tcall);
+ }
lck_mtx_unlock(&inp->input_lck);
dlil_terminate_input_thread(inp);
m_size = qsize(&inp->rcvq_pkts);
/* Packets for this interface */
- m = _getq_all(&inp->rcvq_pkts);
+ m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
VERIFY(m != NULL || m_cnt == 0);
nanouptime(&now);
}
/*
- * NOTE warning %%% attention !!!!
- * We should think about putting some thread starvation
- * safeguards if we deal with long chains of packets.
- */
+ * NOTE warning %%% attention !!!!
+ * We should think about putting some thread starvation
+ * safeguards if we deal with long chains of packets.
+ */
if (m != NULL)
dlil_input_packet_list_extended(NULL, m, m_cnt, mode);
}
if (!locked)
lck_mtx_lock(&inp->input_lck);
- lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_OWNED);
/*
* Normally, we'd reset the parameters to the auto-tuned values
ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail,
const struct ifnet_stat_increment_param *s, boolean_t ext, boolean_t poll)
{
- struct thread *tp = current_thread();
- struct mbuf *last;
- struct dlil_threading_info *inp;
+ dlil_input_func input_func;
+ struct ifnet_stat_increment_param _s;
u_int32_t m_cnt = 0, m_size = 0;
+ struct mbuf *last;
+ errno_t err = 0;
if ((m_head == NULL && !poll) || (s == NULL && ext)) {
if (m_head != NULL)
return (EINVAL);
}
+ input_func = ifp->if_input_dlil;
+ VERIFY(input_func != NULL);
+
if (m_tail == NULL) {
last = m_head;
while (m_head != NULL) {
s->packets_in, m_cnt);
}
+ if (s == NULL) {
+ bzero(&_s, sizeof (_s));
+ s = &_s;
+ } else {
+ _s = *s;
+ }
+ _s.packets_in = m_cnt;
+ _s.bytes_in = m_size;
+
+ err = (*input_func)(ifp, m_head, m_tail, s, poll, current_thread());
+
+ if (ifp != lo_ifp) {
+ /* Release the IO refcnt */
+ ifnet_decr_iorefcnt(ifp);
+ }
+
+ return (err);
+}
+
+
+errno_t
+dlil_output_handler(struct ifnet *ifp, struct mbuf *m)
+{
+ return (ifp->if_output(ifp, m));
+}
+
+errno_t
+dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head,
+ struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
+ boolean_t poll, struct thread *tp)
+{
+ struct dlil_threading_info *inp;
+ u_int32_t m_cnt = s->packets_in;
+ u_int32_t m_size = s->bytes_in;
+
if ((inp = ifp->if_inp) == NULL)
inp = dlil_main_input_thread;
* will only do this once.
*/
lck_mtx_lock_spin(&inp->input_lck);
- if (inp != dlil_main_input_thread && inp->net_affinity &&
+ if (inp != dlil_main_input_thread && inp->net_affinity && tp != NULL &&
((!poll && inp->wloop_thr == THREAD_NULL) ||
(poll && inp->poll_thr == THREAD_NULL))) {
u_int32_t tag = inp->tag;
/*
* Take a reference on the current thread; during detach,
- * we will need to refer to it in order ot tear down its
+ * we will need to refer to it in order to tear down its
* affinity.
*/
thread_reference(tp);
VERIFY(m_head != NULL || (m_tail == NULL && m_cnt == 0));
- /*
+ /*
* Because of loopbacked multicast we cannot stuff the ifp in
* the rcvif of the packet header: loopback (lo0) packets use a
* dedicated list so that we can later associate them with lo_ifp
}
#endif /* IFNET_INPUT_SANITY_CHK */
- if (s != NULL) {
- dlil_input_stats_add(s, inp, poll);
- /*
- * If we're using the main input thread, synchronize the
- * stats now since we have the interface context. All
- * other cases involving dedicated input threads will
- * have their stats synchronized there.
- */
- if (inp == dlil_main_input_thread)
- dlil_input_stats_sync(ifp, inp);
- }
+ dlil_input_stats_add(s, inp, poll);
+ /*
+ * If we're using the main input thread, synchronize the
+ * stats now since we have the interface context. All
+ * other cases involving dedicated input threads will
+ * have their stats synchronized there.
+ */
+ if (inp == dlil_main_input_thread)
+ dlil_input_stats_sync(ifp, inp);
- inp->input_waiting |= DLIL_INPUT_WAITING;
- if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
- inp->wtot++;
- wakeup_one((caddr_t)&inp->input_waiting);
+ if (inp->input_mit_tcall &&
+ qlen(&inp->rcvq_pkts) >= dlil_rcv_mit_pkts_min &&
+ qlen(&inp->rcvq_pkts) < dlil_rcv_mit_pkts_max &&
+ (ifp->if_family == IFNET_FAMILY_ETHERNET ||
+ ifp->if_type == IFT_CELLULAR)
+ ) {
+ if (!thread_call_isactive(inp->input_mit_tcall)) {
+ uint64_t deadline;
+ clock_interval_to_deadline(dlil_rcv_mit_interval,
+ 1, &deadline);
+ (void) thread_call_enter_delayed(
+ inp->input_mit_tcall, deadline);
+ }
+ } else {
+ inp->input_waiting |= DLIL_INPUT_WAITING;
+ if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
+ inp->wtot++;
+ wakeup_one((caddr_t)&inp->input_waiting);
+ }
}
lck_mtx_unlock(&inp->input_lck);
- if (ifp != lo_ifp) {
- /* Release the IO refcnt */
- ifnet_decr_iorefcnt(ifp);
- }
-
return (0);
}
+
static void
-ifnet_start_common(struct ifnet *ifp, int resetfc)
+ifnet_start_common(struct ifnet *ifp, boolean_t resetfc)
{
if (!(ifp->if_eflags & IFEF_TXSTART))
return;
* If the starter thread is inactive, signal it to do work,
* unless the interface is being flow controlled from below,
* e.g. a virtual interface being flow controlled by a real
- * network interface beneath it.
+ * network interface beneath it, or it's been disabled via
+ * a call to ifnet_disable_output().
*/
lck_mtx_lock_spin(&ifp->if_start_lock);
if (resetfc) {
ifp->if_start_req++;
if (!ifp->if_start_active && ifp->if_start_thread != THREAD_NULL &&
(resetfc || !(ifp->if_eflags & IFEF_ENQUEUE_MULTI) ||
- IFCQ_LEN(&ifp->if_snd) >= ifp->if_start_delay_qlen
- || ifp->if_start_delayed == 0)) {
- wakeup_one((caddr_t)&ifp->if_start_thread);
+ IFCQ_LEN(&ifp->if_snd) >= ifp->if_start_delay_qlen ||
+ ifp->if_start_delayed == 0)) {
+ (void) thread_wakeup_thread((caddr_t)&ifp->if_start_thread,
+ ifp->if_start_thread);
}
lck_mtx_unlock(&ifp->if_start_lock);
}
void
ifnet_start(struct ifnet *ifp)
{
- ifnet_start_common(ifp, 0);
+ ifnet_start_common(ifp, FALSE);
}
static void
#pragma unused(w)
struct ifnet *ifp = v;
char ifname[IFNAMSIZ + 1];
+ char thread_name[MAXTHREADNAMESIZE];
struct timespec *ts = NULL;
struct ifclassq *ifq = &ifp->if_snd;
struct timespec delay_start_ts;
+ /* Construct the name for this thread, and then apply it. */
+ bzero(thread_name, sizeof(thread_name));
+ (void) snprintf(thread_name, sizeof (thread_name),
+ "ifnet_start_%s", ifp->if_xname);
+ thread_set_thread_name(ifp->if_start_thread, thread_name);
+
/*
* Treat the dedicated starter thread for lo0 as equivalent to
* the driver workloop thread; if net_affinity is enabled for
}
}
- snprintf(ifname, sizeof (ifname), "%s_starter",
- if_name(ifp));
+ (void) snprintf(ifname, sizeof (ifname), "%s_starter", if_name(ifp));
lck_mtx_lock_spin(&ifp->if_start_lock);
for (;;) {
- if (ifp->if_start_thread != NULL)
- (void) msleep(&ifp->if_start_thread, &ifp->if_start_lock,
+ if (ifp->if_start_thread != NULL) {
+ (void) msleep(&ifp->if_start_thread,
+ &ifp->if_start_lock,
(PZERO - 1) | PSPIN, ifname, ts);
-
+ }
/* interface is detached? */
if (ifp->if_start_thread == THREAD_NULL) {
ifnet_set_start_cycle(ifp, NULL);
lck_mtx_lock_spin(&ifp->if_start_lock);
- /* if there's no pending request, we're done */
- if (req == ifp->if_start_req)
+ /*
+ * If there's no pending request or if the
+ * interface has been disabled, we're done.
+ */
+ if (req == ifp->if_start_req ||
+ (ifp->if_start_flags & IFSF_FLOW_CONTROLLED)) {
break;
+ }
}
ifp->if_start_req = 0;
lck_mtx_lock_spin(&ifp->if_poll_lock);
/* if there's no pending request, we're done */
- if (req == ifp->if_poll_req)
+ if (req == ifp->if_poll_req) {
break;
+ }
}
ifp->if_poll_req = 0;
ifp->if_poll_active = 0;
u_int32_t omodel;
errno_t err;
- if (ifp == NULL || (model != IFNET_SCHED_MODEL_DRIVER_MANAGED &&
- model != IFNET_SCHED_MODEL_NORMAL))
+ if (ifp == NULL || model >= IFNET_SCHED_MODEL_MAX)
return (EINVAL);
else if (!(ifp->if_eflags & IFEF_TXSTART))
return (ENXIO);
return (0);
}
-errno_t
-ifnet_enqueue(struct ifnet *ifp, struct mbuf *m)
+void
+ifnet_enqueue_multi_setup(struct ifnet *ifp, uint16_t delay_qlen,
+ uint16_t delay_timeout)
+{
+ if (delay_qlen > 0 && delay_timeout > 0) {
+ ifp->if_eflags |= IFEF_ENQUEUE_MULTI;
+ ifp->if_start_delay_qlen = min(100, delay_qlen);
+ ifp->if_start_delay_timeout = min(20000, delay_timeout);
+ /* convert timeout to nanoseconds */
+ ifp->if_start_delay_timeout *= 1000;
+ kprintf("%s: forced IFEF_ENQUEUE_MULTI qlen %u timeout %u\n",
+ ifp->if_xname, (uint32_t)delay_qlen,
+ (uint32_t)delay_timeout);
+ } else {
+ ifp->if_eflags &= ~IFEF_ENQUEUE_MULTI;
+ }
+}
+
+static inline errno_t
+ifnet_enqueue_common(struct ifnet *ifp, void *p, classq_pkt_type_t ptype,
+ boolean_t flush, boolean_t *pdrop)
{
- int error;
+ volatile uint64_t *fg_ts = NULL;
+ volatile uint64_t *rt_ts = NULL;
+ struct mbuf *m = p;
struct timespec now;
- u_int64_t now_nsec;
+ u_int64_t now_nsec = 0;
+ int error = 0;
- if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) ||
- m->m_nextpkt != NULL) {
- if (m != NULL)
- m_freem_list(m);
- return (EINVAL);
- } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- !(ifp->if_refflags & IFRF_ATTACHED)) {
- /* flag tested without lock for performance */
- m_freem(m);
- return (ENXIO);
- } else if (!(ifp->if_flags & IFF_UP)) {
- m_freem(m);
- return (ENETDOWN);
- }
+ ASSERT(ifp->if_eflags & IFEF_TXSTART);
- nanouptime(&now);
- net_timernsec(&now, &now_nsec);
- m->m_pkthdr.pkt_enqueue_ts = now_nsec;
+ /*
+ * If packet already carries a timestamp, either from dlil_output()
+ * or from flowswitch, use it here. Otherwise, record timestamp.
+ * PKTF_TS_VALID is always cleared prior to entering classq, i.e.
+ * the timestamp value is used internally there.
+ */
+ switch (ptype) {
+ case QP_MBUF:
+ ASSERT(m->m_flags & M_PKTHDR);
+ ASSERT(m->m_nextpkt == NULL);
+
+ if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ||
+ m->m_pkthdr.pkt_timestamp == 0) {
+ nanouptime(&now);
+ net_timernsec(&now, &now_nsec);
+ m->m_pkthdr.pkt_timestamp = now_nsec;
+ }
+ m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID;
+ /*
+ * If the packet service class is not background,
+ * update the timestamp to indicate recent activity
+ * on a foreground socket.
+ */
+ if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) &&
+ m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
+ if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) {
+ ifp->if_fg_sendts = _net_uptime;
+ if (fg_ts != NULL)
+ *fg_ts = _net_uptime;
+ }
+ if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) {
+ ifp->if_rt_sendts = _net_uptime;
+ if (rt_ts != NULL)
+ *rt_ts = _net_uptime;
+ }
+ }
+ break;
+
+
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ }
if (ifp->if_eflags & IFEF_ENQUEUE_MULTI) {
+ if (now_nsec == 0) {
+ nanouptime(&now);
+ net_timernsec(&now, &now_nsec);
+ }
/*
* If the driver chose to delay start callback for
* coalescing multiple packets, Then use the following
* 3. If the time elapsed since last enqueue is more
* than 200ms we disable delaying start callback. This is
* is to take idle time into account.
- */
+ */
u_int64_t dwin = (ifp->if_start_delay_timeout << 1);
if (ifp->if_start_delay_swin > 0) {
if ((ifp->if_start_delay_swin + dwin) > now_nsec) {
} else {
ifp->if_start_delay_idle++;
}
- }
+ }
ifp->if_start_delay_swin = now_nsec;
ifp->if_start_delay_cnt = 1;
}
ifp->if_eflags &= ~(IFEF_DELAY_START);
}
- /* enqueue the packet */
- error = ifclassq_enqueue(&ifp->if_snd, m);
+ switch (ptype) {
+ case QP_MBUF:
+ /* enqueue the packet (caller consumes object) */
+ error = ifclassq_enqueue(&ifp->if_snd, m, QP_MBUF, pdrop);
+ m = NULL;
+ break;
+
+
+ default:
+ break;
+ }
/*
* Tell the driver to start dequeueing; do this even when the queue
* be dequeueing from other unsuspended queues.
*/
if (!(ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
- (error == 0 || error == EQFULL || error == EQSUSPENDED))
+ ((error == 0 && flush) || error == EQFULL || error == EQSUSPENDED))
ifnet_start(ifp);
return (error);
}
+errno_t
+ifnet_enqueue(struct ifnet *ifp, struct mbuf *m)
+{
+ boolean_t pdrop;
+ return (ifnet_enqueue_mbuf(ifp, m, TRUE, &pdrop));
+}
+
+errno_t
+ifnet_enqueue_mbuf(struct ifnet *ifp, struct mbuf *m, boolean_t flush,
+ boolean_t *pdrop)
+{
+ if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) ||
+ m->m_nextpkt != NULL) {
+ if (m != NULL) {
+ m_freem_list(m);
+ *pdrop = TRUE;
+ }
+ return (EINVAL);
+ } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
+ !IF_FULLY_ATTACHED(ifp)) {
+ /* flag tested without lock for performance */
+ m_freem(m);
+ *pdrop = TRUE;
+ return (ENXIO);
+ } else if (!(ifp->if_flags & IFF_UP)) {
+ m_freem(m);
+ *pdrop = TRUE;
+ return (ENETDOWN);
+ }
+
+ return (ifnet_enqueue_common(ifp, m, QP_MBUF, flush, pdrop));
+}
+
+
errno_t
ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp)
{
errno_t rc;
+ classq_pkt_type_t ptype;
if (ifp == NULL || mp == NULL)
return (EINVAL);
else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- (ifp->if_output_sched_model != IFNET_SCHED_MODEL_NORMAL))
+ ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
return (ENXIO);
if (!ifnet_is_attached(ifp, 1))
return (ENXIO);
- rc = ifclassq_dequeue(&ifp->if_snd, 1, mp, NULL, NULL, NULL);
+
+ rc = ifclassq_dequeue(&ifp->if_snd, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT,
+ (void **)mp, NULL, NULL, NULL, &ptype);
+ VERIFY((*mp == NULL) || (ptype == QP_MBUF));
ifnet_decr_iorefcnt(ifp);
return (rc);
struct mbuf **mp)
{
errno_t rc;
+ classq_pkt_type_t ptype;
if (ifp == NULL || mp == NULL || !MBUF_VALID_SC(sc))
return (EINVAL);
else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- (ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED))
+ ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
return (ENXIO);
if (!ifnet_is_attached(ifp, 1))
return (ENXIO);
-
- rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, mp, NULL, NULL, NULL);
+
+ rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1,
+ CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)mp, NULL, NULL,
+ NULL, &ptype);
+ VERIFY((*mp == NULL) || (ptype == QP_MBUF));
ifnet_decr_iorefcnt(ifp);
return (rc);
}
errno_t
-ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t limit, struct mbuf **head,
- struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
+ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit,
+ struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
{
errno_t rc;
- if (ifp == NULL || head == NULL || limit < 1)
+ classq_pkt_type_t ptype;
+ if (ifp == NULL || head == NULL || pkt_limit < 1)
return (EINVAL);
else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- (ifp->if_output_sched_model != IFNET_SCHED_MODEL_NORMAL))
+ ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
return (ENXIO);
if (!ifnet_is_attached(ifp, 1))
return (ENXIO);
-
- rc = ifclassq_dequeue(&ifp->if_snd, limit, head, tail, cnt, len);
+
+ rc = ifclassq_dequeue(&ifp->if_snd, pkt_limit,
+ CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, (void **)tail, cnt,
+ len, &ptype);
+ VERIFY((*head == NULL) || (ptype == QP_MBUF));
+ ifnet_decr_iorefcnt(ifp);
+ return (rc);
+}
+
+errno_t
+ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit,
+ struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
+{
+ errno_t rc;
+ classq_pkt_type_t ptype;
+ if (ifp == NULL || head == NULL || byte_limit < 1)
+ return (EINVAL);
+ else if (!(ifp->if_eflags & IFEF_TXSTART) ||
+ ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
+ return (ENXIO);
+ if (!ifnet_is_attached(ifp, 1))
+ return (ENXIO);
+
+ rc = ifclassq_dequeue(&ifp->if_snd, CLASSQ_DEQUEUE_MAX_PKT_LIMIT,
+ byte_limit, (void **)head, (void **)tail, cnt, len, &ptype);
+ VERIFY((*head == NULL) || (ptype == QP_MBUF));
ifnet_decr_iorefcnt(ifp);
return (rc);
}
errno_t
ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc,
- u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
+ u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
u_int32_t *len)
{
errno_t rc;
- if (ifp == NULL || head == NULL || limit < 1 || !MBUF_VALID_SC(sc))
+ classq_pkt_type_t ptype;
+ if (ifp == NULL || head == NULL || pkt_limit < 1 ||
+ !MBUF_VALID_SC(sc))
return (EINVAL);
else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- (ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED))
+ ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
return (ENXIO);
if (!ifnet_is_attached(ifp, 1))
return (ENXIO);
- rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, limit, head,
- tail, cnt, len);
+
+ rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit,
+ CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head,
+ (void **)tail, cnt, len, &ptype);
+ VERIFY((*head == NULL) || (ptype == QP_MBUF));
ifnet_decr_iorefcnt(ifp);
return (rc);
}
+#if !CONFIG_EMBEDDED
errno_t
ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m,
const struct sockaddr *dest, const char *dest_linkaddr,
return (ifp->if_framer_legacy(ifp, m, dest, dest_linkaddr, frame_type));
}
+#endif /* !CONFIG_EMBEDDED */
static int
dlil_interface_filters_input(struct ifnet *ifp, struct mbuf **m_p,
if (error != 0 && error != EJUSTRETURN)
m_freem_list(m);
}
- return;
}
static void
atomic_add_64(&ifp->if_data.ifi_iqdrops, s->dropped);
s->dropped = 0;
}
- /*
- * If we went over the threshold, notify NetworkStatistics.
- */
- if (ifp->if_data_threshold &&
- (ifp->if_ibytes + ifp->if_obytes) - ifp->if_dt_bytes >
- ifp->if_data_threshold) {
- ifp->if_dt_bytes = ifp->if_ibytes + ifp->if_obytes;
- nstat_ifnet_threshold_reached(ifp->if_index);
+
+ if (ifp->if_data_threshold != 0) {
+ lck_mtx_convert_spin(&inp->input_lck);
+ ifnet_notify_data_threshold(ifp);
}
+
/*
* No need for atomic operations as they are modified here
* only from within the DLIL input thread context.
mbuf_t * pkt_next = NULL;
u_int32_t poll_thresh = 0, poll_ival = 0;
- KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START,0,0,0,0,0);
+ KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
if (ext && mode == IFNET_MODEL_INPUT_POLL_ON && cnt > 1 &&
(poll_ival = if_rxpoll_interval_pkts) > 0)
goto next;
}
iorefcnt = 1;
- pktf_mask = 0;
+ /*
+ * Preserve the time stamp if it was set.
+ */
+ pktf_mask = PKTF_TS_VALID;
} else {
/*
* If this arrived on lo0, preserve interface addr
goto next;
}
}
- if (error != 0 || ((m->m_flags & M_PROMISC) != 0) ) {
+ if (error != 0 || ((m->m_flags & M_PROMISC) != 0)) {
m_freem(m);
goto next;
}
ifnet_decr_iorefcnt(ifp);
}
- KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_END,0,0,0,0,0);
+ KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
}
errno_t
return (0);
}
-
-#define TMP_IF_PROTO_ARR_SIZE 10
+/* If ifp is set, we will increment the generation for the interface */
+int
+dlil_post_complete_msg(struct ifnet *ifp, struct kev_msg *event)
+{
+ if (ifp != NULL) {
+ ifnet_increment_generation(ifp);
+ }
+
+#if NECP
+ necp_update_all_clients();
+#endif /* NECP */
+
+ return (kev_post_msg(event));
+}
+
+__private_extern__ void
+dlil_post_sifflags_msg(struct ifnet * ifp)
+{
+ struct kev_msg ev_msg;
+ struct net_event_data ev_data;
+
+ bzero(&ev_data, sizeof (ev_data));
+ bzero(&ev_msg, sizeof (ev_msg));
+ ev_msg.vendor_code = KEV_VENDOR_APPLE;
+ ev_msg.kev_class = KEV_NETWORK_CLASS;
+ ev_msg.kev_subclass = KEV_DL_SUBCLASS;
+ ev_msg.event_code = KEV_DL_SIFFLAGS;
+ strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+ ev_data.if_family = ifp->if_family;
+ ev_data.if_unit = (u_int32_t) ifp->if_unit;
+ ev_msg.dv[0].data_length = sizeof(struct net_event_data);
+ ev_msg.dv[0].data_ptr = &ev_data;
+ ev_msg.dv[1].data_length = 0;
+ dlil_post_complete_msg(ifp, &ev_msg);
+}
+
+#define TMP_IF_PROTO_ARR_SIZE 10
static int
-dlil_event_internal(struct ifnet *ifp, struct kev_msg *event)
+dlil_event_internal(struct ifnet *ifp, struct kev_msg *event, bool update_generation)
{
struct ifnet_filter *filter = NULL;
struct if_proto *proto = NULL;
* therefore we are avoiding embedded pointers here.
*/
ifnet_lock_shared(ifp);
- if_proto_count = dlil_ifp_proto_count(ifp);
+ if_proto_count = dlil_ifp_protolist(ifp, NULL, 0);
if (if_proto_count) {
int i;
VERIFY(ifp->if_proto_hash != NULL);
if_proto_free(proto);
}
-cleanup:
+cleanup:
if (tmp_malloc) {
FREE(tmp_ifproto_arr, M_TEMP);
}
/* Release the io ref count */
ifnet_decr_iorefcnt(ifp);
done:
- return (kev_post_msg(event));
+ return (dlil_post_complete_msg(update_generation ? ifp : NULL, event));
}
errno_t
ifnet_event(ifnet_t ifp, struct kern_event_msg *event)
{
- struct kev_msg kev_msg;
+ struct kev_msg kev_msg;
int result = 0;
if (ifp == NULL || event == NULL)
return (EINVAL);
bzero(&kev_msg, sizeof (kev_msg));
- kev_msg.vendor_code = event->vendor_code;
- kev_msg.kev_class = event->kev_class;
- kev_msg.kev_subclass = event->kev_subclass;
- kev_msg.event_code = event->event_code;
+ kev_msg.vendor_code = event->vendor_code;
+ kev_msg.kev_class = event->kev_class;
+ kev_msg.kev_subclass = event->kev_subclass;
+ kev_msg.event_code = event->event_code;
kev_msg.dv[0].data_ptr = &event->event_data[0];
kev_msg.dv[0].data_length = event->total_size - KEV_MSG_HEADER_SIZE;
kev_msg.dv[1].data_length = 0;
- result = dlil_event_internal(ifp, &kev_msg);
+ result = dlil_event_internal(ifp, &kev_msg, TRUE);
return (result);
}
}
#endif
-/*
- * This is mostly called from the context of the DLIL input thread;
- * because of that there is no need for atomic operations.
- */
-static __inline void
-ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m)
-{
- if (!(m->m_flags & M_PKTHDR))
- return;
-
- switch (m_get_traffic_class(m)) {
- case MBUF_TC_BE:
- ifp->if_tc.ifi_ibepackets++;
- ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_BK:
- ifp->if_tc.ifi_ibkpackets++;
- ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_VI:
- ifp->if_tc.ifi_ivipackets++;
- ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_VO:
- ifp->if_tc.ifi_ivopackets++;
- ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len;
- break;
- default:
- break;
- }
-
- if (mbuf_is_traffic_class_privileged(m)) {
- ifp->if_tc.ifi_ipvpackets++;
- ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len;
- }
-}
-
-/*
- * This is called from DLIL output, hence multiple threads could end
- * up modifying the statistics. We trade off acccuracy for performance
- * by not using atomic operations here.
- */
-static __inline void
-ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m)
-{
- if (!(m->m_flags & M_PKTHDR))
- return;
-
- switch (m_get_traffic_class(m)) {
- case MBUF_TC_BE:
- ifp->if_tc.ifi_obepackets++;
- ifp->if_tc.ifi_obebytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_BK:
- ifp->if_tc.ifi_obkpackets++;
- ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_VI:
- ifp->if_tc.ifi_ovipackets++;
- ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len;
- break;
- case MBUF_TC_VO:
- ifp->if_tc.ifi_ovopackets++;
- ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len;
- break;
- default:
- break;
- }
-
- if (mbuf_is_traffic_class_privileged(m)) {
- ifp->if_tc.ifi_opvpackets++;
- ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len;
- }
-}
-
static void
dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls)
{
u_int32_t pre = 0, post = 0;
u_int32_t fpkts = 0, fbytes = 0;
int32_t flen = 0;
+ struct timespec now;
+ u_int64_t now_nsec;
KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
- /* Get an io refcnt if the interface is attached to prevent ifnet_detach
- * from happening while this operation is in progress */
+ /*
+ * Get an io refcnt if the interface is attached to prevent ifnet_detach
+ * from happening while this operation is in progress
+ */
if (!ifnet_is_attached(ifp, 1)) {
retval = ENXIO;
goto cleanup;
}
iorefcnt = 1;
+ VERIFY(ifp->if_output_dlil != NULL);
+
/* update the driver's multicast filter, if needed */
if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0)
ifp->if_updatemcasts = 0;
do {
#if CONFIG_DTRACE
if (!raw && proto_family == PF_INET) {
- struct ip *ip = mtod(m, struct ip*);
- DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
+ struct ip *ip = mtod(m, struct ip *);
+ DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
struct ip *, ip, struct ifnet *, ifp,
struct ip *, ip, struct ip6_hdr *, NULL);
} else if (!raw && proto_family == PF_INET6) {
- struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr*);
- DTRACE_IP6(send, struct mbuf*, m, struct inpcb *, NULL,
- struct ip6_hdr *, ip6, struct ifnet*, ifp,
- struct ip*, NULL, struct ip6_hdr *, ip6);
+ struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
+ DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
+ struct ip6_hdr *, ip6, struct ifnet *, ifp,
+ struct ip *, NULL, struct ip6_hdr *, ip6);
}
#endif /* CONFIG_DTRACE */
goto cleanup;
}
- /*
- * If the packet service class is not background,
- * update the timestamp to indicate recent activity
- * on a foreground socket.
- */
- if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) &&
- m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
- if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND))
- ifp->if_fg_sendts = net_uptime();
-
- if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME)
- ifp->if_rt_sendts = net_uptime();
- }
-
ifp_inc_traffic_class_out(ifp, m);
pktap_output(ifp, proto_family, m, pre, post);
dlil_count_chain_len(m, &tx_chain_len_stats);
}
+ /*
+ * Record timestamp; ifnet_enqueue() will use this info
+ * rather than redoing the work. An optimization could
+ * involve doing this just once at the top, if there are
+ * no interface filters attached, but that's probably
+ * not a big deal.
+ */
+ nanouptime(&now);
+ net_timernsec(&now, &now_nsec);
+ (void) mbuf_set_timestamp(m, now_nsec, TRUE);
+
+ /*
+ * Discard partial sum information if this packet originated
+ * from another interface; the packet would already have the
+ * final checksum and we shouldn't recompute it.
+ */
+ if ((m->m_pkthdr.pkt_flags & PKTF_FORWARDED) &&
+ (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) ==
+ (CSUM_DATA_VALID|CSUM_PARTIAL)) {
+ m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS;
+ m->m_pkthdr.csum_data = 0;
+ }
+
/*
* Finally, call the driver.
*/
}
KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
0, 0, 0, 0, 0);
- retval = (*ifp->if_output)(ifp, m);
+ retval = (*ifp->if_output_dlil)(ifp, m);
if (retval == EQFULL || retval == EQSUSPENDED) {
if (adv != NULL && adv->code == FADV_SUCCESS) {
adv->code = (retval == EQFULL ?
KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
0, 0, 0, 0, 0);
if (ifp->if_eflags & IFEF_SENDLIST) {
- retval = (*ifp->if_output)(ifp, send_head);
+ retval = (*ifp->if_output_dlil)(ifp, send_head);
if (retval == EQFULL || retval == EQSUSPENDED) {
if (adv != NULL) {
adv->code = (retval == EQFULL ?
send_m = send_head;
send_head = send_m->m_nextpkt;
send_m->m_nextpkt = NULL;
- retval = (*ifp->if_output)(ifp, send_m);
+ retval = (*ifp->if_output_dlil)(ifp, send_m);
if (retval == EQFULL || retval == EQSUSPENDED) {
if (adv != NULL) {
adv->code = (retval == EQFULL ?
fpkts++;
}
if (retval != 0 && dlil_verbose) {
- printf("%s: output error on %s retval = %d\n",
+ printf("%s: output error on %s "
+ "retval = %d\n",
__func__, if_name(ifp), retval);
}
}
if (!ifnet_is_attached(ifp, 1))
return (EOPNOTSUPP);
- /* Run the interface filters first.
+ /*
+ * Run the interface filters first.
* We want to run all filters before calling the protocol,
* interface family, or interface.
*/
if (ifp->if_set_bpf_tap) {
/* Get an io reference on the interface if it is attached */
if (!ifnet_is_attached(ifp, 1))
- return ENXIO;
+ return (ENXIO);
error = ifp->if_set_bpf_tap(ifp, mode, callback);
ifnet_decr_iorefcnt(ifp);
}
proto_media_resolve_multi resolvep;
if (!ifnet_is_attached(ifp, 1))
- return result;
+ return (result);
bzero(ll_addr, ll_len);
proto->kpi.v1.resolve_multi : proto->kpi.v2.resolve_multi);
if (resolvep != NULL)
result = resolvep(ifp, proto_addr,
- (struct sockaddr_dl*)(void *)ll_addr, ll_len);
+ (struct sockaddr_dl *)(void *)ll_addr, ll_len);
if_proto_free(proto);
}
__private_extern__ errno_t
dlil_send_arp_internal(ifnet_t ifp, u_short arpop,
- const struct sockaddr_dl* sender_hw, const struct sockaddr* sender_proto,
- const struct sockaddr_dl* target_hw, const struct sockaddr* target_proto)
+ const struct sockaddr_dl *sender_hw, const struct sockaddr *sender_proto,
+ const struct sockaddr_dl *target_hw, const struct sockaddr *target_proto)
{
struct if_proto *proto;
errno_t result = 0;
static const struct net_thread_marks net_thread_marks_base = { };
__private_extern__ const net_thread_marks_t net_thread_marks_none =
- &net_thread_marks_base;
+ &net_thread_marks_base;
__private_extern__ net_thread_marks_t
net_thread_marks_push(u_int32_t push)
}
__private_extern__ errno_t
-dlil_send_arp(ifnet_t ifp, u_short arpop, const struct sockaddr_dl* sender_hw,
- const struct sockaddr* sender_proto, const struct sockaddr_dl* target_hw,
- const struct sockaddr* target_proto0, u_int32_t rtflags)
+dlil_send_arp(ifnet_t ifp, u_short arpop, const struct sockaddr_dl *sender_hw,
+ const struct sockaddr *sender_proto, const struct sockaddr_dl *target_hw,
+ const struct sockaddr *target_proto0, u_int32_t rtflags)
{
errno_t result = 0;
const struct sockaddr_in * sender_sin;
{
struct ifnet *_ifp;
- lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_HELD);
+ LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_HELD);
TAILQ_FOREACH(_ifp, &ifnet_head, if_link) {
if (_ifp == ifp)
break;
}
return (_ifp != NULL);
}
+
/*
* Caller has to pass a non-zero refio argument to get a
* IO reference count. This will prevent ifnet_detach from
- * being called when there are outstanding io reference counts.
+ * being called when there are outstanding io reference counts.
*/
int
ifnet_is_attached(struct ifnet *ifp, int refio)
int ret;
lck_mtx_lock_spin(&ifp->if_ref_lock);
- if ((ret = ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) ==
- IFRF_ATTACHED))) {
+ if ((ret = IF_FULLY_ATTACHED(ifp))) {
if (refio > 0)
ifp->if_refio++;
}
return (ret);
}
+/*
+ * Caller must ensure the interface is attached; the assumption is that
+ * there is at least an outstanding IO reference count held already.
+ * Most callers would call ifnet_is_attached() instead.
+ */
+void
+ifnet_incr_iorefcnt(struct ifnet *ifp)
+{
+ lck_mtx_lock_spin(&ifp->if_ref_lock);
+ VERIFY(IF_FULLY_ATTACHED(ifp));
+ VERIFY(ifp->if_refio > 0);
+ ifp->if_refio++;
+ lck_mtx_unlock(&ifp->if_ref_lock);
+}
+
void
ifnet_decr_iorefcnt(struct ifnet *ifp)
{
lck_mtx_lock_spin(&ifp->if_ref_lock);
VERIFY(ifp->if_refio > 0);
- VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != 0);
+ VERIFY(ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING));
ifp->if_refio--;
- /* if there are no more outstanding io references, wakeup the
+ /*
+ * if there are no more outstanding io references, wakeup the
* ifnet_detach thread if detaching flag is set.
*/
- if (ifp->if_refio == 0 &&
- (ifp->if_refflags & IFRF_DETACHING) != 0) {
+ if (ifp->if_refio == 0 && (ifp->if_refflags & IFRF_DETACHING))
wakeup(&(ifp->if_refio));
- }
+
lck_mtx_unlock(&ifp->if_ref_lock);
}
dlil_if_free(struct ifnet *ifp)
{
struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
+ bool need_release = FALSE;
if (dl_if == NULL)
return (EINVAL);
lck_mtx_lock_spin(&dl_if->dl_if_lock);
- if (dl_if->dl_if_refcnt == 0) {
+ switch (dl_if->dl_if_refcnt) {
+ case 0:
panic("%s: negative refcnt for ifp=%p", __func__, ifp);
/* NOTREACHED */
+ break;
+ case 1:
+ if ((ifp->if_refflags & IFRF_EMBRYONIC) != 0) {
+ need_release = TRUE;
+ }
+ break;
+ default:
+ break;
}
--dl_if->dl_if_refcnt;
if (dl_if->dl_if_trace != NULL)
(*dl_if->dl_if_trace)(dl_if, FALSE);
lck_mtx_unlock(&dl_if->dl_if_lock);
-
+ if (need_release) {
+ dlil_if_release(ifp);
+ }
return (0);
}
static errno_t
dlil_attach_protocol_internal(struct if_proto *proto,
- const struct ifnet_demux_desc *demux_list, u_int32_t demux_count)
+ const struct ifnet_demux_desc *demux_list, u_int32_t demux_count,
+ uint32_t * proto_count)
{
struct kev_dl_proto_data ev_pr_data;
struct ifnet *ifp = proto->ifp;
* (subject to change)
*/
ev_pr_data.proto_family = proto->protocol_family;
- ev_pr_data.proto_remaining_count = dlil_ifp_proto_count(ifp);
+ ev_pr_data.proto_remaining_count = dlil_ifp_protolist(ifp, NULL, 0);
+
ifnet_lock_done(ifp);
dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED,
(struct net_event_data *)&ev_pr_data,
sizeof (struct kev_dl_proto_data));
+ if (proto_count != NULL) {
+ *proto_count = ev_pr_data.proto_remaining_count;
+ }
return (retval);
}
{
int retval = 0;
struct if_proto *ifproto = NULL;
+ uint32_t proto_count = 0;
ifnet_head_lock_shared();
if (ifp == NULL || protocol == 0 || proto_details == NULL) {
ifproto->kpi.v1.send_arp = proto_details->send_arp;
retval = dlil_attach_protocol_internal(ifproto,
- proto_details->demux_list, proto_details->demux_count);
-
- if (dlil_verbose) {
- printf("%s: attached v1 protocol %d\n", if_name(ifp),
- protocol);
- }
+ proto_details->demux_list, proto_details->demux_count,
+ &proto_count);
end:
if (retval != 0 && retval != EEXIST && ifp != NULL) {
DLIL_PRINTF("%s: failed to attach v1 protocol %d (err=%d)\n",
if_name(ifp), protocol, retval);
+ } else {
+ if (dlil_verbose) {
+ printf("%s: attached v1 protocol %d (count = %d)\n",
+ if_name(ifp),
+ protocol, proto_count);
+ }
}
ifnet_head_done();
- if (retval != 0 && ifproto != NULL)
+ if (retval == 0) {
+ /*
+ * A protocol has been attached, mark the interface up.
+ * This used to be done by configd.KernelEventMonitor, but that
+ * is inherently prone to races (rdar://problem/30810208).
+ */
+ (void) ifnet_set_flags(ifp, IFF_UP, IFF_UP);
+ (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
+ dlil_post_sifflags_msg(ifp);
+ } else if (ifproto != NULL) {
zfree(dlif_proto_zone, ifproto);
+ }
return (retval);
}
{
int retval = 0;
struct if_proto *ifproto = NULL;
+ uint32_t proto_count = 0;
ifnet_head_lock_shared();
if (ifp == NULL || protocol == 0 || proto_details == NULL) {
ifproto->kpi.v2.send_arp = proto_details->send_arp;
retval = dlil_attach_protocol_internal(ifproto,
- proto_details->demux_list, proto_details->demux_count);
-
- if (dlil_verbose) {
- printf("%s: attached v2 protocol %d\n", if_name(ifp),
- protocol);
- }
+ proto_details->demux_list, proto_details->demux_count,
+ &proto_count);
end:
if (retval != 0 && retval != EEXIST && ifp != NULL) {
DLIL_PRINTF("%s: failed to attach v2 protocol %d (err=%d)\n",
if_name(ifp), protocol, retval);
+ } else {
+ if (dlil_verbose) {
+ printf("%s: attached v2 protocol %d (count = %d)\n",
+ if_name(ifp),
+ protocol, proto_count);
+ }
}
ifnet_head_done();
- if (retval != 0 && ifproto != NULL)
+ if (retval == 0) {
+ /*
+ * A protocol has been attached, mark the interface up.
+ * This used to be done by configd.KernelEventMonitor, but that
+ * is inherently prone to races (rdar://problem/30810208).
+ */
+ (void) ifnet_set_flags(ifp, IFF_UP, IFF_UP);
+ (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
+ dlil_post_sifflags_msg(ifp);
+ } else if (ifproto != NULL) {
zfree(dlif_proto_zone, ifproto);
+ }
return (retval);
}
if (proto->proto_kpi == kProtoKPI_v1) {
proto->kpi.v1.input = ifproto_media_input_v1;
- proto->kpi.v1.pre_output= ifproto_media_preout;
+ proto->kpi.v1.pre_output = ifproto_media_preout;
proto->kpi.v1.event = ifproto_media_event;
proto->kpi.v1.ioctl = ifproto_media_ioctl;
proto->kpi.v1.resolve_multi = ifproto_media_resolve_multi;
}
lck_mtx_lock_spin(&ifp->if_ref_lock);
- if (ifp->if_refflags & IFRF_ATTACHED) {
- panic_plain("%s: flags mismatch (attached set) ifp=%p",
+ if (!(ifp->if_refflags & IFRF_EMBRYONIC)) {
+ panic_plain("%s: flags mismatch (embryonic not set) ifp=%p",
__func__, ifp);
/* NOTREACHED */
}
VERIFY(ifp->if_flt_waiters == 0);
lck_mtx_unlock(&ifp->if_flt_lock);
- VERIFY(TAILQ_EMPTY(&ifp->if_prefixhead));
- TAILQ_INIT(&ifp->if_prefixhead);
-
if (!(dl_if->dl_if_flags & DLIF_REUSE)) {
VERIFY(LIST_EMPTY(&ifp->if_multiaddrs));
LIST_INIT(&ifp->if_multiaddrs);
VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
/* allocate (if needed) and initialize a link address */
- VERIFY(!(dl_if->dl_if_flags & DLIF_REUSE) || ifp->if_lladdr != NULL);
ifa = dlil_alloc_lladdr(ifp, ll_addr);
if (ifa == NULL) {
ifnet_lock_done(ifp);
ifnet_touch_lastchange(ifp);
VERIFY(ifp->if_output_sched_model == IFNET_SCHED_MODEL_NORMAL ||
- ifp->if_output_sched_model == IFNET_SCHED_MODEL_DRIVER_MANAGED);
+ ifp->if_output_sched_model == IFNET_SCHED_MODEL_DRIVER_MANAGED ||
+ ifp->if_output_sched_model == IFNET_SCHED_MODEL_FQ_CODEL);
/* By default, use SFB and enable flow advisory */
sflags = PKTSCHEDF_QALG_SFB;
if (if_delaybased_queue)
sflags |= PKTSCHEDF_QALG_DELAYBASED;
+ if (ifp->if_output_sched_model ==
+ IFNET_SCHED_MODEL_DRIVER_MANAGED)
+ sflags |= PKTSCHEDF_QALG_DRIVER_MANAGED;
+
/* Initialize transmit queue(s) */
err = ifclassq_setup(ifp, sflags, (dl_if->dl_if_flags & DLIF_REUSE));
if (err != 0) {
}
}
+ if (ifp->if_inp != NULL && ifp->if_inp->input_mit_tcall == NULL) {
+ ifp->if_inp->input_mit_tcall =
+ thread_call_allocate_with_priority(dlil_mit_tcall_fn,
+ ifp, THREAD_CALL_PRIORITY_KERNEL);
+ }
+
/*
* If the driver supports the new transmit model, calculate flow hash
* and create a workloop starter thread to invoke the if_start callback
if (ifp->if_eflags & IFEF_TXSTART) {
ifp->if_flowhash = ifnet_calc_flowhash(ifp);
VERIFY(ifp->if_flowhash != 0);
-
- VERIFY(ifp->if_start != NULL);
VERIFY(ifp->if_start_thread == THREAD_NULL);
ifnet_set_start_cycle(ifp, NULL);
ifp->if_start_active = 0;
ifp->if_start_req = 0;
ifp->if_start_flags = 0;
- if ((err = kernel_thread_start(ifnet_start_thread_fn, ifp,
- &ifp->if_start_thread)) != KERN_SUCCESS) {
- panic_plain("%s: ifp=%p couldn't get a start thread; "
+ VERIFY(ifp->if_start != NULL);
+ if ((err = kernel_thread_start(ifnet_start_thread_fn,
+ ifp, &ifp->if_start_thread)) != KERN_SUCCESS) {
+ panic_plain("%s: "
+ "ifp=%p couldn't get a start thread; "
"err=%d", __func__, ifp, err);
- /* NOTREACHED */
+ /* NOTREACHED */
}
ml_thread_policy(ifp->if_start_thread, MACHINE_GROUP,
(MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP));
/* Clear logging parameters */
bzero(&ifp->if_log, sizeof (ifp->if_log));
+
+ /* Clear foreground/realtime activity timestamps */
ifp->if_fg_sendts = 0;
+ ifp->if_rt_sendts = 0;
VERIFY(ifp->if_delegated.ifp == NULL);
VERIFY(ifp->if_delegated.type == 0);
VERIFY(ifp->if_delegated.subfamily == 0);
VERIFY(ifp->if_delegated.expensive == 0);
- bzero(&ifp->if_agentids, sizeof(ifp->if_agentids));
+ VERIFY(ifp->if_agentids == NULL);
+ VERIFY(ifp->if_agentcount == 0);
/* Reset interface state */
bzero(&ifp->if_interface_state, sizeof(ifp->if_interface_state));
- ifp->if_interface_state.valid_bitmask |=
+ ifp->if_interface_state.valid_bitmask |=
IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID;
ifp->if_interface_state.interface_availability =
IF_INTERFACE_STATE_INTERFACE_AVAILABLE;
ifp->if_eflags &= ~IFEF_ECN_DISABLE;
}
+ /*
+ * Built-in Cyclops always on policy for WiFi infra
+ */
+ if (IFNET_IS_WIFI_INFRA(ifp) && net_qos_policy_wifi_enabled != 0) {
+ errno_t error;
+
+ error = if_set_qosmarking_mode(ifp,
+ IFRTYPE_QOSMARKING_FASTLANE);
+ if (error != 0) {
+ printf("%s if_set_qosmarking_mode(%s) error %d\n",
+ __func__, ifp->if_xname, error);
+ } else {
+ ifp->if_eflags |= IFEF_QOSMARKING_ENABLED;
+#if (DEVELOPMENT || DEBUG)
+ printf("%s fastlane enabled on %s\n",
+ __func__, ifp->if_xname);
+#endif /* (DEVELOPMENT || DEBUG) */
+ }
+ }
+
ifnet_lock_done(ifp);
ifnet_head_done();
+
lck_mtx_lock(&ifp->if_cached_route_lock);
/* Enable forwarding cached route */
ifp->if_fwd_cacheok = 1;
#endif /* INET6 */
VERIFY(ifp->if_data_threshold == 0);
+ VERIFY(ifp->if_dt_tcall != NULL);
/*
* Finally, mark this ifnet as attached.
lck_mtx_lock(rnh_lock);
ifnet_lock_exclusive(ifp);
lck_mtx_lock_spin(&ifp->if_ref_lock);
- ifp->if_refflags = IFRF_ATTACHED;
+ ifp->if_refflags = IFRF_ATTACHED; /* clears embryonic */
lck_mtx_unlock(&ifp->if_ref_lock);
if (net_rtref) {
/* boot-args override; enable idle notification */
namelen = snprintf(workbuf, sizeof (workbuf), "%s",
if_name(ifp));
- masklen = offsetof(struct sockaddr_dl, sdl_data[0]) + namelen;
+ masklen = offsetof(struct sockaddr_dl, sdl_data[0])
+ + ((namelen > 0) ? namelen : 0);
socksize = masklen + ifp->if_addrlen;
-#define ROUNDUP(a) (1 + (((a) - 1) | (sizeof (u_int32_t) - 1)))
+#define ROUNDUP(a) (1 + (((a) - 1) | (sizeof (u_int32_t) - 1)))
if ((u_int32_t)socksize < sizeof (struct sockaddr_dl))
socksize = sizeof(struct sockaddr_dl);
socksize = ROUNDUP(socksize);
ifa->ifa_addr = (struct sockaddr *)asdl;
asdl->sdl_len = socksize;
asdl->sdl_family = AF_LINK;
- bcopy(workbuf, asdl->sdl_data, namelen);
- asdl->sdl_nlen = namelen;
+ if (namelen > 0) {
+ bcopy(workbuf, asdl->sdl_data, min(namelen,
+ sizeof (asdl->sdl_data)));
+ asdl->sdl_nlen = namelen;
+ } else {
+ asdl->sdl_nlen = 0;
+ }
asdl->sdl_index = ifp->if_index;
asdl->sdl_type = ifp->if_type;
if (ll_addr != NULL) {
} else {
asdl->sdl_alen = 0;
}
- ifa->ifa_netmask = (struct sockaddr*)msdl;
+ ifa->ifa_netmask = (struct sockaddr *)msdl;
msdl->sdl_len = masklen;
- while (namelen != 0)
+ while (namelen > 0)
msdl->sdl_data[--namelen] = 0xff;
IFA_UNLOCK(ifa);
ifnet_detach(ifnet_t ifp)
{
struct ifnet *delegated_ifp;
+ struct nd_ifinfo *ndi = NULL;
if (ifp == NULL)
return (EINVAL);
+ ndi = ND_IFINFO(ifp);
+ if (NULL != ndi)
+ ndi->cga_initialized = FALSE;
+
lck_mtx_lock(rnh_lock);
ifnet_head_lock_exclusive();
ifnet_lock_exclusive(ifp);
(void) ifnet_set_idle_flags_locked(ifp, 0, ~0);
lck_mtx_lock_spin(&ifp->if_ref_lock);
- if (!(ifp->if_refflags & IFRF_ATTACHED)) {
+ if (!(ifp->if_refflags & IFRF_ATTACHED)) {
lck_mtx_unlock(&ifp->if_ref_lock);
ifnet_lock_done(ifp);
ifnet_head_done();
lck_mtx_unlock(rnh_lock);
return (ENXIO);
}
+ VERIFY(!(ifp->if_refflags & IFRF_EMBRYONIC));
/* Indicate this interface is being detached */
ifp->if_refflags &= ~IFRF_ATTACHED;
ifp->if_refflags |= IFRF_DETACHING;
lck_mtx_unlock(&ifp->if_ref_lock);
- if (dlil_verbose)
+ if (dlil_verbose) {
printf("%s: detaching\n", if_name(ifp));
+ }
+
+ /* clean up flow control entry object if there's any */
+ if (ifp->if_eflags & IFEF_TXSTART) {
+ ifnet_flowadv(ifp->if_flowhash);
+ }
/* Reset ECN enable/disable flags */
ifp->if_eflags &= ~IFEF_ECN_DISABLE;
TAILQ_REMOVE(&ifnet_head, ifp, if_link);
ifp->if_link.tqe_next = NULL;
ifp->if_link.tqe_prev = NULL;
+ if (ifp->if_ordered_link.tqe_next != NULL ||
+ ifp->if_ordered_link.tqe_prev != NULL) {
+ ifnet_remove_from_ordered_list(ifp);
+ }
ifindex2ifnet[ifp->if_index] = NULL;
/* 18717626 - reset IFEF_IPV4_ROUTER and IFEF_IPV6_ROUTER */
ifnet_head_done();
lck_mtx_unlock(rnh_lock);
+
/* Release reference held on the delegated interface */
if (delegated_ifp != NULL)
ifnet_release(delegated_ifp);
ifp->if_link_status = NULL;
}
+ /* Clear agent IDs */
+ if (ifp->if_agentids != NULL) {
+ FREE(ifp->if_agentids, M_NETAGENT);
+ ifp->if_agentids = NULL;
+ }
+ ifp->if_agentcount = 0;
+
+
/* Let BPF know we're detaching */
bpfdetach(ifp);
ifp->if_fwd_cacheok = 0;
lck_mtx_unlock(&ifp->if_cached_route_lock);
+ /* Disable data threshold and wait for any pending event posting */
ifp->if_data_threshold = 0;
+ VERIFY(ifp->if_dt_tcall != NULL);
+ (void) thread_call_cancel_wait(ifp->if_dt_tcall);
+
/*
* Drain any deferred IGMPv3/MLDv2 query responses, but keep the
* references to the info structures and leave them attached to
dlil_if_lock();
}
}
- /* NOTREACHED */
- return (0);
}
static void
lck_mtx_lock(&ifp->if_flt_lock);
if_flt_monitor_enter(ifp);
- lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
fhead = ifp->if_flt_head;
TAILQ_INIT(&ifp->if_flt_head);
VERIFY(ifp->if_link.tqe_prev == NULL);
VERIFY(ifp->if_detaching_link.tqe_next == NULL);
VERIFY(ifp->if_detaching_link.tqe_prev == NULL);
-
- /* Prefix list should be empty by now */
- VERIFY(TAILQ_EMPTY(&ifp->if_prefixhead));
+ VERIFY(ifp->if_ordered_link.tqe_next == NULL);
+ VERIFY(ifp->if_ordered_link.tqe_prev == NULL);
/* The slot should have been emptied */
VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
/* disassociate ifp DLIL input thread */
ifp->if_inp = NULL;
+ /* tell the input thread to terminate */
lck_mtx_lock_spin(&inp->input_lck);
inp->input_waiting |= DLIL_INPUT_TERMINATE;
if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
wakeup_one((caddr_t)&inp->input_waiting);
}
lck_mtx_unlock(&inp->input_lck);
+ ifnet_lock_done(ifp);
+
+ /* wait for the input thread to terminate */
+ lck_mtx_lock_spin(&inp->input_lck);
+ while ((inp->input_waiting & DLIL_INPUT_TERMINATE_COMPLETE)
+ == 0) {
+ (void) msleep(&inp->input_waiting, &inp->input_lck,
+ (PZERO - 1) | PSPIN, inp->input_name, NULL);
+ }
+ lck_mtx_unlock(&inp->input_lck);
+ ifnet_lock_exclusive(ifp);
+
+ /* clean-up input thread state */
+ dlil_clean_threading_info(inp);
+
}
/* The driver might unload, so point these to ourselves */
if_free = ifp->if_free;
+ ifp->if_output_dlil = ifp_if_output;
ifp->if_output = ifp_if_output;
ifp->if_pre_enqueue = ifp_if_output;
ifp->if_start = ifp_if_start;
ifp->if_output_ctl = ifp_if_ctl;
+ ifp->if_input_dlil = ifp_if_input;
ifp->if_input_poll = ifp_if_input_poll;
ifp->if_input_ctl = ifp_if_ctl;
ifp->if_ioctl = ifp_if_ioctl;
VERIFY(ifp->if_delegated.subfamily == 0);
VERIFY(ifp->if_delegated.expensive == 0);
+ /* QoS marking get cleared */
+ ifp->if_eflags &= ~IFEF_QOSMARKING_ENABLED;
+ if_set_qosmarking_mode(ifp, IFRTYPE_QOSMARKING_MODE_NONE);
+
+
ifnet_lock_done(ifp);
#if PF
lck_mtx_unlock(&ifp->if_cached_route_lock);
VERIFY(ifp->if_data_threshold == 0);
+ VERIFY(ifp->if_dt_tcall != NULL);
+ VERIFY(!thread_call_isactive(ifp->if_dt_tcall));
ifnet_llreach_ifdetach(ifp);
dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHED, NULL, 0);
- if (if_free != NULL)
- if_free(ifp);
-
/*
* Finally, mark this ifnet as detached.
*/
}
ifp->if_refflags &= ~IFRF_DETACHING;
lck_mtx_unlock(&ifp->if_ref_lock);
+ if (if_free != NULL)
+ if_free(ifp);
if (dlil_verbose)
printf("%s: detached\n", if_name(ifp));
ifnet_release(ifp);
}
-static errno_t
+errno_t
ifp_if_output(struct ifnet *ifp, struct mbuf *m)
{
#pragma unused(ifp)
- m_freem(m);
+ m_freem_list(m);
return (0);
}
-static void
+void
ifp_if_start(struct ifnet *ifp)
{
ifnet_purge(ifp);
}
+static errno_t
+ifp_if_input(struct ifnet *ifp, struct mbuf *m_head,
+ struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
+ boolean_t poll, struct thread *tp)
+{
+#pragma unused(ifp, m_tail, s, poll, tp)
+ m_freem_list(m_head);
+ return (ENXIO);
+}
+
static void
ifp_if_input_poll(struct ifnet *ifp, u_int32_t flags, u_int32_t max_cnt,
struct mbuf **m_head, struct mbuf **m_tail, u_int32_t *cnt, u_int32_t *len)
return (EOPNOTSUPP);
}
+#if CONFIG_EMBEDDED
+static errno_t
+ifp_if_framer(struct ifnet *ifp, struct mbuf **m,
+ const struct sockaddr *sa, const char *ll, const char *t,
+ u_int32_t *pre, u_int32_t *post)
+#else
static errno_t
ifp_if_framer(struct ifnet *ifp, struct mbuf **m,
const struct sockaddr *sa, const char *ll, const char *t)
+#endif /* !CONFIG_EMBEDDED */
{
#pragma unused(ifp, m, sa, ll, t)
+#if CONFIG_EMBEDDED
+ return (ifp_if_framer_extended(ifp, m, sa, ll, t, pre, post));
+#else
return (ifp_if_framer_extended(ifp, m, sa, ll, t, NULL, NULL));
+#endif /* !CONFIG_EMBEDDED */
}
static errno_t
#pragma unused(ifp, e)
}
-__private_extern__
int dlil_if_acquire(u_int32_t family, const void *uniqueid,
- size_t uniqueid_len, struct ifnet **ifp)
+ size_t uniqueid_len, const char *ifxname, struct ifnet **ifp)
{
struct ifnet *ifp1 = NULL;
struct dlil_ifnet *dlifp1 = NULL;
void *buf, *base, **pbuf;
int ret = 0;
+ VERIFY(*ifp == NULL);
dlil_if_lock();
+ /*
+ * We absolutely can't have an interface with the same name
+ * in in-use state.
+ * To make sure of that list has to be traversed completely
+ */
TAILQ_FOREACH(dlifp1, &dlil_ifnet_head, dl_if_link) {
ifp1 = (struct ifnet *)dlifp1;
if (ifp1->if_family != family)
continue;
+ /*
+ * If interface is in use, return EBUSY if either unique id
+ * or interface extended names are the same
+ */
lck_mtx_lock(&dlifp1->dl_if_lock);
- /* same uniqueid and same len or no unique id specified */
- if ((uniqueid_len == dlifp1->dl_if_uniqueid_len) &&
- !bcmp(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len)) {
- /* check for matching interface in use */
+ if (strncmp(ifxname, ifp1->if_xname, IFXNAMSIZ) == 0) {
if (dlifp1->dl_if_flags & DLIF_INUSE) {
- if (uniqueid_len) {
- ret = EBUSY;
+ lck_mtx_unlock(&dlifp1->dl_if_lock);
+ ret = EBUSY;
+ goto end;
+ }
+ }
+
+ if (uniqueid_len) {
+ if (uniqueid_len == dlifp1->dl_if_uniqueid_len &&
+ bcmp(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len) == 0) {
+ if (dlifp1->dl_if_flags & DLIF_INUSE) {
lck_mtx_unlock(&dlifp1->dl_if_lock);
+ ret = EBUSY;
goto end;
+ } else {
+ dlifp1->dl_if_flags |= (DLIF_INUSE|DLIF_REUSE);
+ /* Cache the first interface that can be recycled */
+ if (*ifp == NULL)
+ *ifp = ifp1;
+ /*
+ * XXX Do not break or jump to end as we have to traverse
+ * the whole list to ensure there are no name collisions
+ */
}
- } else {
- dlifp1->dl_if_flags |= (DLIF_INUSE|DLIF_REUSE);
- lck_mtx_unlock(&dlifp1->dl_if_lock);
- *ifp = ifp1;
- goto end;
}
}
lck_mtx_unlock(&dlifp1->dl_if_lock);
}
+ /* If there's an interface that can be recycled, use that */
+ if (*ifp != NULL)
+ goto end;
+
/* no interface found, allocate a new one */
buf = zalloc(dlif_zone);
if (buf == NULL) {
MALLOC(dlifp1->dl_if_uniqueid, void *, uniqueid_len,
M_NKE, M_WAITOK);
if (dlifp1->dl_if_uniqueid == NULL) {
- zfree(dlif_zone, dlifp1);
+ zfree(dlif_zone, buf);
ret = ENOMEM;
goto end;
}
ifp1->if_desc.ifd_len = 0;
ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage;
+
#if CONFIG_MACF_NET
mac_ifnet_label_init(ifp1);
#endif
lck_mtx_init(&ifp1->if_poll_lock, ifnet_rcv_lock_group,
ifnet_lock_attr);
+ /* thread call allocation is done with sleeping zalloc */
+ ifp1->if_dt_tcall = thread_call_allocate_with_options(dlil_dt_tcall_fn,
+ ifp1, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE);
+ if (ifp1->if_dt_tcall == NULL) {
+ panic_plain("%s: couldn't create if_dt_tcall", __func__);
+ /* NOTREACHED */
+ }
+
TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link);
*ifp = ifp1;
{
struct dlil_ifnet *dlifp = (struct dlil_ifnet *)ifp;
+ VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_count) > 0);
+ if (!(ifp->if_xflags & IFXF_ALLOC_KPI)) {
+ VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_os_count) > 0);
+ }
+
ifnet_lock_exclusive(ifp);
lck_mtx_lock(&dlifp->dl_if_lock);
dlifp->dl_if_flags &= ~DLIF_INUSE;
ifp->if_name = dlifp->dl_if_namestorage;
/* Reset external name (name + unit) */
ifp->if_xname = dlifp->dl_if_xnamestorage;
- snprintf(__DECONST(char *, ifp->if_xname), IFXNAMSIZ,
+ snprintf(__DECONST(char *, ifp->if_xname), IFXNAMSIZ,
"%s?", ifp->if_name);
lck_mtx_unlock(&dlifp->dl_if_lock);
#if CONFIG_MACF_NET
/*
- * We can either recycle the MAC label here or in dlil_if_acquire().
- * It seems logical to do it here but this means that anything that
- * still has a handle on ifp will now see it as unlabeled.
- * Since the interface is "dead" that may be OK. Revisit later.
- */
+ * We can either recycle the MAC label here or in dlil_if_acquire().
+ * It seems logical to do it here but this means that anything that
+ * still has a handle on ifp will now see it as unlabeled.
+ * Since the interface is "dead" that may be OK. Revisit later.
+ */
mac_ifnet_label_recycle(ifp);
#endif
ifnet_lock_done(ifp);
__private_extern__ void
dlil_if_lock_assert(void)
{
- lck_mtx_assert(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED);
+ LCK_MTX_ASSERT(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED);
}
__private_extern__ void
}
dst->sin_addr = src_ip;
- if (src_rt.ro_rt == NULL) {
- src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst,
- 0, 0, ifp->if_index);
+ VERIFY(src_rt.ro_rt == NULL);
+ src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst,
+ 0, 0, ifp->if_index);
- if (src_rt.ro_rt != NULL) {
- /* retain a ref, copyin consumes one */
- struct rtentry *rte = src_rt.ro_rt;
- RT_ADDREF(rte);
- ifp_src_route_copyin(ifp, &src_rt);
- src_rt.ro_rt = rte;
- }
+ if (src_rt.ro_rt != NULL) {
+ /* retain a ref, copyin consumes one */
+ struct rtentry *rte = src_rt.ro_rt;
+ RT_ADDREF(rte);
+ ifp_src_route_copyin(ifp, &src_rt);
+ src_rt.ro_rt = rte;
}
}
}
#if INET6
-struct rtentry*
+struct rtentry *
ifnet_cached_rtlookup_inet6(struct ifnet *ifp, struct in6_addr *src_ip6)
{
struct route_in6 src_rt;
VERIFY(lqm >= IFNET_LQM_MIN && lqm <= IFNET_LQM_MAX);
/* Normalize to edge */
- if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_BAD)
- lqm = IFNET_LQM_THRESH_BAD;
- else if (lqm > IFNET_LQM_THRESH_BAD && lqm <= IFNET_LQM_THRESH_POOR)
+ if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_ABORT) {
+ lqm = IFNET_LQM_THRESH_ABORT;
+ atomic_bitset_32(&tcbinfo.ipi_flags,
+ INPCBINFO_HANDLE_LQM_ABORT);
+ inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST);
+ } else if (lqm > IFNET_LQM_THRESH_ABORT &&
+ lqm <= IFNET_LQM_THRESH_MINIMALLY_VIABLE) {
+ lqm = IFNET_LQM_THRESH_MINIMALLY_VIABLE;
+ } else if (lqm > IFNET_LQM_THRESH_MINIMALLY_VIABLE &&
+ lqm <= IFNET_LQM_THRESH_POOR) {
lqm = IFNET_LQM_THRESH_POOR;
- else if (lqm > IFNET_LQM_THRESH_POOR && lqm <= IFNET_LQM_THRESH_GOOD)
+ } else if (lqm > IFNET_LQM_THRESH_POOR &&
+ lqm <= IFNET_LQM_THRESH_GOOD) {
lqm = IFNET_LQM_THRESH_GOOD;
+ }
/*
* Take the lock if needed
ifnet_lock_exclusive(ifp);
if (lqm == ifp->if_interface_state.lqm_state &&
- (ifp->if_interface_state.valid_bitmask &
+ (ifp->if_interface_state.valid_bitmask &
IF_INTERFACE_STATE_LQM_STATE_VALID)) {
/*
* Release the lock if was not held by the caller
if_rrc_state_update(struct ifnet *ifp, unsigned int rrc_state)
{
struct kev_dl_rrc_state kev;
-
+
if (rrc_state == ifp->if_interface_state.rrc_state &&
(ifp->if_interface_state.valid_bitmask &
IF_INTERFACE_STATE_RRC_STATE_VALID))
errno_t
if_state_update(struct ifnet *ifp,
- struct if_interface_state* if_interface_state)
+ struct if_interface_state *if_interface_state)
{
u_short if_index_available = 0;
void
if_get_state(struct ifnet *ifp,
- struct if_interface_state* if_interface_state)
+ struct if_interface_state *if_interface_state)
{
ifnet_lock_shared(ifp);
ifp->if_eflags |= IFEF_PROBE_CONNECTIVITY;
ifnet_lock_done(ifp);
+#if NECP
+ necp_update_all_clients();
+#endif /* NECP */
+
tcp_probe_connectivity(ifp, conn_probe);
return (0);
}
[0] = 2
};
- switch (sdl->sdl_type) {
- case IFT_ETHER:
- VERIFY(size == ETHER_ADDR_LEN);
- bytes = unspec;
- break;
- case IFT_IEEE1394:
- VERIFY(size == FIREWIRE_EUI64_LEN);
- bytes = unspec;
- break;
- default:
- VERIFY(FALSE);
- break;
- };
+ bytes = unspec;
}
}
#else
uint32_t flags = 0;
flags |= (cmd == SIOCSIFOPPORTUNISTIC) ?
INPCB_OPPORTUNISTIC_SETCMD : 0;
- flags |= (level == IFNET_THROTTLE_OPPORTUNISTIC) ?
+ flags |= (level == IFNET_THROTTLE_OPPORTUNISTIC) ?
INPCB_OPPORTUNISTIC_THROTTLEON : 0;
ifr->ifr_opportunistic.ifo_inuse =
udp_count_opportunistic(ifp->if_index, flags) +
switch (level) {
case IFNET_THROTTLE_OFF:
case IFNET_THROTTLE_OPPORTUNISTIC:
-#if PF_ALTQ
- /* Throttling works only for IFCQ, not ALTQ instances */
- if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq)))
- return (ENXIO);
-#endif /* PF_ALTQ */
break;
default:
return (EINVAL);
if (ifp == NULL || flowid == NULL) {
return (EINVAL);
} else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- !(ifp->if_refflags & IFRF_ATTACHED)) {
+ !IF_FULLY_ATTACHED(ifp)) {
return (ENXIO);
}
if (ifp == NULL) {
return (EINVAL);
} else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- !(ifp->if_refflags & IFRF_ATTACHED)) {
+ !IF_FULLY_ATTACHED(ifp)) {
return (ENXIO);
}
if (ifp == NULL) {
return (EINVAL);
} else if (!(ifp->if_eflags & IFEF_TXSTART) ||
- !(ifp->if_refflags & IFRF_ATTACHED)) {
+ !IF_FULLY_ATTACHED(ifp)) {
return (ENXIO);
}
- ifnet_start_common(ifp, 1);
+ ifnet_start_common(ifp, TRUE);
return (0);
}
/* become regular mutex */
lck_mtx_convert_spin(&ifnet_fc_lock);
- ifce = zalloc_noblock(ifnet_fc_zone);
+ ifce = zalloc(ifnet_fc_zone);
if (ifce == NULL) {
/* memory allocation failed */
lck_mtx_unlock(&ifnet_fc_lock);
{
int error = 0;
- if (ifp == NULL || len == NULL || flags == NULL || data == NULL)
+ if (ifp == NULL || len == NULL || data == NULL)
return (EINVAL);
switch (family) {
break;
}
- if (error == 0)
+ if (error == 0 && flags != NULL)
*flags = 0;
return (error);
}
+#if INET6
+int
+ifnet_set_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes)
+{
+ int i, error = 0, one_set = 0;
+
+ if_inet6data_lock_exclusive(ifp);
+
+ if (IN6_IFEXTRA(ifp) == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) {
+ uint32_t prefix_len =
+ prefixes[i].prefix_len;
+ struct in6_addr *prefix =
+ &prefixes[i].ipv6_prefix;
+
+ if (prefix_len == 0) {
+ /* Allow clearing the signature */
+ IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = 0;
+ bzero(&IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix,
+ sizeof(struct in6_addr));
+
+ continue;
+ } else if (prefix_len != NAT64_PREFIX_LEN_32 &&
+ prefix_len != NAT64_PREFIX_LEN_40 &&
+ prefix_len != NAT64_PREFIX_LEN_48 &&
+ prefix_len != NAT64_PREFIX_LEN_56 &&
+ prefix_len != NAT64_PREFIX_LEN_64 &&
+ prefix_len != NAT64_PREFIX_LEN_96) {
+ error = EINVAL;
+ goto out;
+ }
+
+ if (IN6_IS_SCOPE_EMBED(prefix)) {
+ error = EINVAL;
+ goto out;
+ }
+
+ IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = prefix_len;
+ bcopy(prefix, &IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix,
+ sizeof(struct in6_addr));
+ one_set = 1;
+ }
+
+out:
+ if_inet6data_lock_done(ifp);
+
+ if (error == 0 && one_set != 0)
+ necp_update_all_clients();
+
+ return (error);
+}
+
+int
+ifnet_get_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes)
+{
+ int i, found_one = 0, error = 0;
+
+ if (ifp == NULL)
+ return (EINVAL);
+
+ if_inet6data_lock_shared(ifp);
+
+ if (IN6_IFEXTRA(ifp) == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
+
+ for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) {
+ if (IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len != 0)
+ found_one = 1;
+ }
+
+ if (found_one == 0) {
+ error = ENOENT;
+ goto out;
+ }
+
+ if (prefixes)
+ bcopy(IN6_IFEXTRA(ifp)->nat64_prefixes, prefixes,
+ sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes));
+
+out:
+ if_inet6data_lock_done(ifp);
+
+ return (error);
+}
+#endif
+
static void
dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff,
protocol_family_t pf)
dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header,
protocol_family_t pf)
{
- uint16_t sum;
+ uint16_t sum = 0;
uint32_t hlen;
if (frame_header == NULL ||
}
return;
}
- rxoff -=hlen;
+ rxoff -= hlen;
if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED)) {
/*
if (aoff == rxoff || aoff > (uint32_t)m->m_pkthdr.len)
return;
- sum = m_adj_sum16(m, rxoff, aoff, sum);
+ sum = m_adj_sum16(m, rxoff, aoff,
+ m_pktlen(m) - aoff, sum);
m->m_pkthdr.csum_rx_val = sum;
m->m_pkthdr.csum_rx_start = (aoff + hlen);
{
#pragma unused(oidp, arg1, arg2)
int err;
-
+
if (req->oldptr == USER_ADDR_NULL) {
-
+
}
if (req->newptr != USER_ADDR_NULL) {
return (EPERM);
}
-#if DEBUG
+#if DEBUG || DEVELOPMENT
/* Blob for sum16 verification */
static uint8_t sumdata[] = {
0x1f, 0x8b, 0x08, 0x08, 0x4c, 0xe5, 0x9a, 0x4f, 0x00, 0x03,
/* Precomputed 16-bit 1's complement sums for various spans of the above data */
static struct {
- int len;
- uint16_t sum;
+ boolean_t init;
+ uint16_t len;
+ uint16_t sumr; /* reference */
+ uint16_t sumrp; /* reference, precomputed */
} sumtbl[] = {
- { 11, 0xcb6d },
- { 20, 0x20dd },
- { 27, 0xbabd },
- { 32, 0xf3e8 },
- { 37, 0x197d },
- { 43, 0x9eae },
- { 64, 0x4678 },
- { 127, 0x9399 },
- { 256, 0xd147 },
- { 325, 0x0358 }
+ { FALSE, 0, 0, 0x0000 },
+ { FALSE, 1, 0, 0x001f },
+ { FALSE, 2, 0, 0x8b1f },
+ { FALSE, 3, 0, 0x8b27 },
+ { FALSE, 7, 0, 0x790e },
+ { FALSE, 11, 0, 0xcb6d },
+ { FALSE, 20, 0, 0x20dd },
+ { FALSE, 27, 0, 0xbabd },
+ { FALSE, 32, 0, 0xf3e8 },
+ { FALSE, 37, 0, 0x197d },
+ { FALSE, 43, 0, 0x9eae },
+ { FALSE, 64, 0, 0x4678 },
+ { FALSE, 127, 0, 0x9399 },
+ { FALSE, 256, 0, 0xd147 },
+ { FALSE, 325, 0, 0x0358 },
};
#define SUMTBL_MAX ((int)sizeof (sumtbl) / (int)sizeof (sumtbl[0]))
/* Make sure test data plus extra room for alignment fits in cluster */
_CASSERT((sizeof (sumdata) + (sizeof (uint64_t) * 2)) <= MCLBYTES);
+ kprintf("DLIL: running SUM16 self-tests ... ");
+
m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
MH_ALIGN(m, sizeof (uint32_t)); /* 32-bit starting alignment */
buf = mtod(m, uint8_t *); /* base address */
/* Verify for all possible alignments */
for (i = 0; i < (int)sizeof (uint64_t); i++) {
- uint16_t sum;
+ uint16_t sum, sumr;
uint8_t *c;
/* Copy over test data to mbuf */
m->m_len = len;
sum = m_sum16(m, 0, len);
+ if (!sumtbl[n].init) {
+ sumr = in_cksum_mbuf_ref(m, len, 0, 0);
+ sumtbl[n].sumr = sumr;
+ sumtbl[n].init = TRUE;
+ } else {
+ sumr = sumtbl[n].sumr;
+ }
+
/* Something is horribly broken; stop now */
- if (sum != sumtbl[n].sum) {
- panic("%s: broken m_sum16 for len=%d align=%d "
- "sum=0x%04x [expected=0x%04x]\n", __func__,
- len, i, sum, sumtbl[n].sum);
+ if (sumr != sumtbl[n].sumrp) {
+ panic_plain("\n%s: broken in_cksum_mbuf_ref() "
+ "for len=%d align=%d sum=0x%04x "
+ "[expected=0x%04x]\n", __func__,
+ len, i, sum, sumr);
+ /* NOTREACHED */
+ } else if (sum != sumr) {
+ panic_plain("\n%s: broken m_sum16() for len=%d "
+ "align=%d sum=0x%04x [expected=0x%04x]\n",
+ __func__, len, i, sum, sumr);
/* NOTREACHED */
}
sum = m_sum16(m, i, len);
/* Something is horribly broken; stop now */
- if (sum != sumtbl[n].sum) {
- panic("%s: broken m_sum16 for len=%d offset=%d "
- "sum=0x%04x [expected=0x%04x]\n", __func__,
- len, i, sum, sumtbl[n].sum);
+ if (sum != sumr) {
+ panic_plain("\n%s: broken m_sum16() for len=%d "
+ "offset=%d sum=0x%04x [expected=0x%04x]\n",
+ __func__, len, i, sum, sumr);
/* NOTREACHED */
}
#if INET
sum = b_sum16(c, len);
/* Something is horribly broken; stop now */
- if (sum != sumtbl[n].sum) {
- panic("%s: broken b_sum16 for len=%d align=%d "
- "sum=0x%04x [expected=0x%04x]\n", __func__,
- len, i, sum, sumtbl[n].sum);
+ if (sum != sumr) {
+ panic_plain("\n%s: broken b_sum16() for len=%d "
+ "align=%d sum=0x%04x [expected=0x%04x]\n",
+ __func__, len, i, sum, sumr);
/* NOTREACHED */
}
#endif /* INET */
}
m_freem(m);
- printf("DLIL: SUM16 self-tests PASSED\n");
+ kprintf("PASSED\n");
}
-#endif /* DEBUG */
+#endif /* DEBUG || DEVELOPMENT */
#define CASE_STRINGIFY(x) case x: return #x
return ("");
}
+static void
+dlil_dt_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1)
+{
+#pragma unused(arg1)
+ struct ifnet *ifp = arg0;
+
+ if (ifnet_is_attached(ifp, 1)) {
+ nstat_ifnet_threshold_reached(ifp->if_index);
+ ifnet_decr_iorefcnt(ifp);
+ }
+}
+
+void
+ifnet_notify_data_threshold(struct ifnet *ifp)
+{
+ uint64_t bytes = (ifp->if_ibytes + ifp->if_obytes);
+ uint64_t oldbytes = ifp->if_dt_bytes;
+
+ ASSERT(ifp->if_dt_tcall != NULL);
+
+ /*
+ * If we went over the threshold, notify NetworkStatistics.
+ * We rate-limit it based on the threshold interval value.
+ */
+ if (threshold_notify && (bytes - oldbytes) > ifp->if_data_threshold &&
+ OSCompareAndSwap64(oldbytes, bytes, &ifp->if_dt_bytes) &&
+ !thread_call_isactive(ifp->if_dt_tcall)) {
+ uint64_t tival = (threshold_interval * NSEC_PER_SEC);
+ uint64_t now = mach_absolute_time(), deadline = now;
+ uint64_t ival;
+
+ if (tival != 0) {
+ nanoseconds_to_absolutetime(tival, &ival);
+ clock_deadline_for_periodic_event(ival, now, &deadline);
+ (void) thread_call_enter_delayed(ifp->if_dt_tcall,
+ deadline);
+ } else {
+ (void) thread_call_enter(ifp->if_dt_tcall);
+ }
+ }
+}
+
+#if (DEVELOPMENT || DEBUG)
/*
- * Mirror the arguments of ifnet_get_local_ports_extended()
- * ifindex
- * protocol
- * flags
+ * The sysctl variable name contains the input parameters of
+ * ifnet_get_keepalive_offload_frames()
+ * ifp (interface index): name[0]
+ * frames_array_count: name[1]
+ * frame_data_offset: name[2]
+ * The return length gives used_frames_count
*/
static int
-sysctl_get_ports_used SYSCTL_HANDLER_ARGS
+sysctl_get_kao_frames SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp)
int *name = (int *)arg1;
- int namelen = arg2;
- int error = 0;
+ u_int namelen = arg2;
int idx;
- protocol_family_t protocol;
- u_int32_t flags;
ifnet_t ifp = NULL;
- u_int8_t *bitfield = NULL;
+ u_int32_t frames_array_count;
+ size_t frame_data_offset;
+ u_int32_t used_frames_count;
+ struct ifnet_keepalive_offload_frame *frames_array = NULL;
+ int error = 0;
+ u_int32_t i;
- if (req->newptr) {
+ /*
+ * Only root can get look at other people TCP frames
+ */
+ error = proc_suser(current_proc());
+ if (error != 0)
+ goto done;
+ /*
+ * Validate the input parameters
+ */
+ if (req->newptr != USER_ADDR_NULL) {
error = EPERM;
goto done;
}
if (namelen != 3) {
- error = ENOENT;
+ error = EINVAL;
goto done;
}
-
if (req->oldptr == USER_ADDR_NULL) {
- req->oldidx = bitstr_size(65536);
+ error = EINVAL;
goto done;
}
- if (req->oldlen < bitstr_size(65536)) {
- error = ENOMEM;
+ if (req->oldlen == 0) {
+ error = EINVAL;
goto done;
}
-
idx = name[0];
- protocol = name[1];
- flags = name[2];
-
-
+ frames_array_count = name[1];
+ frame_data_offset = name[2];
+
+ /* Make sure the passed buffer is large enough */
+ if (frames_array_count * sizeof(struct ifnet_keepalive_offload_frame) >
+ req->oldlen) {
+ error = ENOMEM;
+ goto done;
+ }
+
ifnet_head_lock_shared();
- if (idx > if_index) {
+ if (!IF_INDEX_IN_RANGE(idx)) {
ifnet_head_done();
error = ENOENT;
goto done;
}
ifp = ifindex2ifnet[idx];
ifnet_head_done();
-
- bitfield = _MALLOC(bitstr_size(65536), M_TEMP, M_WAITOK);
- if (bitfield == NULL) {
+
+ frames_array = _MALLOC(frames_array_count *
+ sizeof(struct ifnet_keepalive_offload_frame), M_TEMP, M_WAITOK);
+ if (frames_array == NULL) {
error = ENOMEM;
goto done;
}
- error = ifnet_get_local_ports_extended(ifp, protocol, flags, bitfield);
+
+ error = ifnet_get_keepalive_offload_frames(ifp, frames_array,
+ frames_array_count, frame_data_offset, &used_frames_count);
if (error != 0) {
- printf("%s: ifnet_get_local_ports_extended() error %d\n",
+ printf("%s: ifnet_get_keepalive_offload_frames error %d\n",
__func__, error);
goto done;
}
- error = SYSCTL_OUT(req, bitfield, bitstr_size(65536));
+
+ for (i = 0; i < used_frames_count; i++) {
+ error = SYSCTL_OUT(req, frames_array + i,
+ sizeof(struct ifnet_keepalive_offload_frame));
+ if (error != 0) {
+ goto done;
+ }
+ }
done:
- if (bitfield != NULL)
- _FREE(bitfield, M_TEMP);
+ if (frames_array != NULL)
+ _FREE(frames_array, M_TEMP);
return (error);
}
+#endif /* DEVELOPMENT || DEBUG */
+
+void
+ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *ifs,
+ struct ifnet *ifp)
+{
+ tcp_update_stats_per_flow(ifs, ifp);
+}
+static void
+dlil_mit_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1)
+{
+#pragma unused(arg1)
+ struct ifnet *ifp = (struct ifnet *)arg0;
+ struct dlil_threading_info *inp = ifp->if_inp;
+
+ ifnet_lock_shared(ifp);
+ if (!IF_FULLY_ATTACHED(ifp) || inp == NULL) {
+ ifnet_lock_done(ifp);
+ return;
+ }
+
+ lck_mtx_lock_spin(&inp->input_lck);
+ inp->input_waiting |= DLIL_INPUT_WAITING;
+ if (!(inp->input_waiting & DLIL_INPUT_RUNNING) ||
+ !qempty(&inp->rcvq_pkts)) {
+ inp->wtot++;
+ wakeup_one((caddr_t)&inp->input_waiting);
+ }
+ lck_mtx_unlock(&inp->input_lck);
+ ifnet_lock_done(ifp);
+}
projects / apple / xnu.git / blobdiff