/*
- * Copyright (c) 1999-2017 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
#ifndef DLIL_H
-#define DLIL_H
+#define DLIL_H
#ifdef KERNEL
#include <sys/kernel_types.h>
* variants.native_type_length.
*/
/* Ethernet specific types */
-#define DLIL_DESC_ETYPE2 4
-#define DLIL_DESC_SAP 5
-#define DLIL_DESC_SNAP 6
+#define DLIL_DESC_ETYPE2 4
+#define DLIL_DESC_SAP 5
+#define DLIL_DESC_SNAP 6
#ifdef KERNEL_PRIVATE
#include <net/if.h>
#ifdef BSD_KERNEL_PRIVATE
/* Operations on timespecs. */
-#define net_timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_nsec = 0
+#define net_timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_nsec = 0
-#define net_timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_nsec)
+#define net_timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_nsec)
-#define net_timercmp(tvp, uvp, cmp) \
- (((tvp)->tv_sec == (uvp)->tv_sec) ? \
- ((tvp)->tv_nsec cmp (uvp)->tv_nsec) : \
+#define net_timercmp(tvp, uvp, cmp) \
+ (((tvp)->tv_sec == (uvp)->tv_sec) ? \
+ ((tvp)->tv_nsec cmp (uvp)->tv_nsec) : \
((tvp)->tv_sec cmp (uvp)->tv_sec))
-#define net_timeradd(tvp, uvp, vvp) do { \
- (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \
- (vvp)->tv_nsec = (tvp)->tv_nsec + (uvp)->tv_nsec; \
- if ((vvp)->tv_nsec >= (long)NSEC_PER_SEC) { \
- (vvp)->tv_sec++; \
- (vvp)->tv_nsec -= NSEC_PER_SEC; \
- } \
+#define net_timeradd(tvp, uvp, vvp) do { \
+ (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \
+ (vvp)->tv_nsec = (tvp)->tv_nsec + (uvp)->tv_nsec; \
+ if ((vvp)->tv_nsec >= (long)NSEC_PER_SEC) { \
+ (vvp)->tv_sec++; \
+ (vvp)->tv_nsec -= NSEC_PER_SEC; \
+ } \
} while (0)
-#define net_timersub(tvp, uvp, vvp) do { \
- (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
- (vvp)->tv_nsec = (tvp)->tv_nsec - (uvp)->tv_nsec; \
- if ((vvp)->tv_nsec < 0) { \
- (vvp)->tv_sec--; \
- (vvp)->tv_nsec += NSEC_PER_SEC; \
- } \
+#define net_timersub(tvp, uvp, vvp) do { \
+ (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
+ (vvp)->tv_nsec = (tvp)->tv_nsec - (uvp)->tv_nsec; \
+ if ((vvp)->tv_nsec < 0) { \
+ (vvp)->tv_sec--; \
+ (vvp)->tv_nsec += NSEC_PER_SEC; \
+ } \
} while (0)
-#define net_timernsec(tvp, nsp) do { \
- *(nsp) = (tvp)->tv_nsec; \
- if ((tvp)->tv_sec > 0) \
- *(nsp) += ((tvp)->tv_sec * NSEC_PER_SEC); \
+#define net_timerusec(tvp, nsp) do { \
+ *(nsp) = (tvp)->tv_nsec / NSEC_PER_USEC; \
+ if ((tvp)->tv_sec > 0) \
+ *(nsp) += ((tvp)->tv_sec * USEC_PER_SEC); \
+} while (0)
+
+#define net_timernsec(tvp, nsp) do { \
+ *(nsp) = (tvp)->tv_nsec; \
+ if ((tvp)->tv_sec > 0) \
+ *(nsp) += ((tvp)->tv_sec * NSEC_PER_SEC); \
} while (0)
#if defined(__x86_64__) || defined(__arm64__)
-#define net_nsectimer(nsp, tvp) do { \
- u_int64_t __nsp = *(nsp); \
- net_timerclear(tvp); \
- uint64_t __sec = __nsp / NSEC_PER_SEC; \
- (tvp)->tv_sec = (__darwin_time_t)__sec; \
- (tvp)->tv_nsec = (long)(__nsp - __sec * NSEC_PER_SEC); \
+#define net_nsectimer(nsp, tvp) do { \
+ u_int64_t __nsp = *(nsp); \
+ net_timerclear(tvp); \
+ uint64_t __sec = __nsp / NSEC_PER_SEC; \
+ (tvp)->tv_sec = (__darwin_time_t)__sec; \
+ (tvp)->tv_nsec = (long)(__nsp - __sec * NSEC_PER_SEC); \
} while (0)
#else /* 32 bit */
/*
* The approximation of seconds is correct or too low by 1 unit.
* So we fix it by computing the remainder.
*/
-#define net_nsectimer(nsp, tvp) do { \
- u_int64_t __nsp = *(nsp); \
- net_timerclear(tvp); \
- uint32_t __a = (uint32_t)(__nsp >> 29); \
- const uint32_t __inv = 0x89705F41; \
- uint32_t __sec = (uint32_t)(((uint64_t)__a * __inv) >> 32); \
- uint32_t __rem = (uint32_t)(__nsp - __sec * NSEC_PER_SEC); \
- __sec += ((__rem >= NSEC_PER_SEC) ? 1 : 0); \
- (tvp)->tv_sec = (__darwin_time_t)__sec; \
- (tvp)->tv_nsec = \
- (long)((__rem >= NSEC_PER_SEC) ? (__rem - NSEC_PER_SEC) : __rem); \
+#define net_nsectimer(nsp, tvp) do { \
+ u_int64_t __nsp = *(nsp); \
+ net_timerclear(tvp); \
+ uint32_t __a = (uint32_t)(__nsp >> 29); \
+ const uint32_t __inv = 0x89705F41; \
+ uint32_t __sec = (uint32_t)(((uint64_t)__a * __inv) >> 32); \
+ uint32_t __rem = (uint32_t)(__nsp - __sec * NSEC_PER_SEC); \
+ __sec += ((__rem >= NSEC_PER_SEC) ? 1 : 0); \
+ (tvp)->tv_sec = (__darwin_time_t)__sec; \
+ (tvp)->tv_nsec = \
+ (long)((__rem >= NSEC_PER_SEC) ? (__rem - NSEC_PER_SEC) : __rem); \
} while(0)
#endif /* 32 bit */
struct sockaddr_dl;
struct iff_filter;
-#define DLIL_THREADNAME_LEN 32
+#define DLIL_THREADNAME_LEN 32
/*
- * DLIL input thread info
+ * DLIL threading info
*/
struct dlil_threading_info {
- decl_lck_mtx_data(, input_lck);
- lck_grp_t *lck_grp; /* lock group (for lock stats) */
- u_int32_t input_waiting; /* DLIL condition of thread */
- u_int32_t wtot; /* # of wakeup requests */
- char input_name[DLIL_THREADNAME_LEN]; /* name storage */
- struct ifnet *ifp; /* pointer to interface */
- class_queue_t rcvq_pkts; /* queue of pkts */
- struct ifnet_stat_increment_param stats; /* incremental statistics */
+ decl_lck_mtx_data(, dlth_lock);
+ class_queue_t dlth_pkts; /* queue of pkts */
+ struct ifnet *dlth_ifp; /* pointer to interface */
+ struct ifnet_stat_increment_param dlth_stats; /* incremental stats */
+ uint32_t dlth_flags; /* thread flags (see below) */
+ uint32_t dlth_wtot; /* # of wakeup requests */
+
+ /* strategy (sync or async) */
+ errno_t (*dlth_strategy)(struct dlil_threading_info *,
+ struct ifnet *, struct mbuf *, struct mbuf *,
+ const struct ifnet_stat_increment_param *, boolean_t,
+ struct thread *);
+
/*
* Thread affinity (workloop and DLIL threads).
*/
- boolean_t net_affinity; /* affinity set is available */
- struct thread *input_thr; /* input thread */
- struct thread *wloop_thr; /* workloop thread */
- struct thread *poll_thr; /* poll thread */
- u_int32_t tag; /* affinity tag */
- /*
- * Opportunistic polling.
- */
- ifnet_model_t mode; /* current mode */
- struct pktcntr tstats; /* incremental polling statistics */
- struct if_rxpoll_stats pstats; /* polling statistics */
-#define rxpoll_offreq pstats.ifi_poll_off_req
-#define rxpoll_offerr pstats.ifi_poll_off_err
-#define rxpoll_onreq pstats.ifi_poll_on_req
-#define rxpoll_onerr pstats.ifi_poll_on_err
-#define rxpoll_wavg pstats.ifi_poll_wakeups_avg
-#define rxpoll_wlowat pstats.ifi_poll_wakeups_lowat
-#define rxpoll_whiwat pstats.ifi_poll_wakeups_hiwat
-#define rxpoll_pavg pstats.ifi_poll_packets_avg
-#define rxpoll_pmin pstats.ifi_poll_packets_min
-#define rxpoll_pmax pstats.ifi_poll_packets_max
-#define rxpoll_plowat pstats.ifi_poll_packets_lowat
-#define rxpoll_phiwat pstats.ifi_poll_packets_hiwat
-#define rxpoll_bavg pstats.ifi_poll_bytes_avg
-#define rxpoll_bmin pstats.ifi_poll_bytes_min
-#define rxpoll_bmax pstats.ifi_poll_bytes_max
-#define rxpoll_blowat pstats.ifi_poll_bytes_lowat
-#define rxpoll_bhiwat pstats.ifi_poll_bytes_hiwat
-#define rxpoll_plim pstats.ifi_poll_packets_limit
-#define rxpoll_ival pstats.ifi_poll_interval_time
- struct pktcntr sstats; /* packets and bytes per sampling */
- struct timespec mode_holdtime; /* mode holdtime in nsec */
- struct timespec mode_lasttime; /* last mode change time in nsec */
- struct timespec sample_holdtime; /* sampling holdtime in nsec */
- struct timespec sample_lasttime; /* last sampling time in nsec */
- struct timespec dbg_lasttime; /* last debug message time in nsec */
+ boolean_t dlth_affinity; /* affinity set is available */
+ uint32_t dlth_affinity_tag; /* affinity tag */
+ struct thread *dlth_thread; /* DLIL worker thread */
+ struct thread *dlth_driver_thread; /* driver/workloop thread */
+ struct thread *dlth_poller_thread; /* poll thread */
+
+ lck_grp_t *dlth_lock_grp; /* lock group (for lock stats) */
+ char dlth_name[DLIL_THREADNAME_LEN]; /* name storage */
+
#if IFNET_INPUT_SANITY_CHK
/*
* For debugging.
*/
- u_int64_t input_mbuf_cnt; /* total # of packets processed */
+ uint64_t dlth_pkts_cnt; /* total # of packets */
#endif
- thread_call_t input_mit_tcall; /* coalescing input processing */
};
/*
* DLIL input thread info (for main/loopback input thread)
*/
struct dlil_main_threading_info {
- struct dlil_threading_info inp;
- class_queue_t lo_rcvq_pkts; /* queue of lo0 pkts */
+ struct dlil_threading_info inp;
+ class_queue_t lo_rcvq_pkts; /* queue of lo0 pkts */
};
/*
+ * Valid values for dlth_flags.
+ *
* The following are shared with kpi_protocol.c so that it may wakeup
* the input thread to run through packets queued for protocol input.
-*/
-#define DLIL_INPUT_RUNNING 0x80000000
-#define DLIL_INPUT_WAITING 0x40000000
-#define DLIL_PROTO_REGISTER 0x20000000
-#define DLIL_PROTO_WAITING 0x10000000
-#define DLIL_INPUT_TERMINATE 0x08000000
-#define DLIL_INPUT_TERMINATE_COMPLETE 0x04000000
+ */
+#define DLIL_INPUT_RUNNING 0x80000000
+#define DLIL_INPUT_WAITING 0x40000000
+#define DLIL_PROTO_REGISTER 0x20000000
+#define DLIL_PROTO_WAITING 0x10000000
+#define DLIL_INPUT_TERMINATE 0x08000000
+#define DLIL_INPUT_TERMINATE_COMPLETE 0x04000000
+#define DLIL_INPUT_EMBRYONIC 0x00000001
/*
* Flags for dlil_attach_filter()
*/
#define DLIL_IFF_TSO 0x01 /* Interface filter supports TSO */
-#define DLIL_IFF_INTERNAL 0x02 /* Apple internal -- do not count towards stats */
+#define DLIL_IFF_INTERNAL 0x02 /* Apple internal -- do not count towards stats */
+
+/* Input poll interval definitions */
+#define IF_RXPOLL_INTERVALTIME_MIN (1ULL * 1000) /* 1 us */
+#define IF_RXPOLL_INTERVALTIME (1ULL * 1000 * 1000) /* 1 ms */
extern int dlil_verbose;
extern uint32_t hwcksum_dbg;
extern uint32_t hwcksum_tx;
extern uint32_t hwcksum_rx;
extern struct dlil_threading_info *dlil_main_input_thread;
+extern unsigned int net_rxpoll;
+extern uint32_t if_rxpoll;
+extern uint32_t if_rxpoll_decay;
+extern uint32_t if_rxpoll_interval_pkts;
+extern uint32_t if_rcvq_maxlen;
extern void dlil_init(void);
* net_thread_is_unmarked functions to control the bits in the uu_network_marks
* field of the uthread structure.
*/
-#define NET_THREAD_HELD_PF 0x1 /* thread is holding PF lock */
-#define NET_THREAD_HELD_DOMAIN 0x2 /* thread is holding domain_proto_mtx */
-#define NET_THREAD_CKREQ_LLADDR 0x4 /* thread reqs MACF check for LLADDR */
+#define NET_THREAD_HELD_PF 0x1 /* thread is holding PF lock */
+#define NET_THREAD_HELD_DOMAIN 0x2 /* thread is holding domain_proto_mtx */
+#define NET_THREAD_CKREQ_LLADDR 0x4 /* thread reqs MACF check for LLADDR */
/*
* net_thread_marks_t is a pointer to a phantom structure type used for
extern int dlil_attach_filter(ifnet_t, const struct iff_filter *,
interface_filter_t *, u_int32_t);
extern void dlil_detach_filter(interface_filter_t);
+extern boolean_t dlil_has_ip_filter(void);
+extern boolean_t dlil_has_if_filter(struct ifnet *);
extern void dlil_proto_unplumb_all(ifnet_t);
-extern void dlil_post_msg(struct ifnet *, u_int32_t, u_int32_t,
+extern int dlil_post_msg(struct ifnet *, u_int32_t, u_int32_t,
struct net_event_data *, u_int32_t);
+extern void dlil_post_sifflags_msg(struct ifnet *);
+
extern int dlil_post_complete_msg(struct ifnet *, struct kev_msg *);
extern int dlil_alloc_local_stats(struct ifnet *);
+extern void ifnet_filter_update_tso(boolean_t filter_enable);
+extern errno_t dlil_rxpoll_validate_params(struct ifnet_poll_params *);
+extern void dlil_rxpoll_update_params(struct ifnet *,
+ struct ifnet_poll_params *);
+extern void ifnet_poll(struct ifnet *);
+extern errno_t ifnet_input_poll(struct ifnet *, struct mbuf *,
+ struct mbuf *, const struct ifnet_stat_increment_param *);
+
/*
* dlil_if_acquire is obsolete. Use ifnet_allocate.
*/
-extern int dlil_if_acquire(u_int32_t, const void *, size_t, struct ifnet **);
+extern int dlil_if_acquire(u_int32_t, const void *, size_t, const char *, struct ifnet **);
/*
* dlil_if_release is obsolete. The equivalent is called automatically when
* an interface is detached.
extern errno_t dlil_if_ref(struct ifnet *);
extern errno_t dlil_if_free(struct ifnet *);
-extern void dlil_node_present(struct ifnet *, struct sockaddr *, int32_t, int,
+extern int dlil_node_present(struct ifnet *, struct sockaddr *, int32_t, int,
int, u_int8_t[48]);
extern void dlil_node_absent(struct ifnet *, struct sockaddr *);
+extern int dlil_node_present_v2(struct ifnet *, struct sockaddr *, struct sockaddr_dl *, int32_t, int,
+ int, u_int8_t[48]);
extern const void *dlil_ifaddr_bytes(const struct sockaddr_dl *, size_t *,
kauth_cred_t *);
extern void dlil_report_issues(struct ifnet *, u_int8_t[DLIL_MODIDLEN],
u_int8_t[DLIL_MODARGLEN]);
-#define PROTO_HASH_SLOTS 4
+#define PROTO_HASH_SLOTS 5
extern int proto_hash_value(u_int32_t);
static inline void
ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m)
{
- if (!(m->m_flags & M_PKTHDR))
+ if (!(m->m_flags & M_PKTHDR)) {
return;
+ }
switch (m_get_traffic_class(m)) {
case MBUF_TC_BE:
ifp->if_tc.ifi_ibepackets++;
- ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ibebytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_BK:
ifp->if_tc.ifi_ibkpackets++;
- ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ibkbytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_VI:
ifp->if_tc.ifi_ivipackets++;
- ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ivibytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_VO:
ifp->if_tc.ifi_ivopackets++;
- ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ivobytes += (u_int64_t)m->m_pkthdr.len;
break;
default:
break;
if (mbuf_is_traffic_class_privileged(m)) {
ifp->if_tc.ifi_ipvpackets++;
- ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ipvbytes += (u_int64_t)m->m_pkthdr.len;
}
}
static inline void
ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m)
{
- if (!(m->m_flags & M_PKTHDR))
+ if (!(m->m_flags & M_PKTHDR)) {
return;
+ }
switch (m_get_traffic_class(m)) {
case MBUF_TC_BE:
ifp->if_tc.ifi_obepackets++;
- ifp->if_tc.ifi_obebytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_obebytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_BK:
ifp->if_tc.ifi_obkpackets++;
- ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_obkbytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_VI:
ifp->if_tc.ifi_ovipackets++;
- ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ovibytes += (u_int64_t)m->m_pkthdr.len;
break;
case MBUF_TC_VO:
ifp->if_tc.ifi_ovopackets++;
- ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_ovobytes += (u_int64_t)m->m_pkthdr.len;
break;
default:
break;
if (mbuf_is_traffic_class_privileged(m)) {
ifp->if_tc.ifi_opvpackets++;
- ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len;
+ ifp->if_tc.ifi_opvbytes += (u_int64_t)m->m_pkthdr.len;
}
}
#endif /* BSD_KERNEL_PRIVATE */
projects / apple / xnu.git / blobdiff