/*
- * Copyright (c) 1999-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2017 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <sys/cdefs.h>
#include <sys/appleapiopts.h>
+#include <sys/_types/_u_int32_t.h> /* u_int32_t */
+#include <sys/_types/_u_int64_t.h> /* u_int64_t */
+#include <sys/_types/_u_short.h> /* u_short */
#ifdef XNU_KERNEL_PRIVATE
#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */
#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
-#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
-#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
+#define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT)
+#define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */
-#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
-#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
+#define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT)
+#define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */
+
+#define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT)
+#define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */
-#define NMBPCLSHIFT (NMBPBGSHIFT - NCLPBGSHIFT)
+#define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT)
#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */
-#define NCLPJCLSHIFT ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
+#define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT)
#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
+#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
+#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
+
+#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
+#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
+
/*
* Macros for type conversion
* mtod(m,t) - convert mbuf pointer to data pointer of correct type
+ * mtodo(m, o) -- Same as above but with offset 'o' into data.
* dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX)
*/
#define mtod(m, t) ((t)m_mtod(m))
+#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o)))
#define dtom(x) m_dtom(x)
/* header at beginning of each mbuf: */
int32_t mh_len; /* amount of data in this mbuf */
u_int16_t mh_type; /* type of data in this mbuf */
u_int16_t mh_flags; /* flags; see below */
+#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
+/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
+ * should be defined using the offsetof(struct mbuf, M_dat), since there is
+ * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
+ * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
+ * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
+ * in the definition of mbuf.
+ */
+} __attribute__((aligned(8)));
+#else
};
+#endif
/*
* Packet tag structure (see below for details).
u_int16_t pftag_rtableid; /* alternate routing table id */
u_int16_t pftag_tag;
u_int16_t pftag_routed;
-#if PF_ALTQ
- u_int32_t pftag_qid;
-#endif /* PF_ALTQ */
#if PF_ECN
void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */
#endif /* PF_ECN */
union {
struct {
u_int32_t segsz; /* segment size (actual MSS) */
+ u_int32_t start_seq; /* start seq of this packet */
} __tx;
struct {
u_int16_t lro_pktlen; /* max seg size encountered */
u_int32_t seq; /* recv msg sequence # */
} __msgattr;
#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
+#define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
*/
struct mptcp_pktinfo {
u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */
- union {
- u_int64_t mtpi_dan; /* MPTCP Data Ack Number */
- struct {
- u_int32_t mtpi_rel_seq; /* Relative Seq Number */
- u_int32_t mtpi_length; /* Length of mapping */
- } mtpi_subf;
- };
+ u_int32_t mtpi_rel_seq; /* Relative Seq Number */
+ u_int16_t mtpi_length; /* Length of mapping */
+ u_int16_t mtpi_csum;
#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
-#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq
-#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length
-#define mp_dack proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan
+#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
+#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
+#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
};
/*
};
};
-/*
- * IPSec mbuf tag
- */
-struct ipsec_mtag {
- uint32_t policy_id;
-#define ipsec_policy proto_mtag.__pr_u.ipsec.policy_id
+struct driver_mtag_ {
+ uintptr_t _drv_tx_compl_arg;
+ uintptr_t _drv_tx_compl_data;
+ kern_return_t _drv_tx_status;
+ uint16_t _drv_flowid;
+#define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg
+#define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data
+#define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status
+#define drv_flowid builtin_mtag._drv_mtag._drv_flowid
};
/*
* that the former is used on the virtual ipsec interface that does
* not advertise the TSO capability.)
*/
-struct proto_mtag {
+struct proto_mtag_ {
union {
struct tcp_mtag tcp; /* TCP specific */
- struct ipsec_mtag ipsec; /* IPSec specific */
} __pr_u;
};
+/*
+ * NECP specific mbuf tag.
+ */
+struct necp_mtag_ {
+ u_int32_t necp_policy_id;
+ u_int32_t necp_last_interface_index;
+ u_int32_t necp_route_rule_id;
+ u_int32_t necp_app_id;
+};
+
+union builtin_mtag {
+ struct {
+ struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
+ struct pf_mtag _pf_mtag; /* built-in PF tag */
+ struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
+ } _net_mtag;
+ struct driver_mtag_ _drv_mtag;
+#define necp_mtag builtin_mtag._net_mtag._necp_mtag
+#define proto_mtag builtin_mtag._net_mtag._proto_mtag
+#define driver_mtag builtin_mtag._drv_mtag
+};
+
/*
* Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
*/
-struct pkthdr {
- struct ifnet *rcvif; /* rcv interface */
+struct pkthdr {
+ struct ifnet *rcvif; /* rcv interface */
/* variables for ip and tcp reassembly */
void *pkt_hdr; /* pointer to packet header */
int32_t len; /* total packet length */
u_int32_t pkt_flowid; /* flow ID */
u_int32_t pkt_flags; /* PKTF flags (see below) */
u_int32_t pkt_svc; /* MBUF_SVC value */
+
+ u_int32_t pkt_compl_context; /* Packet completion context */
+
union {
struct {
u_int16_t src; /* ifindex of src addr i/f */
#define dst_ifindex _pkt_iaif.dst
#define dst_iff _pkt_iaif.dst_flags
u_int64_t pkt_ifainfo; /* data field used by ifainfo */
+ struct {
+ u_int32_t if_data; /* bytes in interface queue */
+ u_int32_t sndbuf_data; /* bytes in socket buffer */
+ } _pkt_bsr; /* Buffer status report used by cellular interface */
+#define bufstatus_if _pkt_bsr.if_data
+#define bufstatus_sndbuf _pkt_bsr.sndbuf_data
};
-#if MEASURE_BW
- u_int64_t pkt_bwseq; /* sequence # */
-#endif /* MEASURE_BW */
+ u_int64_t pkt_timestamp; /* enqueue time */
+
/*
* Tags (external and built-in)
*/
SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
- struct proto_mtag proto_mtag; /* built-in protocol-specific tag */
- struct pf_mtag pf_mtag; /* built-in PF tag */
+ union builtin_mtag builtin_mtag;
/*
* Module private scratch space (32-bit aligned), currently 16-bytes
- * large. Anything stored here is not guaranteed to survive across
- * modules. This should be the penultimate structure right before
- * the red zone. Add new fields above this.
+ * large. Anything stored here is not guaranteed to survive across
+ * modules. The AQM layer (outbound) uses all 16-bytes for both
+ * packet scheduling and flow advisory information.
*/
struct {
union {
u_int64_t __mpriv64[2];
} __mpriv_u;
} pkt_mpriv __attribute__((aligned(4)));
+#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
+#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
+#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
+#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
+
u_int32_t redzone; /* red zone */
+ u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */
};
/*
#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
#define FLOWSRC_IFNET 2 /* flow ID generated by interface */
#define FLOWSRC_PF 3 /* flow ID generated by PF */
+#define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */
/*
* Packet flags. Unlike m_flags, all packet flags are copied along when
#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
+#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */
+#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */
+#define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */
+#define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */
+#define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */
+#define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */
+#define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */
+#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */
+#define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */
+#define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */
+#define PKTF_START_SEQ 0x8000000 /* valid start sequence */
+#define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */
+#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */
+#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */
+#define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */
+
/* flags related to flow control/advisory and identification */
#define PKTF_FLOW_MASK \
(PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
/*
* Description of external storage mapped into mbuf, valid only if M_EXT set.
*/
+typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);
struct m_ext {
caddr_t ext_buf; /* start of buffer */
- void (*ext_free) /* free routine if not the usual */
- (caddr_t, u_int, caddr_t);
+ m_ext_free_func_t ext_free; /* free routine if not the usual */
u_int ext_size; /* size of buffer, for ext_free */
caddr_t ext_arg; /* additional ext_free argument */
- struct ext_refsq { /* references held */
- struct ext_refsq *forward, *backward;
- } ext_refs;
struct ext_ref {
- u_int32_t refcnt;
- u_int32_t flags;
+ struct mbuf *paired;
+ u_int16_t minref;
+ u_int16_t refcnt;
+ u_int16_t prefcnt;
+ u_int16_t flags;
+ u_int32_t priv;
+ uintptr_t ext_token;
} *ext_refflags;
};
* The mbuf object
*/
struct mbuf {
- struct m_hdr m_hdr;
+ struct m_hdr m_hdr;
union {
struct {
- struct pkthdr MH_pkthdr; /* M_PKTHDR set */
+ struct pkthdr MH_pkthdr; /* M_PKTHDR set */
union {
- struct m_ext MH_ext; /* M_EXT set */
+ struct m_ext MH_ext; /* M_EXT set */
char MH_databuf[_MHLEN];
} MH_dat;
} MH;
#define m_pktdat M_dat.MH.MH_dat.MH_databuf
#define m_dat M_dat.M_databuf
#define m_pktlen(_m) ((_m)->m_pkthdr.len)
-#define m_pftag(_m) (&(_m)->m_pkthdr.pf_mtag)
+#define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)
/* mbuf flags (private) */
#define M_EXT 0x0001 /* has associated external storage */
#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
+#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */
#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
#define CSUM_TX_FLAGS \
(CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
- CSUM_DATA_VALID | CSUM_PARTIAL)
+ CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)
#define CSUM_RX_FLAGS \
(CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
* If how is M_DONTWAIT and allocation fails, the original mbuf chain
* is freed and m is set to NULL.
*/
-#define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how)))
+#define M_PREPEND(m, plen, how, align) \
+ ((m) = m_prepend_2((m), (plen), (how), (align)))
/* change mbuf to new type */
#define MCHTYPE(m, t) m_mchtype(m, t)
#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
#define MBUFQ_FIRST(q) ((q)->mq_first)
#define MBUFQ_NEXT(m) ((m)->m_nextpkt)
-#define MBUFQ_LAST(q) (*(q)->mq_last)
+/*
+ * mq_last is initialized to point to mq_first, so check if they're
+ * equal and return NULL when the list is empty. Otherwise, we need
+ * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
+ * to the base mbuf address to return to caller.
+ */
+#define MBUFQ_LAST(head) \
+ (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \
+ ((struct mbuf *)(void *)((char *)(head)->mq_last - \
+ (size_t)(&MBUFQ_NEXT((struct mbuf *)0)))))
#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
u_int64_t mbcl_purge_cnt; /* # of purges so far */
u_int64_t mbcl_fail_cnt; /* # of allocation failures */
u_int32_t mbcl_ctotal; /* total only for this class */
+ u_int32_t mbcl_release_cnt; /* amount of memory returned */
/*
* Cache layer statistics
*/
u_int64_t mbcl_purge_cnt; /* # of purges so far */
u_int64_t mbcl_fail_cnt; /* # of allocation failures */
u_int32_t mbcl_ctotal; /* total only for this class */
+ u_int32_t mbcl_release_cnt; /* amount of memory returned */
/*
* Cache layer statistics
*/
u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
- u_int64_t mbcl_reserved[4]; /* for future use */
+ u_int32_t mbcl_peak_reported; /* last usage peak reported */
+ u_int32_t mbcl_reserved[7]; /* for future use */
} mb_class_stat_t;
#define MCS_DISABLED 0 /* cache is permanently disabled */
#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
+#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */
+#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */
/*
* These macros are mapped to the appropriate KPIs, so that private code
extern struct mbuf *m_getpackets(int, int, int);
extern struct mbuf *m_mclget(struct mbuf *, int);
extern void *m_mtod(struct mbuf *);
-extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
+extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);
extern struct mbuf *m_pullup(struct mbuf *, int);
extern struct mbuf *m_split(struct mbuf *, int, int);
extern void m_mclfree(caddr_t p);
#define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
((_sc) == MBUF_SC_BK))
+#define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
+#define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \
+ (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)
#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO || \
c == SCVAL_CTL)
-extern union mbigcluster *mbutl; /* start VA of mbuf pool */
-extern union mbigcluster *embutl; /* end VA of mbuf pool */
+extern unsigned char *mbutl; /* start VA of mbuf pool */
+extern unsigned char *embutl; /* end VA of mbuf pool */
extern unsigned int nmbclusters; /* number of mapped clusters */
extern int njcl; /* # of jumbo clusters */
extern int njclbytes; /* size of a jumbo cluster */
__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
__private_extern__ void mbinit(void);
__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
- void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
+ void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int);
__private_extern__ caddr_t m_bigalloc(int);
__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
unsigned int *, int, int, size_t);
+__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
+__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
+__private_extern__ int m_ext_paired_is_active(struct mbuf *);
+__private_extern__ void m_ext_paired_activate(struct mbuf *);
+
+__private_extern__ void m_drain(void);
+
/*
* Packets may have annotations attached by affixing a list of "packet
* tags" to the pkthdr structure. Packet tags are dynamically allocated
KERNEL_TAG_TYPE_ENCAP = 8,
KERNEL_TAG_TYPE_INET6 = 9,
KERNEL_TAG_TYPE_IPSEC = 10,
- KERNEL_TAG_TYPE_DRVAUX = 11
+ KERNEL_TAG_TYPE_DRVAUX = 11,
};
/* Packet tag routines */
} while (0)
__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
- u_int32_t, u_int32_t);
+ u_int32_t, u_int32_t, u_int32_t);
__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
+__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
+ m_ext_free_func_t, caddr_t);
+__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
+__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
+__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);
+
+extern void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);
+
__END_DECLS
#endif /* XNU_KERNEL_PRIVATE */
#endif /* KERNEL */
projects / apple / xnu.git / blobdiff