/*
- * Copyright (c) 1999-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* Version 2.0.
*/
-#ifndef _SYS_MBUF_H_
-#define _SYS_MBUF_H_
+#ifndef _SYS_MBUF_H_
+#define _SYS_MBUF_H_
-#include <sys/cdefs.h>
#include <sys/appleapiopts.h>
+#include <sys/cdefs.h>
+#include <sys/_types/_u_int32_t.h> /* u_int32_t */
+#include <sys/_types/_u_int64_t.h> /* u_int64_t */
+#include <sys/_types/_u_short.h> /* u_short */
-#ifdef XNU_KERNEL_PRIVATE
+#ifdef KERNEL
+#include <sys/kpi_mbuf.h>
+#endif
+#ifdef XNU_KERNEL_PRIVATE
#include <sys/lock.h>
#include <sys/queue.h>
#include <machine/endian.h>
* that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
* the sizes of the structures are dependent upon specific xnu configs.
*/
-#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */
-#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
+#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */
+#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
+
+#define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT)
+#define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */
+
+#define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT)
+#define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */
+
+#define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT)
+#define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */
-#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
-#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
+#define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT)
+#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */
-#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
-#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
+#define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT)
+#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
-#define NMBPCLSHIFT (NMBPBGSHIFT - NCLPBGSHIFT)
-#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */
+#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
+#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
-#define NCLPJCLSHIFT ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
-#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
+#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
+#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
/*
* Macros for type conversion
* mtod(m,t) - convert mbuf pointer to data pointer of correct type
+ * mtodo(m, o) -- Same as above but with offset 'o' into data.
* dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX)
*/
-#define mtod(m, t) ((t)m_mtod(m))
-#define dtom(x) m_dtom(x)
+#define mtod(m, t) ((t)m_mtod(m))
+#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o)))
+#define dtom(x) m_dtom(x)
/* header at beginning of each mbuf: */
struct m_hdr {
- struct mbuf *mh_next; /* next buffer in chain */
- struct mbuf *mh_nextpkt; /* next chain in queue/record */
- caddr_t mh_data; /* location of data */
- int32_t mh_len; /* amount of data in this mbuf */
- u_int16_t mh_type; /* type of data in this mbuf */
- u_int16_t mh_flags; /* flags; see below */
+ struct mbuf *mh_next; /* next buffer in chain */
+ struct mbuf *mh_nextpkt; /* next chain in queue/record */
+ caddr_t mh_data; /* location of data */
+ int32_t mh_len; /* amount of data in this mbuf */
+ u_int16_t mh_type; /* type of data in this mbuf */
+ u_int16_t mh_flags; /* flags; see below */
+#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
+/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
+ * should be defined using the offsetof(struct mbuf, M_dat), since there is
+ * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
+ * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
+ * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
+ * in the definition of mbuf.
+ */
+} __attribute__((aligned(8)));
+#else
};
+#endif
/*
* Packet tag structure (see below for details).
*/
struct m_tag {
- u_int64_t m_tag_cookie; /* Error checking */
+ u_int64_t m_tag_cookie; /* Error checking */
#ifndef __LP64__
- u_int32_t pad; /* For structure alignment */
+ u_int32_t pad; /* For structure alignment */
#endif /* !__LP64__ */
- SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
- u_int16_t m_tag_type; /* Module specific type */
- u_int16_t m_tag_len; /* Length of data */
- u_int32_t m_tag_id; /* Module ID */
+ SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
+ u_int16_t m_tag_type; /* Module specific type */
+ u_int16_t m_tag_len; /* Length of data */
+ u_int32_t m_tag_id; /* Module ID */
};
-#define M_TAG_ALIGN(len) \
+#define M_TAG_ALIGN(len) \
(P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
-#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL
-#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL
+#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL
+#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL
/*
* Packet tag header structure (at the top of mbuf). Pointers are
*/
struct m_taghdr {
#ifndef __LP64__
- u_int32_t pad; /* For structure alignment */
+ u_int32_t pad; /* For structure alignment */
#endif /* !__LP64__ */
- u_int64_t refcnt; /* Number of tags in this mbuf */
+ u_int64_t refcnt; /* Number of tags in this mbuf */
};
/*
* Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
*/
struct m_drvaux_tag {
- u_int32_t da_family; /* IFNET_FAMILY values */
- u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */
- u_int32_t da_reserved; /* for future */
- u_int32_t da_length; /* length of following data */
+ u_int32_t da_family; /* IFNET_FAMILY values */
+ u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */
+ u_int32_t da_reserved; /* for future */
+ u_int32_t da_length; /* length of following data */
};
/* Values for pftag_flags (16-bit wide) */
-#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */
-#define PF_TAG_FRAGCACHE 0x2
-#define PF_TAG_TRANSLATE_LOCALHOST 0x4
+#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */
+#define PF_TAG_FRAGCACHE 0x2
+#define PF_TAG_TRANSLATE_LOCALHOST 0x4
#if PF_ECN
-#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */
-#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */
+#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */
+#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */
#endif /* PF_ECN */
+#define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */
+#define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */
/*
* PF mbuf tag
*/
struct pf_mtag {
- u_int16_t pftag_flags; /* PF_TAG flags */
- u_int16_t pftag_rtableid; /* alternate routing table id */
- u_int16_t pftag_tag;
- u_int16_t pftag_routed;
-#if PF_ALTQ
- u_int32_t pftag_qid;
-#endif /* PF_ALTQ */
+ u_int16_t pftag_flags; /* PF_TAG flags */
+ u_int16_t pftag_rtableid; /* alternate routing table id */
+ u_int16_t pftag_tag;
+ u_int16_t pftag_routed;
#if PF_ECN
- void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */
+ void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */
#endif /* PF_ECN */
};
+/*
+ * PF fragment tag
+ */
+struct pf_fragment_tag {
+ uint32_t ft_id; /* fragment id */
+ uint16_t ft_hdrlen; /* header length of reassembled pkt */
+ uint16_t ft_unfragpartlen; /* length of the per-fragment headers */
+ uint16_t ft_extoff; /* last extension header offset or 0 */
+ uint16_t ft_maxlen; /* maximum fragment payload length */
+};
+
/*
* TCP mbuf tag
*/
struct tcp_pktinfo {
union {
struct {
- u_int32_t segsz; /* segment size (actual MSS) */
+ u_int32_t segsz; /* segment size (actual MSS) */
+ u_int32_t start_seq; /* start seq of this packet */
+ pid_t pid;
+ pid_t e_pid;
} __tx;
struct {
- u_int16_t lro_pktlen; /* max seg size encountered */
- u_int8_t lro_npkts; /* # of coalesced TCP pkts */
- u_int8_t lro_timediff; /* time spent in LRO */
+ u_int16_t lro_pktlen; /* max seg size encountered */
+ u_int8_t lro_npkts; /* # of coalesced TCP pkts */
+ u_int8_t lro_timediff; /* time spent in LRO */
} __rx;
} __offload;
union {
- u_int32_t pri; /* send msg priority */
- u_int32_t seq; /* recv msg sequence # */
+ u_int32_t pri; /* send msg priority */
+ u_int32_t seq; /* recv msg sequence # */
} __msgattr;
-#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
-#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
-#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
-#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
-#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri
-#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq
+#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
+#define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
+#define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid
+#define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid
+#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
+#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
+#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
+#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri
+#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq
};
/*
* MPTCP mbuf tag
*/
struct mptcp_pktinfo {
- u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */
- union {
- u_int64_t mtpi_dan; /* MPTCP Data Ack Number */
- struct {
- u_int32_t mtpi_rel_seq; /* Relative Seq Number */
- u_int32_t mtpi_length; /* Length of mapping */
- } mtpi_subf;
- };
-#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
-#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq
-#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length
-#define mp_dack proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan
+ u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */
+ u_int32_t mtpi_rel_seq; /* Relative Seq Number */
+ u_int16_t mtpi_length; /* Length of mapping */
+ u_int16_t mtpi_csum;
+#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
+#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
+#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
+#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
};
/*
*/
struct tcp_mtag {
union {
- struct tcp_pktinfo tm_tcp; /* TCP and below */
- struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */
+ struct tcp_pktinfo tm_tcp; /* TCP and below */
+ struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */
};
};
-/*
- * IPSec mbuf tag
- */
-struct ipsec_mtag {
- uint32_t policy_id;
-#define ipsec_policy proto_mtag.__pr_u.ipsec.policy_id
+struct udp_mtag {
+ pid_t _pid;
+ pid_t _e_pid;
+#define tx_udp_pid proto_mtag.__pr_u.udp._pid
+#define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid
+};
+
+struct rawip_mtag {
+ pid_t _pid;
+ pid_t _e_pid;
+#define tx_rawip_pid proto_mtag.__pr_u.rawip._pid
+#define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid
+};
+
+struct driver_mtag_ {
+ uintptr_t _drv_tx_compl_arg;
+ uintptr_t _drv_tx_compl_data;
+ kern_return_t _drv_tx_status;
+ uint16_t _drv_flowid;
+#define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg
+#define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data
+#define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status
+#define drv_flowid builtin_mtag._drv_mtag._drv_flowid
};
/*
* Protocol specific mbuf tag (at most one protocol metadata per mbuf).
*
* Care must be taken to ensure that they are mutually exclusive, e.g.
- * IPSec policy ID implies no TCP segment offload (which is fine given
+ * IPsec policy ID implies no TCP segment offload (which is fine given
* that the former is used on the virtual ipsec interface that does
* not advertise the TSO capability.)
*/
-struct proto_mtag {
+struct proto_mtag_ {
union {
- struct tcp_mtag tcp; /* TCP specific */
- struct ipsec_mtag ipsec; /* IPSec specific */
+ struct tcp_mtag tcp; /* TCP specific */
+ struct udp_mtag udp; /* UDP specific */
+ struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */
} __pr_u;
};
+/*
+ * NECP specific mbuf tag.
+ */
+struct necp_mtag_ {
+ u_int32_t necp_policy_id;
+ u_int32_t necp_skip_policy_id;
+ u_int32_t necp_route_rule_id;
+ u_int16_t necp_last_interface_index;
+ u_int16_t necp_app_id;
+};
+
+union builtin_mtag {
+ struct {
+ struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
+ struct pf_mtag _pf_mtag; /* built-in PF tag */
+ struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
+ } _net_mtag;
+ struct driver_mtag_ _drv_mtag;
+#define necp_mtag builtin_mtag._net_mtag._necp_mtag
+#define proto_mtag builtin_mtag._net_mtag._proto_mtag
+#define driver_mtag builtin_mtag._drv_mtag
+};
+
/*
* Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
*/
-struct pkthdr {
- struct ifnet *rcvif; /* rcv interface */
+struct pkthdr {
+ struct ifnet *rcvif; /* rcv interface */
/* variables for ip and tcp reassembly */
- void *pkt_hdr; /* pointer to packet header */
- int32_t len; /* total packet length */
+ void *pkt_hdr; /* pointer to packet header */
+ int32_t len; /* total packet length */
/* variables for hardware checksum */
/* Note: csum_flags is used for hardware checksum and VLAN */
- u_int32_t csum_flags; /* flags regarding checksum */
+ u_int32_t csum_flags; /* flags regarding checksum */
union {
struct {
- u_int16_t val; /* checksum value */
+ u_int16_t val; /* checksum value */
u_int16_t start; /* checksum start offset */
} _csum_rx;
-#define csum_rx_val _csum_rx.val
-#define csum_rx_start _csum_rx.start
+#define csum_rx_val _csum_rx.val
+#define csum_rx_start _csum_rx.start
struct {
u_int16_t start; /* checksum start offset */
u_int16_t stuff; /* checksum stuff offset */
} _csum_tx;
-#define csum_tx_start _csum_tx.start
-#define csum_tx_stuff _csum_tx.stuff
- u_int32_t csum_data; /* data field used by csum routines */
+#define csum_tx_start _csum_tx.start
+#define csum_tx_stuff _csum_tx.stuff
+ /*
+ * Generic data field used by csum routines.
+ * It gets used differently in different contexts.
+ */
+ u_int32_t csum_data;
};
- u_int16_t vlan_tag; /* VLAN tag, host byte order */
+ u_int16_t vlan_tag; /* VLAN tag, host byte order */
/*
* Packet classifier info
*
* to achieve this. For now, we will just rely on the address family
* related code paths examining this mbuf to interpret the flags.
*/
- u_int8_t pkt_proto; /* IPPROTO value */
- u_int8_t pkt_flowsrc; /* FLOWSRC values */
- u_int32_t pkt_flowid; /* flow ID */
- u_int32_t pkt_flags; /* PKTF flags (see below) */
- u_int32_t pkt_svc; /* MBUF_SVC value */
+ u_int8_t pkt_proto; /* IPPROTO value */
+ u_int8_t pkt_flowsrc; /* FLOWSRC values */
+ u_int32_t pkt_flowid; /* flow ID */
+ u_int32_t pkt_flags; /* PKTF flags (see below) */
+ u_int32_t pkt_svc; /* MBUF_SVC value */
+
+ u_int32_t pkt_compl_context; /* Packet completion context */
+
union {
struct {
- u_int16_t src; /* ifindex of src addr i/f */
- u_int16_t src_flags; /* src PKT_IFAIFF flags */
- u_int16_t dst; /* ifindex of dst addr i/f */
- u_int16_t dst_flags; /* dst PKT_IFAIFF flags */
+ u_int16_t src; /* ifindex of src addr i/f */
+ u_int16_t src_flags; /* src PKT_IFAIFF flags */
+ u_int16_t dst; /* ifindex of dst addr i/f */
+ u_int16_t dst_flags; /* dst PKT_IFAIFF flags */
} _pkt_iaif;
-#define src_ifindex _pkt_iaif.src
-#define src_iff _pkt_iaif.src_flags
-#define dst_ifindex _pkt_iaif.dst
-#define dst_iff _pkt_iaif.dst_flags
- u_int64_t pkt_ifainfo; /* data field used by ifainfo */
+#define src_ifindex _pkt_iaif.src
+#define src_iff _pkt_iaif.src_flags
+#define dst_ifindex _pkt_iaif.dst
+#define dst_iff _pkt_iaif.dst_flags
+ u_int64_t pkt_ifainfo; /* data field used by ifainfo */
+ struct {
+ u_int32_t if_data; /* bytes in interface queue */
+ u_int32_t sndbuf_data; /* bytes in socket buffer */
+ } _pkt_bsr; /* Buffer status report used by cellular interface */
+#define bufstatus_if _pkt_bsr.if_data
+#define bufstatus_sndbuf _pkt_bsr.sndbuf_data
};
-#if MEASURE_BW
- u_int64_t pkt_bwseq; /* sequence # */
-#endif /* MEASURE_BW */
+ u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */
+
/*
* Tags (external and built-in)
*/
SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
- struct proto_mtag proto_mtag; /* built-in protocol-specific tag */
- struct pf_mtag pf_mtag; /* built-in PF tag */
+ union builtin_mtag builtin_mtag;
/*
* Module private scratch space (32-bit aligned), currently 16-bytes
- * large. Anything stored here is not guaranteed to survive across
- * modules. This should be the penultimate structure right before
- * the red zone. Add new fields above this.
+ * large. Anything stored here is not guaranteed to survive across
+ * modules. The AQM layer (outbound) uses all 16-bytes for both
+ * packet scheduling and flow advisory information.
*/
struct {
union {
- u_int8_t __mpriv8[16];
- u_int16_t __mpriv16[8];
+ u_int8_t __mpriv8[16];
+ u_int16_t __mpriv16[8];
struct {
union {
- u_int8_t __val8[4];
- u_int16_t __val16[2];
- u_int32_t __val32;
+ u_int8_t __val8[4];
+ u_int16_t __val16[2];
+ u_int32_t __val32;
} __mpriv32_u;
- } __mpriv32[4];
- u_int64_t __mpriv64[2];
+ } __mpriv32[4];
+ u_int64_t __mpriv64[2];
} __mpriv_u;
} pkt_mpriv __attribute__((aligned(4)));
- u_int32_t redzone; /* red zone */
+#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
+#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
+#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
+#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
+
+ u_int32_t redzone; /* red zone */
+ u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */
};
/*
* to identify the data source object and inform that it can resume its
* transmission (in the event it was flow controlled.)
*/
-#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
-#define FLOWSRC_IFNET 2 /* flow ID generated by interface */
-#define FLOWSRC_PF 3 /* flow ID generated by PF */
+#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
+#define FLOWSRC_IFNET 2 /* flow ID generated by interface */
+#define FLOWSRC_PF 3 /* flow ID generated by PF */
+#define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */
+
+/*
+ * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for
+ * multi-layer packet logging. We're usurping the pkt_flowsrc field because
+ * the mbuf packet header ran out of space and pkt_flowsrc is normally
+ * used on output so we assume we can safely overwrite the normal semantic of
+ * the field.
+ * This value is meant to be used on incoming packet from a lower level protocol
+ * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT
+ * is set, the following fields are used:
+ * - pkt_proto: the IP protocol ID of the lower level protocol
+ * - pkt_flowid: the identifier of the packet at the lower protocol.
+ * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI.
+ */
/*
* Packet flags. Unlike m_flags, all packet flags are copied along when
* if the packet was looped back to the system. This flag should be
* used instead for newer code.
*/
-#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */
-#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */
-#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */
-#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */
-#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */
-#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */
-#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */
-#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */
-#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */
-#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */
-#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */
-#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */
-#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */
-#define PKTF_LOOP 0x2000 /* loopbacked packet */
-#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
-#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
-#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
+#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */
+#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */
+#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */
+#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */
+#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */
+#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */
+#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */
+#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */
+#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */
+#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */
+#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */
+#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */
+#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */
+#define PKTF_LOOP 0x2000 /* loopbacked packet */
+#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
+#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
+#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
+#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */
+#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */
+#define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */
+#define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */
+#define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */
+#define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */
+#define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */
+#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */
+#define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */
+#define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */
+#define PKTF_START_SEQ 0x8000000 /* valid start sequence */
+#define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */
+#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */
+#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */
+#define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */
+
/* flags related to flow control/advisory and identification */
-#define PKTF_FLOW_MASK \
+#define PKTF_FLOW_MASK \
(PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
/*
* Description of external storage mapped into mbuf, valid only if M_EXT set.
*/
+typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);
struct m_ext {
- caddr_t ext_buf; /* start of buffer */
- void (*ext_free) /* free routine if not the usual */
- (caddr_t, u_int, caddr_t);
- u_int ext_size; /* size of buffer, for ext_free */
- caddr_t ext_arg; /* additional ext_free argument */
- struct ext_refsq { /* references held */
- struct ext_refsq *forward, *backward;
- } ext_refs;
+ caddr_t ext_buf; /* start of buffer */
+ m_ext_free_func_t ext_free; /* free routine if not the usual */
+ u_int ext_size; /* size of buffer, for ext_free */
+ caddr_t ext_arg; /* additional ext_free argument */
struct ext_ref {
- u_int32_t refcnt;
- u_int32_t flags;
+ struct mbuf *paired;
+ u_int16_t minref;
+ u_int16_t refcnt;
+ u_int16_t prefcnt;
+ u_int16_t flags;
+ u_int32_t priv;
+ uintptr_t ext_token;
} *ext_refflags;
};
* The mbuf object
*/
struct mbuf {
- struct m_hdr m_hdr;
+ struct m_hdr m_hdr;
union {
struct {
- struct pkthdr MH_pkthdr; /* M_PKTHDR set */
+ struct pkthdr MH_pkthdr; /* M_PKTHDR set */
union {
- struct m_ext MH_ext; /* M_EXT set */
- char MH_databuf[_MHLEN];
+ struct m_ext MH_ext; /* M_EXT set */
+ char MH_databuf[_MHLEN];
} MH_dat;
} MH;
- char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */
+ char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */
} M_dat;
};
-#define m_next m_hdr.mh_next
-#define m_len m_hdr.mh_len
-#define m_data m_hdr.mh_data
-#define m_type m_hdr.mh_type
-#define m_flags m_hdr.mh_flags
-#define m_nextpkt m_hdr.mh_nextpkt
-#define m_act m_nextpkt
-#define m_pkthdr M_dat.MH.MH_pkthdr
-#define m_ext M_dat.MH.MH_dat.MH_ext
-#define m_pktdat M_dat.MH.MH_dat.MH_databuf
-#define m_dat M_dat.M_databuf
-#define m_pktlen(_m) ((_m)->m_pkthdr.len)
-#define m_pftag(_m) (&(_m)->m_pkthdr.pf_mtag)
+#define m_next m_hdr.mh_next
+#define m_len m_hdr.mh_len
+#define m_data m_hdr.mh_data
+#define m_type m_hdr.mh_type
+#define m_flags m_hdr.mh_flags
+#define m_nextpkt m_hdr.mh_nextpkt
+#define m_act m_nextpkt
+#define m_pkthdr M_dat.MH.MH_pkthdr
+#define m_ext M_dat.MH.MH_dat.MH_ext
+#define m_pktdat M_dat.MH.MH_dat.MH_databuf
+#define m_dat M_dat.M_databuf
+#define m_pktlen(_m) ((_m)->m_pkthdr.len)
+#define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)
/* mbuf flags (private) */
-#define M_EXT 0x0001 /* has associated external storage */
-#define M_PKTHDR 0x0002 /* start of record */
-#define M_EOR 0x0004 /* end of record */
-#define M_PROTO1 0x0008 /* protocol-specific */
-#define M_PROTO2 0x0010 /* protocol-specific */
-#define M_PROTO3 0x0020 /* protocol-specific */
-#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */
-#define M_PROTO5 0x0080 /* protocol-specific */
+#define M_EXT 0x0001 /* has associated external storage */
+#define M_PKTHDR 0x0002 /* start of record */
+#define M_EOR 0x0004 /* end of record */
+#define M_PROTO1 0x0008 /* protocol-specific */
+#define M_PROTO2 0x0010 /* protocol-specific */
+#define M_PROTO3 0x0020 /* protocol-specific */
+#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */
+#define M_PROTO5 0x0080 /* protocol-specific */
/* mbuf pkthdr flags, also in m_flags (private) */
-#define M_BCAST 0x0100 /* send/received as link-level broadcast */
-#define M_MCAST 0x0200 /* send/received as link-level multicast */
-#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */
-#define M_FIRSTFRAG 0x0800 /* packet is first fragment */
-#define M_LASTFRAG 0x1000 /* packet is last fragment */
-#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */
-#define M_HASFCS 0x4000 /* packet has FCS */
-#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */
+#define M_BCAST 0x0100 /* send/received as link-level broadcast */
+#define M_MCAST 0x0200 /* send/received as link-level multicast */
+#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */
+#define M_FIRSTFRAG 0x0800 /* packet is first fragment */
+#define M_LASTFRAG 0x1000 /* packet is last fragment */
+#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */
+#define M_HASFCS 0x4000 /* packet has FCS */
+#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */
/*
* Flags to purge when crossing layers.
*/
-#define M_PROTOFLAGS \
+#define M_PROTOFLAGS \
(M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
/* flags copied when copying m_pkthdr */
-#define M_COPYFLAGS \
- (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \
- M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \
+#define M_COPYFLAGS \
+ (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \
+ M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \
M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
/* flags indicating hw checksum support and sw checksum requirements */
-#define CSUM_IP 0x0001 /* will csum IP */
-#define CSUM_TCP 0x0002 /* will csum TCP */
-#define CSUM_UDP 0x0004 /* will csum UDP */
-#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */
-#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */
-#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */
-#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */
-#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */
-
-#define CSUM_IP_CHECKED 0x0100 /* did csum IP */
-#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */
-#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
-#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
-#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
-
-#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
-#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
-#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6)
-#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */
-
-#define CSUM_TX_FLAGS \
- (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
- CSUM_DATA_VALID | CSUM_PARTIAL)
-
-#define CSUM_RX_FLAGS \
- (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
+#define CSUM_IP 0x0001 /* will csum IP */
+#define CSUM_TCP 0x0002 /* will csum TCP */
+#define CSUM_UDP 0x0004 /* will csum UDP */
+#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */
+#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */
+#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */
+#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */
+#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */
+
+#define CSUM_IP_CHECKED 0x0100 /* did csum IP */
+#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */
+#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
+#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
+#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
+#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */
+
+#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
+#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
+#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6)
+#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */
+
+#define CSUM_TX_FLAGS \
+ (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
+ CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)
+
+#define CSUM_RX_FLAGS \
+ (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
CSUM_DATA_VALID | CSUM_PARTIAL)
/*
*/
/* VLAN tag present */
-#define CSUM_VLAN_TAG_VALID 0x10000 /* vlan_tag field is valid */
+#define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */
+
+/* checksum start adjustment has been done */
+#define CSUM_ADJUST_DONE 0x00020000
/* TCP Segment Offloading requested on this mbuf */
-#define CSUM_TSO_IPV4 0x100000 /* This mbuf needs to be segmented by the NIC */
-#define CSUM_TSO_IPV6 0x200000 /* This mbuf needs to be segmented by the NIC */
+#define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */
+#define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */
-#define TSO_IPV4_OK(_ifp, _m) \
- (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
- ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
+#define TSO_IPV4_OK(_ifp, _m) \
+ (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
+ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
-#define TSO_IPV4_NOTOK(_ifp, _m) \
- (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
- ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
+#define TSO_IPV4_NOTOK(_ifp, _m) \
+ (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
+ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
-#define TSO_IPV6_OK(_ifp, _m) \
- (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
- ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
+#define TSO_IPV6_OK(_ifp, _m) \
+ (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
+ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
-#define TSO_IPV6_NOTOK(_ifp, _m) \
- (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
- ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
+#define TSO_IPV6_NOTOK(_ifp, _m) \
+ (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
+ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
#endif /* XNU_KERNEL_PRIVATE */
/* mbuf types */
-#define MT_FREE 0 /* should be on free list */
-#define MT_DATA 1 /* dynamic (data) allocation */
-#define MT_HEADER 2 /* packet header */
-#define MT_SOCKET 3 /* socket structure */
-#define MT_PCB 4 /* protocol control block */
-#define MT_RTABLE 5 /* routing tables */
-#define MT_HTABLE 6 /* IMP host tables */
-#define MT_ATABLE 7 /* address resolution tables */
-#define MT_SONAME 8 /* socket name */
-#define MT_SOOPTS 10 /* socket options */
-#define MT_FTABLE 11 /* fragment reassembly header */
-#define MT_RIGHTS 12 /* access rights */
-#define MT_IFADDR 13 /* interface address */
-#define MT_CONTROL 14 /* extra-data protocol message */
-#define MT_OOBDATA 15 /* expedited data */
-#define MT_TAG 16 /* volatile metadata associated to pkts */
-#define MT_MAX 32 /* enough? */
+#define MT_FREE 0 /* should be on free list */
+#define MT_DATA 1 /* dynamic (data) allocation */
+#define MT_HEADER 2 /* packet header */
+#define MT_SOCKET 3 /* socket structure */
+#define MT_PCB 4 /* protocol control block */
+#define MT_RTABLE 5 /* routing tables */
+#define MT_HTABLE 6 /* IMP host tables */
+#define MT_ATABLE 7 /* address resolution tables */
+#define MT_SONAME 8 /* socket name */
+#define MT_SOOPTS 10 /* socket options */
+#define MT_FTABLE 11 /* fragment reassembly header */
+#define MT_RIGHTS 12 /* access rights */
+#define MT_IFADDR 13 /* interface address */
+#define MT_CONTROL 14 /* extra-data protocol message */
+#define MT_OOBDATA 15 /* expedited data */
+#define MT_TAG 16 /* volatile metadata associated to pkts */
+#define MT_MAX 32 /* enough? */
#ifdef XNU_KERNEL_PRIVATE
/*
*/
#if 1
-#define MCHECK(m) m_mcheck(m)
+#define MCHECK(m) m_mcheck(m)
#else
-#define MCHECK(m)
+#define MCHECK(m)
#endif
-#define MGET(m, how, type) ((m) = m_get((how), (type)))
+#define MGET(m, how, type) ((m) = m_get((how), (type)))
-#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
+#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
/*
* Mbuf cluster macros.
* pointer while on the free list.
*/
union mcluster {
- union mcluster *mcl_next;
- char mcl_buf[MCLBYTES];
+ union mcluster *mcl_next;
+ char mcl_buf[MCLBYTES];
};
-#define MCLALLOC(p, how) ((p) = m_mclalloc(how))
+#define MCLALLOC(p, how) ((p) = m_mclalloc(how))
-#define MCLFREE(p) m_mclfree(p)
+#define MCLFREE(p) m_mclfree(p)
-#define MCLGET(m, how) ((m) = m_mclget(m, how))
+#define MCLGET(m, how) ((m) = m_mclget(m, how))
/*
* Mbuf big cluster
*/
union mbigcluster {
- union mbigcluster *mbc_next;
- char mbc_buf[MBIGCLBYTES];
+ union mbigcluster *mbc_next;
+ char mbc_buf[MBIGCLBYTES];
};
/*
* Mbuf jumbo cluster
*/
union m16kcluster {
- union m16kcluster *m16kcl_next;
- char m16kcl_buf[M16KCLBYTES];
+ union m16kcluster *m16kcl_next;
+ char m16kcl_buf[M16KCLBYTES];
};
-#define MCLHASREFERENCE(m) m_mclhasreference(m)
+#define MCLHASREFERENCE(m) m_mclhasreference(m)
/*
* MFREE(struct mbuf *m, struct mbuf *n)
* Place the successor, if any, in n.
*/
-#define MFREE(m, n) ((n) = m_free(m))
+#define MFREE(m, n) ((n) = m_free(m))
/*
* Copy mbuf pkthdr from from to to.
* from must have M_PKTHDR set, and to must be empty.
* aux pointer will be moved to `to'.
*/
-#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from)
+#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from)
-#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from)
+#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from)
-#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from)
+#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from)
/*
- * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
- * an object of the specified size at the end of the mbuf, longword aligned.
+ * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
+ * be both the local data payload, or an external buffer area, depending on
+ * whether M_EXT is set).
*/
-#define M_ALIGN(m, len) \
-do { \
- (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1); \
-} while (0)
+#define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m))
/*
- * As above, for mbufs allocated with m_gethdr/MGETHDR
- * or initialized by M_COPY_PKTHDR.
+ * These macros are mapped to the appropriate KPIs, so that private code
+ * can be simply recompiled in order to be forward-compatible with future
+ * changes toward the struture sizes.
*/
-#define MH_ALIGN(m, len) \
-do { \
- (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1); \
-} while (0)
+#define MLEN mbuf_get_mlen() /* normal mbuf data len */
+#define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */
+#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */
+/*
+ * Return the address of the start of the buffer associated with an mbuf,
+ * handling external storage, packet-header mbufs, and regular data mbufs.
+ */
+#define M_START(m) \
+ (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \
+ ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \
+ &(m)->m_dat[0])
+
+/*
+ * Return the size of the buffer associated with an mbuf, handling external
+ * storage, packet-header mbufs, and regular data mbufs.
+ */
+#define M_SIZE(m) \
+ (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \
+ ((m)->m_flags & M_PKTHDR) ? MHLEN : \
+ MLEN)
+
+#define M_ALIGN(m, len) m_align(m, len)
+#define MH_ALIGN(m, len) m_align(m, len)
+#define MEXT_ALIGN(m, len) m_align(m, len)
/*
- * Compute the amount of space available
- * before the current start of data in an mbuf.
- * Subroutine - data not available if certain references.
+ * Compute the amount of space available before the current start of data in
+ * an mbuf.
+ *
+ * The M_WRITABLE() is a temporary, conservative safety measure: the burden
+ * of checking writability of the mbuf data area rests solely with the caller.
*/
-#define M_LEADINGSPACE(m) m_leadingspace(m)
+#define M_LEADINGSPACE(m) \
+ (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0)
/*
- * Compute the amount of space available
- * after the end of data in an mbuf.
- * Subroutine - data not available if certain references.
+ * Compute the amount of space available after the end of data in an mbuf.
+ *
+ * The M_WRITABLE() is a temporary, conservative safety measure: the burden
+ * of checking writability of the mbuf data area rests solely with the caller.
*/
-#define M_TRAILINGSPACE(m) m_trailingspace(m)
+#define M_TRAILINGSPACE(m) \
+ (M_WRITABLE(m) ? \
+ ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0)
/*
* Arrange to prepend space of size plen to mbuf m.
* If how is M_DONTWAIT and allocation fails, the original mbuf chain
* is freed and m is set to NULL.
*/
-#define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how)))
+#define M_PREPEND(m, plen, how, align) \
+ ((m) = m_prepend_2((m), (plen), (how), (align)))
/* change mbuf to new type */
-#define MCHTYPE(m, t) m_mchtype(m, t)
+#define MCHTYPE(m, t) m_mchtype(m, t)
/* compatiblity with 4.3 */
-#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
+#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
-#define MBSHIFT 20 /* 1MB */
-#define MBSIZE (1 << MBSHIFT)
-#define GBSHIFT 30 /* 1GB */
-#define GBSIZE (1 << GBSHIFT)
+#define MBSHIFT 20 /* 1MB */
+#define MBSIZE (1 << MBSHIFT)
+#define GBSHIFT 30 /* 1GB */
+#define GBSIZE (1 << GBSHIFT)
/*
* M_STRUCT_GET ensures that intermediate protocol header (from "off" to
* M_STRUCT_GET0 does the same, except that it aligns the structure at
* very top of mbuf. GET0 is likely to make memory copy than GET.
*/
-#define M_STRUCT_GET(val, typ, m, off, len) \
-do { \
- struct mbuf *t; \
- int tmp; \
- \
- if ((m)->m_len >= (off) + (len)) { \
- (val) = (typ)(mtod((m), caddr_t) + (off)); \
- } else { \
- t = m_pulldown((m), (off), (len), &tmp); \
- if (t != NULL) { \
- if (t->m_len < tmp + (len)) \
- panic("m_pulldown malfunction"); \
- (val) = (typ)(mtod(t, caddr_t) + tmp); \
- } else { \
- (val) = (typ)NULL; \
- (m) = NULL; \
- } \
- } \
+#define M_STRUCT_GET(val, typ, m, off, len) \
+do { \
+ struct mbuf *t; \
+ int tmp; \
+ \
+ if ((m)->m_len >= (off) + (len)) { \
+ (val) = (typ)(mtod((m), caddr_t) + (off)); \
+ } else { \
+ t = m_pulldown((m), (off), (len), &tmp); \
+ if (t != NULL) { \
+ if (t->m_len < tmp + (len)) \
+ panic("m_pulldown malfunction"); \
+ (val) = (typ)(mtod(t, caddr_t) + tmp); \
+ } else { \
+ (val) = (typ)NULL; \
+ (m) = NULL; \
+ } \
+ } \
} while (0)
-#define M_STRUCT_GET0(val, typ, m, off, len) \
-do { \
- struct mbuf *t; \
- \
- if ((off) == 0 && ((m)->m_len >= (len))) { \
- (val) = (typ)(void *)mtod(m, caddr_t); \
- } else { \
- t = m_pulldown((m), (off), (len), NULL); \
- if (t != NULL) { \
- if (t->m_len < (len)) \
- panic("m_pulldown malfunction"); \
- (val) = (typ)(void *)mtod(t, caddr_t); \
- } else { \
- (val) = (typ)NULL; \
- (m) = NULL; \
- } \
- } \
+#define M_STRUCT_GET0(val, typ, m, off, len) \
+do { \
+ struct mbuf *t; \
+ \
+ if ((off) == 0 && ((m)->m_len >= (len))) { \
+ (val) = (typ)(void *)mtod(m, caddr_t); \
+ } else { \
+ t = m_pulldown((m), (off), (len), NULL); \
+ if (t != NULL) { \
+ if (t->m_len < (len)) \
+ panic("m_pulldown malfunction"); \
+ (val) = (typ)(void *)mtod(t, caddr_t); \
+ } else { \
+ (val) = (typ)NULL; \
+ (m) = NULL; \
+ } \
+ } \
} while (0)
-#define MBUF_INPUT_CHECK(m, rcvif) \
-do { \
- if (!(m->m_flags & MBUF_PKTHDR) || \
- m->m_len < 0 || \
- m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \
- m->m_type == MT_FREE || \
- ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \
- panic_plain("Failed mbuf validity check: mbuf %p len %d " \
- "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \
- m, m->m_len, m->m_type, m->m_flags, \
- ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
- if_name(rcvif), \
- (rcvif->if_flags & 0xffff)); \
- } \
+#define MBUF_INPUT_CHECK(m, rcvif) \
+do { \
+ if (!(m->m_flags & MBUF_PKTHDR) || \
+ m->m_len < 0 || \
+ m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \
+ m->m_type == MT_FREE || \
+ ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \
+ panic_plain("Failed mbuf validity check: mbuf %p len %d " \
+ "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \
+ m, m->m_len, m->m_type, m->m_flags, \
+ ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
+ if_name(rcvif), \
+ (rcvif->if_flags & 0xffff)); \
+ } \
} while (0)
/*
*
* m_next is ignored, so queueing chains of mbufs is possible
*/
-#define MBUFQ_HEAD(name) \
-struct name { \
- struct mbuf *mq_first; /* first packet */ \
- struct mbuf **mq_last; /* addr of last next packet */ \
+#define MBUFQ_HEAD(name) \
+struct name { \
+ struct mbuf *mq_first; /* first packet */ \
+ struct mbuf **mq_last; /* addr of last next packet */ \
}
-#define MBUFQ_INIT(q) do { \
- MBUFQ_FIRST(q) = NULL; \
- (q)->mq_last = &MBUFQ_FIRST(q); \
+#define MBUFQ_INIT(q) do { \
+ MBUFQ_FIRST(q) = NULL; \
+ (q)->mq_last = &MBUFQ_FIRST(q); \
} while (0)
-#define MBUFQ_PREPEND(q, m) do { \
- if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \
- (q)->mq_last = &MBUFQ_NEXT(m); \
- MBUFQ_FIRST(q) = (m); \
+#define MBUFQ_PREPEND(q, m) do { \
+ if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \
+ (q)->mq_last = &MBUFQ_NEXT(m); \
+ MBUFQ_FIRST(q) = (m); \
} while (0)
-#define MBUFQ_ENQUEUE(q, m) do { \
- MBUFQ_NEXT(m) = NULL; \
- *(q)->mq_last = (m); \
- (q)->mq_last = &MBUFQ_NEXT(m); \
+#define MBUFQ_ENQUEUE(q, m) do { \
+ MBUFQ_NEXT(m) = NULL; \
+ *(q)->mq_last = (m); \
+ (q)->mq_last = &MBUFQ_NEXT(m); \
} while (0)
-#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \
- MBUFQ_NEXT(n) = NULL; \
- *(q)->mq_last = (m); \
- (q)->mq_last = &MBUFQ_NEXT(n); \
+#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \
+ MBUFQ_NEXT(n) = NULL; \
+ *(q)->mq_last = (m); \
+ (q)->mq_last = &MBUFQ_NEXT(n); \
} while (0)
-#define MBUFQ_DEQUEUE(q, m) do { \
- if (((m) = MBUFQ_FIRST(q)) != NULL) { \
- if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \
- (q)->mq_last = &MBUFQ_FIRST(q); \
- else \
- MBUFQ_NEXT(m) = NULL; \
- } \
+#define MBUFQ_DEQUEUE(q, m) do { \
+ if (((m) = MBUFQ_FIRST(q)) != NULL) { \
+ if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \
+ (q)->mq_last = &MBUFQ_FIRST(q); \
+ else \
+ MBUFQ_NEXT(m) = NULL; \
+ } \
} while (0)
-#define MBUFQ_REMOVE(q, m) do { \
- if (MBUFQ_FIRST(q) == (m)) { \
- MBUFQ_DEQUEUE(q, m); \
- } else { \
- struct mbuf *_m = MBUFQ_FIRST(q); \
- while (MBUFQ_NEXT(_m) != (m)) \
- _m = MBUFQ_NEXT(_m); \
- if ((MBUFQ_NEXT(_m) = \
- MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \
- (q)->mq_last = &MBUFQ_NEXT(_m); \
- } \
+#define MBUFQ_REMOVE(q, m) do { \
+ if (MBUFQ_FIRST(q) == (m)) { \
+ MBUFQ_DEQUEUE(q, m); \
+ } else { \
+ struct mbuf *_m = MBUFQ_FIRST(q); \
+ while (MBUFQ_NEXT(_m) != (m)) \
+ _m = MBUFQ_NEXT(_m); \
+ if ((MBUFQ_NEXT(_m) = \
+ MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \
+ (q)->mq_last = &MBUFQ_NEXT(_m); \
+ } \
} while (0)
-#define MBUFQ_DRAIN(q) do { \
- struct mbuf *__m0; \
- while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \
- MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \
- MBUFQ_NEXT(__m0) = NULL; \
- m_freem(__m0); \
- } \
- (q)->mq_last = &MBUFQ_FIRST(q); \
+#define MBUFQ_DRAIN(q) do { \
+ struct mbuf *__m0; \
+ while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \
+ MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \
+ MBUFQ_NEXT(__m0) = NULL; \
+ m_freem(__m0); \
+ } \
+ (q)->mq_last = &MBUFQ_FIRST(q); \
} while (0)
-#define MBUFQ_FOREACH(m, q) \
- for ((m) = MBUFQ_FIRST(q); \
- (m); \
+#define MBUFQ_FOREACH(m, q) \
+ for ((m) = MBUFQ_FIRST(q); \
+ (m); \
(m) = MBUFQ_NEXT(m))
-#define MBUFQ_FOREACH_SAFE(m, q, tvar) \
- for ((m) = MBUFQ_FIRST(q); \
- (m) && ((tvar) = MBUFQ_NEXT(m), 1); \
+#define MBUFQ_FOREACH_SAFE(m, q, tvar) \
+ for ((m) = MBUFQ_FIRST(q); \
+ (m) && ((tvar) = MBUFQ_NEXT(m), 1); \
(m) = (tvar))
-#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
-#define MBUFQ_FIRST(q) ((q)->mq_first)
-#define MBUFQ_NEXT(m) ((m)->m_nextpkt)
-#define MBUFQ_LAST(q) (*(q)->mq_last)
+#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
+#define MBUFQ_FIRST(q) ((q)->mq_first)
+#define MBUFQ_NEXT(m) ((m)->m_nextpkt)
+/*
+ * mq_last is initialized to point to mq_first, so check if they're
+ * equal and return NULL when the list is empty. Otherwise, we need
+ * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
+ * to the base mbuf address to return to caller.
+ */
+#define MBUFQ_LAST(head) \
+ (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \
+ ((struct mbuf *)(void *)((char *)(head)->mq_last - \
+ __builtin_offsetof(struct mbuf, m_nextpkt))))
-#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
-#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
+#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
+#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
#endif /* XNU_KERNEL_PRIVATE */
/*
* Mbuf statistics (legacy).
*/
struct mbstat {
- u_int32_t m_mbufs; /* mbufs obtained from page pool */
- u_int32_t m_clusters; /* clusters obtained from page pool */
- u_int32_t m_spare; /* spare field */
- u_int32_t m_clfree; /* free clusters */
- u_int32_t m_drops; /* times failed to find space */
- u_int32_t m_wait; /* times waited for space */
- u_int32_t m_drain; /* times drained protocols for space */
- u_short m_mtypes[256]; /* type specific mbuf allocations */
- u_int32_t m_mcfail; /* times m_copym failed */
- u_int32_t m_mpfail; /* times m_pullup failed */
- u_int32_t m_msize; /* length of an mbuf */
- u_int32_t m_mclbytes; /* length of an mbuf cluster */
- u_int32_t m_minclsize; /* min length of data to allocate a cluster */
- u_int32_t m_mlen; /* length of data in an mbuf */
- u_int32_t m_mhlen; /* length of data in a header mbuf */
- u_int32_t m_bigclusters; /* clusters obtained from page pool */
- u_int32_t m_bigclfree; /* free clusters */
- u_int32_t m_bigmclbytes; /* length of an mbuf cluster */
+ u_int32_t m_mbufs; /* mbufs obtained from page pool */
+ u_int32_t m_clusters; /* clusters obtained from page pool */
+ u_int32_t m_spare; /* spare field */
+ u_int32_t m_clfree; /* free clusters */
+ u_int32_t m_drops; /* times failed to find space */
+ u_int32_t m_wait; /* times waited for space */
+ u_int32_t m_drain; /* times drained protocols for space */
+ u_short m_mtypes[256]; /* type specific mbuf allocations */
+ u_int32_t m_mcfail; /* times m_copym failed */
+ u_int32_t m_mpfail; /* times m_pullup failed */
+ u_int32_t m_msize; /* length of an mbuf */
+ u_int32_t m_mclbytes; /* length of an mbuf cluster */
+ u_int32_t m_minclsize; /* min length of data to allocate a cluster */
+ u_int32_t m_mlen; /* length of data in an mbuf */
+ u_int32_t m_mhlen; /* length of data in a header mbuf */
+ u_int32_t m_bigclusters; /* clusters obtained from page pool */
+ u_int32_t m_bigclfree; /* free clusters */
+ u_int32_t m_bigmclbytes; /* length of an mbuf cluster */
};
/* Compatibillity with 10.3 */
struct ombstat {
- u_int32_t m_mbufs; /* mbufs obtained from page pool */
- u_int32_t m_clusters; /* clusters obtained from page pool */
- u_int32_t m_spare; /* spare field */
- u_int32_t m_clfree; /* free clusters */
- u_int32_t m_drops; /* times failed to find space */
- u_int32_t m_wait; /* times waited for space */
- u_int32_t m_drain; /* times drained protocols for space */
- u_short m_mtypes[256]; /* type specific mbuf allocations */
- u_int32_t m_mcfail; /* times m_copym failed */
- u_int32_t m_mpfail; /* times m_pullup failed */
- u_int32_t m_msize; /* length of an mbuf */
- u_int32_t m_mclbytes; /* length of an mbuf cluster */
- u_int32_t m_minclsize; /* min length of data to allocate a cluster */
- u_int32_t m_mlen; /* length of data in an mbuf */
- u_int32_t m_mhlen; /* length of data in a header mbuf */
+ u_int32_t m_mbufs; /* mbufs obtained from page pool */
+ u_int32_t m_clusters; /* clusters obtained from page pool */
+ u_int32_t m_spare; /* spare field */
+ u_int32_t m_clfree; /* free clusters */
+ u_int32_t m_drops; /* times failed to find space */
+ u_int32_t m_wait; /* times waited for space */
+ u_int32_t m_drain; /* times drained protocols for space */
+ u_short m_mtypes[256]; /* type specific mbuf allocations */
+ u_int32_t m_mcfail; /* times m_copym failed */
+ u_int32_t m_mpfail; /* times m_pullup failed */
+ u_int32_t m_msize; /* length of an mbuf */
+ u_int32_t m_mclbytes; /* length of an mbuf cluster */
+ u_int32_t m_minclsize; /* min length of data to allocate a cluster */
+ u_int32_t m_mlen; /* length of data in an mbuf */
+ u_int32_t m_mhlen; /* length of data in a header mbuf */
};
/*
* mbuf class statistics.
*/
-#define MAX_MBUF_CNAME 15
+#define MAX_MBUF_CNAME 15
#if defined(XNU_KERNEL_PRIVATE)
/* For backwards compatibility with 32-bit userland process */
struct omb_class_stat {
- char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
- u_int32_t mbcl_size; /* buffer size */
- u_int32_t mbcl_total; /* # of buffers created */
- u_int32_t mbcl_active; /* # of active buffers */
- u_int32_t mbcl_infree; /* # of available buffers */
- u_int32_t mbcl_slab_cnt; /* # of available slabs */
- u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
- u_int64_t mbcl_free_cnt; /* # of times free is called */
- u_int64_t mbcl_notified; /* # of notified wakeups */
- u_int64_t mbcl_purge_cnt; /* # of purges so far */
- u_int64_t mbcl_fail_cnt; /* # of allocation failures */
- u_int32_t mbcl_ctotal; /* total only for this class */
+ char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
+ u_int32_t mbcl_size; /* buffer size */
+ u_int32_t mbcl_total; /* # of buffers created */
+ u_int32_t mbcl_active; /* # of active buffers */
+ u_int32_t mbcl_infree; /* # of available buffers */
+ u_int32_t mbcl_slab_cnt; /* # of available slabs */
+ u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
+ u_int64_t mbcl_free_cnt; /* # of times free is called */
+ u_int64_t mbcl_notified; /* # of notified wakeups */
+ u_int64_t mbcl_purge_cnt; /* # of purges so far */
+ u_int64_t mbcl_fail_cnt; /* # of allocation failures */
+ u_int32_t mbcl_ctotal; /* total only for this class */
+ u_int32_t mbcl_release_cnt; /* amount of memory returned */
/*
* Cache layer statistics
*/
- u_int32_t mbcl_mc_state; /* cache state (see below) */
- u_int32_t mbcl_mc_cached; /* # of cached buffers */
- u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
- u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
- u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
- u_int64_t mbcl_reserved[4]; /* for future use */
+ u_int32_t mbcl_mc_state; /* cache state (see below) */
+ u_int32_t mbcl_mc_cached; /* # of cached buffers */
+ u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
+ u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
+ u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
+ u_int64_t mbcl_reserved[4]; /* for future use */
} __attribute__((__packed__));
#endif /* XNU_KERNEL_PRIVATE */
typedef struct mb_class_stat {
- char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
- u_int32_t mbcl_size; /* buffer size */
- u_int32_t mbcl_total; /* # of buffers created */
- u_int32_t mbcl_active; /* # of active buffers */
- u_int32_t mbcl_infree; /* # of available buffers */
- u_int32_t mbcl_slab_cnt; /* # of available slabs */
+ char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
+ u_int32_t mbcl_size; /* buffer size */
+ u_int32_t mbcl_total; /* # of buffers created */
+ u_int32_t mbcl_active; /* # of active buffers */
+ u_int32_t mbcl_infree; /* # of available buffers */
+ u_int32_t mbcl_slab_cnt; /* # of available slabs */
#if defined(KERNEL) || defined(__LP64__)
- u_int32_t mbcl_pad; /* padding */
+ u_int32_t mbcl_pad; /* padding */
#endif /* KERNEL || __LP64__ */
- u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
- u_int64_t mbcl_free_cnt; /* # of times free is called */
- u_int64_t mbcl_notified; /* # of notified wakeups */
- u_int64_t mbcl_purge_cnt; /* # of purges so far */
- u_int64_t mbcl_fail_cnt; /* # of allocation failures */
- u_int32_t mbcl_ctotal; /* total only for this class */
+ u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
+ u_int64_t mbcl_free_cnt; /* # of times free is called */
+ u_int64_t mbcl_notified; /* # of notified wakeups */
+ u_int64_t mbcl_purge_cnt; /* # of purges so far */
+ u_int64_t mbcl_fail_cnt; /* # of allocation failures */
+ u_int32_t mbcl_ctotal; /* total only for this class */
+ u_int32_t mbcl_release_cnt; /* amount of memory returned */
/*
* Cache layer statistics
*/
- u_int32_t mbcl_mc_state; /* cache state (see below) */
- u_int32_t mbcl_mc_cached; /* # of cached buffers */
- u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
- u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
- u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
- u_int64_t mbcl_reserved[4]; /* for future use */
+ u_int32_t mbcl_mc_state; /* cache state (see below) */
+ u_int32_t mbcl_mc_cached; /* # of cached buffers */
+ u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
+ u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
+ u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
+ u_int32_t mbcl_peak_reported; /* last usage peak reported */
+ u_int32_t mbcl_reserved[7]; /* for future use */
} mb_class_stat_t;
-#define MCS_DISABLED 0 /* cache is permanently disabled */
-#define MCS_ONLINE 1 /* cache is online */
-#define MCS_PURGING 2 /* cache is being purged */
-#define MCS_OFFLINE 3 /* cache is offline (resizing) */
+#define MCS_DISABLED 0 /* cache is permanently disabled */
+#define MCS_ONLINE 1 /* cache is online */
+#define MCS_PURGING 2 /* cache is being purged */
+#define MCS_OFFLINE 3 /* cache is offline (resizing) */
#if defined(XNU_KERNEL_PRIVATE)
/* For backwards compatibility with 32-bit userland process */
struct omb_stat {
- u_int32_t mbs_cnt; /* number of classes */
- struct omb_class_stat mbs_class[1]; /* class array */
+ u_int32_t mbs_cnt; /* number of classes */
+ struct omb_class_stat mbs_class[1]; /* class array */
} __attribute__((__packed__));
#endif /* XNU_KERNEL_PRIVATE */
typedef struct mb_stat {
- u_int32_t mbs_cnt; /* number of classes */
+ u_int32_t mbs_cnt; /* number of classes */
#if defined(KERNEL) || defined(__LP64__)
- u_int32_t mbs_pad; /* padding */
+ u_int32_t mbs_pad; /* padding */
#endif /* KERNEL || __LP64__ */
- mb_class_stat_t mbs_class[1]; /* class array */
+ mb_class_stat_t mbs_class[1]; /* class array */
} mb_stat_t;
#ifdef PRIVATE
-#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */
+#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */
typedef struct mleak_trace_stat {
- u_int64_t mltr_collisions;
- u_int64_t mltr_hitcount;
- u_int64_t mltr_allocs;
- u_int64_t mltr_depth;
- u_int64_t mltr_addr[MLEAK_STACK_DEPTH];
+ u_int64_t mltr_collisions;
+ u_int64_t mltr_hitcount;
+ u_int64_t mltr_allocs;
+ u_int64_t mltr_depth;
+ u_int64_t mltr_addr[MLEAK_STACK_DEPTH];
} mleak_trace_stat_t;
typedef struct mleak_stat {
- u_int32_t ml_isaddr64; /* 64-bit KVA? */
- u_int32_t ml_cnt; /* number of traces */
- mleak_trace_stat_t ml_trace[1]; /* trace array */
+ u_int32_t ml_isaddr64; /* 64-bit KVA? */
+ u_int32_t ml_cnt; /* number of traces */
+ mleak_trace_stat_t ml_trace[1]; /* trace array */
} mleak_stat_t;
struct mleak_table {
- u_int32_t mleak_capture; /* sampling capture counter */
- u_int32_t mleak_sample_factor; /* sample factor */
+ u_int32_t mleak_capture; /* sampling capture counter */
+ u_int32_t mleak_sample_factor; /* sample factor */
/* Times two active records want to occupy the same spot */
u_int64_t alloc_collisions;
* Exported (private)
*/
-extern struct mbstat mbstat; /* statistics */
+extern struct mbstat mbstat; /* statistics */
__END_DECLS
#endif /* KERNEL_PRIVATE */
/* flags to m_get/MGET */
/* Need to include malloc.h to get right options for malloc */
-#include <sys/malloc.h>
+#include <sys/malloc.h>
struct mbuf;
/* length to m_copy to copy all */
-#define M_COPYALL 1000000000
+#define M_COPYALL 1000000000
-#define M_DONTWAIT M_NOWAIT
-#define M_WAIT M_WAITOK
+#define M_DONTWAIT M_NOWAIT
+#define M_WAIT M_WAITOK
/* modes for m_copym and variants */
-#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
-#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
-#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
+#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
+#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
+#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
+#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */
+#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */
-/*
- * These macros are mapped to the appropriate KPIs, so that private code
- * can be simply recompiled in order to be forward-compatible with future
- * changes toward the struture sizes.
- */
-#define MLEN mbuf_get_mlen() /* normal data len */
-#define MHLEN mbuf_get_mhlen() /* data len w/pkthdr */
-
-#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */
-
-extern void m_freem(struct mbuf *);
+extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem);
extern u_int64_t mcl_to_paddr(char *);
extern void m_adj(struct mbuf *, int);
extern void m_cat(struct mbuf *, struct mbuf *);
extern struct mbuf *m_getpackets(int, int, int);
extern struct mbuf *m_mclget(struct mbuf *, int);
extern void *m_mtod(struct mbuf *);
-extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
+extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);
extern struct mbuf *m_pullup(struct mbuf *, int);
extern struct mbuf *m_split(struct mbuf *, int, int);
extern void m_mclfree(caddr_t p);
* headers), and assert otherwise.
*/
#if defined(__i386__) || defined(__x86_64__)
-#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
+#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
#else /* !__i386__ && !__x86_64__ */
-#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \
- if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \
- if (((_m)->m_flags & M_PKTHDR) && \
- (_m)->m_pkthdr.rcvif != NULL) { \
- panic_plain("\n%s: mbuf %p data ptr %p is not " \
- "32-bit aligned [%s: alignerrs=%lld]\n", \
- __func__, (_m), (_m)->m_data, \
- if_name((_m)->m_pkthdr.rcvif), \
- (_m)->m_pkthdr.rcvif->if_alignerrs); \
- } else { \
- panic_plain("\n%s: mbuf %p data ptr %p is not " \
- "32-bit aligned\n", \
- __func__, (_m), (_m)->m_data); \
- } \
- } \
+#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \
+ if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \
+ if (((_m)->m_flags & M_PKTHDR) && \
+ (_m)->m_pkthdr.rcvif != NULL) { \
+ panic_plain("\n%s: mbuf %p data ptr %p is not " \
+ "32-bit aligned [%s: alignerrs=%lld]\n", \
+ __func__, (_m), (_m)->m_data, \
+ if_name((_m)->m_pkthdr.rcvif), \
+ (_m)->m_pkthdr.rcvif->if_alignerrs); \
+ } else { \
+ panic_plain("\n%s: mbuf %p data ptr %p is not " \
+ "32-bit aligned\n", \
+ __func__, (_m), (_m)->m_data); \
+ } \
+ } \
} while (0)
#endif /* !__i386__ && !__x86_64__ */
/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
-#define MBUF_SC_MAX_CLASSES 10
+#define MBUF_SC_MAX_CLASSES 10
/*
* These conversion macros rely on the corresponding MBUF_SC and
* MBUF_SC_AV ] ==> MBUF_TC_VI
* MBUF_SC_RV ]
* MBUF_SC_VI ]
+ * MBUF_SC_SIG ]
*
* MBUF_SC_VO ] ==> MBUF_TC_VO
* MBUF_SC_CTL ]
* corresponding portion, and never assume that a higher class corresponds
* to a higher index.
*/
-#define MBUF_SCVAL(x) ((x) & 0xffff)
-#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3)
-#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7)
-#define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
+#define MBUF_SCVAL(x) ((x) & 0xffff)
+#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3)
+#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7)
+#define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
((_sc) == MBUF_SC_BK))
-
-#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
-#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
-#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE)
-#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD)
-#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM)
-#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV)
-#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV)
-#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI)
-#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO)
-#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL)
-
-#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS)
-#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK)
-#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE)
-#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD)
-#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM)
-#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV)
-#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV)
-#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI)
-#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO)
-#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL)
-
-#define MBUF_VALID_SC(c) \
- (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \
- c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \
- c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO || \
- c == MBUF_SC_CTL)
-
-#define MBUF_VALID_SCIDX(c) \
- (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \
- c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \
- c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO || \
- c == SCIDX_CTL)
-
-#define MBUF_VALID_SCVAL(c) \
- (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \
- c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \
- c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO || \
- c == SCVAL_CTL)
-
-extern union mbigcluster *mbutl; /* start VA of mbuf pool */
-extern union mbigcluster *embutl; /* end VA of mbuf pool */
-extern unsigned int nmbclusters; /* number of mapped clusters */
-extern int njcl; /* # of jumbo clusters */
-extern int njclbytes; /* size of a jumbo cluster */
-extern int max_hdr; /* largest link+protocol header */
-extern int max_datalen; /* MHLEN - max_hdr */
+#define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
+#define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \
+ (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)
+
+#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
+#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
+#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE)
+#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD)
+#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM)
+#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV)
+#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV)
+#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI)
+#define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG)
+#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO)
+#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL)
+
+#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS)
+#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK)
+#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE)
+#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD)
+#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM)
+#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV)
+#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV)
+#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI)
+#define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG)
+#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO)
+#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL)
+
+#define MBUF_VALID_SC(c) \
+ (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \
+ c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \
+ c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \
+ c == MBUF_SC_VO || c == MBUF_SC_CTL)
+
+#define MBUF_VALID_SCIDX(c) \
+ (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \
+ c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \
+ c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \
+ c == SCIDX_VO || c == SCIDX_CTL)
+
+#define MBUF_VALID_SCVAL(c) \
+ (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \
+ c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \
+ c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \
+ c == SCVAL_VO || SCVAL_CTL)
+
+extern unsigned char *mbutl; /* start VA of mbuf pool */
+extern unsigned char *embutl; /* end VA of mbuf pool */
+extern unsigned int nmbclusters; /* number of mapped clusters */
+extern int njcl; /* # of jumbo clusters */
+extern int njclbytes; /* size of a jumbo cluster */
+extern int max_hdr; /* largest link+protocol header */
+extern int max_datalen; /* MHLEN - max_hdr */
/* Use max_linkhdr instead of _max_linkhdr */
-extern int _max_linkhdr; /* largest link-level header */
+extern int _max_linkhdr; /* largest link-level header */
/* Use max_protohdr instead of _max_protohdr */
-extern int _max_protohdr; /* largest protocol header */
+extern int _max_protohdr; /* largest protocol header */
__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
__private_extern__ void mbinit(void);
__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
- void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
+ void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int);
__private_extern__ caddr_t m_bigalloc(int);
__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
__private_extern__ int m_reinit(struct mbuf *, int);
-__private_extern__ struct mbuf *m_free(struct mbuf *);
+__private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free);
__private_extern__ struct mbuf *m_getclr(int, int);
__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
__private_extern__ unsigned int m_length(struct mbuf *);
__private_extern__ int m_mtocl(void *);
__private_extern__ union mcluster *m_cltom(int);
-__private_extern__ int m_trailingspace(struct mbuf *);
-__private_extern__ int m_leadingspace(struct mbuf *);
+__private_extern__ void m_align(struct mbuf *, int);
__private_extern__ struct mbuf *m_normalize(struct mbuf *m);
__private_extern__ void m_mchtype(struct mbuf *m, int t);
__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
unsigned int *, int, int, size_t);
+__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
+__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
+__private_extern__ int m_ext_paired_is_active(struct mbuf *);
+__private_extern__ void m_ext_paired_activate(struct mbuf *);
+
+__private_extern__ void mbuf_drain(boolean_t);
+
/*
* Packets may have annotations attached by affixing a list of "packet
* tags" to the pkthdr structure. Packet tags are dynamically allocated
* struct m_tag *mtag = &p->tag;
*/
-#define KERNEL_MODULE_TAG_ID 0
+#define KERNEL_MODULE_TAG_ID 0
enum {
- KERNEL_TAG_TYPE_NONE = 0,
- KERNEL_TAG_TYPE_DUMMYNET = 1,
- KERNEL_TAG_TYPE_DIVERT = 2,
- KERNEL_TAG_TYPE_IPFORWARD = 3,
- KERNEL_TAG_TYPE_IPFILT = 4,
- KERNEL_TAG_TYPE_MACLABEL = 5,
- KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6,
- KERNEL_TAG_TYPE_ENCAP = 8,
- KERNEL_TAG_TYPE_INET6 = 9,
- KERNEL_TAG_TYPE_IPSEC = 10,
- KERNEL_TAG_TYPE_DRVAUX = 11
+ KERNEL_TAG_TYPE_NONE = 0,
+ KERNEL_TAG_TYPE_DUMMYNET = 1,
+ KERNEL_TAG_TYPE_DIVERT = 2,
+ KERNEL_TAG_TYPE_IPFORWARD = 3,
+ KERNEL_TAG_TYPE_IPFILT = 4,
+ KERNEL_TAG_TYPE_MACLABEL = 5,
+ KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6,
+ KERNEL_TAG_TYPE_ENCAP = 8,
+ KERNEL_TAG_TYPE_INET6 = 9,
+ KERNEL_TAG_TYPE_IPSEC = 10,
+ KERNEL_TAG_TYPE_DRVAUX = 11,
+ KERNEL_TAG_TYPE_CFIL_UDP = 13,
+ KERNEL_TAG_TYPE_PF_REASS = 14,
};
/* Packet tag routines */
__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
__private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
- struct mbuf *);
+ struct mbuf *);
__private_extern__ void m_tag_free(struct m_tag *);
__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
__private_extern__ struct m_tag *m_tag_first(struct mbuf *);
__private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
-__END_DECLS
-#endif /* XNU_KERNEL_PRIVATE */
-#ifdef KERNEL
-#include <sys/kpi_mbuf.h>
-#ifdef XNU_KERNEL_PRIVATE
-__BEGIN_DECLS
-
__private_extern__ void m_scratch_init(struct mbuf *);
__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **);
__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
-#define ADDCARRY(_x) do { \
- while (((_x) >> 16) != 0) \
- (_x) = ((_x) >> 16) + ((_x) & 0xffff); \
+#define ADDCARRY(_x) do { \
+ while (((_x) >> 16) != 0) \
+ (_x) = ((_x) >> 16) + ((_x) & 0xffff); \
} while (0)
__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
- u_int32_t, u_int32_t);
+ u_int32_t, u_int32_t, u_int32_t);
__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
+__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
+ m_ext_free_func_t, caddr_t);
+__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
+__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
+__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);
+
+__private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);
+__private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t);
+
__END_DECLS
#endif /* XNU_KERNEL_PRIVATE */
-#endif /* KERNEL */
-#endif /* !_SYS_MBUF_H_ */
+#endif /* !_SYS_MBUF_H_ */
projects / apple / xnu.git / blobdiff