X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/39236c6e673c41db228275375ab7fdb0f837b292..eb6b6ca394357805f2bdba989abae309f718b4d8:/bsd/sys/mbuf.h diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h index 8a849338b..ff2653d96 100644 --- a/bsd/sys/mbuf.h +++ b/bsd/sys/mbuf.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2013 Apple Inc. All rights reserved. + * Copyright (c) 1999-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -76,14 +76,20 @@ * Version 2.0. */ -#ifndef _SYS_MBUF_H_ -#define _SYS_MBUF_H_ +#ifndef _SYS_MBUF_H_ +#define _SYS_MBUF_H_ -#include #include +#include +#include /* u_int32_t */ +#include /* u_int64_t */ +#include /* u_short */ -#ifdef XNU_KERNEL_PRIVATE +#ifdef KERNEL +#include +#endif +#ifdef XNU_KERNEL_PRIVATE #include #include #include @@ -100,58 +106,80 @@ * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since * the sizes of the structures are dependent upon specific xnu configs. */ -#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ -#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ +#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ +#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ + +#define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT) +#define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */ + +#define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT) +#define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */ + +#define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT) +#define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */ -#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT) -#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */ +#define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT) +#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */ -#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) -#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ +#define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT) +#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ -#define NMBPCLSHIFT (NMBPBGSHIFT - NCLPBGSHIFT) -#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */ +#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) +#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ -#define NCLPJCLSHIFT ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT) -#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ +#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT) +#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */ /* * Macros for type conversion * mtod(m,t) - convert mbuf pointer to data pointer of correct type + * mtodo(m, o) -- Same as above but with offset 'o' into data. * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) */ -#define mtod(m, t) ((t)m_mtod(m)) -#define dtom(x) m_dtom(x) +#define mtod(m, t) ((t)m_mtod(m)) +#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o))) +#define dtom(x) m_dtom(x) /* header at beginning of each mbuf: */ struct m_hdr { - struct mbuf *mh_next; /* next buffer in chain */ - struct mbuf *mh_nextpkt; /* next chain in queue/record */ - caddr_t mh_data; /* location of data */ - int32_t mh_len; /* amount of data in this mbuf */ - u_int16_t mh_type; /* type of data in this mbuf */ - u_int16_t mh_flags; /* flags; see below */ + struct mbuf *mh_next; /* next buffer in chain */ + struct mbuf *mh_nextpkt; /* next chain in queue/record */ + caddr_t mh_data; /* location of data */ + int32_t mh_len; /* amount of data in this mbuf */ + u_int16_t mh_type; /* type of data in this mbuf */ + u_int16_t mh_flags; /* flags; see below */ +#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) +/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN + * should be defined using the offsetof(struct mbuf, M_dat), since there is + * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler + * may (and does in the armv7k case) insert padding between m_hdr and M_dat in + * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced + * in the definition of mbuf. + */ +} __attribute__((aligned(8))); +#else }; +#endif /* * Packet tag structure (see below for details). */ struct m_tag { - u_int64_t m_tag_cookie; /* Error checking */ + u_int64_t m_tag_cookie; /* Error checking */ #ifndef __LP64__ - u_int32_t pad; /* For structure alignment */ + u_int32_t pad; /* For structure alignment */ #endif /* !__LP64__ */ - SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ - u_int16_t m_tag_type; /* Module specific type */ - u_int16_t m_tag_len; /* Length of data */ - u_int32_t m_tag_id; /* Module ID */ + SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ + u_int16_t m_tag_type; /* Module specific type */ + u_int16_t m_tag_len; /* Length of data */ + u_int32_t m_tag_id; /* Module ID */ }; -#define M_TAG_ALIGN(len) \ +#define M_TAG_ALIGN(len) \ (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)) -#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL -#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL +#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL +#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL /* * Packet tag header structure (at the top of mbuf). Pointers are @@ -159,87 +187,99 @@ struct m_tag { */ struct m_taghdr { #ifndef __LP64__ - u_int32_t pad; /* For structure alignment */ + u_int32_t pad; /* For structure alignment */ #endif /* !__LP64__ */ - u_int64_t refcnt; /* Number of tags in this mbuf */ + u_int64_t refcnt; /* Number of tags in this mbuf */ }; /* * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX). */ struct m_drvaux_tag { - u_int32_t da_family; /* IFNET_FAMILY values */ - u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ - u_int32_t da_reserved; /* for future */ - u_int32_t da_length; /* length of following data */ + u_int32_t da_family; /* IFNET_FAMILY values */ + u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ + u_int32_t da_reserved; /* for future */ + u_int32_t da_length; /* length of following data */ }; /* Values for pftag_flags (16-bit wide) */ -#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ -#define PF_TAG_FRAGCACHE 0x2 -#define PF_TAG_TRANSLATE_LOCALHOST 0x4 +#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ +#define PF_TAG_FRAGCACHE 0x2 +#define PF_TAG_TRANSLATE_LOCALHOST 0x4 #if PF_ECN -#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ -#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ +#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ +#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ #endif /* PF_ECN */ +#define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */ +#define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */ /* * PF mbuf tag */ struct pf_mtag { - u_int16_t pftag_flags; /* PF_TAG flags */ - u_int16_t pftag_rtableid; /* alternate routing table id */ - u_int16_t pftag_tag; - u_int16_t pftag_routed; -#if PF_ALTQ - u_int32_t pftag_qid; -#endif /* PF_ALTQ */ + u_int16_t pftag_flags; /* PF_TAG flags */ + u_int16_t pftag_rtableid; /* alternate routing table id */ + u_int16_t pftag_tag; + u_int16_t pftag_routed; #if PF_ECN - void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ + void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ #endif /* PF_ECN */ }; +/* + * PF fragment tag + */ +struct pf_fragment_tag { + uint32_t ft_id; /* fragment id */ + uint16_t ft_hdrlen; /* header length of reassembled pkt */ + uint16_t ft_unfragpartlen; /* length of the per-fragment headers */ + uint16_t ft_extoff; /* last extension header offset or 0 */ + uint16_t ft_maxlen; /* maximum fragment payload length */ +}; + /* * TCP mbuf tag */ struct tcp_pktinfo { union { struct { - u_int32_t segsz; /* segment size (actual MSS) */ + u_int32_t segsz; /* segment size (actual MSS) */ + u_int32_t start_seq; /* start seq of this packet */ + pid_t pid; + pid_t e_pid; } __tx; struct { - u_int16_t lro_pktlen; /* max seg size encountered */ - u_int8_t lro_npkts; /* # of coalesced TCP pkts */ - u_int8_t lro_timediff; /* time spent in LRO */ + u_int16_t lro_pktlen; /* max seg size encountered */ + u_int8_t lro_npkts; /* # of coalesced TCP pkts */ + u_int8_t lro_timediff; /* time spent in LRO */ } __rx; } __offload; union { - u_int32_t pri; /* send msg priority */ - u_int32_t seq; /* recv msg sequence # */ + u_int32_t pri; /* send msg priority */ + u_int32_t seq; /* recv msg sequence # */ } __msgattr; -#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz -#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen -#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts -#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff -#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri -#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq +#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz +#define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq +#define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid +#define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid +#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen +#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts +#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff +#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri +#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq }; /* * MPTCP mbuf tag */ struct mptcp_pktinfo { - u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ - union { - u_int64_t mtpi_dan; /* MPTCP Data Ack Number */ - struct { - u_int32_t mtpi_rel_seq; /* Relative Seq Number */ - u_int32_t mtpi_length; /* Length of mapping */ - } mtpi_subf; - }; -#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn -#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq -#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length -#define mp_dack proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan + u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ + u_int32_t mtpi_rel_seq; /* Relative Seq Number */ + u_int16_t mtpi_length; /* Length of mapping */ + u_int16_t mtpi_csum; +#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn +#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq +#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length +#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum }; /* @@ -250,61 +290,106 @@ struct mptcp_pktinfo { */ struct tcp_mtag { union { - struct tcp_pktinfo tm_tcp; /* TCP and below */ - struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ + struct tcp_pktinfo tm_tcp; /* TCP and below */ + struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ }; }; -/* - * IPSec mbuf tag - */ -struct ipsec_mtag { - uint32_t policy_id; -#define ipsec_policy proto_mtag.__pr_u.ipsec.policy_id +struct udp_mtag { + pid_t _pid; + pid_t _e_pid; +#define tx_udp_pid proto_mtag.__pr_u.udp._pid +#define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid +}; + +struct rawip_mtag { + pid_t _pid; + pid_t _e_pid; +#define tx_rawip_pid proto_mtag.__pr_u.rawip._pid +#define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid +}; + +struct driver_mtag_ { + uintptr_t _drv_tx_compl_arg; + uintptr_t _drv_tx_compl_data; + kern_return_t _drv_tx_status; + uint16_t _drv_flowid; +#define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg +#define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data +#define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status +#define drv_flowid builtin_mtag._drv_mtag._drv_flowid }; /* * Protocol specific mbuf tag (at most one protocol metadata per mbuf). * * Care must be taken to ensure that they are mutually exclusive, e.g. - * IPSec policy ID implies no TCP segment offload (which is fine given + * IPsec policy ID implies no TCP segment offload (which is fine given * that the former is used on the virtual ipsec interface that does * not advertise the TSO capability.) */ -struct proto_mtag { +struct proto_mtag_ { union { - struct tcp_mtag tcp; /* TCP specific */ - struct ipsec_mtag ipsec; /* IPSec specific */ + struct tcp_mtag tcp; /* TCP specific */ + struct udp_mtag udp; /* UDP specific */ + struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */ } __pr_u; }; +/* + * NECP specific mbuf tag. + */ +struct necp_mtag_ { + u_int32_t necp_policy_id; + u_int32_t necp_skip_policy_id; + u_int32_t necp_route_rule_id; + u_int16_t necp_last_interface_index; + u_int16_t necp_app_id; +}; + +union builtin_mtag { + struct { + struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */ + struct pf_mtag _pf_mtag; /* built-in PF tag */ + struct necp_mtag_ _necp_mtag; /* built-in NECP tag */ + } _net_mtag; + struct driver_mtag_ _drv_mtag; +#define necp_mtag builtin_mtag._net_mtag._necp_mtag +#define proto_mtag builtin_mtag._net_mtag._proto_mtag +#define driver_mtag builtin_mtag._drv_mtag +}; + /* * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set. */ -struct pkthdr { - struct ifnet *rcvif; /* rcv interface */ +struct pkthdr { + struct ifnet *rcvif; /* rcv interface */ /* variables for ip and tcp reassembly */ - void *pkt_hdr; /* pointer to packet header */ - int32_t len; /* total packet length */ + void *pkt_hdr; /* pointer to packet header */ + int32_t len; /* total packet length */ /* variables for hardware checksum */ /* Note: csum_flags is used for hardware checksum and VLAN */ - u_int32_t csum_flags; /* flags regarding checksum */ + u_int32_t csum_flags; /* flags regarding checksum */ union { struct { - u_int16_t val; /* checksum value */ + u_int16_t val; /* checksum value */ u_int16_t start; /* checksum start offset */ } _csum_rx; -#define csum_rx_val _csum_rx.val -#define csum_rx_start _csum_rx.start +#define csum_rx_val _csum_rx.val +#define csum_rx_start _csum_rx.start struct { u_int16_t start; /* checksum start offset */ u_int16_t stuff; /* checksum stuff offset */ } _csum_tx; -#define csum_tx_start _csum_tx.start -#define csum_tx_stuff _csum_tx.stuff - u_int32_t csum_data; /* data field used by csum routines */ +#define csum_tx_start _csum_tx.start +#define csum_tx_stuff _csum_tx.stuff + /* + * Generic data field used by csum routines. + * It gets used differently in different contexts. + */ + u_int32_t csum_data; }; - u_int16_t vlan_tag; /* VLAN tag, host byte order */ + u_int16_t vlan_tag; /* VLAN tag, host byte order */ /* * Packet classifier info * @@ -334,54 +419,67 @@ struct pkthdr { * to achieve this. For now, we will just rely on the address family * related code paths examining this mbuf to interpret the flags. */ - u_int8_t pkt_proto; /* IPPROTO value */ - u_int8_t pkt_flowsrc; /* FLOWSRC values */ - u_int32_t pkt_flowid; /* flow ID */ - u_int32_t pkt_flags; /* PKTF flags (see below) */ - u_int32_t pkt_svc; /* MBUF_SVC value */ + u_int8_t pkt_proto; /* IPPROTO value */ + u_int8_t pkt_flowsrc; /* FLOWSRC values */ + u_int32_t pkt_flowid; /* flow ID */ + u_int32_t pkt_flags; /* PKTF flags (see below) */ + u_int32_t pkt_svc; /* MBUF_SVC value */ + + u_int32_t pkt_compl_context; /* Packet completion context */ + union { struct { - u_int16_t src; /* ifindex of src addr i/f */ - u_int16_t src_flags; /* src PKT_IFAIFF flags */ - u_int16_t dst; /* ifindex of dst addr i/f */ - u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ + u_int16_t src; /* ifindex of src addr i/f */ + u_int16_t src_flags; /* src PKT_IFAIFF flags */ + u_int16_t dst; /* ifindex of dst addr i/f */ + u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ } _pkt_iaif; -#define src_ifindex _pkt_iaif.src -#define src_iff _pkt_iaif.src_flags -#define dst_ifindex _pkt_iaif.dst -#define dst_iff _pkt_iaif.dst_flags - u_int64_t pkt_ifainfo; /* data field used by ifainfo */ +#define src_ifindex _pkt_iaif.src +#define src_iff _pkt_iaif.src_flags +#define dst_ifindex _pkt_iaif.dst +#define dst_iff _pkt_iaif.dst_flags + u_int64_t pkt_ifainfo; /* data field used by ifainfo */ + struct { + u_int32_t if_data; /* bytes in interface queue */ + u_int32_t sndbuf_data; /* bytes in socket buffer */ + } _pkt_bsr; /* Buffer status report used by cellular interface */ +#define bufstatus_if _pkt_bsr.if_data +#define bufstatus_sndbuf _pkt_bsr.sndbuf_data }; -#if MEASURE_BW - u_int64_t pkt_bwseq; /* sequence # */ -#endif /* MEASURE_BW */ + u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */ + /* * Tags (external and built-in) */ SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */ - struct proto_mtag proto_mtag; /* built-in protocol-specific tag */ - struct pf_mtag pf_mtag; /* built-in PF tag */ + union builtin_mtag builtin_mtag; /* * Module private scratch space (32-bit aligned), currently 16-bytes - * large. Anything stored here is not guaranteed to survive across - * modules. This should be the penultimate structure right before - * the red zone. Add new fields above this. + * large. Anything stored here is not guaranteed to survive across + * modules. The AQM layer (outbound) uses all 16-bytes for both + * packet scheduling and flow advisory information. */ struct { union { - u_int8_t __mpriv8[16]; - u_int16_t __mpriv16[8]; + u_int8_t __mpriv8[16]; + u_int16_t __mpriv16[8]; struct { union { - u_int8_t __val8[4]; - u_int16_t __val16[2]; - u_int32_t __val32; + u_int8_t __val8[4]; + u_int16_t __val16[2]; + u_int32_t __val32; } __mpriv32_u; - } __mpriv32[4]; - u_int64_t __mpriv64[2]; + } __mpriv32[4]; + u_int64_t __mpriv64[2]; } __mpriv_u; } pkt_mpriv __attribute__((aligned(4))); - u_int32_t redzone; /* red zone */ +#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 +#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 +#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32 +#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32 + + u_int32_t redzone; /* red zone */ + u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */ }; /* @@ -391,9 +489,24 @@ struct pkthdr { * to identify the data source object and inform that it can resume its * transmission (in the event it was flow controlled.) */ -#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ -#define FLOWSRC_IFNET 2 /* flow ID generated by interface */ -#define FLOWSRC_PF 3 /* flow ID generated by PF */ +#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ +#define FLOWSRC_IFNET 2 /* flow ID generated by interface */ +#define FLOWSRC_PF 3 /* flow ID generated by PF */ +#define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */ + +/* + * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for + * multi-layer packet logging. We're usurping the pkt_flowsrc field because + * the mbuf packet header ran out of space and pkt_flowsrc is normally + * used on output so we assume we can safely overwrite the normal semantic of + * the field. + * This value is meant to be used on incoming packet from a lower level protocol + * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT + * is set, the following fields are used: + * - pkt_proto: the IP protocol ID of the lower level protocol + * - pkt_flowid: the identifier of the packet at the lower protocol. + * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI. + */ /* * Packet flags. Unlike m_flags, all packet flags are copied along when @@ -416,42 +529,60 @@ struct pkthdr { * if the packet was looped back to the system. This flag should be * used instead for newer code. */ -#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ -#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ -#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ -#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ -#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ -#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ -#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ -#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ -#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ -#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */ -#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */ -#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ -#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ -#define PKTF_LOOP 0x2000 /* loopbacked packet */ -#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ -#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ -#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ +#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ +#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ +#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ +#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ +#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ +#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ +#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ +#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ +#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ +#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */ +#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */ +#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ +#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ +#define PKTF_LOOP 0x2000 /* loopbacked packet */ +#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ +#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ +#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ +#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */ +#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */ +#define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */ +#define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */ +#define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */ +#define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */ +#define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */ +#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */ +#define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */ +#define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */ +#define PKTF_START_SEQ 0x8000000 /* valid start sequence */ +#define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ +#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ +#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */ +#define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */ + /* flags related to flow control/advisory and identification */ -#define PKTF_FLOW_MASK \ +#define PKTF_FLOW_MASK \ (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK) /* * Description of external storage mapped into mbuf, valid only if M_EXT set. */ +typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t); struct m_ext { - caddr_t ext_buf; /* start of buffer */ - void (*ext_free) /* free routine if not the usual */ - (caddr_t, u_int, caddr_t); - u_int ext_size; /* size of buffer, for ext_free */ - caddr_t ext_arg; /* additional ext_free argument */ - struct ext_refsq { /* references held */ - struct ext_refsq *forward, *backward; - } ext_refs; + caddr_t ext_buf; /* start of buffer */ + m_ext_free_func_t ext_free; /* free routine if not the usual */ + u_int ext_size; /* size of buffer, for ext_free */ + caddr_t ext_arg; /* additional ext_free argument */ struct ext_ref { - u_int32_t refcnt; - u_int32_t flags; + struct mbuf *paired; + u_int16_t minref; + u_int16_t refcnt; + u_int16_t prefcnt; + u_int16_t flags; + u_int32_t priv; + uintptr_t ext_token; } *ext_refflags; }; @@ -462,92 +593,93 @@ typedef struct m_ext _m_ext_t; * The mbuf object */ struct mbuf { - struct m_hdr m_hdr; + struct m_hdr m_hdr; union { struct { - struct pkthdr MH_pkthdr; /* M_PKTHDR set */ + struct pkthdr MH_pkthdr; /* M_PKTHDR set */ union { - struct m_ext MH_ext; /* M_EXT set */ - char MH_databuf[_MHLEN]; + struct m_ext MH_ext; /* M_EXT set */ + char MH_databuf[_MHLEN]; } MH_dat; } MH; - char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ + char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ } M_dat; }; -#define m_next m_hdr.mh_next -#define m_len m_hdr.mh_len -#define m_data m_hdr.mh_data -#define m_type m_hdr.mh_type -#define m_flags m_hdr.mh_flags -#define m_nextpkt m_hdr.mh_nextpkt -#define m_act m_nextpkt -#define m_pkthdr M_dat.MH.MH_pkthdr -#define m_ext M_dat.MH.MH_dat.MH_ext -#define m_pktdat M_dat.MH.MH_dat.MH_databuf -#define m_dat M_dat.M_databuf -#define m_pktlen(_m) ((_m)->m_pkthdr.len) -#define m_pftag(_m) (&(_m)->m_pkthdr.pf_mtag) +#define m_next m_hdr.mh_next +#define m_len m_hdr.mh_len +#define m_data m_hdr.mh_data +#define m_type m_hdr.mh_type +#define m_flags m_hdr.mh_flags +#define m_nextpkt m_hdr.mh_nextpkt +#define m_act m_nextpkt +#define m_pkthdr M_dat.MH.MH_pkthdr +#define m_ext M_dat.MH.MH_dat.MH_ext +#define m_pktdat M_dat.MH.MH_dat.MH_databuf +#define m_dat M_dat.M_databuf +#define m_pktlen(_m) ((_m)->m_pkthdr.len) +#define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag) /* mbuf flags (private) */ -#define M_EXT 0x0001 /* has associated external storage */ -#define M_PKTHDR 0x0002 /* start of record */ -#define M_EOR 0x0004 /* end of record */ -#define M_PROTO1 0x0008 /* protocol-specific */ -#define M_PROTO2 0x0010 /* protocol-specific */ -#define M_PROTO3 0x0020 /* protocol-specific */ -#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ -#define M_PROTO5 0x0080 /* protocol-specific */ +#define M_EXT 0x0001 /* has associated external storage */ +#define M_PKTHDR 0x0002 /* start of record */ +#define M_EOR 0x0004 /* end of record */ +#define M_PROTO1 0x0008 /* protocol-specific */ +#define M_PROTO2 0x0010 /* protocol-specific */ +#define M_PROTO3 0x0020 /* protocol-specific */ +#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ +#define M_PROTO5 0x0080 /* protocol-specific */ /* mbuf pkthdr flags, also in m_flags (private) */ -#define M_BCAST 0x0100 /* send/received as link-level broadcast */ -#define M_MCAST 0x0200 /* send/received as link-level multicast */ -#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ -#define M_FIRSTFRAG 0x0800 /* packet is first fragment */ -#define M_LASTFRAG 0x1000 /* packet is last fragment */ -#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ -#define M_HASFCS 0x4000 /* packet has FCS */ -#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ +#define M_BCAST 0x0100 /* send/received as link-level broadcast */ +#define M_MCAST 0x0200 /* send/received as link-level multicast */ +#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ +#define M_FIRSTFRAG 0x0800 /* packet is first fragment */ +#define M_LASTFRAG 0x1000 /* packet is last fragment */ +#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ +#define M_HASFCS 0x4000 /* packet has FCS */ +#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ /* * Flags to purge when crossing layers. */ -#define M_PROTOFLAGS \ +#define M_PROTOFLAGS \ (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5) /* flags copied when copying m_pkthdr */ -#define M_COPYFLAGS \ - (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ - M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ +#define M_COPYFLAGS \ + (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ + M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS) /* flags indicating hw checksum support and sw checksum requirements */ -#define CSUM_IP 0x0001 /* will csum IP */ -#define CSUM_TCP 0x0002 /* will csum TCP */ -#define CSUM_UDP 0x0004 /* will csum UDP */ -#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ -#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ -#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ -#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ -#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ - -#define CSUM_IP_CHECKED 0x0100 /* did csum IP */ -#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ -#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ -#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ -#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ - -#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) -#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ -#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) -#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ - -#define CSUM_TX_FLAGS \ - (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ - CSUM_DATA_VALID | CSUM_PARTIAL) - -#define CSUM_RX_FLAGS \ - (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ +#define CSUM_IP 0x0001 /* will csum IP */ +#define CSUM_TCP 0x0002 /* will csum TCP */ +#define CSUM_UDP 0x0004 /* will csum UDP */ +#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ +#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ +#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ +#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ +#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ + +#define CSUM_IP_CHECKED 0x0100 /* did csum IP */ +#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ +#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ +#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ +#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ +#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */ + +#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) +#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ +#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) +#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ + +#define CSUM_TX_FLAGS \ + (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ + CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT) + +#define CSUM_RX_FLAGS \ + (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ CSUM_DATA_VALID | CSUM_PARTIAL) /* @@ -555,48 +687,51 @@ struct mbuf { */ /* VLAN tag present */ -#define CSUM_VLAN_TAG_VALID 0x10000 /* vlan_tag field is valid */ +#define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */ + +/* checksum start adjustment has been done */ +#define CSUM_ADJUST_DONE 0x00020000 /* TCP Segment Offloading requested on this mbuf */ -#define CSUM_TSO_IPV4 0x100000 /* This mbuf needs to be segmented by the NIC */ -#define CSUM_TSO_IPV6 0x200000 /* This mbuf needs to be segmented by the NIC */ +#define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */ +#define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */ -#define TSO_IPV4_OK(_ifp, _m) \ - (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ - ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ +#define TSO_IPV4_OK(_ifp, _m) \ + (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ + ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ -#define TSO_IPV4_NOTOK(_ifp, _m) \ - (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ - ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ +#define TSO_IPV4_NOTOK(_ifp, _m) \ + (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ + ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ -#define TSO_IPV6_OK(_ifp, _m) \ - (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ - ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ +#define TSO_IPV6_OK(_ifp, _m) \ + (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ + ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ -#define TSO_IPV6_NOTOK(_ifp, _m) \ - (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ - ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ +#define TSO_IPV6_NOTOK(_ifp, _m) \ + (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ + ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ #endif /* XNU_KERNEL_PRIVATE */ /* mbuf types */ -#define MT_FREE 0 /* should be on free list */ -#define MT_DATA 1 /* dynamic (data) allocation */ -#define MT_HEADER 2 /* packet header */ -#define MT_SOCKET 3 /* socket structure */ -#define MT_PCB 4 /* protocol control block */ -#define MT_RTABLE 5 /* routing tables */ -#define MT_HTABLE 6 /* IMP host tables */ -#define MT_ATABLE 7 /* address resolution tables */ -#define MT_SONAME 8 /* socket name */ -#define MT_SOOPTS 10 /* socket options */ -#define MT_FTABLE 11 /* fragment reassembly header */ -#define MT_RIGHTS 12 /* access rights */ -#define MT_IFADDR 13 /* interface address */ -#define MT_CONTROL 14 /* extra-data protocol message */ -#define MT_OOBDATA 15 /* expedited data */ -#define MT_TAG 16 /* volatile metadata associated to pkts */ -#define MT_MAX 32 /* enough? */ +#define MT_FREE 0 /* should be on free list */ +#define MT_DATA 1 /* dynamic (data) allocation */ +#define MT_HEADER 2 /* packet header */ +#define MT_SOCKET 3 /* socket structure */ +#define MT_PCB 4 /* protocol control block */ +#define MT_RTABLE 5 /* routing tables */ +#define MT_HTABLE 6 /* IMP host tables */ +#define MT_ATABLE 7 /* address resolution tables */ +#define MT_SONAME 8 /* socket name */ +#define MT_SOOPTS 10 /* socket options */ +#define MT_FTABLE 11 /* fragment reassembly header */ +#define MT_RIGHTS 12 /* access rights */ +#define MT_IFADDR 13 /* interface address */ +#define MT_CONTROL 14 /* extra-data protocol message */ +#define MT_OOBDATA 15 /* expedited data */ +#define MT_TAG 16 /* volatile metadata associated to pkts */ +#define MT_MAX 32 /* enough? */ #ifdef XNU_KERNEL_PRIVATE /* @@ -611,14 +746,14 @@ struct mbuf { */ #if 1 -#define MCHECK(m) m_mcheck(m) +#define MCHECK(m) m_mcheck(m) #else -#define MCHECK(m) +#define MCHECK(m) #endif -#define MGET(m, how, type) ((m) = m_get((how), (type))) +#define MGET(m, how, type) ((m) = m_get((how), (type))) -#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) +#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) /* * Mbuf cluster macros. @@ -633,33 +768,33 @@ struct mbuf { * pointer while on the free list. */ union mcluster { - union mcluster *mcl_next; - char mcl_buf[MCLBYTES]; + union mcluster *mcl_next; + char mcl_buf[MCLBYTES]; }; -#define MCLALLOC(p, how) ((p) = m_mclalloc(how)) +#define MCLALLOC(p, how) ((p) = m_mclalloc(how)) -#define MCLFREE(p) m_mclfree(p) +#define MCLFREE(p) m_mclfree(p) -#define MCLGET(m, how) ((m) = m_mclget(m, how)) +#define MCLGET(m, how) ((m) = m_mclget(m, how)) /* * Mbuf big cluster */ union mbigcluster { - union mbigcluster *mbc_next; - char mbc_buf[MBIGCLBYTES]; + union mbigcluster *mbc_next; + char mbc_buf[MBIGCLBYTES]; }; /* * Mbuf jumbo cluster */ union m16kcluster { - union m16kcluster *m16kcl_next; - char m16kcl_buf[M16KCLBYTES]; + union m16kcluster *m16kcl_next; + char m16kcl_buf[M16KCLBYTES]; }; -#define MCLHASREFERENCE(m) m_mclhasreference(m) +#define MCLHASREFERENCE(m) m_mclhasreference(m) /* * MFREE(struct mbuf *m, struct mbuf *n) @@ -667,50 +802,75 @@ union m16kcluster { * Place the successor, if any, in n. */ -#define MFREE(m, n) ((n) = m_free(m)) +#define MFREE(m, n) ((n) = m_free(m)) /* * Copy mbuf pkthdr from from to to. * from must have M_PKTHDR set, and to must be empty. * aux pointer will be moved to `to'. */ -#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) +#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) -#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) +#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) -#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) +#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) /* - * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place - * an object of the specified size at the end of the mbuf, longword aligned. + * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can + * be both the local data payload, or an external buffer area, depending on + * whether M_EXT is set). */ -#define M_ALIGN(m, len) \ -do { \ - (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1); \ -} while (0) +#define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m)) /* - * As above, for mbufs allocated with m_gethdr/MGETHDR - * or initialized by M_COPY_PKTHDR. + * These macros are mapped to the appropriate KPIs, so that private code + * can be simply recompiled in order to be forward-compatible with future + * changes toward the struture sizes. */ -#define MH_ALIGN(m, len) \ -do { \ - (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1); \ -} while (0) +#define MLEN mbuf_get_mlen() /* normal mbuf data len */ +#define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */ +#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ +/* + * Return the address of the start of the buffer associated with an mbuf, + * handling external storage, packet-header mbufs, and regular data mbufs. + */ +#define M_START(m) \ + (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \ + ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \ + &(m)->m_dat[0]) + +/* + * Return the size of the buffer associated with an mbuf, handling external + * storage, packet-header mbufs, and regular data mbufs. + */ +#define M_SIZE(m) \ + (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \ + ((m)->m_flags & M_PKTHDR) ? MHLEN : \ + MLEN) + +#define M_ALIGN(m, len) m_align(m, len) +#define MH_ALIGN(m, len) m_align(m, len) +#define MEXT_ALIGN(m, len) m_align(m, len) /* - * Compute the amount of space available - * before the current start of data in an mbuf. - * Subroutine - data not available if certain references. + * Compute the amount of space available before the current start of data in + * an mbuf. + * + * The M_WRITABLE() is a temporary, conservative safety measure: the burden + * of checking writability of the mbuf data area rests solely with the caller. */ -#define M_LEADINGSPACE(m) m_leadingspace(m) +#define M_LEADINGSPACE(m) \ + (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0) /* - * Compute the amount of space available - * after the end of data in an mbuf. - * Subroutine - data not available if certain references. + * Compute the amount of space available after the end of data in an mbuf. + * + * The M_WRITABLE() is a temporary, conservative safety measure: the burden + * of checking writability of the mbuf data area rests solely with the caller. */ -#define M_TRAILINGSPACE(m) m_trailingspace(m) +#define M_TRAILINGSPACE(m) \ + (M_WRITABLE(m) ? \ + ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0) /* * Arrange to prepend space of size plen to mbuf m. @@ -718,18 +878,19 @@ do { \ * If how is M_DONTWAIT and allocation fails, the original mbuf chain * is freed and m is set to NULL. */ -#define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how))) +#define M_PREPEND(m, plen, how, align) \ + ((m) = m_prepend_2((m), (plen), (how), (align))) /* change mbuf to new type */ -#define MCHTYPE(m, t) m_mchtype(m, t) +#define MCHTYPE(m, t) m_mchtype(m, t) /* compatiblity with 4.3 */ -#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) +#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) -#define MBSHIFT 20 /* 1MB */ -#define MBSIZE (1 << MBSHIFT) -#define GBSHIFT 30 /* 1GB */ -#define GBSIZE (1 << GBSHIFT) +#define MBSHIFT 20 /* 1MB */ +#define MBSIZE (1 << MBSHIFT) +#define GBSHIFT 30 /* 1GB */ +#define GBSIZE (1 << GBSHIFT) /* * M_STRUCT_GET ensures that intermediate protocol header (from "off" to @@ -740,59 +901,59 @@ do { \ * M_STRUCT_GET0 does the same, except that it aligns the structure at * very top of mbuf. GET0 is likely to make memory copy than GET. */ -#define M_STRUCT_GET(val, typ, m, off, len) \ -do { \ - struct mbuf *t; \ - int tmp; \ - \ - if ((m)->m_len >= (off) + (len)) { \ - (val) = (typ)(mtod((m), caddr_t) + (off)); \ - } else { \ - t = m_pulldown((m), (off), (len), &tmp); \ - if (t != NULL) { \ - if (t->m_len < tmp + (len)) \ - panic("m_pulldown malfunction"); \ - (val) = (typ)(mtod(t, caddr_t) + tmp); \ - } else { \ - (val) = (typ)NULL; \ - (m) = NULL; \ - } \ - } \ +#define M_STRUCT_GET(val, typ, m, off, len) \ +do { \ + struct mbuf *t; \ + int tmp; \ + \ + if ((m)->m_len >= (off) + (len)) { \ + (val) = (typ)(mtod((m), caddr_t) + (off)); \ + } else { \ + t = m_pulldown((m), (off), (len), &tmp); \ + if (t != NULL) { \ + if (t->m_len < tmp + (len)) \ + panic("m_pulldown malfunction"); \ + (val) = (typ)(mtod(t, caddr_t) + tmp); \ + } else { \ + (val) = (typ)NULL; \ + (m) = NULL; \ + } \ + } \ } while (0) -#define M_STRUCT_GET0(val, typ, m, off, len) \ -do { \ - struct mbuf *t; \ - \ - if ((off) == 0 && ((m)->m_len >= (len))) { \ - (val) = (typ)(void *)mtod(m, caddr_t); \ - } else { \ - t = m_pulldown((m), (off), (len), NULL); \ - if (t != NULL) { \ - if (t->m_len < (len)) \ - panic("m_pulldown malfunction"); \ - (val) = (typ)(void *)mtod(t, caddr_t); \ - } else { \ - (val) = (typ)NULL; \ - (m) = NULL; \ - } \ - } \ +#define M_STRUCT_GET0(val, typ, m, off, len) \ +do { \ + struct mbuf *t; \ + \ + if ((off) == 0 && ((m)->m_len >= (len))) { \ + (val) = (typ)(void *)mtod(m, caddr_t); \ + } else { \ + t = m_pulldown((m), (off), (len), NULL); \ + if (t != NULL) { \ + if (t->m_len < (len)) \ + panic("m_pulldown malfunction"); \ + (val) = (typ)(void *)mtod(t, caddr_t); \ + } else { \ + (val) = (typ)NULL; \ + (m) = NULL; \ + } \ + } \ } while (0) -#define MBUF_INPUT_CHECK(m, rcvif) \ -do { \ - if (!(m->m_flags & MBUF_PKTHDR) || \ - m->m_len < 0 || \ - m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \ - m->m_type == MT_FREE || \ - ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ - panic_plain("Failed mbuf validity check: mbuf %p len %d " \ - "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ - m, m->m_len, m->m_type, m->m_flags, \ - ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \ - if_name(rcvif), \ - (rcvif->if_flags & 0xffff)); \ - } \ +#define MBUF_INPUT_CHECK(m, rcvif) \ +do { \ + if (!(m->m_flags & MBUF_PKTHDR) || \ + m->m_len < 0 || \ + m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \ + m->m_type == MT_FREE || \ + ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ + panic_plain("Failed mbuf validity check: mbuf %p len %d " \ + "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ + m, m->m_len, m->m_type, m->m_flags, \ + ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \ + if_name(rcvif), \ + (rcvif->if_flags & 0xffff)); \ + } \ } while (0) /* @@ -803,229 +964,241 @@ do { \ * * m_next is ignored, so queueing chains of mbufs is possible */ -#define MBUFQ_HEAD(name) \ -struct name { \ - struct mbuf *mq_first; /* first packet */ \ - struct mbuf **mq_last; /* addr of last next packet */ \ +#define MBUFQ_HEAD(name) \ +struct name { \ + struct mbuf *mq_first; /* first packet */ \ + struct mbuf **mq_last; /* addr of last next packet */ \ } -#define MBUFQ_INIT(q) do { \ - MBUFQ_FIRST(q) = NULL; \ - (q)->mq_last = &MBUFQ_FIRST(q); \ +#define MBUFQ_INIT(q) do { \ + MBUFQ_FIRST(q) = NULL; \ + (q)->mq_last = &MBUFQ_FIRST(q); \ } while (0) -#define MBUFQ_PREPEND(q, m) do { \ - if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ - (q)->mq_last = &MBUFQ_NEXT(m); \ - MBUFQ_FIRST(q) = (m); \ +#define MBUFQ_PREPEND(q, m) do { \ + if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ + (q)->mq_last = &MBUFQ_NEXT(m); \ + MBUFQ_FIRST(q) = (m); \ } while (0) -#define MBUFQ_ENQUEUE(q, m) do { \ - MBUFQ_NEXT(m) = NULL; \ - *(q)->mq_last = (m); \ - (q)->mq_last = &MBUFQ_NEXT(m); \ +#define MBUFQ_ENQUEUE(q, m) do { \ + MBUFQ_NEXT(m) = NULL; \ + *(q)->mq_last = (m); \ + (q)->mq_last = &MBUFQ_NEXT(m); \ } while (0) -#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ - MBUFQ_NEXT(n) = NULL; \ - *(q)->mq_last = (m); \ - (q)->mq_last = &MBUFQ_NEXT(n); \ +#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ + MBUFQ_NEXT(n) = NULL; \ + *(q)->mq_last = (m); \ + (q)->mq_last = &MBUFQ_NEXT(n); \ } while (0) -#define MBUFQ_DEQUEUE(q, m) do { \ - if (((m) = MBUFQ_FIRST(q)) != NULL) { \ - if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ - (q)->mq_last = &MBUFQ_FIRST(q); \ - else \ - MBUFQ_NEXT(m) = NULL; \ - } \ +#define MBUFQ_DEQUEUE(q, m) do { \ + if (((m) = MBUFQ_FIRST(q)) != NULL) { \ + if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ + (q)->mq_last = &MBUFQ_FIRST(q); \ + else \ + MBUFQ_NEXT(m) = NULL; \ + } \ } while (0) -#define MBUFQ_REMOVE(q, m) do { \ - if (MBUFQ_FIRST(q) == (m)) { \ - MBUFQ_DEQUEUE(q, m); \ - } else { \ - struct mbuf *_m = MBUFQ_FIRST(q); \ - while (MBUFQ_NEXT(_m) != (m)) \ - _m = MBUFQ_NEXT(_m); \ - if ((MBUFQ_NEXT(_m) = \ - MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ - (q)->mq_last = &MBUFQ_NEXT(_m); \ - } \ +#define MBUFQ_REMOVE(q, m) do { \ + if (MBUFQ_FIRST(q) == (m)) { \ + MBUFQ_DEQUEUE(q, m); \ + } else { \ + struct mbuf *_m = MBUFQ_FIRST(q); \ + while (MBUFQ_NEXT(_m) != (m)) \ + _m = MBUFQ_NEXT(_m); \ + if ((MBUFQ_NEXT(_m) = \ + MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ + (q)->mq_last = &MBUFQ_NEXT(_m); \ + } \ } while (0) -#define MBUFQ_DRAIN(q) do { \ - struct mbuf *__m0; \ - while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ - MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ - MBUFQ_NEXT(__m0) = NULL; \ - m_freem(__m0); \ - } \ - (q)->mq_last = &MBUFQ_FIRST(q); \ +#define MBUFQ_DRAIN(q) do { \ + struct mbuf *__m0; \ + while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ + MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ + MBUFQ_NEXT(__m0) = NULL; \ + m_freem(__m0); \ + } \ + (q)->mq_last = &MBUFQ_FIRST(q); \ } while (0) -#define MBUFQ_FOREACH(m, q) \ - for ((m) = MBUFQ_FIRST(q); \ - (m); \ +#define MBUFQ_FOREACH(m, q) \ + for ((m) = MBUFQ_FIRST(q); \ + (m); \ (m) = MBUFQ_NEXT(m)) -#define MBUFQ_FOREACH_SAFE(m, q, tvar) \ - for ((m) = MBUFQ_FIRST(q); \ - (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ +#define MBUFQ_FOREACH_SAFE(m, q, tvar) \ + for ((m) = MBUFQ_FIRST(q); \ + (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ (m) = (tvar)) -#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) -#define MBUFQ_FIRST(q) ((q)->mq_first) -#define MBUFQ_NEXT(m) ((m)->m_nextpkt) -#define MBUFQ_LAST(q) (*(q)->mq_last) +#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) +#define MBUFQ_FIRST(q) ((q)->mq_first) +#define MBUFQ_NEXT(m) ((m)->m_nextpkt) +/* + * mq_last is initialized to point to mq_first, so check if they're + * equal and return NULL when the list is empty. Otherwise, we need + * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get + * to the base mbuf address to return to caller. + */ +#define MBUFQ_LAST(head) \ + (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \ + ((struct mbuf *)(void *)((char *)(head)->mq_last - \ + __builtin_offsetof(struct mbuf, m_nextpkt)))) -#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t)) -#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t)) +#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t)) +#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t)) #endif /* XNU_KERNEL_PRIVATE */ /* * Mbuf statistics (legacy). */ struct mbstat { - u_int32_t m_mbufs; /* mbufs obtained from page pool */ - u_int32_t m_clusters; /* clusters obtained from page pool */ - u_int32_t m_spare; /* spare field */ - u_int32_t m_clfree; /* free clusters */ - u_int32_t m_drops; /* times failed to find space */ - u_int32_t m_wait; /* times waited for space */ - u_int32_t m_drain; /* times drained protocols for space */ - u_short m_mtypes[256]; /* type specific mbuf allocations */ - u_int32_t m_mcfail; /* times m_copym failed */ - u_int32_t m_mpfail; /* times m_pullup failed */ - u_int32_t m_msize; /* length of an mbuf */ - u_int32_t m_mclbytes; /* length of an mbuf cluster */ - u_int32_t m_minclsize; /* min length of data to allocate a cluster */ - u_int32_t m_mlen; /* length of data in an mbuf */ - u_int32_t m_mhlen; /* length of data in a header mbuf */ - u_int32_t m_bigclusters; /* clusters obtained from page pool */ - u_int32_t m_bigclfree; /* free clusters */ - u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ + u_int32_t m_mbufs; /* mbufs obtained from page pool */ + u_int32_t m_clusters; /* clusters obtained from page pool */ + u_int32_t m_spare; /* spare field */ + u_int32_t m_clfree; /* free clusters */ + u_int32_t m_drops; /* times failed to find space */ + u_int32_t m_wait; /* times waited for space */ + u_int32_t m_drain; /* times drained protocols for space */ + u_short m_mtypes[256]; /* type specific mbuf allocations */ + u_int32_t m_mcfail; /* times m_copym failed */ + u_int32_t m_mpfail; /* times m_pullup failed */ + u_int32_t m_msize; /* length of an mbuf */ + u_int32_t m_mclbytes; /* length of an mbuf cluster */ + u_int32_t m_minclsize; /* min length of data to allocate a cluster */ + u_int32_t m_mlen; /* length of data in an mbuf */ + u_int32_t m_mhlen; /* length of data in a header mbuf */ + u_int32_t m_bigclusters; /* clusters obtained from page pool */ + u_int32_t m_bigclfree; /* free clusters */ + u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ }; /* Compatibillity with 10.3 */ struct ombstat { - u_int32_t m_mbufs; /* mbufs obtained from page pool */ - u_int32_t m_clusters; /* clusters obtained from page pool */ - u_int32_t m_spare; /* spare field */ - u_int32_t m_clfree; /* free clusters */ - u_int32_t m_drops; /* times failed to find space */ - u_int32_t m_wait; /* times waited for space */ - u_int32_t m_drain; /* times drained protocols for space */ - u_short m_mtypes[256]; /* type specific mbuf allocations */ - u_int32_t m_mcfail; /* times m_copym failed */ - u_int32_t m_mpfail; /* times m_pullup failed */ - u_int32_t m_msize; /* length of an mbuf */ - u_int32_t m_mclbytes; /* length of an mbuf cluster */ - u_int32_t m_minclsize; /* min length of data to allocate a cluster */ - u_int32_t m_mlen; /* length of data in an mbuf */ - u_int32_t m_mhlen; /* length of data in a header mbuf */ + u_int32_t m_mbufs; /* mbufs obtained from page pool */ + u_int32_t m_clusters; /* clusters obtained from page pool */ + u_int32_t m_spare; /* spare field */ + u_int32_t m_clfree; /* free clusters */ + u_int32_t m_drops; /* times failed to find space */ + u_int32_t m_wait; /* times waited for space */ + u_int32_t m_drain; /* times drained protocols for space */ + u_short m_mtypes[256]; /* type specific mbuf allocations */ + u_int32_t m_mcfail; /* times m_copym failed */ + u_int32_t m_mpfail; /* times m_pullup failed */ + u_int32_t m_msize; /* length of an mbuf */ + u_int32_t m_mclbytes; /* length of an mbuf cluster */ + u_int32_t m_minclsize; /* min length of data to allocate a cluster */ + u_int32_t m_mlen; /* length of data in an mbuf */ + u_int32_t m_mhlen; /* length of data in a header mbuf */ }; /* * mbuf class statistics. */ -#define MAX_MBUF_CNAME 15 +#define MAX_MBUF_CNAME 15 #if defined(XNU_KERNEL_PRIVATE) /* For backwards compatibility with 32-bit userland process */ struct omb_class_stat { - char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ - u_int32_t mbcl_size; /* buffer size */ - u_int32_t mbcl_total; /* # of buffers created */ - u_int32_t mbcl_active; /* # of active buffers */ - u_int32_t mbcl_infree; /* # of available buffers */ - u_int32_t mbcl_slab_cnt; /* # of available slabs */ - u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ - u_int64_t mbcl_free_cnt; /* # of times free is called */ - u_int64_t mbcl_notified; /* # of notified wakeups */ - u_int64_t mbcl_purge_cnt; /* # of purges so far */ - u_int64_t mbcl_fail_cnt; /* # of allocation failures */ - u_int32_t mbcl_ctotal; /* total only for this class */ + char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ + u_int32_t mbcl_size; /* buffer size */ + u_int32_t mbcl_total; /* # of buffers created */ + u_int32_t mbcl_active; /* # of active buffers */ + u_int32_t mbcl_infree; /* # of available buffers */ + u_int32_t mbcl_slab_cnt; /* # of available slabs */ + u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ + u_int64_t mbcl_free_cnt; /* # of times free is called */ + u_int64_t mbcl_notified; /* # of notified wakeups */ + u_int64_t mbcl_purge_cnt; /* # of purges so far */ + u_int64_t mbcl_fail_cnt; /* # of allocation failures */ + u_int32_t mbcl_ctotal; /* total only for this class */ + u_int32_t mbcl_release_cnt; /* amount of memory returned */ /* * Cache layer statistics */ - u_int32_t mbcl_mc_state; /* cache state (see below) */ - u_int32_t mbcl_mc_cached; /* # of cached buffers */ - u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ - u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ - u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ - u_int64_t mbcl_reserved[4]; /* for future use */ + u_int32_t mbcl_mc_state; /* cache state (see below) */ + u_int32_t mbcl_mc_cached; /* # of cached buffers */ + u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ + u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ + u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ + u_int64_t mbcl_reserved[4]; /* for future use */ } __attribute__((__packed__)); #endif /* XNU_KERNEL_PRIVATE */ typedef struct mb_class_stat { - char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ - u_int32_t mbcl_size; /* buffer size */ - u_int32_t mbcl_total; /* # of buffers created */ - u_int32_t mbcl_active; /* # of active buffers */ - u_int32_t mbcl_infree; /* # of available buffers */ - u_int32_t mbcl_slab_cnt; /* # of available slabs */ + char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ + u_int32_t mbcl_size; /* buffer size */ + u_int32_t mbcl_total; /* # of buffers created */ + u_int32_t mbcl_active; /* # of active buffers */ + u_int32_t mbcl_infree; /* # of available buffers */ + u_int32_t mbcl_slab_cnt; /* # of available slabs */ #if defined(KERNEL) || defined(__LP64__) - u_int32_t mbcl_pad; /* padding */ + u_int32_t mbcl_pad; /* padding */ #endif /* KERNEL || __LP64__ */ - u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ - u_int64_t mbcl_free_cnt; /* # of times free is called */ - u_int64_t mbcl_notified; /* # of notified wakeups */ - u_int64_t mbcl_purge_cnt; /* # of purges so far */ - u_int64_t mbcl_fail_cnt; /* # of allocation failures */ - u_int32_t mbcl_ctotal; /* total only for this class */ + u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ + u_int64_t mbcl_free_cnt; /* # of times free is called */ + u_int64_t mbcl_notified; /* # of notified wakeups */ + u_int64_t mbcl_purge_cnt; /* # of purges so far */ + u_int64_t mbcl_fail_cnt; /* # of allocation failures */ + u_int32_t mbcl_ctotal; /* total only for this class */ + u_int32_t mbcl_release_cnt; /* amount of memory returned */ /* * Cache layer statistics */ - u_int32_t mbcl_mc_state; /* cache state (see below) */ - u_int32_t mbcl_mc_cached; /* # of cached buffers */ - u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ - u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ - u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ - u_int64_t mbcl_reserved[4]; /* for future use */ + u_int32_t mbcl_mc_state; /* cache state (see below) */ + u_int32_t mbcl_mc_cached; /* # of cached buffers */ + u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ + u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ + u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ + u_int32_t mbcl_peak_reported; /* last usage peak reported */ + u_int32_t mbcl_reserved[7]; /* for future use */ } mb_class_stat_t; -#define MCS_DISABLED 0 /* cache is permanently disabled */ -#define MCS_ONLINE 1 /* cache is online */ -#define MCS_PURGING 2 /* cache is being purged */ -#define MCS_OFFLINE 3 /* cache is offline (resizing) */ +#define MCS_DISABLED 0 /* cache is permanently disabled */ +#define MCS_ONLINE 1 /* cache is online */ +#define MCS_PURGING 2 /* cache is being purged */ +#define MCS_OFFLINE 3 /* cache is offline (resizing) */ #if defined(XNU_KERNEL_PRIVATE) /* For backwards compatibility with 32-bit userland process */ struct omb_stat { - u_int32_t mbs_cnt; /* number of classes */ - struct omb_class_stat mbs_class[1]; /* class array */ + u_int32_t mbs_cnt; /* number of classes */ + struct omb_class_stat mbs_class[1]; /* class array */ } __attribute__((__packed__)); #endif /* XNU_KERNEL_PRIVATE */ typedef struct mb_stat { - u_int32_t mbs_cnt; /* number of classes */ + u_int32_t mbs_cnt; /* number of classes */ #if defined(KERNEL) || defined(__LP64__) - u_int32_t mbs_pad; /* padding */ + u_int32_t mbs_pad; /* padding */ #endif /* KERNEL || __LP64__ */ - mb_class_stat_t mbs_class[1]; /* class array */ + mb_class_stat_t mbs_class[1]; /* class array */ } mb_stat_t; #ifdef PRIVATE -#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ +#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ typedef struct mleak_trace_stat { - u_int64_t mltr_collisions; - u_int64_t mltr_hitcount; - u_int64_t mltr_allocs; - u_int64_t mltr_depth; - u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; + u_int64_t mltr_collisions; + u_int64_t mltr_hitcount; + u_int64_t mltr_allocs; + u_int64_t mltr_depth; + u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; } mleak_trace_stat_t; typedef struct mleak_stat { - u_int32_t ml_isaddr64; /* 64-bit KVA? */ - u_int32_t ml_cnt; /* number of traces */ - mleak_trace_stat_t ml_trace[1]; /* trace array */ + u_int32_t ml_isaddr64; /* 64-bit KVA? */ + u_int32_t ml_cnt; /* number of traces */ + mleak_trace_stat_t ml_trace[1]; /* trace array */ } mleak_stat_t; struct mleak_table { - u_int32_t mleak_capture; /* sampling capture counter */ - u_int32_t mleak_sample_factor; /* sample factor */ + u_int32_t mleak_capture; /* sampling capture counter */ + u_int32_t mleak_sample_factor; /* sample factor */ /* Times two active records want to occupy the same spot */ u_int64_t alloc_collisions; @@ -1054,7 +1227,7 @@ __BEGIN_DECLS * Exported (private) */ -extern struct mbstat mbstat; /* statistics */ +extern struct mbstat mbstat; /* statistics */ __END_DECLS #endif /* KERNEL_PRIVATE */ @@ -1068,32 +1241,24 @@ __BEGIN_DECLS /* flags to m_get/MGET */ /* Need to include malloc.h to get right options for malloc */ -#include +#include struct mbuf; /* length to m_copy to copy all */ -#define M_COPYALL 1000000000 +#define M_COPYALL 1000000000 -#define M_DONTWAIT M_NOWAIT -#define M_WAIT M_WAITOK +#define M_DONTWAIT M_NOWAIT +#define M_WAIT M_WAITOK /* modes for m_copym and variants */ -#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ -#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ -#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ +#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ +#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ +#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ +#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */ +#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */ -/* - * These macros are mapped to the appropriate KPIs, so that private code - * can be simply recompiled in order to be forward-compatible with future - * changes toward the struture sizes. - */ -#define MLEN mbuf_get_mlen() /* normal data len */ -#define MHLEN mbuf_get_mhlen() /* data len w/pkthdr */ - -#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ - -extern void m_freem(struct mbuf *); +extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem); extern u_int64_t mcl_to_paddr(char *); extern void m_adj(struct mbuf *, int); extern void m_cat(struct mbuf *, struct mbuf *); @@ -1106,7 +1271,7 @@ extern struct mbuf *m_getpacket(void); extern struct mbuf *m_getpackets(int, int, int); extern struct mbuf *m_mclget(struct mbuf *, int); extern void *m_mtod(struct mbuf *); -extern struct mbuf *m_prepend_2(struct mbuf *, int, int); +extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int); extern struct mbuf *m_pullup(struct mbuf *, int); extern struct mbuf *m_split(struct mbuf *, int, int); extern void m_mclfree(caddr_t p); @@ -1118,28 +1283,28 @@ extern void m_mclfree(caddr_t p); * headers), and assert otherwise. */ #if defined(__i386__) || defined(__x86_64__) -#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) +#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) #else /* !__i386__ && !__x86_64__ */ -#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ - if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ - if (((_m)->m_flags & M_PKTHDR) && \ - (_m)->m_pkthdr.rcvif != NULL) { \ - panic_plain("\n%s: mbuf %p data ptr %p is not " \ - "32-bit aligned [%s: alignerrs=%lld]\n", \ - __func__, (_m), (_m)->m_data, \ - if_name((_m)->m_pkthdr.rcvif), \ - (_m)->m_pkthdr.rcvif->if_alignerrs); \ - } else { \ - panic_plain("\n%s: mbuf %p data ptr %p is not " \ - "32-bit aligned\n", \ - __func__, (_m), (_m)->m_data); \ - } \ - } \ +#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ + if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ + if (((_m)->m_flags & M_PKTHDR) && \ + (_m)->m_pkthdr.rcvif != NULL) { \ + panic_plain("\n%s: mbuf %p data ptr %p is not " \ + "32-bit aligned [%s: alignerrs=%lld]\n", \ + __func__, (_m), (_m)->m_data, \ + if_name((_m)->m_pkthdr.rcvif), \ + (_m)->m_pkthdr.rcvif->if_alignerrs); \ + } else { \ + panic_plain("\n%s: mbuf %p data ptr %p is not " \ + "32-bit aligned\n", \ + __func__, (_m), (_m)->m_data); \ + } \ + } \ } while (0) #endif /* !__i386__ && !__x86_64__ */ /* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */ -#define MBUF_SC_MAX_CLASSES 10 +#define MBUF_SC_MAX_CLASSES 10 /* * These conversion macros rely on the corresponding MBUF_SC and @@ -1155,6 +1320,7 @@ extern void m_mclfree(caddr_t p); * MBUF_SC_AV ] ==> MBUF_TC_VI * MBUF_SC_RV ] * MBUF_SC_VI ] + * MBUF_SC_SIG ] * * MBUF_SC_VO ] ==> MBUF_TC_VO * MBUF_SC_CTL ] @@ -1166,71 +1332,76 @@ extern void m_mclfree(caddr_t p); * corresponding portion, and never assume that a higher class corresponds * to a higher index. */ -#define MBUF_SCVAL(x) ((x) & 0xffff) -#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) -#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) -#define MBUF_TC2SCVAL(_tc) ((_tc) << 7) +#define MBUF_SCVAL(x) ((x) & 0xffff) +#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) +#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) +#define MBUF_TC2SCVAL(_tc) ((_tc) << 7) #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \ ((_sc) == MBUF_SC_BK)) - -#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) -#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) -#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) -#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) -#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) -#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) -#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) -#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) -#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) -#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) - -#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) -#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) -#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) -#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) -#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) -#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) -#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) -#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) -#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) -#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) - -#define MBUF_VALID_SC(c) \ - (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ - c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ - c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO || \ - c == MBUF_SC_CTL) - -#define MBUF_VALID_SCIDX(c) \ - (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ - c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ - c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO || \ - c == SCIDX_CTL) - -#define MBUF_VALID_SCVAL(c) \ - (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ - c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ - c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO || \ - c == SCVAL_CTL) - -extern union mbigcluster *mbutl; /* start VA of mbuf pool */ -extern union mbigcluster *embutl; /* end VA of mbuf pool */ -extern unsigned int nmbclusters; /* number of mapped clusters */ -extern int njcl; /* # of jumbo clusters */ -extern int njclbytes; /* size of a jumbo cluster */ -extern int max_hdr; /* largest link+protocol header */ -extern int max_datalen; /* MHLEN - max_hdr */ +#define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO) +#define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \ + (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM) + +#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) +#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) +#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) +#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) +#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) +#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) +#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) +#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) +#define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG) +#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) +#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) + +#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) +#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) +#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) +#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) +#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) +#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) +#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) +#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) +#define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG) +#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) +#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) + +#define MBUF_VALID_SC(c) \ + (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ + c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ + c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \ + c == MBUF_SC_VO || c == MBUF_SC_CTL) + +#define MBUF_VALID_SCIDX(c) \ + (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ + c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ + c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \ + c == SCIDX_VO || c == SCIDX_CTL) + +#define MBUF_VALID_SCVAL(c) \ + (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ + c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ + c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \ + c == SCVAL_VO || SCVAL_CTL) + +extern unsigned char *mbutl; /* start VA of mbuf pool */ +extern unsigned char *embutl; /* end VA of mbuf pool */ +extern unsigned int nmbclusters; /* number of mapped clusters */ +extern int njcl; /* # of jumbo clusters */ +extern int njclbytes; /* size of a jumbo cluster */ +extern int max_hdr; /* largest link+protocol header */ +extern int max_datalen; /* MHLEN - max_hdr */ /* Use max_linkhdr instead of _max_linkhdr */ -extern int _max_linkhdr; /* largest link-level header */ +extern int _max_linkhdr; /* largest link-level header */ /* Use max_protohdr instead of _max_protohdr */ -extern int _max_protohdr; /* largest protocol header */ +extern int _max_protohdr; /* largest protocol header */ __private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t); __private_extern__ void mbinit(void); __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t, - void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int); + void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int); __private_extern__ caddr_t m_bigalloc(int); __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t); __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int); @@ -1238,7 +1409,7 @@ __private_extern__ caddr_t m_16kalloc(int); __private_extern__ void m_16kfree(caddr_t, u_int, caddr_t); __private_extern__ struct mbuf *m_m16kget(struct mbuf *, int); __private_extern__ int m_reinit(struct mbuf *, int); -__private_extern__ struct mbuf *m_free(struct mbuf *); +__private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free); __private_extern__ struct mbuf *m_getclr(int, int); __private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *); __private_extern__ unsigned int m_length(struct mbuf *); @@ -1268,8 +1439,7 @@ __private_extern__ struct mbuf *m_dtom(void *); __private_extern__ int m_mtocl(void *); __private_extern__ union mcluster *m_cltom(int); -__private_extern__ int m_trailingspace(struct mbuf *); -__private_extern__ int m_leadingspace(struct mbuf *); +__private_extern__ void m_align(struct mbuf *, int); __private_extern__ struct mbuf *m_normalize(struct mbuf *m); __private_extern__ void m_mchtype(struct mbuf *m, int t); @@ -1289,6 +1459,13 @@ __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int, __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, unsigned int *, int, int, size_t); +__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t); +__private_extern__ uint32_t m_ext_get_prop(struct mbuf *); +__private_extern__ int m_ext_paired_is_active(struct mbuf *); +__private_extern__ void m_ext_paired_activate(struct mbuf *); + +__private_extern__ void mbuf_drain(boolean_t); + /* * Packets may have annotations attached by affixing a list of "packet * tags" to the pkthdr structure. Packet tags are dynamically allocated @@ -1316,26 +1493,28 @@ __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, * struct m_tag *mtag = &p->tag; */ -#define KERNEL_MODULE_TAG_ID 0 +#define KERNEL_MODULE_TAG_ID 0 enum { - KERNEL_TAG_TYPE_NONE = 0, - KERNEL_TAG_TYPE_DUMMYNET = 1, - KERNEL_TAG_TYPE_DIVERT = 2, - KERNEL_TAG_TYPE_IPFORWARD = 3, - KERNEL_TAG_TYPE_IPFILT = 4, - KERNEL_TAG_TYPE_MACLABEL = 5, - KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6, - KERNEL_TAG_TYPE_ENCAP = 8, - KERNEL_TAG_TYPE_INET6 = 9, - KERNEL_TAG_TYPE_IPSEC = 10, - KERNEL_TAG_TYPE_DRVAUX = 11 + KERNEL_TAG_TYPE_NONE = 0, + KERNEL_TAG_TYPE_DUMMYNET = 1, + KERNEL_TAG_TYPE_DIVERT = 2, + KERNEL_TAG_TYPE_IPFORWARD = 3, + KERNEL_TAG_TYPE_IPFILT = 4, + KERNEL_TAG_TYPE_MACLABEL = 5, + KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6, + KERNEL_TAG_TYPE_ENCAP = 8, + KERNEL_TAG_TYPE_INET6 = 9, + KERNEL_TAG_TYPE_IPSEC = 10, + KERNEL_TAG_TYPE_DRVAUX = 11, + KERNEL_TAG_TYPE_CFIL_UDP = 13, + KERNEL_TAG_TYPE_PF_REASS = 14, }; /* Packet tag routines */ __private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int); __private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int, - struct mbuf *); + struct mbuf *); __private_extern__ void m_tag_free(struct m_tag *); __private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *); __private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *); @@ -1349,13 +1528,6 @@ __private_extern__ void m_tag_init(struct mbuf *, int); __private_extern__ struct m_tag *m_tag_first(struct mbuf *); __private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); -__END_DECLS -#endif /* XNU_KERNEL_PRIVATE */ -#ifdef KERNEL -#include -#ifdef XNU_KERNEL_PRIVATE -__BEGIN_DECLS - __private_extern__ void m_scratch_init(struct mbuf *); __private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **); @@ -1368,16 +1540,24 @@ __private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t); __private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t); __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); -#define ADDCARRY(_x) do { \ - while (((_x) >> 16) != 0) \ - (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ +#define ADDCARRY(_x) do { \ + while (((_x) >> 16) != 0) \ + (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ } while (0) __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, - u_int32_t, u_int32_t); + u_int32_t, u_int32_t, u_int32_t); __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); +__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, + m_ext_free_func_t, caddr_t); +__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *); +__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *); +__private_extern__ caddr_t m_get_ext_arg(struct mbuf *); + +__private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *); +__private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t); + __END_DECLS #endif /* XNU_KERNEL_PRIVATE */ -#endif /* KERNEL */ -#endif /* !_SYS_MBUF_H_ */ +#endif /* !_SYS_MBUF_H_ */