xnu-4570.51.1.tar.gz

[apple/xnu.git] / bsd / sys / mbuf.h

diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h
index 8a849338b8d6f4035e72956a4c0a81cea08120fd..8dafca387c015f37a77fa4a0b35e7d344b4857f8 100644 (file)
--- a/bsd/sys/mbuf.h
+++ b/bsd/sys/mbuf.h
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 1999-2013 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -81,6 +81,9 @@
 
 #include <sys/cdefs.h>
 #include <sys/appleapiopts.h>
 
 #include <sys/cdefs.h>
 #include <sys/appleapiopts.h>

+#include <sys/_types/_u_int32_t.h> /* u_int32_t */
+#include <sys/_types/_u_int64_t.h> /* u_int64_t */
+#include <sys/_types/_u_short.h> /* u_short */

 
 #ifdef XNU_KERNEL_PRIVATE
 
 
 #ifdef XNU_KERNEL_PRIVATE
 


@@ -103,24 +106,35 @@
 #define        _MLEN           (MSIZE - sizeof(struct m_hdr))  /* normal data len */
 #define        _MHLEN          (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
 
 #define        _MLEN           (MSIZE - sizeof(struct m_hdr))  /* normal data len */
 #define        _MHLEN          (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
 

-#define        NMBPBGSHIFT     (MBIGCLSHIFT - MSIZESHIFT)
-#define        NMBPBG          (1 << NMBPBGSHIFT)      /* # of mbufs per big cl */
+#define        NMBPGSHIFT      (PAGE_SHIFT - MSIZESHIFT)
+#define        NMBPG           (1 << NMBPGSHIFT)       /* # of mbufs per page */

 
 

-#define        NCLPBGSHIFT     (MBIGCLSHIFT - MCLSHIFT)
-#define        NCLPBG          (1 << NCLPBGSHIFT)      /* # of cl per big cl */
+#define        NCLPGSHIFT      (PAGE_SHIFT - MCLSHIFT)
+#define        NCLPG           (1 << NCLPGSHIFT)       /* # of cl per page */
+
+#define        NBCLPGSHIFT     (PAGE_SHIFT - MBIGCLSHIFT)
+#define NBCLPG         (1 << NBCLPGSHIFT)      /* # of big cl per page */

 
 

-#define        NMBPCLSHIFT     (NMBPBGSHIFT - NCLPBGSHIFT)
+#define        NMBPCLSHIFT     (MCLSHIFT - MSIZESHIFT)

 #define        NMBPCL          (1 << NMBPCLSHIFT)      /* # of mbufs per cl */
 
 #define        NMBPCL          (1 << NMBPCLSHIFT)      /* # of mbufs per cl */
 

-#define        NCLPJCLSHIFT    ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
+#define        NCLPJCLSHIFT    (M16KCLSHIFT - MCLSHIFT)

 #define        NCLPJCL         (1 << NCLPJCLSHIFT)     /* # of cl per jumbo cl */
 
 #define        NCLPJCL         (1 << NCLPJCLSHIFT)     /* # of cl per jumbo cl */
 

+#define        NCLPBGSHIFT     (MBIGCLSHIFT - MCLSHIFT)
+#define        NCLPBG          (1 << NCLPBGSHIFT)      /* # of cl per big cl */
+
+#define        NMBPBGSHIFT     (MBIGCLSHIFT - MSIZESHIFT)
+#define        NMBPBG          (1 << NMBPBGSHIFT)      /* # of mbufs per big cl */
+

 /*
  * Macros for type conversion
  * mtod(m,t) - convert mbuf pointer to data pointer of correct type
 /*
  * Macros for type conversion
  * mtod(m,t) - convert mbuf pointer to data pointer of correct type

+ * mtodo(m, o) -- Same as above but with offset 'o' into data.

  * dtom(x) -   convert data pointer within mbuf to mbuf pointer (XXX)
  */
 #define        mtod(m, t)      ((t)m_mtod(m))
  * dtom(x) -   convert data pointer within mbuf to mbuf pointer (XXX)
  */
 #define        mtod(m, t)      ((t)m_mtod(m))

+#define mtodo(m, o)     ((void *)(mtod(m, uint8_t *) + (o)))

 #define        dtom(x)         m_dtom(x)
 
 /* header at beginning of each mbuf: */
 #define        dtom(x)         m_dtom(x)
 
 /* header at beginning of each mbuf: */


@@ -131,7 +145,18 @@ struct m_hdr {
        int32_t         mh_len;         /* amount of data in this mbuf */
        u_int16_t       mh_type;        /* type of data in this mbuf */
        u_int16_t       mh_flags;       /* flags; see below */
        int32_t         mh_len;         /* amount of data in this mbuf */
        u_int16_t       mh_type;        /* type of data in this mbuf */
        u_int16_t       mh_flags;       /* flags; see below */

+#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
+/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
+ * should be defined using the offsetof(struct mbuf, M_dat), since there is
+ * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
+ * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
+ * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
+ * in the definition of mbuf.
+ */
+} __attribute__((aligned(8)));
+#else

 };
 };

+#endif

 
 /*
  * Packet tag structure (see below for details).
 
 /*
  * Packet tag structure (see below for details).


@@ -190,9 +215,6 @@ struct pf_mtag {
        u_int16_t       pftag_rtableid; /* alternate routing table id */
        u_int16_t       pftag_tag;
        u_int16_t       pftag_routed;
        u_int16_t       pftag_rtableid; /* alternate routing table id */
        u_int16_t       pftag_tag;
        u_int16_t       pftag_routed;

-#if PF_ALTQ
-       u_int32_t       pftag_qid;
-#endif /* PF_ALTQ */

 #if PF_ECN
        void            *pftag_hdr;     /* saved hdr pos in mbuf, for ECN */
 #endif /* PF_ECN */
 #if PF_ECN
        void            *pftag_hdr;     /* saved hdr pos in mbuf, for ECN */
 #endif /* PF_ECN */


@@ -205,6 +227,7 @@ struct tcp_pktinfo {
        union {
                struct {
                        u_int32_t segsz;        /* segment size (actual MSS) */
        union {
                struct {
                        u_int32_t segsz;        /* segment size (actual MSS) */

+                       u_int32_t start_seq;    /* start seq of this packet */

                } __tx;
                struct {
                        u_int16_t lro_pktlen;   /* max seg size encountered */
                } __tx;
                struct {
                        u_int16_t lro_pktlen;   /* max seg size encountered */


@@ -217,6 +240,7 @@ struct tcp_pktinfo {
                u_int32_t       seq;            /* recv msg sequence # */
        } __msgattr;
 #define tso_segsz      proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
                u_int32_t       seq;            /* recv msg sequence # */
        } __msgattr;
 #define tso_segsz      proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz

+#define        tx_start_seq    proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq

 #define lro_pktlen     proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
 #define lro_npkts      proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
 #define lro_elapsed    proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
 #define lro_pktlen     proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
 #define lro_npkts      proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
 #define lro_elapsed    proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff


@@ -229,17 +253,13 @@ struct tcp_pktinfo {
  */
 struct mptcp_pktinfo {
        u_int64_t       mtpi_dsn;       /* MPTCP Data Sequence Number */
  */
 struct mptcp_pktinfo {
        u_int64_t       mtpi_dsn;       /* MPTCP Data Sequence Number */

-       union {
-               u_int64_t       mtpi_dan;       /* MPTCP Data Ack Number */
-               struct {
-                       u_int32_t mtpi_rel_seq; /* Relative Seq Number */
-                       u_int32_t mtpi_length;  /* Length of mapping */
-               } mtpi_subf;
-       };
+       u_int32_t       mtpi_rel_seq;   /* Relative Seq Number */
+       u_int16_t       mtpi_length;    /* Length of mapping */
+       u_int16_t       mtpi_csum;

 #define        mp_dsn          proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
 #define        mp_dsn          proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn

-#define        mp_rseq         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq
-#define        mp_rlen         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length
-#define        mp_dack         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan
+#define        mp_rseq         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
+#define        mp_rlen         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
+#define        mp_csum         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum

 };
 
 /*
 };
 
 /*


@@ -255,12 +275,15 @@ struct tcp_mtag {
        };
 };
 
        };
 };
 

-/*
- * IPSec mbuf tag
- */
-struct ipsec_mtag {
-       uint32_t policy_id;
-#define        ipsec_policy    proto_mtag.__pr_u.ipsec.policy_id
+struct driver_mtag_ {
+       uintptr_t               _drv_tx_compl_arg;
+       uintptr_t               _drv_tx_compl_data;
+       kern_return_t           _drv_tx_status;
+       uint16_t                _drv_flowid;
+#define drv_tx_compl_arg       builtin_mtag._drv_mtag._drv_tx_compl_arg
+#define drv_tx_compl_data      builtin_mtag._drv_mtag._drv_tx_compl_data
+#define drv_tx_status          builtin_mtag._drv_mtag._drv_tx_status
+#define drv_flowid             builtin_mtag._drv_mtag._drv_flowid

 };
 
 /*
 };
 
 /*


@@ -271,18 +294,39 @@ struct ipsec_mtag {
  * that the former is used on the virtual ipsec interface that does
  * not advertise the TSO capability.)
  */
  * that the former is used on the virtual ipsec interface that does
  * not advertise the TSO capability.)
  */

-struct proto_mtag {
+struct proto_mtag_ {

        union {
                struct tcp_mtag tcp;            /* TCP specific */
        union {
                struct tcp_mtag tcp;            /* TCP specific */

-               struct ipsec_mtag ipsec;        /* IPSec specific */

        } __pr_u;
 };
 
        } __pr_u;
 };
 

+/*
+ * NECP specific mbuf tag.
+ */
+struct necp_mtag_ {
+       u_int32_t       necp_policy_id;
+       u_int32_t       necp_last_interface_index;
+       u_int32_t       necp_route_rule_id;
+       u_int32_t       necp_app_id;
+};
+
+union builtin_mtag {
+       struct {
+               struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
+               struct pf_mtag  _pf_mtag;       /* built-in PF tag */
+               struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
+       } _net_mtag;
+       struct driver_mtag_ _drv_mtag;
+#define necp_mtag builtin_mtag._net_mtag._necp_mtag
+#define proto_mtag builtin_mtag._net_mtag._proto_mtag
+#define driver_mtag builtin_mtag._drv_mtag
+};
+

 /*
  * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
  */
 /*
  * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
  */

-struct pkthdr {
-       struct  ifnet *rcvif;           /* rcv interface */
+struct pkthdr {
+       struct ifnet *rcvif;            /* rcv interface */

        /* variables for ip and tcp reassembly */
        void    *pkt_hdr;               /* pointer to packet header */
        int32_t len;                    /* total packet length */
        /* variables for ip and tcp reassembly */
        void    *pkt_hdr;               /* pointer to packet header */
        int32_t len;                    /* total packet length */


@@ -339,6 +383,9 @@ struct      pkthdr {
        u_int32_t pkt_flowid;           /* flow ID */
        u_int32_t pkt_flags;            /* PKTF flags (see below) */
        u_int32_t pkt_svc;              /* MBUF_SVC value */
        u_int32_t pkt_flowid;           /* flow ID */
        u_int32_t pkt_flags;            /* PKTF flags (see below) */
        u_int32_t pkt_svc;              /* MBUF_SVC value */

+
+       u_int32_t pkt_compl_context;            /* Packet completion context */
+

        union {
                struct {
                        u_int16_t src;          /* ifindex of src addr i/f */
        union {
                struct {
                        u_int16_t src;          /* ifindex of src addr i/f */


@@ -351,21 +398,25 @@ struct    pkthdr {
 #define        dst_ifindex     _pkt_iaif.dst
 #define        dst_iff         _pkt_iaif.dst_flags
                u_int64_t pkt_ifainfo;  /* data field used by ifainfo */
 #define        dst_ifindex     _pkt_iaif.dst
 #define        dst_iff         _pkt_iaif.dst_flags
                u_int64_t pkt_ifainfo;  /* data field used by ifainfo */

+               struct {
+                       u_int32_t if_data; /* bytes in interface queue */
+                       u_int32_t sndbuf_data; /* bytes in socket buffer */
+               } _pkt_bsr;     /* Buffer status report used by cellular interface */
+#define        bufstatus_if    _pkt_bsr.if_data
+#define        bufstatus_sndbuf        _pkt_bsr.sndbuf_data

        };
        };

-#if MEASURE_BW
-       u_int64_t pkt_bwseq;            /* sequence # */
-#endif /* MEASURE_BW */
+       u_int64_t pkt_timestamp;        /* enqueue time */
+

        /*
         * Tags (external and built-in)
         */
        SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
        /*
         * Tags (external and built-in)
         */
        SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */

-       struct proto_mtag proto_mtag;   /* built-in protocol-specific tag */
-       struct pf_mtag  pf_mtag;        /* built-in PF tag */
+       union builtin_mtag builtin_mtag;

        /*
         * Module private scratch space (32-bit aligned), currently 16-bytes
        /*
         * Module private scratch space (32-bit aligned), currently 16-bytes

-        * large.  Anything stored here is not guaranteed to survive across
-        * modules.  This should be the penultimate structure right before
-        * the red zone.  Add new fields above this.
+        * large. Anything stored here is not guaranteed to survive across
+        * modules.  The AQM layer (outbound) uses all 16-bytes for both
+        * packet scheduling and flow advisory information.

         */
        struct {
                union {
         */
        struct {
                union {


@@ -381,7 +432,13 @@ struct     pkthdr {
                        u_int64_t       __mpriv64[2];
                } __mpriv_u;
        } pkt_mpriv __attribute__((aligned(4)));
                        u_int64_t       __mpriv64[2];
                } __mpriv_u;
        } pkt_mpriv __attribute__((aligned(4)));

+#define        pkt_mpriv_hash  pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
+#define        pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
+#define        pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
+#define        pkt_mpriv_fidx  pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
+

        u_int32_t redzone;              /* red zone */
        u_int32_t redzone;              /* red zone */

+       u_int32_t pkt_compl_callbacks;  /* Packet completion callbacks */

 };
 
 /*
 };
 
 /*


@@ -394,6 +451,7 @@ struct      pkthdr {
 #define        FLOWSRC_INPCB           1       /* flow ID generated by INPCB */
 #define        FLOWSRC_IFNET           2       /* flow ID generated by interface */
 #define        FLOWSRC_PF              3       /* flow ID generated by PF */
 #define        FLOWSRC_INPCB           1       /* flow ID generated by INPCB */
 #define        FLOWSRC_IFNET           2       /* flow ID generated by interface */
 #define        FLOWSRC_PF              3       /* flow ID generated by PF */

+#define        FLOWSRC_CHANNEL         4       /* flow ID generated by channel */

 
 /*
  * Packet flags.  Unlike m_flags, all packet flags are copied along when
 
 /*
  * Packet flags.  Unlike m_flags, all packet flags are copied along when


@@ -433,6 +491,22 @@ struct     pkthdr {
 #define        PKTF_IFAINFO            0x4000  /* pkt has valid interface addr info */
 #define        PKTF_SO_BACKGROUND      0x8000  /* data is from background source */
 #define        PKTF_FORWARDED          0x10000 /* pkt was forwarded from another i/f */
 #define        PKTF_IFAINFO            0x4000  /* pkt has valid interface addr info */
 #define        PKTF_SO_BACKGROUND      0x8000  /* data is from background source */
 #define        PKTF_FORWARDED          0x10000 /* pkt was forwarded from another i/f */

+#define        PKTF_PRIV_GUARDED       0x20000 /* pkt_mpriv area guard enabled */
+#define        PKTF_KEEPALIVE          0x40000 /* pkt is kernel-generated keepalive */
+#define        PKTF_SO_REALTIME        0x80000 /* data is realtime traffic */
+#define        PKTF_VALID_UNSENT_DATA  0x100000 /* unsent data is valid */
+#define        PKTF_TCP_REXMT          0x200000 /* packet is TCP retransmission */
+#define        PKTF_REASSEMBLED        0x400000 /* Packet was reassembled */
+#define        PKTF_TX_COMPL_TS_REQ    0x800000 /* tx completion timestamp requested */
+#define        PKTF_TS_VALID           0x1000000 /* pkt timestamp is valid */
+#define        PKTF_DRIVER_MTAG        0x2000000 /* driver mbuf tags fields inited */
+#define        PKTF_NEW_FLOW           0x4000000 /* Data from a new flow */
+#define        PKTF_START_SEQ          0x8000000 /* valid start sequence */
+#define        PKTF_LAST_PKT           0x10000000 /* last packet in the flow */
+#define        PKTF_MPTCP_REINJ        0x20000000 /* Packet has been reinjected for MPTCP */
+#define        PKTF_MPTCP_DFIN         0x40000000 /* Packet is a data-fin */
+#define        PKTF_HBH_CHKED          0x80000000 /* HBH option is checked */
+

 /* flags related to flow control/advisory and identification */
 #define        PKTF_FLOW_MASK  \
        (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
 /* flags related to flow control/advisory and identification */
 #define        PKTF_FLOW_MASK  \
        (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)


@@ -440,18 +514,20 @@ struct    pkthdr {
 /*
  * Description of external storage mapped into mbuf, valid only if M_EXT set.
  */
 /*
  * Description of external storage mapped into mbuf, valid only if M_EXT set.
  */

+typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);

 struct m_ext {
        caddr_t ext_buf;                /* start of buffer */
 struct m_ext {
        caddr_t ext_buf;                /* start of buffer */

-       void    (*ext_free)             /* free routine if not the usual */
-                   (caddr_t, u_int, caddr_t);
+       m_ext_free_func_t ext_free;     /* free routine if not the usual */

        u_int   ext_size;               /* size of buffer, for ext_free */
        caddr_t ext_arg;                /* additional ext_free argument */
        u_int   ext_size;               /* size of buffer, for ext_free */
        caddr_t ext_arg;                /* additional ext_free argument */

-       struct  ext_refsq {             /* references held */
-               struct ext_refsq *forward, *backward;
-       } ext_refs;

        struct ext_ref {
        struct ext_ref {

-               u_int32_t refcnt;
-               u_int32_t flags;
+               struct mbuf *paired;
+               u_int16_t minref;
+               u_int16_t refcnt;
+               u_int16_t prefcnt;
+               u_int16_t flags;
+               u_int32_t priv;
+               uintptr_t ext_token;

        } *ext_refflags;
 };
 
        } *ext_refflags;
 };
 


@@ -462,12 +538,12 @@ typedef struct m_ext _m_ext_t;
  * The mbuf object
  */
 struct mbuf {
  * The mbuf object
  */
 struct mbuf {

-       struct  m_hdr m_hdr;
+       struct m_hdr m_hdr;

        union {
                struct {
        union {
                struct {

-                       struct  pkthdr MH_pkthdr;       /* M_PKTHDR set */
+                       struct pkthdr MH_pkthdr;        /* M_PKTHDR set */

                        union {
                        union {

-                               struct  m_ext MH_ext;   /* M_EXT set */
+                               struct m_ext MH_ext;    /* M_EXT set */

                                char    MH_databuf[_MHLEN];
                        } MH_dat;
                } MH;
                                char    MH_databuf[_MHLEN];
                        } MH_dat;
                } MH;


@@ -487,7 +563,7 @@ struct mbuf {
 #define        m_pktdat        M_dat.MH.MH_dat.MH_databuf
 #define        m_dat           M_dat.M_databuf
 #define        m_pktlen(_m)    ((_m)->m_pkthdr.len)
 #define        m_pktdat        M_dat.MH.MH_dat.MH_databuf
 #define        m_dat           M_dat.M_databuf
 #define        m_pktlen(_m)    ((_m)->m_pkthdr.len)

-#define        m_pftag(_m)     (&(_m)->m_pkthdr.pf_mtag)
+#define        m_pftag(_m)     (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)

 
 /* mbuf flags (private) */
 #define        M_EXT           0x0001  /* has associated external storage */
 
 /* mbuf flags (private) */
 #define        M_EXT           0x0001  /* has associated external storage */


@@ -536,6 +612,7 @@ struct mbuf {
 #define        CSUM_DATA_VALID         0x0400          /* csum_data field is valid */
 #define        CSUM_PSEUDO_HDR         0x0800          /* csum_data has pseudo hdr */
 #define        CSUM_PARTIAL            0x1000          /* simple Sum16 computation */
 #define        CSUM_DATA_VALID         0x0400          /* csum_data field is valid */
 #define        CSUM_PSEUDO_HDR         0x0800          /* csum_data has pseudo hdr */
 #define        CSUM_PARTIAL            0x1000          /* simple Sum16 computation */

+#define        CSUM_ZERO_INVERT        0x2000          /* invert 0 to -0 (0xffff) */

 
 #define        CSUM_DELAY_DATA         (CSUM_TCP | CSUM_UDP)
 #define        CSUM_DELAY_IP           (CSUM_IP)       /* IPv4 only: no IPv6 IP cksum */
 
 #define        CSUM_DELAY_DATA         (CSUM_TCP | CSUM_UDP)
 #define        CSUM_DELAY_IP           (CSUM_IP)       /* IPv4 only: no IPv6 IP cksum */


@@ -544,7 +621,7 @@ struct mbuf {
 
 #define        CSUM_TX_FLAGS                                                   \
        (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA |       \
 
 #define        CSUM_TX_FLAGS                                                   \
        (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA |       \

-       CSUM_DATA_VALID | CSUM_PARTIAL)
+       CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)

 
 #define        CSUM_RX_FLAGS                                                   \
        (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR |            \
 
 #define        CSUM_RX_FLAGS                                                   \
        (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR |            \


@@ -718,7 +795,8 @@ do {                                                                        \
  * If how is M_DONTWAIT and allocation fails, the original mbuf chain
  * is freed and m is set to NULL.
  */
  * If how is M_DONTWAIT and allocation fails, the original mbuf chain
  * is freed and m is set to NULL.
  */

-#define        M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how)))
+#define        M_PREPEND(m, plen, how, align)  \
+    ((m) = m_prepend_2((m), (plen), (how), (align)))

 
 /* change mbuf to new type */
 #define        MCHTYPE(m, t)           m_mchtype(m, t)
 
 /* change mbuf to new type */
 #define        MCHTYPE(m, t)           m_mchtype(m, t)


@@ -877,7 +955,16 @@ struct name {                                                      \
 #define        MBUFQ_EMPTY(q)          ((q)->mq_first == NULL)
 #define MBUFQ_FIRST(q)         ((q)->mq_first)
 #define MBUFQ_NEXT(m)          ((m)->m_nextpkt)
 #define        MBUFQ_EMPTY(q)          ((q)->mq_first == NULL)
 #define MBUFQ_FIRST(q)         ((q)->mq_first)
 #define MBUFQ_NEXT(m)          ((m)->m_nextpkt)

-#define MBUFQ_LAST(q)          (*(q)->mq_last)
+/*
+ * mq_last is initialized to point to mq_first, so check if they're
+ * equal and return NULL when the list is empty.  Otherwise, we need
+ * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
+ * to the base mbuf address to return to caller.
+ */
+#define        MBUFQ_LAST(head)                                        \
+       (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL :       \
+       ((struct mbuf *)(void *)((char *)(head)->mq_last -      \
+           (size_t)(&MBUFQ_NEXT((struct mbuf *)0)))))

 
 #define        max_linkhdr     P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
 #define        max_protohdr    P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
 
 #define        max_linkhdr     P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
 #define        max_protohdr    P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))


@@ -946,6 +1033,7 @@ struct omb_class_stat {
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */

+       u_int32_t       mbcl_release_cnt; /* amount of memory returned */

        /*
         * Cache layer statistics
         */
        /*
         * Cache layer statistics
         */


@@ -974,6 +1062,7 @@ typedef struct mb_class_stat {
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */

+       u_int32_t       mbcl_release_cnt; /* amount of memory returned */

        /*
         * Cache layer statistics
         */
        /*
         * Cache layer statistics
         */


@@ -982,7 +1071,8 @@ typedef struct mb_class_stat {
        u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
        u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
        u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
        u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
        u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
        u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */

-       u_int64_t       mbcl_reserved[4];    /* for future use */
+       u_int32_t       mbcl_peak_reported; /* last usage peak reported */
+       u_int32_t       mbcl_reserved[7];    /* for future use */

 } mb_class_stat_t;
 
 #define        MCS_DISABLED    0       /* cache is permanently disabled */
 } mb_class_stat_t;
 
 #define        MCS_DISABLED    0       /* cache is permanently disabled */


@@ -1082,6 +1172,8 @@ struct mbuf;
 #define        M_COPYM_NOOP_HDR        0       /* don't copy/move pkthdr contents */
 #define        M_COPYM_COPY_HDR        1       /* copy pkthdr from old to new */
 #define        M_COPYM_MOVE_HDR        2       /* move pkthdr from old to new */
 #define        M_COPYM_NOOP_HDR        0       /* don't copy/move pkthdr contents */
 #define        M_COPYM_COPY_HDR        1       /* copy pkthdr from old to new */
 #define        M_COPYM_MOVE_HDR        2       /* move pkthdr from old to new */

+#define        M_COPYM_MUST_COPY_HDR   3       /* MUST copy pkthdr from old to new */
+#define        M_COPYM_MUST_MOVE_HDR   4       /* MUST move pkthdr from old to new */

 
 /*
  * These macros are mapped to the appropriate KPIs, so that private code
 
 /*
  * These macros are mapped to the appropriate KPIs, so that private code


@@ -1106,7 +1198,7 @@ extern struct mbuf *m_getpacket(void);
 extern struct mbuf *m_getpackets(int, int, int);
 extern struct mbuf *m_mclget(struct mbuf *, int);
 extern void *m_mtod(struct mbuf *);
 extern struct mbuf *m_getpackets(int, int, int);
 extern struct mbuf *m_mclget(struct mbuf *, int);
 extern void *m_mtod(struct mbuf *);

-extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
+extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);

 extern struct mbuf *m_pullup(struct mbuf *, int);
 extern struct mbuf *m_split(struct mbuf *, int, int);
 extern void m_mclfree(caddr_t p);
 extern struct mbuf *m_pullup(struct mbuf *, int);
 extern struct mbuf *m_split(struct mbuf *, int, int);
 extern void m_mclfree(caddr_t p);


@@ -1172,6 +1264,9 @@ extern void m_mclfree(caddr_t p);
 #define        MBUF_TC2SCVAL(_tc)      ((_tc) << 7)
 #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
        ((_sc) == MBUF_SC_BK))
 #define        MBUF_TC2SCVAL(_tc)      ((_tc) << 7)
 #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
        ((_sc) == MBUF_SC_BK))

+#define        IS_MBUF_SC_REALTIME(_sc)        ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
+#define IS_MBUF_SC_BESTEFFORT(_sc)     ((_sc) == MBUF_SC_BE || \
+    (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)

 
 #define        SCIDX_BK_SYS            MBUF_SCIDX(MBUF_SC_BK_SYS)
 #define        SCIDX_BK                MBUF_SCIDX(MBUF_SC_BK)
 
 #define        SCIDX_BK_SYS            MBUF_SCIDX(MBUF_SC_BK_SYS)
 #define        SCIDX_BK                MBUF_SCIDX(MBUF_SC_BK)


@@ -1213,8 +1308,8 @@ extern void m_mclfree(caddr_t p);
        c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO ||              \
        c == SCVAL_CTL)
 
        c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO ||              \
        c == SCVAL_CTL)
 

-extern union mbigcluster *mbutl;       /* start VA of mbuf pool */
-extern union mbigcluster *embutl;      /* end VA of mbuf pool */
+extern unsigned char *mbutl;   /* start VA of mbuf pool */
+extern unsigned char *embutl;  /* end VA of mbuf pool */

 extern unsigned int nmbclusters;       /* number of mapped clusters */
 extern int njcl;               /* # of jumbo clusters  */
 extern int njclbytes;  /* size of a jumbo cluster */
 extern unsigned int nmbclusters;       /* number of mapped clusters */
 extern int njcl;               /* # of jumbo clusters  */
 extern int njclbytes;  /* size of a jumbo cluster */


@@ -1230,7 +1325,7 @@ extern int _max_protohdr; /* largest protocol header */
 __private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
 __private_extern__ void mbinit(void);
 __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
 __private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
 __private_extern__ void mbinit(void);
 __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,

-    void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
+    void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int);

 __private_extern__ caddr_t m_bigalloc(int);
 __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
 __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
 __private_extern__ caddr_t m_bigalloc(int);
 __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
 __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);


@@ -1289,6 +1384,13 @@ __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
 __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
     unsigned int *, int, int, size_t);
 
 __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
     unsigned int *, int, int, size_t);
 

+__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
+__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
+__private_extern__ int m_ext_paired_is_active(struct mbuf *);
+__private_extern__ void m_ext_paired_activate(struct mbuf *);
+
+__private_extern__ void m_drain(void);
+

 /*
  * Packets may have annotations attached by affixing a list of "packet
  * tags" to the pkthdr structure.  Packet tags are dynamically allocated
 /*
  * Packets may have annotations attached by affixing a list of "packet
  * tags" to the pkthdr structure.  Packet tags are dynamically allocated


@@ -1329,7 +1431,7 @@ enum {
        KERNEL_TAG_TYPE_ENCAP                   = 8,
        KERNEL_TAG_TYPE_INET6                   = 9,
        KERNEL_TAG_TYPE_IPSEC                   = 10,
        KERNEL_TAG_TYPE_ENCAP                   = 8,
        KERNEL_TAG_TYPE_INET6                   = 9,
        KERNEL_TAG_TYPE_IPSEC                   = 10,

-       KERNEL_TAG_TYPE_DRVAUX                  = 11
+       KERNEL_TAG_TYPE_DRVAUX                  = 11,

 };
 
 /* Packet tag routines */
 };
 
 /* Packet tag routines */


@@ -1374,9 +1476,17 @@ __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
 } while (0)
 
 __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
 } while (0)
 
 __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,

-    u_int32_t, u_int32_t);
+    u_int32_t, u_int32_t, u_int32_t);

 __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
 
 __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
 

+__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
+    m_ext_free_func_t, caddr_t);
+__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
+__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
+__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);
+
+extern void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);
+

 __END_DECLS
 #endif /* XNU_KERNEL_PRIVATE */
 #endif /* KERNEL */
 __END_DECLS
 #endif /* XNU_KERNEL_PRIVATE */
 #endif /* KERNEL */