[apple/xnu.git] / bsd / sys / mbuf.h

/*
 * Copyright (c) 1999-2017 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
/*
 * Mach Operating System
 * Copyright (c) 1987 Carnegie-Mellon University
 * All rights reserved.  The CMU software License Agreement specifies
 * the terms and conditions for use and redistribution.
 */
/*
 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
 *
 * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by the University of
 *      California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)mbuf.h      8.3 (Berkeley) 1/21/94
 */
/*
 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
 * support for mandatory and extensible security protections.  This notice
 * is included in support of clause 2.2 (b) of the Apple Public License,
 * Version 2.0.
 */

#ifndef _SYS_MBUF_H_
#define _SYS_MBUF_H_

#include <sys/cdefs.h>
#include <sys/appleapiopts.h>
#include <sys/_types/_u_int32_t.h> /* u_int32_t */
#include <sys/_types/_u_int64_t.h> /* u_int64_t */
#include <sys/_types/_u_short.h> /* u_short */

#ifdef XNU_KERNEL_PRIVATE

#include <sys/lock.h>
#include <sys/queue.h>
#include <machine/endian.h>
/*
 * Mbufs are of a single size, MSIZE (machine/param.h), which
 * includes overhead.  An mbuf may add a single "mbuf cluster" of size
 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
 * no additional overhead and is used instead of the internal data area;
 * this is done when at least MINCLSIZE of data must be stored.
 */

/*
 * The following _MLEN and _MHLEN macros are private to xnu.  Private code
 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
 * the sizes of the structures are dependent upon specific xnu configs.
 */
#define _MLEN           (MSIZE - sizeof(struct m_hdr))  /* normal data len */
#define _MHLEN          (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */

#define NMBPGSHIFT      (PAGE_SHIFT - MSIZESHIFT)
#define NMBPG           (1 << NMBPGSHIFT)       /* # of mbufs per page */

#define NCLPGSHIFT      (PAGE_SHIFT - MCLSHIFT)
#define NCLPG           (1 << NCLPGSHIFT)       /* # of cl per page */

#define NBCLPGSHIFT     (PAGE_SHIFT - MBIGCLSHIFT)
#define NBCLPG          (1 << NBCLPGSHIFT)      /* # of big cl per page */

#define NMBPCLSHIFT     (MCLSHIFT - MSIZESHIFT)
#define NMBPCL          (1 << NMBPCLSHIFT)      /* # of mbufs per cl */

#define NCLPJCLSHIFT    (M16KCLSHIFT - MCLSHIFT)
#define NCLPJCL         (1 << NCLPJCLSHIFT)     /* # of cl per jumbo cl */

#define NCLPBGSHIFT     (MBIGCLSHIFT - MCLSHIFT)
#define NCLPBG          (1 << NCLPBGSHIFT)      /* # of cl per big cl */

#define NMBPBGSHIFT     (MBIGCLSHIFT - MSIZESHIFT)
#define NMBPBG          (1 << NMBPBGSHIFT)      /* # of mbufs per big cl */

/*
 * Macros for type conversion
 * mtod(m,t) -  convert mbuf pointer to data pointer of correct type
 * mtodo(m, o) -- Same as above but with offset 'o' into data.
 * dtom(x) -    convert data pointer within mbuf to mbuf pointer (XXX)
 */
#define mtod(m, t)      ((t)m_mtod(m))
#define mtodo(m, o)     ((void *)(mtod(m, uint8_t *) + (o)))
#define dtom(x)         m_dtom(x)

/* header at beginning of each mbuf: */
struct m_hdr {
        struct mbuf     *mh_next;       /* next buffer in chain */
        struct mbuf     *mh_nextpkt;    /* next chain in queue/record */
        caddr_t         mh_data;        /* location of data */
        int32_t         mh_len;         /* amount of data in this mbuf */
        u_int16_t       mh_type;        /* type of data in this mbuf */
        u_int16_t       mh_flags;       /* flags; see below */
#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
 * should be defined using the offsetof(struct mbuf, M_dat), since there is
 * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
 * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
 * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
 * in the definition of mbuf.
 */
} __attribute__((aligned(8)));
#else
};
#endif

/*
 * Packet tag structure (see below for details).
 */
struct m_tag {
        u_int64_t               m_tag_cookie;   /* Error checking */
#ifndef __LP64__
        u_int32_t               pad;            /* For structure alignment */
#endif /* !__LP64__ */
        SLIST_ENTRY(m_tag)      m_tag_link;     /* List of packet tags */
        u_int16_t               m_tag_type;     /* Module specific type */
        u_int16_t               m_tag_len;      /* Length of data */
        u_int32_t               m_tag_id;       /* Module ID */
};

#define M_TAG_ALIGN(len) \
        (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))

#define M_TAG_VALID_PATTERN     0xfeedfacefeedfaceULL
#define M_TAG_FREE_PATTERN      0xdeadbeefdeadbeefULL

/*
 * Packet tag header structure (at the top of mbuf).  Pointers are
 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
 */
struct m_taghdr {
#ifndef __LP64__
        u_int32_t               pad;            /* For structure alignment */
#endif /* !__LP64__ */
        u_int64_t               refcnt;         /* Number of tags in this mbuf */
};

/*
 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
 */
struct m_drvaux_tag {
        u_int32_t       da_family;      /* IFNET_FAMILY values */
        u_int32_t       da_subfamily;   /* IFNET_SUBFAMILY values */
        u_int32_t       da_reserved;    /* for future */
        u_int32_t       da_length;      /* length of following data */
};

/* Values for pftag_flags (16-bit wide) */
#define PF_TAG_GENERATED                0x1     /* pkt generated by PF */
#define PF_TAG_FRAGCACHE                0x2
#define PF_TAG_TRANSLATE_LOCALHOST      0x4
#if PF_ECN
#define PF_TAG_HDR_INET                 0x8     /* hdr points to IPv4 */
#define PF_TAG_HDR_INET6                0x10    /* hdr points to IPv6 */
#endif /* PF_ECN */
/*
 * PF mbuf tag
 */
struct pf_mtag {
        u_int16_t       pftag_flags;    /* PF_TAG flags */
        u_int16_t       pftag_rtableid; /* alternate routing table id */
        u_int16_t       pftag_tag;
        u_int16_t       pftag_routed;
#if PF_ECN
        void            *pftag_hdr;     /* saved hdr pos in mbuf, for ECN */
#endif /* PF_ECN */
};

/*
 * TCP mbuf tag
 */
struct tcp_pktinfo {
        union {
                struct {
                        u_int32_t segsz;        /* segment size (actual MSS) */
                        u_int32_t start_seq;    /* start seq of this packet */
                } __tx;
                struct {
                        u_int16_t lro_pktlen;   /* max seg size encountered */
                        u_int8_t  lro_npkts;    /* # of coalesced TCP pkts */
                        u_int8_t  lro_timediff; /* time spent in LRO */
                } __rx;
        } __offload;
        union {
                u_int32_t       pri;            /* send msg priority */
                u_int32_t       seq;            /* recv msg sequence # */
        } __msgattr;
#define tso_segsz       proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
#define tx_start_seq    proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
#define lro_pktlen      proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
#define lro_npkts       proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
#define lro_elapsed     proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
#define msg_pri         proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri
#define msg_seq         proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq
};

/*
 * MPTCP mbuf tag
 */
struct mptcp_pktinfo {
        u_int64_t       mtpi_dsn;       /* MPTCP Data Sequence Number */
        u_int32_t       mtpi_rel_seq;   /* Relative Seq Number */
        u_int16_t       mtpi_length;    /* Length of mapping */
        u_int16_t       mtpi_csum;
#define mp_dsn          proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
#define mp_rseq         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
#define mp_rlen         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
#define mp_csum         proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
};

/*
 * TCP specific mbuf tag.  Note that the current implementation uses
 * MPTCP metadata strictly between MPTCP and the TCP subflow layers,
 * hence tm_tcp and tm_mptcp are mutually exclusive.  This also means
 * that TCP messages functionality is currently incompatible with MPTCP.
 */
struct tcp_mtag {
        union {
                struct tcp_pktinfo      tm_tcp;         /* TCP and below */
                struct mptcp_pktinfo    tm_mptcp;       /* MPTCP-TCP only */
        };
};

struct driver_mtag_ {
        uintptr_t               _drv_tx_compl_arg;
        uintptr_t               _drv_tx_compl_data;
        kern_return_t           _drv_tx_status;
        uint16_t                _drv_flowid;
#define drv_tx_compl_arg        builtin_mtag._drv_mtag._drv_tx_compl_arg
#define drv_tx_compl_data       builtin_mtag._drv_mtag._drv_tx_compl_data
#define drv_tx_status           builtin_mtag._drv_mtag._drv_tx_status
#define drv_flowid              builtin_mtag._drv_mtag._drv_flowid
};

/*
 * Protocol specific mbuf tag (at most one protocol metadata per mbuf).
 *
 * Care must be taken to ensure that they are mutually exclusive, e.g.
 * IPSec policy ID implies no TCP segment offload (which is fine given
 * that the former is used on the virtual ipsec interface that does
 * not advertise the TSO capability.)
 */
struct proto_mtag_ {
        union {
                struct tcp_mtag tcp;            /* TCP specific */
        } __pr_u;
};

/*
 * NECP specific mbuf tag.
 */
struct necp_mtag_ {
        u_int32_t       necp_policy_id;
        u_int32_t       necp_last_interface_index;
        u_int32_t       necp_route_rule_id;
        u_int32_t       necp_app_id;
};

union builtin_mtag {
        struct {
                struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
                struct pf_mtag  _pf_mtag;       /* built-in PF tag */
                struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
        } _net_mtag;
        struct driver_mtag_ _drv_mtag;
#define necp_mtag builtin_mtag._net_mtag._necp_mtag
#define proto_mtag builtin_mtag._net_mtag._proto_mtag
#define driver_mtag builtin_mtag._drv_mtag
};

/*
 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
 */
struct pkthdr {
        struct ifnet *rcvif;            /* rcv interface */
        /* variables for ip and tcp reassembly */
        void    *pkt_hdr;               /* pointer to packet header */
        int32_t len;                    /* total packet length */
        /* variables for hardware checksum */
        /* Note: csum_flags is used for hardware checksum and VLAN */
        u_int32_t csum_flags;           /* flags regarding checksum */
        union {
                struct {
                        u_int16_t val;   /* checksum value */
                        u_int16_t start; /* checksum start offset */
                } _csum_rx;
#define csum_rx_val     _csum_rx.val
#define csum_rx_start   _csum_rx.start
                struct {
                        u_int16_t start; /* checksum start offset */
                        u_int16_t stuff; /* checksum stuff offset */
                } _csum_tx;
#define csum_tx_start   _csum_tx.start
#define csum_tx_stuff   _csum_tx.stuff
                u_int32_t csum_data;    /* data field used by csum routines */
        };
        u_int16_t vlan_tag;             /* VLAN tag, host byte order */
        /*
         * Packet classifier info
         *
         * PKTF_FLOW_ID set means valid flow ID.  A non-zero flow ID value
         * means the packet has been classified by one of the flow sources.
         * It is also a prerequisite for flow control advisory, which is
         * enabled by additionally setting PKTF_FLOW_ADV.
         *
         * The protocol value is a best-effort representation of the payload.
         * It is opportunistically updated and used only for optimization.
         * It is not a substitute for parsing the protocol header(s); use it
         * only as a hint.
         *
         * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the
         * indices of interfaces which own the source and/or destination
         * addresses of the packet.  For the local/loopback case (PKTF_LOOP),
         * both should be valid, and thus allows for the receiving end to
         * quickly determine the actual interfaces used by the the addresses;
         * they may not necessarily be the same or refer to the loopback
         * interface.  Otherwise, in the non-local/loopback case, the indices
         * are opportunistically set, and because of that only one may be set
         * (0 means the index has not been determined.)  In addition, the
         * interface address flags are also recorded.  This allows us to avoid
         * storing the corresponding {in,in6}_ifaddr in an mbuf tag.  Ideally
         * this would be a superset of {ia,ia6}_flags, but the namespaces are
         * overlapping at present, so we'll need a new set of values in future
         * to achieve this.  For now, we will just rely on the address family
         * related code paths examining this mbuf to interpret the flags.
         */
        u_int8_t pkt_proto;             /* IPPROTO value */
        u_int8_t pkt_flowsrc;           /* FLOWSRC values */
        u_int32_t pkt_flowid;           /* flow ID */
        u_int32_t pkt_flags;            /* PKTF flags (see below) */
        u_int32_t pkt_svc;              /* MBUF_SVC value */

        u_int32_t pkt_compl_context;            /* Packet completion context */

        union {
                struct {
                        u_int16_t src;          /* ifindex of src addr i/f */
                        u_int16_t src_flags;    /* src PKT_IFAIFF flags */
                        u_int16_t dst;          /* ifindex of dst addr i/f */
                        u_int16_t dst_flags;    /* dst PKT_IFAIFF flags */
                } _pkt_iaif;
#define src_ifindex     _pkt_iaif.src
#define src_iff         _pkt_iaif.src_flags
#define dst_ifindex     _pkt_iaif.dst
#define dst_iff         _pkt_iaif.dst_flags
                u_int64_t pkt_ifainfo;  /* data field used by ifainfo */
                struct {
                        u_int32_t if_data; /* bytes in interface queue */
                        u_int32_t sndbuf_data; /* bytes in socket buffer */
                } _pkt_bsr;     /* Buffer status report used by cellular interface */
#define bufstatus_if    _pkt_bsr.if_data
#define bufstatus_sndbuf        _pkt_bsr.sndbuf_data
        };
        u_int64_t pkt_timestamp;        /* enqueue time */

        /*
         * Tags (external and built-in)
         */
        SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
        union builtin_mtag builtin_mtag;
        /*
         * Module private scratch space (32-bit aligned), currently 16-bytes
         * large. Anything stored here is not guaranteed to survive across
         * modules.  The AQM layer (outbound) uses all 16-bytes for both
         * packet scheduling and flow advisory information.
         */
        struct {
                union {
                        u_int8_t        __mpriv8[16];
                        u_int16_t       __mpriv16[8];
                        struct {
                                union {
                                        u_int8_t        __val8[4];
                                        u_int16_t       __val16[2];
                                        u_int32_t       __val32;
                                } __mpriv32_u;
                        }               __mpriv32[4];
                        u_int64_t       __mpriv64[2];
                } __mpriv_u;
        } pkt_mpriv __attribute__((aligned(4)));
#define pkt_mpriv_hash  pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
#define pkt_mpriv_fidx  pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32

        u_int32_t redzone;              /* red zone */
        u_int32_t pkt_compl_callbacks;  /* Packet completion callbacks */
};

/*
 * Flow data source type.  A data source module is responsible for generating
 * a unique flow ID and associating it to each data flow as pkt_flowid.
 * This is required for flow control/advisory, as it allows the output queue
 * to identify the data source object and inform that it can resume its
 * transmission (in the event it was flow controlled.)
 */
#define FLOWSRC_INPCB           1       /* flow ID generated by INPCB */
#define FLOWSRC_IFNET           2       /* flow ID generated by interface */
#define FLOWSRC_PF              3       /* flow ID generated by PF */
#define FLOWSRC_CHANNEL         4       /* flow ID generated by channel */

/*
 * Packet flags.  Unlike m_flags, all packet flags are copied along when
 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here.  These flags
 * (and other classifier info) will be cleared during DLIL input.
 *
 * Some notes about M_LOOP and PKTF_LOOP:
 *
 *    - M_LOOP flag is overloaded, and its use is discouraged.  Historically,
 *      that flag was used by the KAME implementation for allowing certain
 *      certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this
 *      was originally meant to be set as the packet is looped back to the
 *      system, and in some circumstances temporarily set in ip6_output().
 *      Over time, this flag was used by the pre-output routines to indicate
 *      to the DLIL frameout and output routines, that the packet may be
 *      looped back to the system under the right conditions.  In addition,
 *      this is an mbuf flag rather than an mbuf packet header flag.
 *
 *    - PKTF_LOOP is an mbuf packet header flag, which is set if and only
 *      if the packet was looped back to the system.  This flag should be
 *      used instead for newer code.
 */
#define PKTF_FLOW_ID            0x1     /* pkt has valid flowid value */
#define PKTF_FLOW_ADV           0x2     /* pkt triggers local flow advisory */
#define PKTF_FLOW_LOCALSRC      0x4     /* pkt is locally originated  */
#define PKTF_FLOW_RAWSOCK       0x8     /* pkt locally generated by raw sock */
#define PKTF_PRIO_PRIVILEGED    0x10    /* packet priority is privileged */
#define PKTF_PROXY_DST          0x20    /* processed but not locally destined */
#define PKTF_INET_RESOLVE       0x40    /* IPv4 resolver packet */
#define PKTF_INET6_RESOLVE      0x80    /* IPv6 resolver packet */
#define PKTF_RESOLVE_RTR        0x100   /* pkt is for resolving router */
#define PKTF_SW_LRO_PKT         0x200   /* pkt is a large coalesced pkt */
#define PKTF_SW_LRO_DID_CSUM    0x400   /* IP and TCP checksums done by LRO */
#define PKTF_MPTCP              0x800   /* TCP with MPTCP metadata */
#define PKTF_MPSO               0x1000  /* MPTCP socket meta data */
#define PKTF_LOOP               0x2000  /* loopbacked packet */
#define PKTF_IFAINFO            0x4000  /* pkt has valid interface addr info */
#define PKTF_SO_BACKGROUND      0x8000  /* data is from background source */
#define PKTF_FORWARDED          0x10000 /* pkt was forwarded from another i/f */
#define PKTF_PRIV_GUARDED       0x20000 /* pkt_mpriv area guard enabled */
#define PKTF_KEEPALIVE          0x40000 /* pkt is kernel-generated keepalive */
#define PKTF_SO_REALTIME        0x80000 /* data is realtime traffic */
#define PKTF_VALID_UNSENT_DATA  0x100000 /* unsent data is valid */
#define PKTF_TCP_REXMT          0x200000 /* packet is TCP retransmission */
#define PKTF_REASSEMBLED        0x400000 /* Packet was reassembled */
#define PKTF_TX_COMPL_TS_REQ    0x800000 /* tx completion timestamp requested */
#define PKTF_TS_VALID           0x1000000 /* pkt timestamp is valid */
#define PKTF_DRIVER_MTAG        0x2000000 /* driver mbuf tags fields inited */
#define PKTF_NEW_FLOW           0x4000000 /* Data from a new flow */
#define PKTF_START_SEQ          0x8000000 /* valid start sequence */
#define PKTF_LAST_PKT           0x10000000 /* last packet in the flow */
#define PKTF_MPTCP_REINJ        0x20000000 /* Packet has been reinjected for MPTCP */

/* flags related to flow control/advisory and identification */
#define PKTF_FLOW_MASK  \
        (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)

/*
 * Description of external storage mapped into mbuf, valid only if M_EXT set.
 */
typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);
struct m_ext {
        caddr_t ext_buf;                /* start of buffer */
        m_ext_free_func_t ext_free;     /* free routine if not the usual */
        u_int   ext_size;               /* size of buffer, for ext_free */
        caddr_t ext_arg;                /* additional ext_free argument */
        struct ext_ref {
                struct mbuf *paired;
                u_int16_t minref;
                u_int16_t refcnt;
                u_int16_t prefcnt;
                u_int16_t flags;
                u_int32_t priv;
                uintptr_t ext_token;
        } *ext_refflags;
};

/* define m_ext to a type since it gets redefined below */
typedef struct m_ext _m_ext_t;

/*
 * The mbuf object
 */
struct mbuf {
        struct m_hdr m_hdr;
        union {
                struct {
                        struct pkthdr MH_pkthdr;        /* M_PKTHDR set */
                        union {
                                struct m_ext MH_ext;    /* M_EXT set */
                                char    MH_databuf[_MHLEN];
                        } MH_dat;
                } MH;
                char    M_databuf[_MLEN];               /* !M_PKTHDR, !M_EXT */
        } M_dat;
};

#define m_next          m_hdr.mh_next
#define m_len           m_hdr.mh_len
#define m_data          m_hdr.mh_data
#define m_type          m_hdr.mh_type
#define m_flags         m_hdr.mh_flags
#define m_nextpkt       m_hdr.mh_nextpkt
#define m_act           m_nextpkt
#define m_pkthdr        M_dat.MH.MH_pkthdr
#define m_ext           M_dat.MH.MH_dat.MH_ext
#define m_pktdat        M_dat.MH.MH_dat.MH_databuf
#define m_dat           M_dat.M_databuf
#define m_pktlen(_m)    ((_m)->m_pkthdr.len)
#define m_pftag(_m)     (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)

/* mbuf flags (private) */
#define M_EXT           0x0001  /* has associated external storage */
#define M_PKTHDR        0x0002  /* start of record */
#define M_EOR           0x0004  /* end of record */
#define M_PROTO1        0x0008  /* protocol-specific */
#define M_PROTO2        0x0010  /* protocol-specific */
#define M_PROTO3        0x0020  /* protocol-specific */
#define M_LOOP          0x0040  /* packet is looped back (also see PKTF_LOOP) */
#define M_PROTO5        0x0080  /* protocol-specific */

/* mbuf pkthdr flags, also in m_flags (private) */
#define M_BCAST         0x0100  /* send/received as link-level broadcast */
#define M_MCAST         0x0200  /* send/received as link-level multicast */
#define M_FRAG          0x0400  /* packet is a fragment of a larger packet */
#define M_FIRSTFRAG     0x0800  /* packet is first fragment */
#define M_LASTFRAG      0x1000  /* packet is last fragment */
#define M_PROMISC       0x2000  /* packet is promiscuous (shouldn't go to stack) */
#define M_HASFCS        0x4000  /* packet has FCS */
#define M_TAGHDR        0x8000  /* m_tag hdr structure at top of mbuf data */

/*
 * Flags to purge when crossing layers.
 */
#define M_PROTOFLAGS \
        (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)

/* flags copied when copying m_pkthdr */
#define M_COPYFLAGS                                                     \
        (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 |                    \
        M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG |                        \
        M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)

/* flags indicating hw checksum support and sw checksum requirements */
#define CSUM_IP                 0x0001          /* will csum IP */
#define CSUM_TCP                0x0002          /* will csum TCP */
#define CSUM_UDP                0x0004          /* will csum UDP */
#define CSUM_IP_FRAGS           0x0008          /* will csum IP fragments */
#define CSUM_FRAGMENT           0x0010          /* will do IP fragmentation */
#define CSUM_TCPIPV6            0x0020          /* will csum TCP for IPv6 */
#define CSUM_UDPIPV6            0x0040          /* will csum UDP for IPv6 */
#define CSUM_FRAGMENT_IPV6      0x0080          /* will do IPv6 fragmentation */

#define CSUM_IP_CHECKED         0x0100          /* did csum IP */
#define CSUM_IP_VALID           0x0200          /*   ... the csum is valid */
#define CSUM_DATA_VALID         0x0400          /* csum_data field is valid */
#define CSUM_PSEUDO_HDR         0x0800          /* csum_data has pseudo hdr */
#define CSUM_PARTIAL            0x1000          /* simple Sum16 computation */
#define CSUM_ZERO_INVERT        0x2000          /* invert 0 to -0 (0xffff) */

#define CSUM_DELAY_DATA         (CSUM_TCP | CSUM_UDP)
#define CSUM_DELAY_IP           (CSUM_IP)       /* IPv4 only: no IPv6 IP cksum */
#define CSUM_DELAY_IPV6_DATA    (CSUM_TCPIPV6 | CSUM_UDPIPV6)
#define CSUM_DATA_IPV6_VALID    CSUM_DATA_VALID /* csum_data field is valid */

#define CSUM_TX_FLAGS                                                   \
        (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA |       \
        CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)

#define CSUM_RX_FLAGS                                                   \
        (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR |            \
        CSUM_DATA_VALID | CSUM_PARTIAL)

/*
 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
 */

/* VLAN tag present */
#define CSUM_VLAN_TAG_VALID     0x10000         /* vlan_tag field is valid */

/* TCP Segment Offloading requested on this mbuf */
#define CSUM_TSO_IPV4           0x100000        /* This mbuf needs to be segmented by the NIC */
#define CSUM_TSO_IPV6           0x200000        /* This mbuf needs to be segmented by the NIC */

#define TSO_IPV4_OK(_ifp, _m)                                           \
    (((_ifp)->if_hwassist & IFNET_TSO_IPV4) &&                          \
    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4))                        \

#define TSO_IPV4_NOTOK(_ifp, _m)                                        \
    (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) &&                         \
    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4))                        \

#define TSO_IPV6_OK(_ifp, _m)                                           \
    (((_ifp)->if_hwassist & IFNET_TSO_IPV6) &&                          \
    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6))                        \

#define TSO_IPV6_NOTOK(_ifp, _m)                                        \
    (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) &&                         \
    ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6))                        \

#endif /* XNU_KERNEL_PRIVATE */

/* mbuf types */
#define MT_FREE         0       /* should be on free list */
#define MT_DATA         1       /* dynamic (data) allocation */
#define MT_HEADER       2       /* packet header */
#define MT_SOCKET       3       /* socket structure */
#define MT_PCB          4       /* protocol control block */
#define MT_RTABLE       5       /* routing tables */
#define MT_HTABLE       6       /* IMP host tables */
#define MT_ATABLE       7       /* address resolution tables */
#define MT_SONAME       8       /* socket name */
#define MT_SOOPTS       10      /* socket options */
#define MT_FTABLE       11      /* fragment reassembly header */
#define MT_RIGHTS       12      /* access rights */
#define MT_IFADDR       13      /* interface address */
#define MT_CONTROL      14      /* extra-data protocol message */
#define MT_OOBDATA      15      /* expedited data  */
#define MT_TAG          16      /* volatile metadata associated to pkts */
#define MT_MAX          32      /* enough? */

#ifdef XNU_KERNEL_PRIVATE
/*
 * mbuf allocation/deallocation macros:
 *
 *      MGET(struct mbuf *m, int how, int type)
 * allocates an mbuf and initializes it to contain internal data.
 *
 *      MGETHDR(struct mbuf *m, int how, int type)
 * allocates an mbuf and initializes it to contain a packet header
 * and internal data.
 */

#if 1
#define MCHECK(m) m_mcheck(m)
#else
#define MCHECK(m)
#endif

#define MGET(m, how, type) ((m) = m_get((how), (type)))

#define MGETHDR(m, how, type)   ((m) = m_gethdr((how), (type)))

/*
 * Mbuf cluster macros.
 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
 * MCLGET adds such clusters to a normal mbuf;
 * the flag M_EXT is set upon success.
 * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
 * freeing the cluster if the reference count has reached 0.
 *
 * Normal mbuf clusters are normally treated as character arrays
 * after allocation, but use the first word of the buffer as a free list
 * pointer while on the free list.
 */
union mcluster {
        union   mcluster *mcl_next;
        char    mcl_buf[MCLBYTES];
};

#define MCLALLOC(p, how)        ((p) = m_mclalloc(how))

#define MCLFREE(p)              m_mclfree(p)

#define MCLGET(m, how)          ((m) = m_mclget(m, how))

/*
 * Mbuf big cluster
 */
union mbigcluster {
        union mbigcluster       *mbc_next;
        char                    mbc_buf[MBIGCLBYTES];
};

/*
 * Mbuf jumbo cluster
 */
union m16kcluster {
        union m16kcluster       *m16kcl_next;
        char                    m16kcl_buf[M16KCLBYTES];
};

#define MCLHASREFERENCE(m)      m_mclhasreference(m)

/*
 * MFREE(struct mbuf *m, struct mbuf *n)
 * Free a single mbuf and associated external storage.
 * Place the successor, if any, in n.
 */

#define MFREE(m, n) ((n) = m_free(m))

/*
 * Copy mbuf pkthdr from from to to.
 * from must have M_PKTHDR set, and to must be empty.
 * aux pointer will be moved to `to'.
 */
#define M_COPY_PKTHDR(to, from)         m_copy_pkthdr(to, from)

#define M_COPY_PFTAG(to, from)          m_copy_pftag(to, from)

#define M_COPY_CLASSIFIER(to, from)     m_copy_classifier(to, from)

/*
 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
 * an object of the specified size at the end of the mbuf, longword aligned.
 */
#define M_ALIGN(m, len)                                                 \
do {                                                                    \
        (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1);           \
} while (0)

/*
 * As above, for mbufs allocated with m_gethdr/MGETHDR
 * or initialized by M_COPY_PKTHDR.
 */
#define MH_ALIGN(m, len)                                                \
do {                                                                    \
        (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1);          \
} while (0)

/*
 * Compute the amount of space available
 * before the current start of data in an mbuf.
 * Subroutine - data not available if certain references.
 */
#define M_LEADINGSPACE(m)       m_leadingspace(m)

/*
 * Compute the amount of space available
 * after the end of data in an mbuf.
 * Subroutine - data not available if certain references.
 */
#define M_TRAILINGSPACE(m)      m_trailingspace(m)

/*
 * Arrange to prepend space of size plen to mbuf m.
 * If a new mbuf must be allocated, how specifies whether to wait.
 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
 * is freed and m is set to NULL.
 */
#define M_PREPEND(m, plen, how, align)  \
    ((m) = m_prepend_2((m), (plen), (how), (align)))

/* change mbuf to new type */
#define MCHTYPE(m, t)           m_mchtype(m, t)

/* compatiblity with 4.3 */
#define m_copy(m, o, l)         m_copym((m), (o), (l), M_DONTWAIT)

#define MBSHIFT         20                              /* 1MB */
#define MBSIZE          (1 << MBSHIFT)
#define GBSHIFT         30                              /* 1GB */
#define GBSIZE          (1 << GBSHIFT)

/*
 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
 * "off+len") is located in single mbuf, on contiguous memory region.
 * The pointer to the region will be returned to pointer variable "val",
 * with type "typ".
 *
 * M_STRUCT_GET0 does the same, except that it aligns the structure at
 * very top of mbuf.  GET0 is likely to make memory copy than GET.
 */
#define M_STRUCT_GET(val, typ, m, off, len)                             \
do {                                                                    \
        struct mbuf *t;                                                 \
        int tmp;                                                        \
                                                                        \
        if ((m)->m_len >= (off) + (len)) {                              \
                (val) = (typ)(mtod((m), caddr_t) + (off));              \
        } else {                                                        \
                t = m_pulldown((m), (off), (len), &tmp);                \
                if (t != NULL) {                                        \
                        if (t->m_len < tmp + (len))                     \
                                panic("m_pulldown malfunction");        \
                        (val) = (typ)(mtod(t, caddr_t) + tmp);          \
                } else {                                                \
                        (val) = (typ)NULL;                              \
                        (m) = NULL;                                     \
                }                                                       \
        }                                                               \
} while (0)

#define M_STRUCT_GET0(val, typ, m, off, len)                            \
do {                                                                    \
        struct mbuf *t;                                                 \
                                                                        \
        if ((off) == 0 && ((m)->m_len >= (len))) {                      \
                (val) = (typ)(void *)mtod(m, caddr_t);                  \
        } else {                                                        \
                t = m_pulldown((m), (off), (len), NULL);                \
                if (t != NULL) {                                        \
                        if (t->m_len < (len))                           \
                                panic("m_pulldown malfunction");        \
                        (val) = (typ)(void *)mtod(t, caddr_t);          \
                } else {                                                \
                        (val) = (typ)NULL;                              \
                        (m) = NULL;                                     \
                }                                                       \
        }                                                               \
} while (0)

#define MBUF_INPUT_CHECK(m, rcvif)                                      \
do {                                                                    \
        if (!(m->m_flags & MBUF_PKTHDR) ||                              \
            m->m_len < 0 ||                                             \
            m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) ||        \
            m->m_type == MT_FREE ||                                     \
            ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) {  \
                panic_plain("Failed mbuf validity check: mbuf %p len %d "  \
                    "type %d flags 0x%x data %p rcvif %s ifflags 0x%x",  \
                    m, m->m_len, m->m_type, m->m_flags,                    \
                    ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
                    if_name(rcvif),                                     \
                    (rcvif->if_flags & 0xffff));                        \
        }                                                               \
} while (0)

/*
 * Simple mbuf queueing system
 *
 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
 * m_nextpkt instead of field.sqe_next).
 *
 * m_next is ignored, so queueing chains of mbufs is possible
 */
#define MBUFQ_HEAD(name)                                        \
struct name {                                                   \
        struct mbuf *mq_first;  /* first packet */              \
        struct mbuf **mq_last;  /* addr of last next packet */  \
}

#define MBUFQ_INIT(q)           do {                            \
        MBUFQ_FIRST(q) = NULL;                                  \
        (q)->mq_last = &MBUFQ_FIRST(q);                         \
} while (0)

#define MBUFQ_PREPEND(q, m)     do {                            \
        if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL)           \
                (q)->mq_last = &MBUFQ_NEXT(m);                  \
        MBUFQ_FIRST(q) = (m);                                   \
} while (0)

#define MBUFQ_ENQUEUE(q, m)     do {                            \
        MBUFQ_NEXT(m) = NULL;                                   \
        *(q)->mq_last = (m);                                    \
        (q)->mq_last = &MBUFQ_NEXT(m);                          \
} while (0)

#define MBUFQ_ENQUEUE_MULTI(q, m, n)    do {                    \
        MBUFQ_NEXT(n) = NULL;                                   \
        *(q)->mq_last = (m);                                    \
        (q)->mq_last = &MBUFQ_NEXT(n);                          \
} while (0)

#define MBUFQ_DEQUEUE(q, m)     do {                            \
        if (((m) = MBUFQ_FIRST(q)) != NULL) {                   \
                if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL)   \
                        (q)->mq_last = &MBUFQ_FIRST(q);         \
                else                                            \
                        MBUFQ_NEXT(m) = NULL;                   \
        }                                                       \
} while (0)

#define MBUFQ_REMOVE(q, m)      do {                            \
        if (MBUFQ_FIRST(q) == (m)) {                            \
                MBUFQ_DEQUEUE(q, m);                            \
        } else {                                                \
                struct mbuf *_m = MBUFQ_FIRST(q);               \
                while (MBUFQ_NEXT(_m) != (m))                   \
                        _m = MBUFQ_NEXT(_m);                    \
                if ((MBUFQ_NEXT(_m) =                           \
                    MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL)        \
                        (q)->mq_last = &MBUFQ_NEXT(_m);         \
        }                                                       \
} while (0)

#define MBUFQ_DRAIN(q)          do {                            \
        struct mbuf *__m0;                                      \
        while ((__m0 = MBUFQ_FIRST(q)) != NULL) {               \
                MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0);              \
                MBUFQ_NEXT(__m0) = NULL;                        \
                m_freem(__m0);                                  \
        }                                                       \
        (q)->mq_last = &MBUFQ_FIRST(q);                         \
} while (0)

#define MBUFQ_FOREACH(m, q)                                     \
        for ((m) = MBUFQ_FIRST(q);                              \
            (m);                                                \
            (m) = MBUFQ_NEXT(m))

#define MBUFQ_FOREACH_SAFE(m, q, tvar)                          \
        for ((m) = MBUFQ_FIRST(q);                              \
            (m) && ((tvar) = MBUFQ_NEXT(m), 1);                 \
            (m) = (tvar))

#define MBUFQ_EMPTY(q)          ((q)->mq_first == NULL)
#define MBUFQ_FIRST(q)          ((q)->mq_first)
#define MBUFQ_NEXT(m)           ((m)->m_nextpkt)
/*
 * mq_last is initialized to point to mq_first, so check if they're
 * equal and return NULL when the list is empty.  Otherwise, we need
 * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
 * to the base mbuf address to return to caller.
 */
#define MBUFQ_LAST(head)                                        \
        (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL :       \
        ((struct mbuf *)(void *)((char *)(head)->mq_last -      \
            (size_t)(&MBUFQ_NEXT((struct mbuf *)0)))))

#define max_linkhdr     P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
#define max_protohdr    P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
#endif /* XNU_KERNEL_PRIVATE */

/*
 * Mbuf statistics (legacy).
 */
struct mbstat {
        u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
        u_int32_t       m_clusters;     /* clusters obtained from page pool */
        u_int32_t       m_spare;        /* spare field */
        u_int32_t       m_clfree;       /* free clusters */
        u_int32_t       m_drops;        /* times failed to find space */
        u_int32_t       m_wait;         /* times waited for space */
        u_int32_t       m_drain;        /* times drained protocols for space */
        u_short         m_mtypes[256];  /* type specific mbuf allocations */
        u_int32_t       m_mcfail;       /* times m_copym failed */
        u_int32_t       m_mpfail;       /* times m_pullup failed */
        u_int32_t       m_msize;        /* length of an mbuf */
        u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
        u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
        u_int32_t       m_mlen;         /* length of data in an mbuf */
        u_int32_t       m_mhlen;        /* length of data in a header mbuf */
        u_int32_t       m_bigclusters;  /* clusters obtained from page pool */
        u_int32_t       m_bigclfree;    /* free clusters */
        u_int32_t       m_bigmclbytes;  /* length of an mbuf cluster */
};

/* Compatibillity with 10.3 */
struct ombstat {
        u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
        u_int32_t       m_clusters;     /* clusters obtained from page pool */
        u_int32_t       m_spare;        /* spare field */
        u_int32_t       m_clfree;       /* free clusters */
        u_int32_t       m_drops;        /* times failed to find space */
        u_int32_t       m_wait;         /* times waited for space */
        u_int32_t       m_drain;        /* times drained protocols for space */
        u_short         m_mtypes[256];  /* type specific mbuf allocations */
        u_int32_t       m_mcfail;       /* times m_copym failed */
        u_int32_t       m_mpfail;       /* times m_pullup failed */
        u_int32_t       m_msize;        /* length of an mbuf */
        u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
        u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
        u_int32_t       m_mlen;         /* length of data in an mbuf */
        u_int32_t       m_mhlen;        /* length of data in a header mbuf */
};

/*
 * mbuf class statistics.
 */
#define MAX_MBUF_CNAME  15

#if defined(XNU_KERNEL_PRIVATE)
/* For backwards compatibility with 32-bit userland process */
struct omb_class_stat {
        char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
        u_int32_t       mbcl_size;      /* buffer size */
        u_int32_t       mbcl_total;     /* # of buffers created */
        u_int32_t       mbcl_active;    /* # of active buffers */
        u_int32_t       mbcl_infree;    /* # of available buffers */
        u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
        u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
        u_int64_t       mbcl_free_cnt;  /* # of times free is called */
        u_int64_t       mbcl_notified;  /* # of notified wakeups */
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */
        u_int32_t       mbcl_release_cnt; /* amount of memory returned */
        /*
         * Cache layer statistics
         */
        u_int32_t       mbcl_mc_state;  /* cache state (see below) */
        u_int32_t       mbcl_mc_cached; /* # of cached buffers */
        u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
        u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
        u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
        u_int64_t       mbcl_reserved[4];    /* for future use */
} __attribute__((__packed__));
#endif /* XNU_KERNEL_PRIVATE */

typedef struct mb_class_stat {
        char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
        u_int32_t       mbcl_size;      /* buffer size */
        u_int32_t       mbcl_total;     /* # of buffers created */
        u_int32_t       mbcl_active;    /* # of active buffers */
        u_int32_t       mbcl_infree;    /* # of available buffers */
        u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
#if defined(KERNEL) || defined(__LP64__)
        u_int32_t       mbcl_pad;       /* padding */
#endif /* KERNEL || __LP64__ */
        u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
        u_int64_t       mbcl_free_cnt;  /* # of times free is called */
        u_int64_t       mbcl_notified;  /* # of notified wakeups */
        u_int64_t       mbcl_purge_cnt; /* # of purges so far */
        u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
        u_int32_t       mbcl_ctotal;    /* total only for this class */
        u_int32_t       mbcl_release_cnt; /* amount of memory returned */
        /*
         * Cache layer statistics
         */
        u_int32_t       mbcl_mc_state;  /* cache state (see below) */
        u_int32_t       mbcl_mc_cached; /* # of cached buffers */
        u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
        u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
        u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
        u_int32_t       mbcl_peak_reported; /* last usage peak reported */
        u_int32_t       mbcl_reserved[7];    /* for future use */
} mb_class_stat_t;

#define MCS_DISABLED    0       /* cache is permanently disabled */
#define MCS_ONLINE      1       /* cache is online */
#define MCS_PURGING     2       /* cache is being purged */
#define MCS_OFFLINE     3       /* cache is offline (resizing) */

#if defined(XNU_KERNEL_PRIVATE)
/* For backwards compatibility with 32-bit userland process */
struct omb_stat {
        u_int32_t               mbs_cnt;        /* number of classes */
        struct omb_class_stat   mbs_class[1];   /* class array */
} __attribute__((__packed__));
#endif /* XNU_KERNEL_PRIVATE */

typedef struct mb_stat {
        u_int32_t       mbs_cnt;        /* number of classes */
#if defined(KERNEL) || defined(__LP64__)
        u_int32_t       mbs_pad;        /* padding */
#endif /* KERNEL || __LP64__ */
        mb_class_stat_t mbs_class[1];   /* class array */
} mb_stat_t;

#ifdef PRIVATE
#define MLEAK_STACK_DEPTH       16      /* Max PC stack depth */

typedef struct mleak_trace_stat {
        u_int64_t       mltr_collisions;
        u_int64_t       mltr_hitcount;
        u_int64_t       mltr_allocs;
        u_int64_t       mltr_depth;
        u_int64_t       mltr_addr[MLEAK_STACK_DEPTH];
} mleak_trace_stat_t;

typedef struct mleak_stat {
        u_int32_t               ml_isaddr64;    /* 64-bit KVA? */
        u_int32_t               ml_cnt;         /* number of traces */
        mleak_trace_stat_t      ml_trace[1];    /* trace array */
} mleak_stat_t;

struct mleak_table {
        u_int32_t mleak_capture;        /* sampling capture counter */
        u_int32_t mleak_sample_factor;  /* sample factor */

        /* Times two active records want to occupy the same spot */
        u_int64_t alloc_collisions;
        u_int64_t trace_collisions;

        /* Times new record lands on spot previously occupied by freed alloc */
        u_int64_t alloc_overwrites;
        u_int64_t trace_overwrites;

        /* Times a new alloc or trace is put into the hash table */
        u_int64_t alloc_recorded;
        u_int64_t trace_recorded;

        /* Total number of outstanding allocs */
        u_int64_t outstanding_allocs;

        /* Times mleak_log returned false because couldn't acquire the lock */
        u_int64_t total_conflicts;
};
#endif /* PRIVATE */

#ifdef KERNEL_PRIVATE
__BEGIN_DECLS

/*
 * Exported (private)
 */

extern struct mbstat mbstat;                    /* statistics */

__END_DECLS
#endif /* KERNEL_PRIVATE */

#ifdef XNU_KERNEL_PRIVATE
__BEGIN_DECLS

/*
 * Not exported (xnu private)
 */

/* flags to m_get/MGET */
/* Need to include malloc.h to get right options for malloc  */
#include        <sys/malloc.h>

struct mbuf;

/* length to m_copy to copy all */
#define M_COPYALL       1000000000

#define M_DONTWAIT      M_NOWAIT
#define M_WAIT          M_WAITOK

/* modes for m_copym and variants */
#define M_COPYM_NOOP_HDR        0       /* don't copy/move pkthdr contents */
#define M_COPYM_COPY_HDR        1       /* copy pkthdr from old to new */
#define M_COPYM_MOVE_HDR        2       /* move pkthdr from old to new */
#define M_COPYM_MUST_COPY_HDR   3       /* MUST copy pkthdr from old to new */
#define M_COPYM_MUST_MOVE_HDR   4       /* MUST move pkthdr from old to new */

/*
 * These macros are mapped to the appropriate KPIs, so that private code
 * can be simply recompiled in order to be forward-compatible with future
 * changes toward the struture sizes.
 */
#define MLEN            mbuf_get_mlen()         /* normal data len */
#define MHLEN           mbuf_get_mhlen()        /* data len w/pkthdr */

#define MINCLSIZE       mbuf_get_minclsize()    /* cluster usage threshold */

extern void m_freem(struct mbuf *);
extern u_int64_t mcl_to_paddr(char *);
extern void m_adj(struct mbuf *, int);
extern void m_cat(struct mbuf *, struct mbuf *);
extern void m_copydata(struct mbuf *, int, int, void *);
extern struct mbuf *m_copym(struct mbuf *, int, int, int);
extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t);
extern struct mbuf *m_get(int, int);
extern struct mbuf *m_gethdr(int, int);
extern struct mbuf *m_getpacket(void);
extern struct mbuf *m_getpackets(int, int, int);
extern struct mbuf *m_mclget(struct mbuf *, int);
extern void *m_mtod(struct mbuf *);
extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);
extern struct mbuf *m_pullup(struct mbuf *, int);
extern struct mbuf *m_split(struct mbuf *, int, int);
extern void m_mclfree(caddr_t p);

/*
 * On platforms which require strict alignment (currently for anything but
 * i386 or x86_64), this macro checks whether the data pointer of an mbuf
 * is 32-bit aligned (this is the expected minimum alignment for protocol
 * headers), and assert otherwise.
 */
#if defined(__i386__) || defined(__x86_64__)
#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
#else /* !__i386__ && !__x86_64__ */
#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do {                    \
        if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) {          \
                if (((_m)->m_flags & M_PKTHDR) &&                       \
                    (_m)->m_pkthdr.rcvif != NULL) {                     \
                        panic_plain("\n%s: mbuf %p data ptr %p is not " \
                            "32-bit aligned [%s: alignerrs=%lld]\n",    \
                            __func__, (_m), (_m)->m_data,               \
                            if_name((_m)->m_pkthdr.rcvif),              \
                            (_m)->m_pkthdr.rcvif->if_alignerrs);        \
                } else {                                                \
                        panic_plain("\n%s: mbuf %p data ptr %p is not " \
                            "32-bit aligned\n",                         \
                            __func__, (_m), (_m)->m_data);              \
                }                                                       \
        }                                                               \
} while (0)
#endif /* !__i386__ && !__x86_64__ */

/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
#define MBUF_SC_MAX_CLASSES     10

/*
 * These conversion macros rely on the corresponding MBUF_SC and
 * MBUF_TC values in order to establish the following mapping:
 *
 *      MBUF_SC_BK_SYS  ] ==>   MBUF_TC_BK
 *      MBUF_SC_BK      ]
 *
 *      MBUF_SC_BE      ] ==>   MBUF_TC_BE
 *      MBUF_SC_RD      ]
 *      MBUF_SC_OAM     ]
 *
 *      MBUF_SC_AV      ] ==>   MBUF_TC_VI
 *      MBUF_SC_RV      ]
 *      MBUF_SC_VI      ]
 *
 *      MBUF_SC_VO      ] ==>   MBUF_TC_VO
 *      MBUF_SC_CTL     ]
 *
 * The values assigned to each service class allows for a fast mapping to
 * the corresponding MBUF_TC traffic class values, as well as to retrieve the
 * assigned index; therefore care must be taken when comparing against these
 * values.  Use the corresponding class and index macros to retrieve the
 * corresponding portion, and never assume that a higher class corresponds
 * to a higher index.
 */
#define MBUF_SCVAL(x)           ((x) & 0xffff)
#define MBUF_SCIDX(x)           ((((x) >> 16) & 0xff) >> 3)
#define MBUF_SC2TC(_sc)         (MBUF_SCVAL(_sc) >> 7)
#define MBUF_TC2SCVAL(_tc)      ((_tc) << 7)
#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
        ((_sc) == MBUF_SC_BK))
#define IS_MBUF_SC_REALTIME(_sc)        ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
#define IS_MBUF_SC_BESTEFFORT(_sc)      ((_sc) == MBUF_SC_BE || \
    (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)

#define SCIDX_BK_SYS            MBUF_SCIDX(MBUF_SC_BK_SYS)
#define SCIDX_BK                MBUF_SCIDX(MBUF_SC_BK)
#define SCIDX_BE                MBUF_SCIDX(MBUF_SC_BE)
#define SCIDX_RD                MBUF_SCIDX(MBUF_SC_RD)
#define SCIDX_OAM               MBUF_SCIDX(MBUF_SC_OAM)
#define SCIDX_AV                MBUF_SCIDX(MBUF_SC_AV)
#define SCIDX_RV                MBUF_SCIDX(MBUF_SC_RV)
#define SCIDX_VI                MBUF_SCIDX(MBUF_SC_VI)
#define SCIDX_VO                MBUF_SCIDX(MBUF_SC_VO)
#define SCIDX_CTL               MBUF_SCIDX(MBUF_SC_CTL)

#define SCVAL_BK_SYS            MBUF_SCVAL(MBUF_SC_BK_SYS)
#define SCVAL_BK                MBUF_SCVAL(MBUF_SC_BK)
#define SCVAL_BE                MBUF_SCVAL(MBUF_SC_BE)
#define SCVAL_RD                MBUF_SCVAL(MBUF_SC_RD)
#define SCVAL_OAM               MBUF_SCVAL(MBUF_SC_OAM)
#define SCVAL_AV                MBUF_SCVAL(MBUF_SC_AV)
#define SCVAL_RV                MBUF_SCVAL(MBUF_SC_RV)
#define SCVAL_VI                MBUF_SCVAL(MBUF_SC_VI)
#define SCVAL_VO                MBUF_SCVAL(MBUF_SC_VO)
#define SCVAL_CTL               MBUF_SCVAL(MBUF_SC_CTL)

#define MBUF_VALID_SC(c)                                                \
        (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE ||   \
        c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV ||       \
        c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO ||        \
        c == MBUF_SC_CTL)

#define MBUF_VALID_SCIDX(c)                                             \
        (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE ||         \
        c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV ||             \
        c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO ||              \
        c == SCIDX_CTL)

#define MBUF_VALID_SCVAL(c)                                             \
        (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE ||         \
        c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV ||             \
        c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO ||              \
        c == SCVAL_CTL)

extern unsigned char *mbutl;    /* start VA of mbuf pool */
extern unsigned char *embutl;   /* end VA of mbuf pool */
extern unsigned int nmbclusters;        /* number of mapped clusters */
extern int njcl;                /* # of jumbo clusters  */
extern int njclbytes;   /* size of a jumbo cluster */
extern int max_hdr;             /* largest link+protocol header */
extern int max_datalen; /* MHLEN - max_hdr */

/* Use max_linkhdr instead of _max_linkhdr */
extern int _max_linkhdr;        /* largest link-level header */

/* Use max_protohdr instead of _max_protohdr */
extern int _max_protohdr;       /* largest protocol header */

__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
__private_extern__ void mbinit(void);
__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
    void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int);
__private_extern__ caddr_t m_bigalloc(int);
__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
__private_extern__ caddr_t m_16kalloc(int);
__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
__private_extern__ int m_reinit(struct mbuf *, int);
__private_extern__ struct mbuf *m_free(struct mbuf *);
__private_extern__ struct mbuf *m_getclr(int, int);
__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
__private_extern__ unsigned int m_length(struct mbuf *);
__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **);
__private_extern__ unsigned int m_fixhdr(struct mbuf *);
__private_extern__ struct mbuf *m_defrag(struct mbuf *, int);
__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int);
__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
__private_extern__ struct mbuf *m_retry(int, int);
__private_extern__ struct mbuf *m_retryhdr(int, int);
__private_extern__ int m_freem_list(struct mbuf *);
__private_extern__ int m_append(struct mbuf *, int, caddr_t);
__private_extern__ struct mbuf *m_last(struct mbuf *);
__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
    void (*)(const void *, void *, size_t));
__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);

__private_extern__ struct mbuf *m_getcl(int, int, int);
__private_extern__ caddr_t m_mclalloc(int);
__private_extern__ int m_mclhasreference(struct mbuf *);
__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *);
__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *);

__private_extern__ struct mbuf *m_dtom(void *);
__private_extern__ int m_mtocl(void *);
__private_extern__ union mcluster *m_cltom(int);

__private_extern__ int m_trailingspace(struct mbuf *);
__private_extern__ int m_leadingspace(struct mbuf *);

__private_extern__ struct mbuf *m_normalize(struct mbuf *m);
__private_extern__ void m_mchtype(struct mbuf *m, int t);
__private_extern__ void m_mcheck(struct mbuf *);

__private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
    const void *, int);
__private_extern__ int m_makewritable(struct mbuf **, int, int, int);
__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
    struct mbuf **, int *, uint32_t);
__private_extern__ struct mbuf *m_getpackethdrs(int, int);
__private_extern__ struct mbuf *m_getpacket_how(int);
__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
    int, int, size_t);
__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
    unsigned int *, int, int, size_t);

__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
__private_extern__ int m_ext_paired_is_active(struct mbuf *);
__private_extern__ void m_ext_paired_activate(struct mbuf *);

__private_extern__ void m_drain(void);

/*
 * Packets may have annotations attached by affixing a list of "packet
 * tags" to the pkthdr structure.  Packet tags are dynamically allocated
 * semi-opaque data structures that have a fixed header (struct m_tag)
 * that specifies the size of the memory block and an <id,type> pair that
 * identifies it. The id identifies the module and the type identifies the
 * type of data for that module. The id of zero is reserved for the kernel.
 *
 * Note that the packet tag returned by m_tag_allocate has the default
 * memory alignment implemented by malloc.  To reference private data one
 * can use a construct like:
 *
 *      struct m_tag *mtag = m_tag_allocate(...);
 *      struct foo *p = (struct foo *)(mtag+1);
 *
 * if the alignment of struct m_tag is sufficient for referencing members
 * of struct foo.  Otherwise it is necessary to embed struct m_tag within
 * the private data structure to insure proper alignment; e.g.
 *
 *      struct foo {
 *              struct m_tag    tag;
 *              ...
 *      };
 *      struct foo *p = (struct foo *) m_tag_allocate(...);
 *      struct m_tag *mtag = &p->tag;
 */

#define KERNEL_MODULE_TAG_ID    0

enum {
        KERNEL_TAG_TYPE_NONE                    = 0,
        KERNEL_TAG_TYPE_DUMMYNET                = 1,
        KERNEL_TAG_TYPE_DIVERT                  = 2,
        KERNEL_TAG_TYPE_IPFORWARD               = 3,
        KERNEL_TAG_TYPE_IPFILT                  = 4,
        KERNEL_TAG_TYPE_MACLABEL                = 5,
        KERNEL_TAG_TYPE_MAC_POLICY_LABEL        = 6,
        KERNEL_TAG_TYPE_ENCAP                   = 8,
        KERNEL_TAG_TYPE_INET6                   = 9,
        KERNEL_TAG_TYPE_IPSEC                   = 10,
        KERNEL_TAG_TYPE_DRVAUX                  = 11,
};

/* Packet tag routines */
__private_extern__ struct  m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
__private_extern__ struct  m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
        struct mbuf *);
__private_extern__ void m_tag_free(struct m_tag *);
__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
__private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *);
__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
    u_int16_t, struct m_tag *);
__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
__private_extern__ void m_tag_init(struct mbuf *, int);
__private_extern__ struct  m_tag *m_tag_first(struct mbuf *);
__private_extern__ struct  m_tag *m_tag_next(struct mbuf *, struct m_tag *);

__END_DECLS
#endif /* XNU_KERNEL_PRIVATE */
#ifdef KERNEL
#include <sys/kpi_mbuf.h>
#ifdef XNU_KERNEL_PRIVATE
__BEGIN_DECLS

__private_extern__ void m_scratch_init(struct mbuf *);
__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **);

__private_extern__ void m_classifier_init(struct mbuf *, uint32_t);

__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t);
__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *);
__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t);
__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t);
__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);

#define ADDCARRY(_x)  do {                                              \
        while (((_x) >> 16) != 0)                                       \
                (_x) = ((_x) >> 16) + ((_x) & 0xffff);                  \
} while (0)

__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
    u_int32_t, u_int32_t, u_int32_t);
__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);

__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
    m_ext_free_func_t, caddr_t);
__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);

extern void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);

__END_DECLS
#endif /* XNU_KERNEL_PRIVATE */
#endif /* KERNEL */
#endif  /* !_SYS_MBUF_H_ */