bsd/sys/mbuf.h

   1 /*
   2  * Copyright (c) 1999-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
  29 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
  30 /*
  31  * Mach Operating System
  32  * Copyright (c) 1987 Carnegie-Mellon University
  33  * All rights reserved.  The CMU software License Agreement specifies
  34  * the terms and conditions for use and redistribution.
  35  */
  36 /*
  37  * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
  38  *
  39  * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
  40  * All rights reserved.
  41  *
  42  * Redistribution and use in source and binary forms, with or without
  43  * modification, are permitted provided that the following conditions
  44  * are met:
  45  * 1. Redistributions of source code must retain the above copyright
  46  *    notice, this list of conditions and the following disclaimer.
  47  * 2. Redistributions in binary form must reproduce the above copyright
  48  *    notice, this list of conditions and the following disclaimer in the
  49  *    documentation and/or other materials provided with the distribution.
  50  * 3. All advertising materials mentioning features or use of this software
  51  *    must display the following acknowledgement:
  52  *      This product includes software developed by the University of
  53  *      California, Berkeley and its contributors.
  54  * 4. Neither the name of the University nor the names of its contributors
  55  *    may be used to endorse or promote products derived from this software
  56  *    without specific prior written permission.
  57  *
  58  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  59  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  60  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  61  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  62  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  63  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  64  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  65  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  66  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  67  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  68  * SUCH DAMAGE.
  69  *
  70  *      @(#)mbuf.h      8.3 (Berkeley) 1/21/94
  71  */
  72 /*
  73  * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
  74  * support for mandatory and extensible security protections.  This notice
  75  * is included in support of clause 2.2 (b) of the Apple Public License,
  76  * Version 2.0.
  77  */
  78
  79 #ifndef _SYS_MBUF_H_
  80 #define _SYS_MBUF_H_
  81
  82 #include <sys/cdefs.h>
  83 #include <sys/appleapiopts.h>
  84
  85 #ifdef XNU_KERNEL_PRIVATE
  86
  87 #include <sys/lock.h>
  88 #include <sys/queue.h>
  89
  90 /*
  91  * Mbufs are of a single size, MSIZE (machine/param.h), which
  92  * includes overhead.  An mbuf may add a single "mbuf cluster" of size
  93  * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
  94  * no additional overhead and is used instead of the internal data area;
  95  * this is done when at least MINCLSIZE of data must be stored.
  96  */
  97
  98 /*
  99  * The following _MLEN and _MHLEN macros are private to xnu.  Private code
 100  * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
 101  * the sizes of the structures are dependent upon specific xnu configs.
 102  */
 103 #define _MLEN           (MSIZE - sizeof(struct m_hdr))  /* normal data len */
 104 #define _MHLEN          (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
 105
 106 #define NMBPBGSHIFT     (MBIGCLSHIFT - MSIZESHIFT)
 107 #define NMBPBG          (1 << NMBPBGSHIFT)      /* # of mbufs per big cl */
 108
 109 #define NCLPBGSHIFT     (MBIGCLSHIFT - MCLSHIFT)
 110 #define NCLPBG          (1 << NCLPBGSHIFT)      /* # of cl per big cl */
 111
 112 #define NMBPCLSHIFT     (NMBPBGSHIFT - NCLPBGSHIFT)
 113 #define NMBPCL          (1 << NMBPCLSHIFT)      /* # of mbufs per cl */
 114
 115 #define NCLPJCLSHIFT    ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
 116 #define NCLPJCL         (1 << NCLPJCLSHIFT)     /* # of cl per jumbo cl */
 117
 118 /*
 119  * Macros for type conversion
 120  * mtod(m,t) -  convert mbuf pointer to data pointer of correct type
 121  * dtom(x) -    convert data pointer within mbuf to mbuf pointer (XXX)
 122  */
 123 #define mtod(m, t)      ((t)m_mtod(m))
 124 #define dtom(x)         m_dtom(x)
 125
 126 /* header at beginning of each mbuf: */
 127 struct m_hdr {
 128         struct  mbuf *mh_next;          /* next buffer in chain */
 129         struct  mbuf *mh_nextpkt;       /* next chain in queue/record */
 130         int32_t mh_len;                 /* amount of data in this mbuf */
 131         caddr_t mh_data;                /* location of data */
 132         short   mh_type;                /* type of data in this mbuf */
 133         short   mh_flags;               /* flags; see below */
 134 };
 135
 136 /*
 137  * Packet tag structure (see below for details).
 138  */
 139 struct m_tag {
 140         u_int64_t               m_tag_cookie;   /* Error checking */
 141 #ifndef __LP64__
 142         u_int32_t               pad;            /* For structure alignment */
 143 #endif /* !__LP64__ */
 144         SLIST_ENTRY(m_tag)      m_tag_link;     /* List of packet tags */
 145         u_int16_t               m_tag_type;     /* Module specific type */
 146         u_int16_t               m_tag_len;      /* Length of data */
 147         u_int32_t               m_tag_id;       /* Module ID */
 148 };
 149
 150 #define M_TAG_ALIGN(len) \
 151         (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
 152
 153 #define M_TAG_VALID_PATTERN     0xfeedfacefeedfaceULL
 154 #define M_TAG_FREE_PATTERN      0xdeadbeefdeadbeefULL
 155
 156 /*
 157  * Packet tag header structure (at the top of mbuf).  Pointers are
 158  * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
 159  */
 160 struct m_taghdr {
 161 #ifndef __LP64__
 162         u_int32_t               pad;            /* For structure alignment */
 163 #endif /* !__LP64__ */
 164         u_int64_t               refcnt;         /* Number of tags in this mbuf */
 165 };
 166
 167 /* Values for pftag_flags */
 168 #define PF_TAG_GENERATED        0x000001        /* pkt generated by PF */
 169 #define PF_TAG_FRAGCACHE        0x000002
 170 #define PF_TAG_TRANSLATE_LOCALHOST 0x000004
 171 #define PF_TAG_FLOWHASH         0x000100        /* valid flowhash value */
 172 #define PF_TAG_HDR_INET         0x000200        /* hdr points to IPv4 */
 173 #define PF_TAG_HDR_INET6        0x000400        /* hdr points to IPv6 */
 174 #define PF_TAG_TCP              0x000800        /* payload is TCP */
 175 #define PF_TAG_FLOWADV          0x010000        /* local flow advisory */
 176 #define PF_TAG_QUEUE1           0x100000        /* queue-specific */
 177
 178 #define IF_PKTSEQ_SHIFT         4
 179
 180 /* PF mbuf tag */
 181 struct pf_mtag {
 182         void            *pftag_hdr;     /* saved hdr pos in mbuf, for ECN */
 183         unsigned int    pftag_rtableid; /* alternate routing table id */
 184         union {
 185                 struct {
 186                         u_int32_t       qid;
 187                         union {
 188                                 u_int8_t        val8[4];
 189                                 u_int16_t       val16[2];
 190                                 u_int32_t       val32;
 191                         } __qpriv_u;    /* for queue-specific use */
 192                 } __pf_data;
 193                 u_int64_t       pktseq;
 194         } __pfifseq_u; /* Used for pf or interface bandwidth measurement */
 195 #define pftag_qid       __pfifseq_u.__pf_data.qid
 196 #define pftag_qpriv8    __pfifseq_u.__pf_data.__qpriv_u.val8
 197 #define pftag_qpriv16   __pfifseq_u.__pf_data.__qpriv_u.val16
 198 #define pftag_qpriv32   __pfifseq_u.__pf_data.__qpriv_u.val32
 199 #define pftag_pktseq    __pfifseq_u.pktseq
 200         u_int32_t       pftag_flowhash;
 201         u_int16_t       pftag_tag;
 202         u_int16_t       pftag_routed;
 203         u_int32_t       pftag_flags;    /* PF_TAG flags */
 204 };
 205
 206 /* TCP specific mbuf tag */
 207 struct tcp_mtag {
 208         u_int           tm_tso_segz;    /* TSO segment size (actual MSS) */
 209         u_int16_t       tm_pktlen;      /* LRO - max segment size encountered */
 210         u_int16_t       tm_npkts;       /* LRO - number of coalesced TCP pkts */
 211 };
 212
 213 /* record/packet header in first mbuf of chain; valid if M_PKTHDR set */
 214 struct  pkthdr {
 215         int     len;                    /* total packet length */
 216         struct  ifnet *rcvif;           /* rcv interface */
 217
 218         /* variables for ip and tcp reassembly */
 219         void    *header;                /* pointer to packet header */
 220         /* variables for hardware checksum */
 221         /* Note: csum_flags is used for hardware checksum and VLAN */
 222         int     csum_flags;             /* flags regarding checksum */
 223         int     csum_data;              /* data field used by csum routines */
 224         u_short vlan_tag;               /* VLAN tag, host byte order */
 225         u_short socket_id;              /* socket id */
 226         SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
 227         struct pf_mtag pf_mtag;         /* built-in PF tag */
 228 #define m_flowhash      pf_mtag.pftag_flowhash
 229 #define m_fhflags       pf_mtag.pftag_flags
 230         u_int32_t svc;                  /* MBUF_SVC value */
 231         u_int16_t vt_nrecs;             /* # of IGMPv3/MLDv2 records */
 232         u_int16_t aux_flags;            /* auxiliary packet flags */
 233         struct tcp_mtag tcp_mtag;       /* tcp related data */
 234 #define tso_segsz       tcp_mtag.tm_tso_segz
 235 #define lro_pktlen      tcp_mtag.tm_pktlen
 236 #define lro_npkts       tcp_mtag.tm_npkts
 237 };
 238
 239 /* description of external storage mapped into mbuf, valid if M_EXT set */
 240 struct m_ext {
 241         caddr_t ext_buf;                /* start of buffer */
 242         void    (*ext_free)(caddr_t, u_int, caddr_t);   /* free routine if not the usual */
 243         u_int   ext_size;               /* size of buffer, for ext_free */
 244         caddr_t ext_arg;                /* additional ext_free argument */
 245         struct  ext_refsq {             /* references held */
 246                 struct ext_refsq *forward, *backward;
 247         } ext_refs;
 248         struct ext_ref {
 249                 u_int32_t refcnt;
 250                 u_int32_t flags;
 251         } *ext_refflags;
 252 };
 253
 254 /* define m_ext to a type since it gets redefined below */
 255 typedef struct m_ext _m_ext_t;
 256
 257 struct mbuf {
 258         struct  m_hdr m_hdr;
 259         union {
 260                 struct {
 261                         struct  pkthdr MH_pkthdr;       /* M_PKTHDR set */
 262                         union {
 263                                 struct  m_ext MH_ext;   /* M_EXT set */
 264                                 char    MH_databuf[_MHLEN];
 265                         } MH_dat;
 266                 } MH;
 267                 char    M_databuf[_MLEN];               /* !M_PKTHDR, !M_EXT */
 268         } M_dat;
 269 };
 270
 271 #define m_next          m_hdr.mh_next
 272 #define m_len           m_hdr.mh_len
 273 #define m_data          m_hdr.mh_data
 274 #define m_type          m_hdr.mh_type
 275 #define m_flags         m_hdr.mh_flags
 276 #define m_nextpkt       m_hdr.mh_nextpkt
 277 #define m_act           m_nextpkt
 278 #define m_pkthdr        M_dat.MH.MH_pkthdr
 279 #define m_ext           M_dat.MH.MH_dat.MH_ext
 280 #define m_pktdat        M_dat.MH.MH_dat.MH_databuf
 281 #define m_dat           M_dat.M_databuf
 282 #define m_pktlen(_m)    ((_m)->m_pkthdr.len)
 283 #define m_pftag(_m)     (&(_m)->m_pkthdr.pf_mtag)
 284
 285 /* mbuf flags (private) */
 286 #define M_EXT           0x0001  /* has associated external storage */
 287 #define M_PKTHDR        0x0002  /* start of record */
 288 #define M_EOR           0x0004  /* end of record */
 289 #define M_PROTO1        0x0008  /* protocol-specific */
 290 #define M_PROTO2        0x0010  /* protocol-specific */
 291 #define M_PROTO3        0x0020  /* protocol-specific */
 292 #define M_LOOP          0x0040  /* packet is looped back */
 293 #define M_PROTO5        0x0080  /* protocol-specific */
 294
 295 /* mbuf pkthdr flags, also in m_flags (private) */
 296 #define M_BCAST         0x0100  /* send/received as link-level broadcast */
 297 #define M_MCAST         0x0200  /* send/received as link-level multicast */
 298 #define M_FRAG          0x0400  /* packet is a fragment of a larger packet */
 299 #define M_FIRSTFRAG     0x0800  /* packet is first fragment */
 300 #define M_LASTFRAG      0x1000  /* packet is last fragment */
 301 #define M_PROMISC       0x2000  /* packet is promiscuous (shouldn't go to stack) */
 302 #define M_HASFCS        0x4000  /* packet has FCS */
 303 #define M_TAGHDR        0x8000  /* m_tag hdr structure at top of mbuf data */
 304
 305 /*
 306  * Flags to purge when crossing layers.
 307  */
 308 #define M_PROTOFLAGS \
 309         (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
 310
 311 /* flags copied when copying m_pkthdr */
 312 #define M_COPYFLAGS                                                     \
 313         (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 |                    \
 314         M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG |                        \
 315         M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
 316
 317 /* flags indicating hw checksum support and sw checksum requirements */
 318 #define CSUM_IP                 0x0001          /* will csum IP */
 319 #define CSUM_TCP                0x0002          /* will csum TCP */
 320 #define CSUM_UDP                0x0004          /* will csum UDP */
 321 #define CSUM_IP_FRAGS           0x0008          /* will csum IP fragments */
 322 #define CSUM_FRAGMENT           0x0010          /* will do IP fragmentation */
 323 #define CSUM_TCPIPV6            0x0020          /* will csum TCP for IPv6 */
 324 #define CSUM_UDPIPV6            0x0040          /* will csum UDP for IPv6 */
 325 #define CSUM_FRAGMENT_IPV6      0x0080          /* will do IPv6 fragmentation */
 326
 327 #define CSUM_IP_CHECKED         0x0100          /* did csum IP */
 328 #define CSUM_IP_VALID           0x0200          /*   ... the csum is valid */
 329 #define CSUM_DATA_VALID         0x0400          /* csum_data field is valid */
 330 #define CSUM_PSEUDO_HDR         0x0800          /* csum_data has pseudo hdr */
 331 #define CSUM_TCP_SUM16          0x1000          /* simple TCP Sum16 computation */
 332
 333 #define CSUM_DELAY_DATA         (CSUM_TCP | CSUM_UDP)
 334 #define CSUM_DELAY_IP           (CSUM_IP)       /* IPv4 only: no IPv6 IP cksum */
 335 #define CSUM_DELAY_IPV6_DATA    (CSUM_TCPIPV6 | CSUM_UDPIPV6)
 336 #define CSUM_DATA_IPV6_VALID    CSUM_DATA_VALID /* csum_data field is valid */
 337 /*
 338  * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
 339  */
 340 /* bottom 16 bits reserved for hardware checksum */
 341 #define CSUM_CHECKSUM_MASK      0xffff
 342
 343 /* VLAN tag present */
 344 #define CSUM_VLAN_TAG_VALID     0x10000         /* vlan_tag field is valid */
 345
 346 /* TCP Segment Offloading requested on this mbuf */
 347 #define CSUM_TSO_IPV4           0x100000        /* This mbuf needs to be segmented by the NIC */
 348 #define CSUM_TSO_IPV6           0x200000        /* This mbuf needs to be segmented by the NIC */
 349
 350 /*
 351  * Auxiliary packet flags.  Unlike m_flags, all auxiliary flags are copied
 352  * along when copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here.
 353  * Note that this flag is 16-bit wide.
 354  */
 355 #define MAUXF_PRIO_PRIVILEGED   0x0001  /* packet priority is privileged */
 356 #define MAUXF_PROXY_DST         0x0002  /* processed but not locally destined */
 357 #define MAUXF_INET_RESOLVE_RTR  0x0004  /* pkt is for resolving IPv4 router */
 358 #define MAUXF_INET6_RESOLVE_RTR 0x0008  /* pkt is for resolving IPv6 router */
 359 #define MAUXF_SW_LRO_PKT        0x0010  /* pkt is a large coalesced pkt */
 360 #define MAUXF_SW_LRO_DID_CSUM   0x0020  /* IP and TCP checksums done by LRO*/
 361 #endif /* XNU_KERNEL_PRIVATE */
 362
 363 /* mbuf types */
 364 #define MT_FREE         0       /* should be on free list */
 365 #define MT_DATA         1       /* dynamic (data) allocation */
 366 #define MT_HEADER       2       /* packet header */
 367 #define MT_SOCKET       3       /* socket structure */
 368 #define MT_PCB          4       /* protocol control block */
 369 #define MT_RTABLE       5       /* routing tables */
 370 #define MT_HTABLE       6       /* IMP host tables */
 371 #define MT_ATABLE       7       /* address resolution tables */
 372 #define MT_SONAME       8       /* socket name */
 373 #define MT_SOOPTS       10      /* socket options */
 374 #define MT_FTABLE       11      /* fragment reassembly header */
 375 #define MT_RIGHTS       12      /* access rights */
 376 #define MT_IFADDR       13      /* interface address */
 377 #define MT_CONTROL      14      /* extra-data protocol message */
 378 #define MT_OOBDATA      15      /* expedited data  */
 379 #define MT_TAG          16      /* volatile metadata associated to pkts */
 380 #define MT_MAX          32      /* enough? */
 381
 382 #ifdef XNU_KERNEL_PRIVATE
 383 /*
 384  * mbuf allocation/deallocation macros:
 385  *
 386  *      MGET(struct mbuf *m, int how, int type)
 387  * allocates an mbuf and initializes it to contain internal data.
 388  *
 389  *      MGETHDR(struct mbuf *m, int how, int type)
 390  * allocates an mbuf and initializes it to contain a packet header
 391  * and internal data.
 392  */
 393
 394 #if 1
 395 #define MCHECK(m) m_mcheck(m)
 396 #else
 397 #define MCHECK(m)
 398 #endif
 399
 400 #define MGET(m, how, type) ((m) = m_get((how), (type)))
 401
 402 #define MGETHDR(m, how, type)   ((m) = m_gethdr((how), (type)))
 403
 404 /*
 405  * Mbuf cluster macros.
 406  * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
 407  * MCLGET adds such clusters to a normal mbuf;
 408  * the flag M_EXT is set upon success.
 409  * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
 410  * freeing the cluster if the reference count has reached 0.
 411  *
 412  * Normal mbuf clusters are normally treated as character arrays
 413  * after allocation, but use the first word of the buffer as a free list
 414  * pointer while on the free list.
 415  */
 416 union mcluster {
 417         union   mcluster *mcl_next;
 418         char    mcl_buf[MCLBYTES];
 419 };
 420
 421 #define MCLALLOC(p, how)        ((p) = m_mclalloc(how))
 422
 423 #define MCLFREE(p)              m_mclfree(p)
 424
 425 #define MCLGET(m, how)          ((m) = m_mclget(m, how))
 426
 427 /*
 428  * Mbuf big cluster
 429  */
 430 union mbigcluster {
 431         union mbigcluster       *mbc_next;
 432         char                    mbc_buf[MBIGCLBYTES];
 433 };
 434
 435 /*
 436  * Mbuf jumbo cluster
 437  */
 438 union m16kcluster {
 439         union m16kcluster       *m16kcl_next;
 440         char                    m16kcl_buf[M16KCLBYTES];
 441 };
 442
 443 #define MCLHASREFERENCE(m)      m_mclhasreference(m)
 444
 445 /*
 446  * MFREE(struct mbuf *m, struct mbuf *n)
 447  * Free a single mbuf and associated external storage.
 448  * Place the successor, if any, in n.
 449  */
 450
 451 #define MFREE(m, n) ((n) = m_free(m))
 452
 453 /*
 454  * Copy mbuf pkthdr from from to to.
 455  * from must have M_PKTHDR set, and to must be empty.
 456  * aux pointer will be moved to `to'.
 457  */
 458 #define M_COPY_PKTHDR(to, from)         m_copy_pkthdr(to, from)
 459
 460 #define M_COPY_PFTAG(to, from)          m_copy_pftag(to, from)
 461
 462 /*
 463  * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
 464  * an object of the specified size at the end of the mbuf, longword aligned.
 465  */
 466 #define M_ALIGN(m, len)                                                 \
 467 do {                                                                    \
 468         (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1);           \
 469 } while (0)
 470
 471 /*
 472  * As above, for mbufs allocated with m_gethdr/MGETHDR
 473  * or initialized by M_COPY_PKTHDR.
 474  */
 475 #define MH_ALIGN(m, len)                                                \
 476 do {                                                                    \
 477         (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1);          \
 478 } while (0)
 479
 480 /*
 481  * Compute the amount of space available
 482  * before the current start of data in an mbuf.
 483  * Subroutine - data not available if certain references.
 484  */
 485 #define M_LEADINGSPACE(m)       m_leadingspace(m)
 486
 487 /*
 488  * Compute the amount of space available
 489  * after the end of data in an mbuf.
 490  * Subroutine - data not available if certain references.
 491  */
 492 #define M_TRAILINGSPACE(m)      m_trailingspace(m)
 493
 494 /*
 495  * Arrange to prepend space of size plen to mbuf m.
 496  * If a new mbuf must be allocated, how specifies whether to wait.
 497  * If how is M_DONTWAIT and allocation fails, the original mbuf chain
 498  * is freed and m is set to NULL.
 499  */
 500 #define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how)))
 501
 502 /* change mbuf to new type */
 503 #define MCHTYPE(m, t)           m_mchtype(m, t)
 504
 505 /* compatiblity with 4.3 */
 506 #define m_copy(m, o, l)         m_copym((m), (o), (l), M_DONTWAIT)
 507
 508 #define MBSHIFT         20                              /* 1MB */
 509 #define MBSIZE          (1 << MBSHIFT)
 510 #define GBSHIFT         30                              /* 1GB */
 511 #define GBSIZE          (1 << GBSHIFT)
 512
 513 /*
 514  * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
 515  * "off+len") is located in single mbuf, on contiguous memory region.
 516  * The pointer to the region will be returned to pointer variable "val",
 517  * with type "typ".
 518  *
 519  * M_STRUCT_GET0 does the same, except that it aligns the structure at
 520  * very top of mbuf.  GET0 is likely to make memory copy than GET.
 521  */
 522 #define M_STRUCT_GET(val, typ, m, off, len)                             \
 523 do {                                                                    \
 524         struct mbuf *t;                                                 \
 525         int tmp;                                                        \
 526                                                                         \
 527         if ((m)->m_len >= (off) + (len)) {                              \
 528                 (val) = (typ)(mtod((m), caddr_t) + (off));              \
 529         } else {                                                        \
 530                 t = m_pulldown((m), (off), (len), &tmp);                \
 531                 if (t != NULL) {                                        \
 532                         if (t->m_len < tmp + (len))                     \
 533                                 panic("m_pulldown malfunction");        \
 534                         (val) = (typ)(mtod(t, caddr_t) + tmp);          \
 535                 } else {                                                \
 536                         (val) = (typ)NULL;                              \
 537                         (m) = NULL;                                     \
 538                 }                                                       \
 539         }                                                               \
 540 } while (0)
 541
 542 #define M_STRUCT_GET0(val, typ, m, off, len)                            \
 543 do {                                                                    \
 544         struct mbuf *t;                                                 \
 545                                                                         \
 546         if ((off) == 0 && ((m)->m_len >= (len))) {                      \
 547                 (val) = (typ)(void *)mtod(m, caddr_t);                  \
 548         } else {                                                        \
 549                 t = m_pulldown((m), (off), (len), NULL);                \
 550                 if (t != NULL) {                                        \
 551                         if (t->m_len < (len))                           \
 552                                 panic("m_pulldown malfunction");        \
 553                         (val) = (typ)(void *)mtod(t, caddr_t);          \
 554                 } else {                                                \
 555                         (val) = (typ)NULL;                              \
 556                         (m) = NULL;                                     \
 557                 }                                                       \
 558         }                                                               \
 559 } while (0)
 560
 561 #define MBUF_INPUT_CHECK(m, rcvif)                                      \
 562 do {                                                                    \
 563         if (!(m->m_flags & MBUF_PKTHDR) ||                              \
 564             m->m_len < 0 ||                                             \
 565             m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) ||        \
 566             m->m_type == MT_FREE ||                                     \
 567             ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) {  \
 568                 panic_plain("Failed mbuf validity check: mbuf %p len %d "  \
 569                     "type %d flags 0x%x data %p rcvif %s%d ifflags 0x%x",  \
 570                     m, m->m_len, m->m_type, m->m_flags,                    \
 571                     ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
 572                     rcvif->if_name, rcvif->if_unit,                     \
 573                     (rcvif->if_flags & 0xffff));                        \
 574         }                                                               \
 575 } while (0)
 576
 577 /*
 578  * Simple mbuf queueing system
 579  *
 580  * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
 581  * m_nextpkt instead of field.sqe_next).
 582  *
 583  * m_next is ignored, so queueing chains of mbufs is possible
 584  */
 585 #define MBUFQ_HEAD(name)                                        \
 586 struct name {                                                   \
 587         struct mbuf *mq_first;  /* first packet */              \
 588         struct mbuf **mq_last;  /* addr of last next packet */  \
 589 }
 590
 591 #define MBUFQ_INIT(q)           do {                            \
 592         MBUFQ_FIRST(q) = NULL;                                  \
 593         (q)->mq_last = &MBUFQ_FIRST(q);                         \
 594 } while (0)
 595
 596 #define MBUFQ_PREPEND(q, m)     do {                            \
 597         if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL)           \
 598                 (q)->mq_last = &MBUFQ_NEXT(m);                  \
 599         MBUFQ_FIRST(q) = (m);                                   \
 600 } while (0)
 601
 602 #define MBUFQ_ENQUEUE(q, m)     do {                            \
 603         MBUFQ_NEXT(m) = NULL;                                   \
 604         *(q)->mq_last = (m);                                    \
 605         (q)->mq_last = &MBUFQ_NEXT(m);                          \
 606 } while (0)
 607
 608 #define MBUFQ_ENQUEUE_MULTI(q, m, n)    do {                    \
 609         MBUFQ_NEXT(n) = NULL;                                   \
 610         *(q)->mq_last = (m);                                    \
 611         (q)->mq_last = &MBUFQ_NEXT(n);                          \
 612 } while (0)
 613
 614 #define MBUFQ_DEQUEUE(q, m)     do {                            \
 615         if (((m) = MBUFQ_FIRST(q)) != NULL) {                   \
 616                 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL)   \
 617                         (q)->mq_last = &MBUFQ_FIRST(q);         \
 618                 else                                            \
 619                         MBUFQ_NEXT(m) = NULL;                   \
 620         }                                                       \
 621 } while (0)
 622
 623 #define MBUFQ_REMOVE(q, m)      do {                            \
 624         if (MBUFQ_FIRST(q) == (m)) {                            \
 625                 MBUFQ_DEQUEUE(q, m);                            \
 626         } else {                                                \
 627                 struct mbuf *_m = MBUFQ_FIRST(q);               \
 628                 while (MBUFQ_NEXT(_m) != (m))                   \
 629                         _m = MBUFQ_NEXT(_m);                    \
 630                 if ((MBUFQ_NEXT(_m) =                           \
 631                     MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL)        \
 632                         (q)->mq_last = &MBUFQ_NEXT(_m);         \
 633         }                                                       \
 634 } while (0)
 635
 636 #define MBUFQ_DRAIN(q)          do {                            \
 637         struct mbuf *__m0;                                      \
 638         while ((__m0 = MBUFQ_FIRST(q)) != NULL) {               \
 639                 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0);              \
 640                 MBUFQ_NEXT(__m0) = NULL;                        \
 641                 m_freem(__m0);                                  \
 642         }                                                       \
 643         (q)->mq_last = &MBUFQ_FIRST(q);                         \
 644 } while (0)
 645
 646 #define MBUFQ_FOREACH(m, q)                                     \
 647         for ((m) = MBUFQ_FIRST(q);                              \
 648             (m);                                                \
 649             (m) = MBUFQ_NEXT(m))
 650
 651 #define MBUFQ_FOREACH_SAFE(m, q, tvar)                          \
 652         for ((m) = MBUFQ_FIRST(q);                              \
 653             (m) && ((tvar) = MBUFQ_NEXT(m), 1);                 \
 654             (m) = (tvar))
 655
 656 #define MBUFQ_EMPTY(q)          ((q)->mq_first == NULL)
 657 #define MBUFQ_FIRST(q)          ((q)->mq_first)
 658 #define MBUFQ_NEXT(m)           ((m)->m_nextpkt)
 659 #define MBUFQ_LAST(q)           (*(q)->mq_last)
 660
 661 #define max_linkhdr     P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
 662 #define max_protohdr    P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
 663 #endif /* XNU_KERNEL_PRIVATE */
 664
 665 /*
 666  * Mbuf statistics (legacy).
 667  */
 668 struct mbstat {
 669         u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
 670         u_int32_t       m_clusters;     /* clusters obtained from page pool */
 671         u_int32_t       m_spare;        /* spare field */
 672         u_int32_t       m_clfree;       /* free clusters */
 673         u_int32_t       m_drops;        /* times failed to find space */
 674         u_int32_t       m_wait;         /* times waited for space */
 675         u_int32_t       m_drain;        /* times drained protocols for space */
 676         u_short         m_mtypes[256];  /* type specific mbuf allocations */
 677         u_int32_t       m_mcfail;       /* times m_copym failed */
 678         u_int32_t       m_mpfail;       /* times m_pullup failed */
 679         u_int32_t       m_msize;        /* length of an mbuf */
 680         u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
 681         u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
 682         u_int32_t       m_mlen;         /* length of data in an mbuf */
 683         u_int32_t       m_mhlen;        /* length of data in a header mbuf */
 684         u_int32_t       m_bigclusters;  /* clusters obtained from page pool */
 685         u_int32_t       m_bigclfree;    /* free clusters */
 686         u_int32_t       m_bigmclbytes;  /* length of an mbuf cluster */
 687 };
 688
 689 /* Compatibillity with 10.3 */
 690 struct ombstat {
 691         u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
 692         u_int32_t       m_clusters;     /* clusters obtained from page pool */
 693         u_int32_t       m_spare;        /* spare field */
 694         u_int32_t       m_clfree;       /* free clusters */
 695         u_int32_t       m_drops;        /* times failed to find space */
 696         u_int32_t       m_wait;         /* times waited for space */
 697         u_int32_t       m_drain;        /* times drained protocols for space */
 698         u_short         m_mtypes[256];  /* type specific mbuf allocations */
 699         u_int32_t       m_mcfail;       /* times m_copym failed */
 700         u_int32_t       m_mpfail;       /* times m_pullup failed */
 701         u_int32_t       m_msize;        /* length of an mbuf */
 702         u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
 703         u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
 704         u_int32_t       m_mlen;         /* length of data in an mbuf */
 705         u_int32_t       m_mhlen;        /* length of data in a header mbuf */
 706 };
 707
 708 /*
 709  * mbuf class statistics.
 710  */
 711 #define MAX_MBUF_CNAME  15
 712
 713 #if defined(XNU_KERNEL_PRIVATE)
 714 /* For backwards compatibility with 32-bit userland process */
 715 struct omb_class_stat {
 716         char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
 717         u_int32_t       mbcl_size;      /* buffer size */
 718         u_int32_t       mbcl_total;     /* # of buffers created */
 719         u_int32_t       mbcl_active;    /* # of active buffers */
 720         u_int32_t       mbcl_infree;    /* # of available buffers */
 721         u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
 722         u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
 723         u_int64_t       mbcl_free_cnt;  /* # of times free is called */
 724         u_int64_t       mbcl_notified;  /* # of notified wakeups */
 725         u_int64_t       mbcl_purge_cnt; /* # of purges so far */
 726         u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
 727         u_int32_t       mbcl_ctotal;    /* total only for this class */
 728         /*
 729          * Cache layer statistics
 730          */
 731         u_int32_t       mbcl_mc_state;  /* cache state (see below) */
 732         u_int32_t       mbcl_mc_cached; /* # of cached buffers */
 733         u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
 734         u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
 735         u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
 736         u_int64_t       mbcl_reserved[4];    /* for future use */
 737 } __attribute__((__packed__));
 738 #endif /* XNU_KERNEL_PRIVATE */
 739
 740 typedef struct mb_class_stat {
 741         char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
 742         u_int32_t       mbcl_size;      /* buffer size */
 743         u_int32_t       mbcl_total;     /* # of buffers created */
 744         u_int32_t       mbcl_active;    /* # of active buffers */
 745         u_int32_t       mbcl_infree;    /* # of available buffers */
 746         u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
 747 #if defined(KERNEL) || defined(__LP64__)
 748         u_int32_t       mbcl_pad;       /* padding */
 749 #endif /* KERNEL || __LP64__ */
 750         u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
 751         u_int64_t       mbcl_free_cnt;  /* # of times free is called */
 752         u_int64_t       mbcl_notified;  /* # of notified wakeups */
 753         u_int64_t       mbcl_purge_cnt; /* # of purges so far */
 754         u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
 755         u_int32_t       mbcl_ctotal;    /* total only for this class */
 756         /*
 757          * Cache layer statistics
 758          */
 759         u_int32_t       mbcl_mc_state;  /* cache state (see below) */
 760         u_int32_t       mbcl_mc_cached; /* # of cached buffers */
 761         u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
 762         u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
 763         u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
 764         u_int64_t       mbcl_reserved[4];    /* for future use */
 765 } mb_class_stat_t;
 766
 767 #define MCS_DISABLED    0       /* cache is permanently disabled */
 768 #define MCS_ONLINE      1       /* cache is online */
 769 #define MCS_PURGING     2       /* cache is being purged */
 770 #define MCS_OFFLINE     3       /* cache is offline (resizing) */
 771
 772 #if defined(XNU_KERNEL_PRIVATE)
 773 /* For backwards compatibility with 32-bit userland process */
 774 struct omb_stat {
 775         u_int32_t               mbs_cnt;        /* number of classes */
 776         struct omb_class_stat   mbs_class[1];   /* class array */
 777 } __attribute__((__packed__));
 778 #endif /* XNU_KERNEL_PRIVATE */
 779
 780 typedef struct mb_stat {
 781         u_int32_t       mbs_cnt;        /* number of classes */
 782 #if defined(KERNEL) || defined(__LP64__)
 783         u_int32_t       mbs_pad;        /* padding */
 784 #endif /* KERNEL || __LP64__ */
 785         mb_class_stat_t mbs_class[1];   /* class array */
 786 } mb_stat_t;
 787
 788 #ifdef PRIVATE
 789 #define MLEAK_STACK_DEPTH       16      /* Max PC stack depth */
 790
 791 typedef struct mleak_trace_stat {
 792         u_int64_t       mltr_collisions;
 793         u_int64_t       mltr_hitcount;
 794         u_int64_t       mltr_allocs;
 795         u_int64_t       mltr_depth;
 796         u_int64_t       mltr_addr[MLEAK_STACK_DEPTH];
 797 } mleak_trace_stat_t;
 798
 799 typedef struct mleak_stat {
 800         u_int32_t               ml_isaddr64;    /* 64-bit KVA? */
 801         u_int32_t               ml_cnt;         /* number of traces */
 802         mleak_trace_stat_t      ml_trace[1];    /* trace array */
 803 } mleak_stat_t;
 804
 805 struct mleak_table {
 806         u_int32_t mleak_capture;        /* sampling capture counter */
 807         u_int32_t mleak_sample_factor;  /* sample factor */
 808
 809         /* Times two active records want to occupy the same spot */
 810         u_int64_t alloc_collisions;
 811         u_int64_t trace_collisions;
 812
 813         /* Times new record lands on spot previously occupied by freed alloc */
 814         u_int64_t alloc_overwrites;
 815         u_int64_t trace_overwrites;
 816
 817         /* Times a new alloc or trace is put into the hash table */
 818         u_int64_t alloc_recorded;
 819         u_int64_t trace_recorded;
 820
 821         /* Total number of outstanding allocs */
 822         u_int64_t outstanding_allocs;
 823
 824         /* Times mleak_log returned false because couldn't acquire the lock */
 825         u_int64_t total_conflicts;
 826 };
 827 #endif /* PRIVATE */
 828
 829 #ifdef KERNEL_PRIVATE
 830 __BEGIN_DECLS
 831
 832 /*
 833  * Exported (private)
 834  */
 835
 836 extern struct mbstat mbstat;                    /* statistics */
 837
 838 __END_DECLS
 839 #endif /* KERNEL_PRIVATE */
 840
 841 #ifdef XNU_KERNEL_PRIVATE
 842 __BEGIN_DECLS
 843
 844 /*
 845  * Not exported (xnu private)
 846  */
 847
 848 /* flags to m_get/MGET */
 849 /* Need to include malloc.h to get right options for malloc  */
 850 #include        <sys/malloc.h>
 851
 852 struct mbuf;
 853
 854 /* length to m_copy to copy all */
 855 #define M_COPYALL       1000000000
 856
 857 #define M_DONTWAIT      M_NOWAIT
 858 #define M_WAIT          M_WAITOK
 859
 860 /*
 861  * These macros are mapped to the appropriate KPIs, so that private code
 862  * can be simply recompiled in order to be forward-compatible with future
 863  * changes toward the struture sizes.
 864  */
 865 #define MLEN            mbuf_get_mlen()         /* normal data len */
 866 #define MHLEN           mbuf_get_mhlen()        /* data len w/pkthdr */
 867
 868 #define MINCLSIZE       mbuf_get_minclsize()    /* cluster usage threshold */
 869
 870 extern void m_freem(struct mbuf *);
 871 extern char *mcl_to_paddr(char *);
 872 extern void m_adj(struct mbuf *, int);
 873 extern void m_cat(struct mbuf *, struct mbuf *);
 874 extern void m_copydata(struct mbuf *, int, int, void *);
 875 extern struct mbuf *m_copym(struct mbuf *, int, int, int);
 876 extern struct mbuf *m_get(int, int);
 877 extern struct mbuf *m_gethdr(int, int);
 878 extern struct mbuf *m_getpacket(void);
 879 extern struct mbuf *m_getpackets(int, int, int);
 880 extern struct mbuf *m_mclget(struct mbuf *, int);
 881 extern void *m_mtod(struct mbuf *);
 882 extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
 883 extern struct mbuf *m_pullup(struct mbuf *, int);
 884 extern struct mbuf *m_split(struct mbuf *, int, int);
 885 extern void m_mclfree(caddr_t p);
 886
 887 /*
 888  * On platforms which require strict alignment (currently for anything but
 889  * i386 or x86_64), this macro checks whether the data pointer of an mbuf
 890  * is 32-bit aligned (this is the expected minimum alignment for protocol
 891  * headers), and assert otherwise.
 892  */
 893 #if defined(__i386__) || defined(__x86_64__)
 894 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
 895 #else /* !__i386__ && !__x86_64__ */
 896 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do {                    \
 897         if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) {          \
 898                 if (((_m)->m_flags & M_PKTHDR) &&                       \
 899                     (_m)->m_pkthdr.rcvif != NULL) {                     \
 900                         panic_plain("\n%s: mbuf %p data ptr %p is not " \
 901                             "32-bit aligned [%s%d: alignerrs=%lld]\n",  \
 902                             __func__, (_m), (_m)->m_data,               \
 903                             (_m)->m_pkthdr.rcvif->if_name,              \
 904                             (_m)->m_pkthdr.rcvif->if_unit,              \
 905                             (_m)->m_pkthdr.rcvif->if_alignerrs);        \
 906                 } else {                                                \
 907                         panic_plain("\n%s: mbuf %p data ptr %p is not " \
 908                             "32-bit aligned\n",                         \
 909                             __func__, (_m), (_m)->m_data);              \
 910                 }                                                       \
 911         }                                                               \
 912 } while (0)
 913 #endif /* !__i386__ && !__x86_64__ */
 914
 915 /* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
 916 #define MBUF_SC_MAX_CLASSES     10
 917
 918 /*
 919  * These conversion macros rely on the corresponding MBUF_SC and
 920  * MBUF_TC values in order to establish the following mapping:
 921  *
 922  *      MBUF_SC_BK_SYS  ] ==>   MBUF_TC_BK
 923  *      MBUF_SC_BK      ]
 924  *
 925  *      MBUF_SC_BE      ] ==>   MBUF_TC_BE
 926  *      MBUF_SC_RD      ]
 927  *      MBUF_SC_OAM     ]
 928  *
 929  *      MBUF_SC_AV      ] ==>   MBUF_TC_VI
 930  *      MBUF_SC_RV      ]
 931  *      MBUF_SC_VI      ]
 932  *
 933  *      MBUF_SC_VO      ] ==>   MBUF_TC_VO
 934  *      MBUF_SC_CTL     ]
 935  *
 936  * The values assigned to each service class allows for a fast mapping to
 937  * the corresponding MBUF_TC traffic class values, as well as to retrieve the
 938  * assigned index; therefore care must be taken when comparing against these
 939  * values.  Use the corresponding class and index macros to retrieve the
 940  * corresponding portion, and never assume that a higher class corresponds
 941  * to a higher index.
 942  */
 943 #define MBUF_SCVAL(x)           ((x) & 0xffff)
 944 #define MBUF_SCIDX(x)           ((((x) >> 16) & 0xff) >> 3)
 945 #define MBUF_SC2TC(_sc)         (MBUF_SCVAL(_sc) >> 7)
 946 #define MBUF_TC2SCVAL(_tc)      ((_tc) << 7)
 947 #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
 948         ((_sc) == MBUF_SC_BK))
 949
 950 #define SCIDX_BK_SYS            MBUF_SCIDX(MBUF_SC_BK_SYS)
 951 #define SCIDX_BK                MBUF_SCIDX(MBUF_SC_BK)
 952 #define SCIDX_BE                MBUF_SCIDX(MBUF_SC_BE)
 953 #define SCIDX_RD                MBUF_SCIDX(MBUF_SC_RD)
 954 #define SCIDX_OAM               MBUF_SCIDX(MBUF_SC_OAM)
 955 #define SCIDX_AV                MBUF_SCIDX(MBUF_SC_AV)
 956 #define SCIDX_RV                MBUF_SCIDX(MBUF_SC_RV)
 957 #define SCIDX_VI                MBUF_SCIDX(MBUF_SC_VI)
 958 #define SCIDX_VO                MBUF_SCIDX(MBUF_SC_VO)
 959 #define SCIDX_CTL               MBUF_SCIDX(MBUF_SC_CTL)
 960
 961 #define SCVAL_BK_SYS            MBUF_SCVAL(MBUF_SC_BK_SYS)
 962 #define SCVAL_BK                MBUF_SCVAL(MBUF_SC_BK)
 963 #define SCVAL_BE                MBUF_SCVAL(MBUF_SC_BE)
 964 #define SCVAL_RD                MBUF_SCVAL(MBUF_SC_RD)
 965 #define SCVAL_OAM               MBUF_SCVAL(MBUF_SC_OAM)
 966 #define SCVAL_AV                MBUF_SCVAL(MBUF_SC_AV)
 967 #define SCVAL_RV                MBUF_SCVAL(MBUF_SC_RV)
 968 #define SCVAL_VI                MBUF_SCVAL(MBUF_SC_VI)
 969 #define SCVAL_VO                MBUF_SCVAL(MBUF_SC_VO)
 970 #define SCVAL_CTL               MBUF_SCVAL(MBUF_SC_CTL)
 971
 972 #define MBUF_VALID_SC(c)                                                \
 973         (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE ||   \
 974         c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV ||       \
 975         c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO ||        \
 976         c == MBUF_SC_CTL)
 977
 978 #define MBUF_VALID_SCIDX(c)                                             \
 979         (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE ||         \
 980         c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV ||             \
 981         c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO ||              \
 982         c == SCIDX_CTL)
 983
 984 #define MBUF_VALID_SCVAL(c)                                             \
 985         (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE ||         \
 986         c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV ||             \
 987         c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO ||              \
 988         c == SCVAL_CTL)
 989
 990 __private_extern__ union mbigcluster *mbutl;    /* start VA of mbuf pool */
 991 __private_extern__ union mbigcluster *embutl;   /* end VA of mbuf pool */
 992 __private_extern__ unsigned int nmbclusters;    /* number of mapped clusters */
 993 __private_extern__ int njcl;            /* # of jumbo clusters  */
 994 __private_extern__ int njclbytes;       /* size of a jumbo cluster */
 995 __private_extern__ int max_hdr;         /* largest link+protocol header */
 996 __private_extern__ int max_datalen;     /* MHLEN - max_hdr */
 997
 998 /* Use max_linkhdr instead of _max_linkhdr */
 999 __private_extern__ int _max_linkhdr;    /* largest link-level header */
1000
1001 /* Use max_protohdr instead of _max_protohdr */
1002 __private_extern__ int _max_protohdr;   /* largest protocol header */
1003
1004 __private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
1005 __private_extern__ void mbinit(void);
1006 __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
1007     void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
1008 __private_extern__ caddr_t m_bigalloc(int);
1009 __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
1010 __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
1011 __private_extern__ caddr_t m_16kalloc(int);
1012 __private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
1013 __private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
1014
1015 __private_extern__ struct mbuf *m_free(struct mbuf *);
1016 __private_extern__ struct mbuf *m_getclr(int, int);
1017 __private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
1018 __private_extern__ unsigned int m_length(struct mbuf *);
1019 __private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **);
1020 __private_extern__ unsigned int m_fixhdr(struct mbuf *);
1021 __private_extern__ struct mbuf *m_defrag(struct mbuf *, int);
1022 __private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int);
1023 __private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
1024 __private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
1025 __private_extern__ struct mbuf *m_retry(int, int);
1026 __private_extern__ struct mbuf *m_retryhdr(int, int);
1027 __private_extern__ int m_freem_list(struct mbuf *);
1028 __private_extern__ int m_append(struct mbuf *, int, caddr_t);
1029 __private_extern__ struct mbuf *m_last(struct mbuf *);
1030 __private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
1031     void (*)(const void *, void *, size_t));
1032 __private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
1033
1034 __private_extern__ struct mbuf *m_getcl(int, int, int);
1035 __private_extern__ caddr_t m_mclalloc(int);
1036 __private_extern__ int m_mclhasreference(struct mbuf *);
1037 __private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
1038 __private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *);
1039
1040 __private_extern__ struct mbuf *m_dtom(void *);
1041 __private_extern__ int m_mtocl(void *);
1042 __private_extern__ union mcluster *m_cltom(int);
1043
1044 __private_extern__ int m_trailingspace(struct mbuf *);
1045 __private_extern__ int m_leadingspace(struct mbuf *);
1046
1047 __private_extern__ struct mbuf *m_normalize(struct mbuf *m);
1048 __private_extern__ void m_mchtype(struct mbuf *m, int t);
1049 __private_extern__ void m_mcheck(struct mbuf *);
1050
1051 __private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
1052 __private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
1053     const void *, int);
1054 __private_extern__ int m_makewritable(struct mbuf **, int, int, int);
1055 __private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
1056 __private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
1057     struct mbuf **, int *);
1058 __private_extern__ struct mbuf *m_getpackethdrs(int, int);
1059 __private_extern__ struct mbuf *m_getpacket_how(int);
1060 __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
1061     int, int, size_t);
1062 __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
1063     unsigned int *, int, int, size_t);
1064
1065 /*
1066  * Packets may have annotations attached by affixing a list of "packet
1067  * tags" to the pkthdr structure.  Packet tags are dynamically allocated
1068  * semi-opaque data structures that have a fixed header (struct m_tag)
1069  * that specifies the size of the memory block and an <id,type> pair that
1070  * identifies it. The id identifies the module and the type identifies the
1071  * type of data for that module. The id of zero is reserved for the kernel.
1072  *
1073  * Note that the packet tag returned by m_tag_allocate has the default
1074  * memory alignment implemented by malloc.  To reference private data one
1075  * can use a construct like:
1076  *
1077  *      struct m_tag *mtag = m_tag_allocate(...);
1078  *      struct foo *p = (struct foo *)(mtag+1);
1079  *
1080  * if the alignment of struct m_tag is sufficient for referencing members
1081  * of struct foo.  Otherwise it is necessary to embed struct m_tag within
1082  * the private data structure to insure proper alignment; e.g.
1083  *
1084  *      struct foo {
1085  *              struct m_tag    tag;
1086  *              ...
1087  *      };
1088  *      struct foo *p = (struct foo *) m_tag_allocate(...);
1089  *      struct m_tag *mtag = &p->tag;
1090  */
1091
1092 #define KERNEL_MODULE_TAG_ID    0
1093
1094 enum {
1095         KERNEL_TAG_TYPE_NONE                    = 0,
1096         KERNEL_TAG_TYPE_DUMMYNET                = 1,
1097         KERNEL_TAG_TYPE_DIVERT                  = 2,
1098         KERNEL_TAG_TYPE_IPFORWARD               = 3,
1099         KERNEL_TAG_TYPE_IPFILT                  = 4,
1100         KERNEL_TAG_TYPE_MACLABEL                = 5,
1101         KERNEL_TAG_TYPE_MAC_POLICY_LABEL        = 6,
1102         KERNEL_TAG_TYPE_ENCAP                   = 8,
1103         KERNEL_TAG_TYPE_INET6                   = 9,
1104         KERNEL_TAG_TYPE_IPSEC                   = 10,
1105         KERNEL_TAG_TYPE_PF                      = 11
1106 };
1107
1108 /* Packet tag routines */
1109 __private_extern__ struct  m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
1110 __private_extern__ struct  m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
1111         struct mbuf *);
1112 __private_extern__ void m_tag_free(struct m_tag *);
1113 __private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
1114 __private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
1115 __private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
1116 __private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *);
1117 __private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
1118     u_int16_t, struct m_tag *);
1119 __private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
1120 __private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
1121 __private_extern__ void m_tag_init(struct mbuf *);
1122 __private_extern__ struct  m_tag *m_tag_first(struct mbuf *);
1123 __private_extern__ struct  m_tag *m_tag_next(struct mbuf *, struct m_tag *);
1124
1125 __END_DECLS
1126 #endif /* XNU_KERNEL_PRIVATE */
1127 #ifdef KERNEL
1128 #include <sys/kpi_mbuf.h>
1129 #ifdef XNU_KERNEL_PRIVATE
1130 __BEGIN_DECLS
1131
1132 __private_extern__ void m_service_class_init(struct mbuf *);
1133 __private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t);
1134 __private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *);
1135 __private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t);
1136 __private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t);
1137 __private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
1138 __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
1139
1140 __END_DECLS
1141 #endif /* XNU_KERNEL_PRIVATE */
1142 #endif /* KERNEL */
1143 #endif  /* !_SYS_MBUF_H_ */