bsd/sys/mbuf.h

   1 /*
   2  * Copyright (c) 1999-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
  29 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
  30 /*
  31  * Mach Operating System
  32  * Copyright (c) 1987 Carnegie-Mellon University
  33  * All rights reserved.  The CMU software License Agreement specifies
  34  * the terms and conditions for use and redistribution.
  35  */
  36 /*
  37  * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
  38  *
  39  * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
  40  * All rights reserved.
  41  *
  42  * Redistribution and use in source and binary forms, with or without
  43  * modification, are permitted provided that the following conditions
  44  * are met:
  45  * 1. Redistributions of source code must retain the above copyright
  46  *    notice, this list of conditions and the following disclaimer.
  47  * 2. Redistributions in binary form must reproduce the above copyright
  48  *    notice, this list of conditions and the following disclaimer in the
  49  *    documentation and/or other materials provided with the distribution.
  50  * 3. All advertising materials mentioning features or use of this software
  51  *    must display the following acknowledgement:
  52  *      This product includes software developed by the University of
  53  *      California, Berkeley and its contributors.
  54  * 4. Neither the name of the University nor the names of its contributors
  55  *    may be used to endorse or promote products derived from this software
  56  *    without specific prior written permission.
  57  *
  58  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  59  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  60  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  61  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  62  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  63  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  64  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  65  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  66  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  67  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  68  * SUCH DAMAGE.
  69  *
  70  *      @(#)mbuf.h      8.3 (Berkeley) 1/21/94
  71  */
  72 /*
  73  * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
  74  * support for mandatory and extensible security protections.  This notice
  75  * is included in support of clause 2.2 (b) of the Apple Public License,
  76  * Version 2.0.
  77  */
  78
  79 #ifndef _SYS_MBUF_H_
  80 #define _SYS_MBUF_H_
  81
  82 #include <sys/cdefs.h>
  83 #include <sys/appleapiopts.h>
  84
  85 #ifdef XNU_KERNEL_PRIVATE
  86
  87 #include <sys/lock.h>
  88 #include <sys/queue.h>
  89
  90 #if PF_PKTHDR
  91 #include <net/pf_mtag.h>
  92 #endif /* PF_PKTHDR */
  93
  94 /*
  95  * Mbufs are of a single size, MSIZE (machine/param.h), which
  96  * includes overhead.  An mbuf may add a single "mbuf cluster" of size
  97  * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
  98  * no additional overhead and is used instead of the internal data area;
  99  * this is done when at least MINCLSIZE of data must be stored.
 100  */
 101
 102 /*
 103  * The following _MLEN and _MHLEN macros are private to xnu.  Private code
 104  * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
 105  * the sizes of the structures are dependent upon specific xnu configs.
 106  */
 107 #define _MLEN           (MSIZE - sizeof(struct m_hdr))  /* normal data len */
 108 #define _MHLEN          (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
 109
 110 #define NMBPBGSHIFT     (MBIGCLSHIFT - MSIZESHIFT)
 111 #define NMBPBG          (1 << NMBPBGSHIFT)      /* # of mbufs per big cl */
 112
 113 #define NCLPBGSHIFT     (MBIGCLSHIFT - MCLSHIFT)
 114 #define NCLPBG          (1 << NCLPBGSHIFT)      /* # of cl per big cl */
 115
 116 #define NMBPCLSHIFT     (NMBPBGSHIFT - NCLPBGSHIFT)
 117 #define NMBPCL          (1 << NMBPCLSHIFT)      /* # of mbufs per cl */
 118
 119 #define NCLPJCLSHIFT    ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT)
 120 #define NCLPJCL         (1 << NCLPJCLSHIFT)     /* # of cl per jumbo cl */
 121
 122 /*
 123  * Macros for type conversion
 124  * mtod(m,t) -  convert mbuf pointer to data pointer of correct type
 125  * dtom(x) -    convert data pointer within mbuf to mbuf pointer (XXX)
 126  */
 127 #define mtod(m, t)      ((t)m_mtod(m))
 128 #define dtom(x)         m_dtom(x)
 129
 130 /* header at beginning of each mbuf: */
 131 struct m_hdr {
 132         struct  mbuf *mh_next;          /* next buffer in chain */
 133         struct  mbuf *mh_nextpkt;       /* next chain in queue/record */
 134         int32_t mh_len;                 /* amount of data in this mbuf */
 135         caddr_t mh_data;                /* location of data */
 136         short   mh_type;                /* type of data in this mbuf */
 137         short   mh_flags;               /* flags; see below */
 138 };
 139
 140 /*
 141  * Packet tag structure (see below for details).
 142  */
 143 struct m_tag {
 144         u_int64_t               m_tag_cookie;   /* Error checking */
 145 #ifndef __LP64__
 146         u_int32_t               pad;            /* For structure alignment */
 147 #endif /* !__LP64__ */
 148         SLIST_ENTRY(m_tag)      m_tag_link;     /* List of packet tags */
 149         u_int16_t               m_tag_type;     /* Module specific type */
 150         u_int16_t               m_tag_len;      /* Length of data */
 151         u_int32_t               m_tag_id;       /* Module ID */
 152 };
 153
 154 #ifdef __LP64__
 155 #define M_TAG_ALIGN(len) \
 156         P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)
 157 #else
 158 #define M_TAG_ALIGN(len) \
 159         P2ROUNDUP(len, sizeof (u_int32_t)) + sizeof (struct m_tag)
 160 #endif /* !__LP64__ */
 161
 162 #define M_TAG_VALID_PATTERN     0xfeedfacefeedfaceULL
 163 #define M_TAG_FREE_PATTERN      0xdeadbeefdeadbeefULL
 164
 165 struct m_taghdr {
 166         u_int64_t               refcnt;         /* Number of tags in this mbuf */
 167 };
 168
 169 /* record/packet header in first mbuf of chain; valid if M_PKTHDR set */
 170 struct  pkthdr {
 171         int     len;                    /* total packet length */
 172         struct  ifnet *rcvif;           /* rcv interface */
 173
 174         /* variables for ip and tcp reassembly */
 175         void    *header;                /* pointer to packet header */
 176         /* variables for hardware checksum */
 177         /* Note: csum_flags is used for hardware checksum and VLAN */
 178         int     csum_flags;             /* flags regarding checksum */
 179         int     csum_data;              /* data field used by csum routines */
 180         u_int   tso_segsz;              /* TSO segment size (actual MSS) */
 181         u_short vlan_tag;               /* VLAN tag, host byte order */
 182         u_short socket_id;              /* socket id */
 183         SLIST_HEAD(packet_tags, m_tag) tags; /* list of packet tags */
 184 #if PF_PKTHDR
 185         /*
 186          * Be careful; {en,dis}abling PF_PKTHDR will require xnu recompile;
 187          * private code outside of xnu must use mbuf_get_mhlen() instead
 188          * of MHLEN.
 189          */
 190         struct pf_mtag pf_mtag;
 191 #endif /* PF_PKTHDR */
 192         u_int32_t prio;                 /* packet priority */
 193         u_short vt_nrecs;               /* # of IGMPv3 records in this chain */
 194         u_short _pad;
 195 };
 196
 197
 198 /* description of external storage mapped into mbuf, valid if M_EXT set */
 199 struct m_ext {
 200         caddr_t ext_buf;                /* start of buffer */
 201         void    (*ext_free)(caddr_t, u_int, caddr_t);   /* free routine if not the usual */
 202         u_int   ext_size;               /* size of buffer, for ext_free */
 203         caddr_t ext_arg;                /* additional ext_free argument */
 204         struct  ext_refsq {             /* references held */
 205                 struct ext_refsq *forward, *backward;
 206         } ext_refs;
 207         struct ext_ref {
 208                 u_int32_t refcnt;
 209                 u_int32_t flags;
 210         } *ext_refflags;
 211 };
 212
 213 /* define m_ext to a type since it gets redefined below */
 214 typedef struct m_ext _m_ext_t;
 215
 216 struct mbuf {
 217         struct  m_hdr m_hdr;
 218         union {
 219                 struct {
 220                         struct  pkthdr MH_pkthdr;       /* M_PKTHDR set */
 221                         union {
 222                                 struct  m_ext MH_ext;   /* M_EXT set */
 223                                 char    MH_databuf[_MHLEN];
 224                         } MH_dat;
 225                 } MH;
 226                 char    M_databuf[_MLEN];               /* !M_PKTHDR, !M_EXT */
 227         } M_dat;
 228 };
 229
 230 #define m_next          m_hdr.mh_next
 231 #define m_len           m_hdr.mh_len
 232 #define m_data          m_hdr.mh_data
 233 #define m_type          m_hdr.mh_type
 234 #define m_flags         m_hdr.mh_flags
 235 #define m_nextpkt       m_hdr.mh_nextpkt
 236 #define m_act           m_nextpkt
 237 #define m_pkthdr        M_dat.MH.MH_pkthdr
 238 #define m_ext           M_dat.MH.MH_dat.MH_ext
 239 #define m_pktdat        M_dat.MH.MH_dat.MH_databuf
 240 #define m_dat           M_dat.M_databuf
 241
 242 /* mbuf flags (private) */
 243 #define M_EXT           0x0001  /* has associated external storage */
 244 #define M_PKTHDR        0x0002  /* start of record */
 245 #define M_EOR           0x0004  /* end of record */
 246 #define M_PROTO1        0x0008  /* protocol-specific */
 247 #define M_PROTO2        0x0010  /* protocol-specific */
 248 #define M_PROTO3        0x0020  /* protocol-specific */
 249 #define M_LOOP          0x0040  /* packet is looped back */
 250 #define M_PROTO5        0x0080  /* protocol-specific */
 251
 252 /* mbuf pkthdr flags, also in m_flags (private) */
 253 #define M_BCAST         0x0100  /* send/received as link-level broadcast */
 254 #define M_MCAST         0x0200  /* send/received as link-level multicast */
 255 #define M_FRAG          0x0400  /* packet is a fragment of a larger packet */
 256 #define M_FIRSTFRAG     0x0800  /* packet is first fragment */
 257 #define M_LASTFRAG      0x1000  /* packet is last fragment */
 258 #define M_PROMISC       0x2000  /* packet is promiscuous (shouldn't go to stack) */
 259 #define M_HASFCS        0x4000  /* packet has FCS */
 260 #define M_TAGHDR        0x8000  /* m_tag hdr structure at top of mbuf data */
 261
 262 /*
 263  * Flags to purge when crossing layers.
 264  */
 265 #define M_PROTOFLAGS \
 266         (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
 267
 268 /* flags copied when copying m_pkthdr */
 269 #define M_COPYFLAGS                                                     \
 270         (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 |                    \
 271         M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG |                        \
 272         M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
 273
 274 /* flags indicating hw checksum support and sw checksum requirements [freebsd4.1] */
 275 #define CSUM_IP                 0x0001          /* will csum IP */
 276 #define CSUM_TCP                0x0002          /* will csum TCP */
 277 #define CSUM_UDP                0x0004          /* will csum UDP */
 278 #define CSUM_IP_FRAGS           0x0008          /* will csum IP fragments */
 279 #define CSUM_FRAGMENT           0x0010          /* will do IP fragmentation */
 280 #define CSUM_TCPIPV6            0x0020          /* will csum TCP for IPv6 */
 281 #define CSUM_UDPIPV6            0x0040          /* will csum UDP for IPv6 */
 282 #define CSUM_FRAGMENT_IPV6      0x0080          /* will do IPv6 fragmentation */
 283
 284 #define CSUM_IP_CHECKED         0x0100          /* did csum IP */
 285 #define CSUM_IP_VALID           0x0200          /*   ... the csum is valid */
 286 #define CSUM_DATA_VALID         0x0400          /* csum_data field is valid */
 287 #define CSUM_PSEUDO_HDR         0x0800          /* csum_data has pseudo hdr */
 288 #define CSUM_TCP_SUM16          0x1000          /* simple TCP Sum16 computation */
 289
 290 #define CSUM_DELAY_DATA         (CSUM_TCP | CSUM_UDP)
 291 #define CSUM_DELAY_IP           (CSUM_IP)       /* IPv4 only: no IPv6 IP cksum */
 292 #define CSUM_DELAY_IPV6_DATA    (CSUM_TCPIPV6 | CSUM_UDPIPV6)
 293 #define CSUM_DATA_IPV6_VALID    CSUM_DATA_VALID /* csum_data field is valid */
 294 /*
 295  * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
 296  */
 297 /* bottom 16 bits reserved for hardware checksum */
 298 #define CSUM_CHECKSUM_MASK      0xffff
 299
 300 /* VLAN tag present */
 301 #define CSUM_VLAN_TAG_VALID     0x10000         /* vlan_tag field is valid */
 302
 303 /* TCP Segment Offloading requested on this mbuf */
 304 #define CSUM_TSO_IPV4           0x100000        /* This mbuf needs to be segmented by the NIC */
 305 #define CSUM_TSO_IPV6           0x200000        /* This mbuf needs to be segmented by the NIC */
 306 #endif /* XNU_KERNEL_PRIVATE */
 307
 308 /* mbuf types */
 309 #define MT_FREE         0       /* should be on free list */
 310 #define MT_DATA         1       /* dynamic (data) allocation */
 311 #define MT_HEADER       2       /* packet header */
 312 #define MT_SOCKET       3       /* socket structure */
 313 #define MT_PCB          4       /* protocol control block */
 314 #define MT_RTABLE       5       /* routing tables */
 315 #define MT_HTABLE       6       /* IMP host tables */
 316 #define MT_ATABLE       7       /* address resolution tables */
 317 #define MT_SONAME       8       /* socket name */
 318 #define MT_SOOPTS       10      /* socket options */
 319 #define MT_FTABLE       11      /* fragment reassembly header */
 320 #define MT_RIGHTS       12      /* access rights */
 321 #define MT_IFADDR       13      /* interface address */
 322 #define MT_CONTROL      14      /* extra-data protocol message */
 323 #define MT_OOBDATA      15      /* expedited data  */
 324 #define MT_TAG          16      /* volatile metadata associated to pkts */
 325 #define MT_MAX          32      /* enough? */
 326
 327 #ifdef XNU_KERNEL_PRIVATE
 328 /*
 329  * mbuf allocation/deallocation macros:
 330  *
 331  *      MGET(struct mbuf *m, int how, int type)
 332  * allocates an mbuf and initializes it to contain internal data.
 333  *
 334  *      MGETHDR(struct mbuf *m, int how, int type)
 335  * allocates an mbuf and initializes it to contain a packet header
 336  * and internal data.
 337  */
 338
 339 #if 1
 340 #define MCHECK(m) m_mcheck(m)
 341 #else
 342 #define MCHECK(m)
 343 #endif
 344
 345 #define MGET(m, how, type) ((m) = m_get((how), (type)))
 346
 347 #define MGETHDR(m, how, type)   ((m) = m_gethdr((how), (type)))
 348
 349 /*
 350  * Mbuf cluster macros.
 351  * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
 352  * MCLGET adds such clusters to a normal mbuf;
 353  * the flag M_EXT is set upon success.
 354  * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
 355  * freeing the cluster if the reference count has reached 0.
 356  *
 357  * Normal mbuf clusters are normally treated as character arrays
 358  * after allocation, but use the first word of the buffer as a free list
 359  * pointer while on the free list.
 360  */
 361 union mcluster {
 362         union   mcluster *mcl_next;
 363         char    mcl_buf[MCLBYTES];
 364 };
 365
 366 #define MCLALLOC(p, how)        ((p) = m_mclalloc(how))
 367
 368 #define MCLFREE(p)              m_mclfree(p)
 369
 370 #define MCLGET(m, how)          ((m) = m_mclget(m, how))
 371
 372 /*
 373  * Mbuf big cluster
 374  */
 375 union mbigcluster {
 376         union mbigcluster       *mbc_next;
 377         char                    mbc_buf[MBIGCLBYTES];
 378 };
 379
 380 /*
 381  * Mbuf jumbo cluster
 382  */
 383 union m16kcluster {
 384         union m16kcluster       *m16kcl_next;
 385         char                    m16kcl_buf[M16KCLBYTES];
 386 };
 387
 388 #define MCLHASREFERENCE(m)      m_mclhasreference(m)
 389
 390 /*
 391  * MFREE(struct mbuf *m, struct mbuf *n)
 392  * Free a single mbuf and associated external storage.
 393  * Place the successor, if any, in n.
 394  */
 395
 396 #define MFREE(m, n) ((n) = m_free(m))
 397
 398 /*
 399  * Copy mbuf pkthdr from from to to.
 400  * from must have M_PKTHDR set, and to must be empty.
 401  * aux pointer will be moved to `to'.
 402  */
 403 #define M_COPY_PKTHDR(to, from)         m_copy_pkthdr(to, from)
 404
 405 /*
 406  * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place
 407  * an object of the specified size at the end of the mbuf, longword aligned.
 408  */
 409 #define M_ALIGN(m, len)                                                 \
 410 do {                                                                    \
 411         (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1);           \
 412 } while (0)
 413
 414 /*
 415  * As above, for mbufs allocated with m_gethdr/MGETHDR
 416  * or initialized by M_COPY_PKTHDR.
 417  */
 418 #define MH_ALIGN(m, len)                                                \
 419 do {                                                                    \
 420         (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1);          \
 421 } while (0)
 422
 423 /*
 424  * Compute the amount of space available
 425  * before the current start of data in an mbuf.
 426  * Subroutine - data not available if certain references.
 427  */
 428 #define M_LEADINGSPACE(m)       m_leadingspace(m)
 429
 430 /*
 431  * Compute the amount of space available
 432  * after the end of data in an mbuf.
 433  * Subroutine - data not available if certain references.
 434  */
 435 #define M_TRAILINGSPACE(m)      m_trailingspace(m)
 436
 437 /*
 438  * Arrange to prepend space of size plen to mbuf m.
 439  * If a new mbuf must be allocated, how specifies whether to wait.
 440  * If how is M_DONTWAIT and allocation fails, the original mbuf chain
 441  * is freed and m is set to NULL.
 442  */
 443 #define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how)))
 444
 445 /* change mbuf to new type */
 446 #define MCHTYPE(m, t)           m_mchtype(m, t)
 447
 448 /* compatiblity with 4.3 */
 449 #define m_copy(m, o, l)         m_copym((m), (o), (l), M_DONTWAIT)
 450
 451 #define MBSHIFT         20                              /* 1MB */
 452 #define MBSIZE          (1 << MBSHIFT)
 453 #define GBSHIFT         30                              /* 1GB */
 454 #define GBSIZE          (1 << GBSHIFT)
 455
 456 /*
 457  * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
 458  * "len") is located in single mbuf, on contiguous memory region.
 459  * The pointer to the region will be returned to pointer variable "val",
 460  * with type "typ".
 461  *
 462  * M_STRUCT_GET0 does the same, except that it aligns the structure at
 463  * very top of mbuf.  GET0 is likely to make memory copy than GET.
 464  */
 465 #define M_STRUCT_GET(val, typ, m, off, len)                             \
 466 do {                                                                    \
 467         struct mbuf *t;                                                 \
 468         int tmp;                                                        \
 469                                                                         \
 470         if ((m)->m_len >= (off) + (len)) {                              \
 471                 (val) = (typ)(mtod((m), caddr_t) + (off));              \
 472         } else {                                                        \
 473                 t = m_pulldown((m), (off), (len), &tmp);                \
 474                 if (t != NULL) {                                        \
 475                         if (t->m_len < tmp + (len))                     \
 476                                 panic("m_pulldown malfunction");        \
 477                         (val) = (typ)(mtod(t, caddr_t) + tmp);          \
 478                 } else {                                                \
 479                         (val) = (typ)NULL;                              \
 480                         (m) = NULL;                                     \
 481                 }                                                       \
 482         }                                                               \
 483 } while (0)
 484
 485 #define M_STRUCT_GET0(val, typ, m, off, len)                            \
 486 do {                                                                    \
 487         struct mbuf *t;                                                 \
 488                                                                         \
 489         if ((off) == 0) {                                               \
 490                 (val) = (typ)mtod(m, caddr_t);                          \
 491         } else {                                                        \
 492                 t = m_pulldown((m), (off), (len), NULL);                \
 493                 if (t != NULL) {                                        \
 494                         if (t->m_len < (len))                           \
 495                                 panic("m_pulldown malfunction");        \
 496                         (val) = (typ)mtod(t, caddr_t);                  \
 497                 } else {                                                \
 498                         (val) = (typ)NULL;                              \
 499                         (m) = NULL;                                     \
 500                 }                                                       \
 501         }                                                               \
 502 } while (0)
 503
 504 #define MBUF_INPUT_CHECK(m, rcvif)                                      \
 505 do {                                                                    \
 506         if (!(m->m_flags & MBUF_PKTHDR) ||                              \
 507             m->m_len < 0 ||                                             \
 508             m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) ||        \
 509             m->m_type == MT_FREE ||                                     \
 510             ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) {  \
 511                 panic("Failed mbuf validity check: mbuf %p len %d "     \
 512                     "type %d flags 0x%x data %p rcvif %s%d ifflags 0x%x",  \
 513                     m, m->m_len, m->m_type, m->m_flags,                    \
 514                     ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
 515                     rcvif->if_name, rcvif->if_unit,                     \
 516                     (rcvif->if_flags & 0xffff));                        \
 517         }                                                               \
 518 } while (0)
 519
 520 #endif /* XNU_KERNEL_PRIVATE */
 521
 522 /*
 523  * Mbuf statistics (legacy).
 524  */
 525 struct mbstat {
 526         u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
 527         u_int32_t       m_clusters;     /* clusters obtained from page pool */
 528         u_int32_t       m_spare;        /* spare field */
 529         u_int32_t       m_clfree;       /* free clusters */
 530         u_int32_t       m_drops;        /* times failed to find space */
 531         u_int32_t       m_wait;         /* times waited for space */
 532         u_int32_t       m_drain;        /* times drained protocols for space */
 533         u_short         m_mtypes[256];  /* type specific mbuf allocations */
 534         u_int32_t       m_mcfail;       /* times m_copym failed */
 535         u_int32_t       m_mpfail;       /* times m_pullup failed */
 536         u_int32_t       m_msize;        /* length of an mbuf */
 537         u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
 538         u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
 539         u_int32_t       m_mlen;         /* length of data in an mbuf */
 540         u_int32_t       m_mhlen;        /* length of data in a header mbuf */
 541         u_int32_t       m_bigclusters;  /* clusters obtained from page pool */
 542         u_int32_t       m_bigclfree;    /* free clusters */
 543         u_int32_t       m_bigmclbytes;  /* length of an mbuf cluster */
 544 };
 545
 546 /* Compatibillity with 10.3 */
 547 struct ombstat {
 548         u_int32_t       m_mbufs;        /* mbufs obtained from page pool */
 549         u_int32_t       m_clusters;     /* clusters obtained from page pool */
 550         u_int32_t       m_spare;        /* spare field */
 551         u_int32_t       m_clfree;       /* free clusters */
 552         u_int32_t       m_drops;        /* times failed to find space */
 553         u_int32_t       m_wait;         /* times waited for space */
 554         u_int32_t       m_drain;        /* times drained protocols for space */
 555         u_short         m_mtypes[256];  /* type specific mbuf allocations */
 556         u_int32_t       m_mcfail;       /* times m_copym failed */
 557         u_int32_t       m_mpfail;       /* times m_pullup failed */
 558         u_int32_t       m_msize;        /* length of an mbuf */
 559         u_int32_t       m_mclbytes;     /* length of an mbuf cluster */
 560         u_int32_t       m_minclsize;    /* min length of data to allocate a cluster */
 561         u_int32_t       m_mlen;         /* length of data in an mbuf */
 562         u_int32_t       m_mhlen;        /* length of data in a header mbuf */
 563 };
 564
 565 /*
 566  * mbuf class statistics.
 567  */
 568 #define MAX_MBUF_CNAME  15
 569
 570 #if defined(XNU_KERNEL_PRIVATE)
 571 /* For backwards compatibility with 32-bit userland process */
 572 struct omb_class_stat {
 573         char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
 574         u_int32_t       mbcl_size;      /* buffer size */
 575         u_int32_t       mbcl_total;     /* # of buffers created */
 576         u_int32_t       mbcl_active;    /* # of active buffers */
 577         u_int32_t       mbcl_infree;    /* # of available buffers */
 578         u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
 579         u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
 580         u_int64_t       mbcl_free_cnt;  /* # of times free is called */
 581         u_int64_t       mbcl_notified;  /* # of notified wakeups */
 582         u_int64_t       mbcl_purge_cnt; /* # of purges so far */
 583         u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
 584         u_int32_t       mbcl_ctotal;    /* total only for this class */
 585         /*
 586          * Cache layer statistics
 587          */
 588         u_int32_t       mbcl_mc_state;  /* cache state (see below) */
 589         u_int32_t       mbcl_mc_cached; /* # of cached buffers */
 590         u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
 591         u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
 592         u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
 593         u_int64_t       mbcl_reserved[4];    /* for future use */
 594 } __attribute__((__packed__));
 595 #endif /* XNU_KERNEL_PRIVATE */
 596
 597 typedef struct mb_class_stat {
 598         char            mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
 599         u_int32_t       mbcl_size;      /* buffer size */
 600         u_int32_t       mbcl_total;     /* # of buffers created */
 601         u_int32_t       mbcl_active;    /* # of active buffers */
 602         u_int32_t       mbcl_infree;    /* # of available buffers */
 603         u_int32_t       mbcl_slab_cnt;  /* # of available slabs */
 604 #if defined(KERNEL) || defined(__LP64__)
 605         u_int32_t       mbcl_pad;       /* padding */
 606 #endif /* KERNEL || __LP64__ */
 607         u_int64_t       mbcl_alloc_cnt; /* # of times alloc is called */
 608         u_int64_t       mbcl_free_cnt;  /* # of times free is called */
 609         u_int64_t       mbcl_notified;  /* # of notified wakeups */
 610         u_int64_t       mbcl_purge_cnt; /* # of purges so far */
 611         u_int64_t       mbcl_fail_cnt;  /* # of allocation failures */
 612         u_int32_t       mbcl_ctotal;    /* total only for this class */
 613         /*
 614          * Cache layer statistics
 615          */
 616         u_int32_t       mbcl_mc_state;  /* cache state (see below) */
 617         u_int32_t       mbcl_mc_cached; /* # of cached buffers */
 618         u_int32_t       mbcl_mc_waiter_cnt;  /* # waiters on the cache */
 619         u_int32_t       mbcl_mc_wretry_cnt;  /* # of wait retries */
 620         u_int32_t       mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
 621         u_int64_t       mbcl_reserved[4];    /* for future use */
 622 } mb_class_stat_t;
 623
 624 #define MCS_DISABLED    0       /* cache is permanently disabled */
 625 #define MCS_ONLINE      1       /* cache is online */
 626 #define MCS_PURGING     2       /* cache is being purged */
 627 #define MCS_OFFLINE     3       /* cache is offline (resizing) */
 628
 629 #if defined(XNU_KERNEL_PRIVATE)
 630 /* For backwards compatibility with 32-bit userland process */
 631 struct omb_stat {
 632         u_int32_t               mbs_cnt;        /* number of classes */
 633         struct omb_class_stat   mbs_class[1];   /* class array */
 634 } __attribute__((__packed__));
 635 #endif /* XNU_KERNEL_PRIVATE */
 636
 637 typedef struct mb_stat {
 638         u_int32_t       mbs_cnt;        /* number of classes */
 639 #if defined(KERNEL) || defined(__LP64__)
 640         u_int32_t       mbs_pad;        /* padding */
 641 #endif /* KERNEL || __LP64__ */
 642         mb_class_stat_t mbs_class[1];   /* class array */
 643 } mb_stat_t;
 644
 645 #ifdef PRIVATE
 646 #define MLEAK_STACK_DEPTH       16      /* Max PC stack depth */
 647
 648 typedef struct mleak_trace_stat {
 649         u_int64_t       mltr_collisions;
 650         u_int64_t       mltr_hitcount;
 651         u_int64_t       mltr_allocs;
 652         u_int64_t       mltr_depth;
 653         u_int64_t       mltr_addr[MLEAK_STACK_DEPTH];
 654 } mleak_trace_stat_t;
 655
 656 typedef struct mleak_stat {
 657         u_int32_t               ml_isaddr64;    /* 64-bit KVA? */
 658         u_int32_t               ml_cnt;         /* number of traces */
 659         mleak_trace_stat_t      ml_trace[1];    /* trace array */
 660 } mleak_stat_t;
 661
 662 struct mleak_table {
 663         u_int32_t mleak_capture;        /* sampling capture counter */
 664         u_int32_t mleak_sample_factor;  /* sample factor */
 665
 666         /* Times two active records want to occupy the same spot */
 667         u_int64_t alloc_collisions;
 668         u_int64_t trace_collisions;
 669
 670         /* Times new record lands on spot previously occupied by freed alloc */
 671         u_int64_t alloc_overwrites;
 672         u_int64_t trace_overwrites;
 673
 674         /* Times a new alloc or trace is put into the hash table */
 675         u_int64_t alloc_recorded;
 676         u_int64_t trace_recorded;
 677
 678         /* Total number of outstanding allocs */
 679         u_int64_t outstanding_allocs;
 680
 681         /* Times mleak_log returned false because couldn't acquire the lock */
 682         u_int64_t total_conflicts;
 683 };
 684 #endif /* PRIVATE */
 685
 686 #ifdef KERNEL_PRIVATE
 687 __BEGIN_DECLS
 688
 689 /*
 690  * Exported (private)
 691  */
 692
 693 extern struct mbstat mbstat;                    /* statistics */
 694
 695 __END_DECLS
 696 #endif /* KERNEL_PRIVATE */
 697
 698 #ifdef XNU_KERNEL_PRIVATE
 699 __BEGIN_DECLS
 700
 701 /*
 702  * Not exported (xnu private)
 703  */
 704
 705 /* flags to m_get/MGET */
 706 /* Need to include malloc.h to get right options for malloc  */
 707 #include        <sys/malloc.h>
 708
 709 struct mbuf;
 710
 711 /* length to m_copy to copy all */
 712 #define M_COPYALL       1000000000
 713
 714 #define M_DONTWAIT      M_NOWAIT
 715 #define M_WAIT          M_WAITOK
 716
 717 /*
 718  * These macros are mapped to the appropriate KPIs, so that private code
 719  * can be simply recompiled in order to be forward-compatible with future
 720  * changes toward the struture sizes.
 721  */
 722 #define MLEN            mbuf_get_mlen()         /* normal data len */
 723 #define MHLEN           mbuf_get_mhlen()        /* data len w/pkthdr */
 724
 725 #define MINCLSIZE       mbuf_get_minclsize()    /* cluster usage threshold */
 726
 727 extern void m_freem(struct mbuf *);
 728 extern char *mcl_to_paddr(char *);
 729 extern void m_adj(struct mbuf *, int);
 730 extern void m_cat(struct mbuf *, struct mbuf *);
 731 extern void m_copydata(struct mbuf *, int, int, void *);
 732 extern struct mbuf *m_copym(struct mbuf *, int, int, int);
 733 extern struct mbuf *m_get(int, int);
 734 extern struct mbuf *m_gethdr(int, int);
 735 extern struct mbuf *m_getpacket(void);
 736 extern struct mbuf *m_getpackets(int, int, int);
 737 extern struct mbuf *m_mclget(struct mbuf *, int);
 738 extern void *m_mtod(struct mbuf *);
 739 extern struct mbuf *m_prepend_2(struct mbuf *, int, int);
 740 extern struct mbuf *m_pullup(struct mbuf *, int);
 741 extern struct mbuf *m_split(struct mbuf *, int, int);
 742 extern void m_mclfree(caddr_t p);
 743
 744 __private_extern__ union mbigcluster *mbutl;    /* start VA of mbuf pool */
 745 __private_extern__ union mbigcluster *embutl;   /* end VA of mbuf pool */
 746 __private_extern__ unsigned int nmbclusters;    /* number of mapped clusters */
 747 __private_extern__ int njcl;            /* # of jumbo clusters  */
 748 __private_extern__ int njclbytes;       /* size of a jumbo cluster */
 749 __private_extern__ int max_linkhdr;     /* largest link-level header */
 750 __private_extern__ int max_protohdr;    /* largest protocol header */
 751 __private_extern__ int max_hdr;         /* largest link+protocol header */
 752 __private_extern__ int max_datalen;     /* MHLEN - max_hdr */
 753
 754 __private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t);
 755 __private_extern__ void mbinit(void);
 756 __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
 757     void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int);
 758 __private_extern__ caddr_t m_bigalloc(int);
 759 __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
 760 __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
 761 __private_extern__ caddr_t m_16kalloc(int);
 762 __private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
 763 __private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
 764
 765 __private_extern__ struct mbuf *m_free(struct mbuf *);
 766 __private_extern__ struct mbuf *m_getclr(int, int);
 767 __private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
 768 __private_extern__ unsigned int m_length(struct mbuf *);
 769 __private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
 770 __private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
 771 __private_extern__ struct mbuf *m_retry(int, int);
 772 __private_extern__ struct mbuf *m_retryhdr(int, int);
 773 __private_extern__ int m_freem_list(struct mbuf *);
 774 __private_extern__ int m_append(struct mbuf *, int, caddr_t);
 775 __private_extern__ struct mbuf *m_last(struct mbuf *);
 776 __private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
 777     void (*)(const void *, void *, size_t));
 778 __private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
 779
 780 __private_extern__ struct mbuf *m_getcl(int, int, int);
 781 __private_extern__ caddr_t m_mclalloc(int);
 782 __private_extern__ int m_mclhasreference(struct mbuf *);
 783 __private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
 784
 785 __private_extern__ struct mbuf *m_dtom(void *);
 786 __private_extern__ int m_mtocl(void *);
 787 __private_extern__ union mcluster *m_cltom(int);
 788
 789 __private_extern__ int m_trailingspace(struct mbuf *);
 790 __private_extern__ int m_leadingspace(struct mbuf *);
 791
 792 __private_extern__ struct mbuf *m_normalize(struct mbuf *m);
 793 __private_extern__ void m_mchtype(struct mbuf *m, int t);
 794 __private_extern__ void m_mcheck(struct mbuf *);
 795
 796 __private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
 797 __private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
 798     const void *, int);
 799 __private_extern__ int m_makewritable(struct mbuf **, int, int, int);
 800 __private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
 801 __private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
 802     struct mbuf **, int *);
 803 __private_extern__ struct mbuf *m_getpackethdrs(int, int);
 804 __private_extern__ struct mbuf *m_getpacket_how(int);
 805 __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
 806     int, int, size_t);
 807 __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
 808     unsigned int *, int, int, size_t);
 809
 810 /*
 811  * Packets may have annotations attached by affixing a list of "packet
 812  * tags" to the pkthdr structure.  Packet tags are dynamically allocated
 813  * semi-opaque data structures that have a fixed header (struct m_tag)
 814  * that specifies the size of the memory block and an <id,type> pair that
 815  * identifies it. The id identifies the module and the type identifies the
 816  * type of data for that module. The id of zero is reserved for the kernel.
 817  *
 818  * Note that the packet tag returned by m_tag_allocate has the default
 819  * memory alignment implemented by malloc.  To reference private data one
 820  * can use a construct like:
 821  *
 822  *      struct m_tag *mtag = m_tag_allocate(...);
 823  *      struct foo *p = (struct foo *)(mtag+1);
 824  *
 825  * if the alignment of struct m_tag is sufficient for referencing members
 826  * of struct foo.  Otherwise it is necessary to embed struct m_tag within
 827  * the private data structure to insure proper alignment; e.g.
 828  *
 829  *      struct foo {
 830  *              struct m_tag    tag;
 831  *              ...
 832  *      };
 833  *      struct foo *p = (struct foo *) m_tag_allocate(...);
 834  *      struct m_tag *mtag = &p->tag;
 835  */
 836
 837 #define KERNEL_MODULE_TAG_ID    0
 838
 839 enum {
 840         KERNEL_TAG_TYPE_NONE                    = 0,
 841         KERNEL_TAG_TYPE_DUMMYNET                = 1,
 842         KERNEL_TAG_TYPE_DIVERT                  = 2,
 843         KERNEL_TAG_TYPE_IPFORWARD               = 3,
 844         KERNEL_TAG_TYPE_IPFILT                  = 4,
 845         KERNEL_TAG_TYPE_MACLABEL                = 5,
 846         KERNEL_TAG_TYPE_MAC_POLICY_LABEL        = 6,
 847         KERNEL_TAG_TYPE_ENCAP                   = 8,
 848         KERNEL_TAG_TYPE_INET6                   = 9,
 849         KERNEL_TAG_TYPE_IPSEC                   = 10,
 850         KERNEL_TAG_TYPE_PF                      = 11
 851 };
 852
 853 /* Packet tag routines */
 854 __private_extern__ struct  m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
 855 __private_extern__ struct  m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
 856         struct mbuf *);
 857 __private_extern__ void m_tag_free(struct m_tag *);
 858 __private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
 859 __private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
 860 __private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
 861 __private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *);
 862 __private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
 863     u_int16_t, struct m_tag *);
 864 __private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
 865 __private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
 866 __private_extern__ void m_tag_init(struct mbuf *);
 867 __private_extern__ struct  m_tag *m_tag_first(struct mbuf *);
 868 __private_extern__ struct  m_tag *m_tag_next(struct mbuf *, struct m_tag *);
 869
 870 __private_extern__ void m_prio_init(struct mbuf *);
 871
 872 __END_DECLS
 873 #endif /* XNU_KERNEL_PRIVATE */
 874 #ifdef KERNEL
 875 #include <sys/kpi_mbuf.h>
 876 #endif /* KERNEL */
 877 #endif  /* !_SYS_MBUF_H_ */