]> git.saurik.com Git - apple/xnu.git/blame - bsd/sys/mbuf.h
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.101.1.tar.gz
[apple/xnu.git] / bsd / sys / mbuf.h
CommitLineData
1c79356b 1/*
f427ee49 2 * Copyright (c) 1999-2020 Apple Inc. All rights reserved.
5d5c5d0d 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6d2010ae 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
6d2010ae 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
6d2010ae 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
6d2010ae 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
29/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
6d2010ae 30/*
1c79356b
A		 31 * Mach Operating System
32 * Copyright (c) 1987 Carnegie-Mellon University
33 * All rights reserved. The CMU software License Agreement specifies
34 * the terms and conditions for use and redistribution.
35 */
36/*
37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
38 *
39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
40 * All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 * may be used to endorse or promote products derived from this software
56 * without specific prior written permission.
57 *
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 * SUCH DAMAGE.
69 *
70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94
1c79356b 71 */
2d21ac55
A		 72/*
73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
74 * support for mandatory and extensible security protections. This notice
75 * is included in support of clause 2.2 (b) of the Apple Public License,
76 * Version 2.0.
77 */
1c79356b 78
0a7de745
A		 79#ifndef _SYS_MBUF_H_
80#define _SYS_MBUF_H_
1c79356b 81
9bccf70c 82#include <sys/appleapiopts.h>
d9a64523 83#include <sys/cdefs.h>
5ba3f43e
A		 84#include <sys/_types/_u_int32_t.h> /* u_int32_t */
85#include <sys/_types/_u_int64_t.h> /* u_int64_t */
86#include <sys/_types/_u_short.h> /* u_short */
91447636 87
d9a64523
A		 88#ifdef KERNEL
89#include <sys/kpi_mbuf.h>
90#endif
91447636 91
d9a64523 92#ifdef XNU_KERNEL_PRIVATE
1c79356b 93#include <sys/lock.h>
91447636 94#include <sys/queue.h>
39236c6e 95#include <machine/endian.h>
1c79356b
A		 96/*
97 * Mbufs are of a single size, MSIZE (machine/param.h), which
98 * includes overhead. An mbuf may add a single "mbuf cluster" of size
6d2010ae
A		 99 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
100 * no additional overhead and is used instead of the internal data area;
101 * this is done when at least MINCLSIZE of data must be stored.
b0d623f7 102 */
b0d623f7
A		 103
104/*
105 * The following _MLEN and _MHLEN macros are private to xnu. Private code
106 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
107 * the sizes of the structures are dependent upon specific xnu configs.
108 */
0a7de745
A		 109#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */
110#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
1c79356b 111
0a7de745
A		 112#define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT)
113#define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */
1c79356b 114
0a7de745
A		 115#define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT)
116#define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */
6d2010ae 117
0a7de745
A		 118#define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT)
119#define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */
3e170ce0 120
0a7de745
A		 121#define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT)
122#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */
6d2010ae 123
0a7de745
A		 124#define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT)
125#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
1c79356b 126
0a7de745
A		 127#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
128#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
3e170ce0 129
0a7de745
A		 130#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
131#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
3e170ce0 132
1c79356b
A		 133/*
134 * Macros for type conversion
135 * mtod(m,t) - convert mbuf pointer to data pointer of correct type
5ba3f43e 136 * mtodo(m, o) -- Same as above but with offset 'o' into data.
1c79356b 137 * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX)
1c79356b 138 */
0a7de745 139#define mtod(m, t) ((t)m_mtod(m))
5ba3f43e 140#define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o)))
0a7de745 141#define dtom(x) m_dtom(x)
1c79356b
A		 142
143/* header at beginning of each mbuf: */
144struct m_hdr {
0a7de745
A		 145 struct mbuf *mh_next; /* next buffer in chain */
146 struct mbuf *mh_nextpkt; /* next chain in queue/record */
147 caddr_t mh_data; /* location of data */
148 int32_t mh_len; /* amount of data in this mbuf */
149 u_int16_t mh_type; /* type of data in this mbuf */
150 u_int16_t mh_flags; /* flags; see below */
5ba3f43e
A		 151#if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
152/* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
153 * should be defined using the offsetof(struct mbuf, M_dat), since there is
154 * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
155 * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
156 * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
157 * in the definition of mbuf.
158 */
159} __attribute__((aligned(8)));
160#else
1c79356b 161};
5ba3f43e 162#endif
1c79356b 163
91447636
A		 164/*
165 * Packet tag structure (see below for details).
166 */
167struct m_tag {
f427ee49 168 uint64_t m_tag_cookie; /* Error checking */
6d2010ae 169#ifndef __LP64__
f427ee49 170 uint32_t pad; /* For structure alignment */
6d2010ae 171#endif /* !__LP64__ */
0a7de745 172 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */
f427ee49
A		 173 uint16_t m_tag_type; /* Module specific type */
174 uint16_t m_tag_len; /* Length of data */
175 uint32_t m_tag_id; /* Module ID */
6d2010ae
A		 176};
177
0a7de745 178#define M_TAG_ALIGN(len) \
316670eb 179 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
6d2010ae 180
0a7de745
A		 181#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL
182#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL
6d2010ae 183
316670eb
A		 184/*
185 * Packet tag header structure (at the top of mbuf). Pointers are
186 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
187 */
6d2010ae 188struct m_taghdr {
316670eb 189#ifndef __LP64__
0a7de745 190 u_int32_t pad; /* For structure alignment */
316670eb 191#endif /* !__LP64__ */
0a7de745 192 u_int64_t refcnt; /* Number of tags in this mbuf */
91447636
A		 193};
194
39236c6e
A		 195/*
196 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
197 */
198struct m_drvaux_tag {
0a7de745
A		 199 u_int32_t da_family; /* IFNET_FAMILY values */
200 u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */
201 u_int32_t da_reserved; /* for future */
202 u_int32_t da_length; /* length of following data */
39236c6e 203};
316670eb 204
39236c6e 205/* Values for pftag_flags (16-bit wide) */
0a7de745
A		 206#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */
207#define PF_TAG_FRAGCACHE 0x2
208#define PF_TAG_TRANSLATE_LOCALHOST 0x4
39236c6e 209#if PF_ECN
0a7de745
A		 210#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */
211#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */
39236c6e 212#endif /* PF_ECN */
cb323159
A		 213#define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */
214#define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */
39236c6e
A		 215/*
216 * PF mbuf tag
217 */
316670eb 218struct pf_mtag {
0a7de745
A		 219 u_int16_t pftag_flags; /* PF_TAG flags */
220 u_int16_t pftag_rtableid; /* alternate routing table id */
221 u_int16_t pftag_tag;
222 u_int16_t pftag_routed;
39236c6e 223#if PF_ECN
0a7de745 224 void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */
39236c6e
A		 225#endif /* PF_ECN */
226};
227
f427ee49
A		 228/* System reserved PF tags */
229#define PF_TAG_ID_SYSTEM_SERVICE 0xff00
230#define PF_TAG_ID_STACK_DROP 0xff01
231
cb323159
A		 232/*
233 * PF fragment tag
234 */
235struct pf_fragment_tag {
236 uint32_t ft_id; /* fragment id */
237 uint16_t ft_hdrlen; /* header length of reassembled pkt */
238 uint16_t ft_unfragpartlen; /* length of the per-fragment headers */
239 uint16_t ft_extoff; /* last extension header offset or 0 */
240 uint16_t ft_maxlen; /* maximum fragment payload length */
241};
242
39236c6e
A		 243/*
244 * TCP mbuf tag
245 */
246struct tcp_pktinfo {
316670eb
A		 247 union {
248 struct {
f427ee49
A		 249 uint32_t segsz; /* segment size (actual MSS) */
250 uint32_t start_seq; /* start seq of this packet */
d9a64523
A		 251 pid_t pid;
252 pid_t e_pid;
39236c6e
A		 253 } __tx;
254 struct {
f427ee49 255 uint8_t seg_cnt; /* # of coalesced TCP pkts */
39236c6e
A		 256 } __rx;
257 } __offload;
0a7de745
A		 258#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
259#define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
260#define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid
261#define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid
f427ee49 262#define seg_cnt proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.seg_cnt
316670eb
A		 263};
264
39236c6e
A		 265/*
266 * MPTCP mbuf tag
267 */
268struct mptcp_pktinfo {
f427ee49
A		 269 uint64_t mtpi_dsn; /* MPTCP Data Sequence Number */
270 uint32_t mtpi_rel_seq; /* Relative Seq Number */
271 uint16_t mtpi_length; /* Length of mapping */
272 uint16_t mtpi_csum;
0a7de745
A		 273#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
274#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
275#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
276#define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
39236c6e
A		 277};
278
279/*
280 * TCP specific mbuf tag. Note that the current implementation uses
281 * MPTCP metadata strictly between MPTCP and the TCP subflow layers,
282 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means
283 * that TCP messages functionality is currently incompatible with MPTCP.
284 */
316670eb 285struct tcp_mtag {
39236c6e 286 union {
0a7de745
A		 287 struct tcp_pktinfo tm_tcp; /* TCP and below */
288 struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */
39236c6e
A		 289 };
290};
291
d9a64523
A		 292struct udp_mtag {
293 pid_t _pid;
294 pid_t _e_pid;
0a7de745
A		 295#define tx_udp_pid proto_mtag.__pr_u.udp._pid
296#define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid
d9a64523
A		 297};
298
299struct rawip_mtag {
300 pid_t _pid;
301 pid_t _e_pid;
0a7de745
A		 302#define tx_rawip_pid proto_mtag.__pr_u.rawip._pid
303#define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid
d9a64523
A		 304};
305
39037602 306struct driver_mtag_ {
0a7de745
A		 307 uintptr_t _drv_tx_compl_arg;
308 uintptr_t _drv_tx_compl_data;
309 kern_return_t _drv_tx_status;
310 uint16_t _drv_flowid;
311#define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg
312#define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data
313#define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status
314#define drv_flowid builtin_mtag._drv_mtag._drv_flowid
39037602
A		 315};
316
39236c6e
A		 317/*
318 * Protocol specific mbuf tag (at most one protocol metadata per mbuf).
319 *
320 * Care must be taken to ensure that they are mutually exclusive, e.g.
cb323159 321 * IPsec policy ID implies no TCP segment offload (which is fine given
39236c6e
A		 322 * that the former is used on the virtual ipsec interface that does
323 * not advertise the TSO capability.)
324 */
39037602 325struct proto_mtag_ {
39236c6e 326 union {
0a7de745
A		 327 struct tcp_mtag tcp; /* TCP specific */
328 struct udp_mtag udp; /* UDP specific */
329 struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */
39236c6e
A		 330 } __pr_u;
331};
332
fe8ab488
A		 333/*
334 * NECP specific mbuf tag.
335 */
39037602 336struct necp_mtag_ {
0a7de745
A		 337 u_int32_t necp_policy_id;
338 u_int32_t necp_skip_policy_id;
339 u_int32_t necp_route_rule_id;
340 u_int16_t necp_last_interface_index;
341 u_int16_t necp_app_id;
39037602
A		 342};
343
344union builtin_mtag {
345 struct {
0a7de745
A		 346 struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */
347 struct pf_mtag _pf_mtag; /* built-in PF tag */
39037602
A		 348 struct necp_mtag_ _necp_mtag; /* built-in NECP tag */
349 } _net_mtag;
350 struct driver_mtag_ _drv_mtag;
351#define necp_mtag builtin_mtag._net_mtag._necp_mtag
352#define proto_mtag builtin_mtag._net_mtag._proto_mtag
353#define driver_mtag builtin_mtag._drv_mtag
fe8ab488
A		 354};
355
39236c6e
A		 356/*
357 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
358 */
39037602 359struct pkthdr {
0a7de745 360 struct ifnet *rcvif; /* rcv interface */
1c79356b 361 /* variables for ip and tcp reassembly */
0a7de745
A		 362 void *pkt_hdr; /* pointer to packet header */
363 int32_t len; /* total packet length */
6d2010ae
A		 364 /* variables for hardware checksum */
365 /* Note: csum_flags is used for hardware checksum and VLAN */
0a7de745 366 u_int32_t csum_flags; /* flags regarding checksum */
39236c6e
A		 367 union {
368 struct {
0a7de745 369 u_int16_t val; /* checksum value */
39236c6e
A		 370 u_int16_t start; /* checksum start offset */
371 } _csum_rx;
0a7de745
A		 372#define csum_rx_val _csum_rx.val
373#define csum_rx_start _csum_rx.start
39236c6e
A		 374 struct {
375 u_int16_t start; /* checksum start offset */
376 u_int16_t stuff; /* checksum stuff offset */
377 } _csum_tx;
0a7de745
A		 378#define csum_tx_start _csum_tx.start
379#define csum_tx_stuff _csum_tx.stuff
d9a64523
A		 380 /*
381 * Generic data field used by csum routines.
382 * It gets used differently in different contexts.
383 */
384 u_int32_t csum_data;
39236c6e 385 };
0a7de745 386 u_int16_t vlan_tag; /* VLAN tag, host byte order */
39236c6e
A		 387 /*
388 * Packet classifier info
389 *
390 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value
391 * means the packet has been classified by one of the flow sources.
392 * It is also a prerequisite for flow control advisory, which is
393 * enabled by additionally setting PKTF_FLOW_ADV.
394 *
395 * The protocol value is a best-effort representation of the payload.
396 * It is opportunistically updated and used only for optimization.
397 * It is not a substitute for parsing the protocol header(s); use it
398 * only as a hint.
399 *
400 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the
401 * indices of interfaces which own the source and/or destination
402 * addresses of the packet. For the local/loopback case (PKTF_LOOP),
403 * both should be valid, and thus allows for the receiving end to
404 * quickly determine the actual interfaces used by the the addresses;
405 * they may not necessarily be the same or refer to the loopback
406 * interface. Otherwise, in the non-local/loopback case, the indices
407 * are opportunistically set, and because of that only one may be set
408 * (0 means the index has not been determined.) In addition, the
409 * interface address flags are also recorded. This allows us to avoid
410 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally
411 * this would be a superset of {ia,ia6}_flags, but the namespaces are
412 * overlapping at present, so we'll need a new set of values in future
413 * to achieve this. For now, we will just rely on the address family
414 * related code paths examining this mbuf to interpret the flags.
415 */
0a7de745
A		 416 u_int8_t pkt_proto; /* IPPROTO value */
417 u_int8_t pkt_flowsrc; /* FLOWSRC values */
418 u_int32_t pkt_flowid; /* flow ID */
419 u_int32_t pkt_flags; /* PKTF flags (see below) */
420 u_int32_t pkt_svc; /* MBUF_SVC value */
39037602 421
0a7de745 422 u_int32_t pkt_compl_context; /* Packet completion context */
39037602 423
39236c6e
A		 424 union {
425 struct {
0a7de745
A		 426 u_int16_t src; /* ifindex of src addr i/f */
427 u_int16_t src_flags; /* src PKT_IFAIFF flags */
428 u_int16_t dst; /* ifindex of dst addr i/f */
429 u_int16_t dst_flags; /* dst PKT_IFAIFF flags */
39236c6e 430 } _pkt_iaif;
0a7de745
A		 431#define src_ifindex _pkt_iaif.src
432#define src_iff _pkt_iaif.src_flags
433#define dst_ifindex _pkt_iaif.dst
434#define dst_iff _pkt_iaif.dst_flags
435 u_int64_t pkt_ifainfo; /* data field used by ifainfo */
39037602
A		 436 struct {
437 u_int32_t if_data; /* bytes in interface queue */
438 u_int32_t sndbuf_data; /* bytes in socket buffer */
0a7de745
A		 439 } _pkt_bsr; /* Buffer status report used by cellular interface */
440#define bufstatus_if _pkt_bsr.if_data
441#define bufstatus_sndbuf _pkt_bsr.sndbuf_data
39236c6e 442 };
cb323159 443 u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */
3e170ce0 444
39236c6e
A		 445 /*
446 * Tags (external and built-in)
447 */
448 SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */
39037602 449 union builtin_mtag builtin_mtag;
f427ee49
A		 450
451 uint32_t comp_gencnt;
452 uint32_t padding;
39236c6e
A		 453 /*
454 * Module private scratch space (32-bit aligned), currently 16-bytes
39037602 455 * large. Anything stored here is not guaranteed to survive across
5ba3f43e
A		 456 * modules. The AQM layer (outbound) uses all 16-bytes for both
457 * packet scheduling and flow advisory information.
39236c6e
A		 458 */
459 struct {
460 union {
0a7de745
A		 461 u_int8_t __mpriv8[16];
462 u_int16_t __mpriv16[8];
39236c6e
A		 463 struct {
464 union {
0a7de745
A		 465 u_int8_t __val8[4];
466 u_int16_t __val16[2];
467 u_int32_t __val32;
39236c6e 468 } __mpriv32_u;
0a7de745
A		 469 } __mpriv32[4];
470 u_int64_t __mpriv64[2];
39236c6e
A		 471 } __mpriv_u;
472 } pkt_mpriv __attribute__((aligned(4)));
0a7de745
A		 473#define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
474#define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
475#define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
476#define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
5ba3f43e 477
0a7de745
A		 478 u_int32_t redzone; /* red zone */
479 u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */
1c79356b
A		 480};
481
39236c6e
A		 482/*
483 * Flow data source type. A data source module is responsible for generating
484 * a unique flow ID and associating it to each data flow as pkt_flowid.
485 * This is required for flow control/advisory, as it allows the output queue
486 * to identify the data source object and inform that it can resume its
487 * transmission (in the event it was flow controlled.)
488 */
0a7de745
A		 489#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
490#define FLOWSRC_IFNET 2 /* flow ID generated by interface */
491#define FLOWSRC_PF 3 /* flow ID generated by PF */
492#define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */
39236c6e 493
cb323159
A		 494/*
495 * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for
496 * multi-layer packet logging. We're usurping the pkt_flowsrc field because
497 * the mbuf packet header ran out of space and pkt_flowsrc is normally
498 * used on output so we assume we can safely overwrite the normal semantic of
499 * the field.
500 * This value is meant to be used on incoming packet from a lower level protocol
501 * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT
502 * is set, the following fields are used:
503 * - pkt_proto: the IP protocol ID of the lower level protocol
504 * - pkt_flowid: the identifier of the packet at the lower protocol.
505 * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI.
506 */
507
39236c6e
A		 508/*
509 * Packet flags. Unlike m_flags, all packet flags are copied along when
510 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags
511 * (and other classifier info) will be cleared during DLIL input.
512 *
513 * Some notes about M_LOOP and PKTF_LOOP:
514 *
515 * - M_LOOP flag is overloaded, and its use is discouraged. Historically,
516 * that flag was used by the KAME implementation for allowing certain
517 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this
518 * was originally meant to be set as the packet is looped back to the
519 * system, and in some circumstances temporarily set in ip6_output().
520 * Over time, this flag was used by the pre-output routines to indicate
521 * to the DLIL frameout and output routines, that the packet may be
522 * looped back to the system under the right conditions. In addition,
523 * this is an mbuf flag rather than an mbuf packet header flag.
524 *
525 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only
526 * if the packet was looped back to the system. This flag should be
527 * used instead for newer code.
528 */
0a7de745
A		 529#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */
530#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */
531#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */
532#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */
533#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */
534#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */
535#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */
536#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */
537#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */
f427ee49 538#define PKTF_SKIP_PKTAP 0x200 /* pkt has already passed through pktap */
0a7de745
A		 539#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */
540#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */
541#define PKTF_LOOP 0x2000 /* loopbacked packet */
542#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
543#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
544#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
545#define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */
546#define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */
547#define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */
548#define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */
549#define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */
550#define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */
551#define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */
552#define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */
553#define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */
554#define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */
555#define PKTF_START_SEQ 0x8000000 /* valid start sequence */
556#define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */
557#define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */
558#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */
559#define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */
3e170ce0 560
39236c6e 561/* flags related to flow control/advisory and identification */
0a7de745 562#define PKTF_FLOW_MASK \
39236c6e
A		 563 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
564
565/*
566 * Description of external storage mapped into mbuf, valid only if M_EXT set.
567 */
813fb2f6 568typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t);
1c79356b 569struct m_ext {
0a7de745
A		 570 caddr_t ext_buf; /* start of buffer */
571 m_ext_free_func_t ext_free; /* free routine if not the usual */
572 u_int ext_size; /* size of buffer, for ext_free */
573 caddr_t ext_arg; /* additional ext_free argument */
2d21ac55 574 struct ext_ref {
39037602
A		 575 struct mbuf *paired;
576 u_int16_t minref;
577 u_int16_t refcnt;
578 u_int16_t prefcnt;
579 u_int16_t flags;
580 u_int32_t priv;
813fb2f6 581 uintptr_t ext_token;
2d21ac55 582 } *ext_refflags;
1c79356b
A		 583};
584
b0d623f7
A		 585/* define m_ext to a type since it gets redefined below */
586typedef struct m_ext _m_ext_t;
587
39236c6e
A		 588/*
589 * The mbuf object
590 */
1c79356b 591struct mbuf {
39037602 592 struct m_hdr m_hdr;
1c79356b
A		 593 union {
594 struct {
0a7de745 595 struct pkthdr MH_pkthdr; /* M_PKTHDR set */
1c79356b 596 union {
0a7de745
A		 597 struct m_ext MH_ext; /* M_EXT set */
598 char MH_databuf[_MHLEN];
1c79356b
A		 599 } MH_dat;
600 } MH;
0a7de745 601 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */
1c79356b
A		 602 } M_dat;
603};
604
0a7de745
A		 605#define m_next m_hdr.mh_next
606#define m_len m_hdr.mh_len
607#define m_data m_hdr.mh_data
608#define m_type m_hdr.mh_type
609#define m_flags m_hdr.mh_flags
610#define m_nextpkt m_hdr.mh_nextpkt
611#define m_act m_nextpkt
612#define m_pkthdr M_dat.MH.MH_pkthdr
613#define m_ext M_dat.MH.MH_dat.MH_ext
614#define m_pktdat M_dat.MH.MH_dat.MH_databuf
615#define m_dat M_dat.M_databuf
616#define m_pktlen(_m) ((_m)->m_pkthdr.len)
617#define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)
2a1bd2d3 618#define m_necptag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._necp_mtag)
1c79356b 619
6d2010ae 620/* mbuf flags (private) */
0a7de745
A		 621#define M_EXT 0x0001 /* has associated external storage */
622#define M_PKTHDR 0x0002 /* start of record */
623#define M_EOR 0x0004 /* end of record */
624#define M_PROTO1 0x0008 /* protocol-specific */
625#define M_PROTO2 0x0010 /* protocol-specific */
626#define M_PROTO3 0x0020 /* protocol-specific */
627#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */
628#define M_PROTO5 0x0080 /* protocol-specific */
1c79356b 629
6d2010ae 630/* mbuf pkthdr flags, also in m_flags (private) */
0a7de745
A		 631#define M_BCAST 0x0100 /* send/received as link-level broadcast */
632#define M_MCAST 0x0200 /* send/received as link-level multicast */
633#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */
634#define M_FIRSTFRAG 0x0800 /* packet is first fragment */
635#define M_LASTFRAG 0x1000 /* packet is last fragment */
636#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */
637#define M_HASFCS 0x4000 /* packet has FCS */
638#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */
6d2010ae
A		 639
640/*
641 * Flags to purge when crossing layers.
642 */
0a7de745 643#define M_PROTOFLAGS \
6d2010ae 644 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
1c79356b
A		 645
646/* flags copied when copying m_pkthdr */
0a7de745
A		 647#define M_COPYFLAGS \
648 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \
649 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \
6d2010ae
A		 650 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
651
316670eb 652/* flags indicating hw checksum support and sw checksum requirements */
0a7de745
A		 653#define CSUM_IP 0x0001 /* will csum IP */
654#define CSUM_TCP 0x0002 /* will csum TCP */
655#define CSUM_UDP 0x0004 /* will csum UDP */
656#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */
657#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */
658#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */
659#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */
660#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */
661
662#define CSUM_IP_CHECKED 0x0100 /* did csum IP */
663#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */
664#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
665#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
666#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
667#define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */
668
669#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
670#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
671#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6)
672#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */
673
674#define CSUM_TX_FLAGS \
675 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
5ba3f43e 676 CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)
39236c6e 677
f427ee49 678#define CSUM_RX_FULL_FLAGS \
0a7de745 679 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
f427ee49
A		 680 CSUM_DATA_VALID)
681
682#define CSUM_RX_FLAGS \
683 (CSUM_RX_FULL_FLAGS | CSUM_PARTIAL)
684
685
39236c6e 686
4a249263
A		 687/*
688 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
689 */
4a249263
A		 690
691/* VLAN tag present */
cb323159
A		 692#define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */
693
694/* checksum start adjustment has been done */
695#define CSUM_ADJUST_DONE 0x00020000
b0d623f7 696
c3c9b80d
A		 697/* VLAN encapsulation present */
698#define CSUM_VLAN_ENCAP_PRESENT 0x00040000 /* mbuf has vlan encapsulation */
699
b0d623f7 700/* TCP Segment Offloading requested on this mbuf */
cb323159
A		 701#define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */
702#define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */
316670eb 703
0a7de745
A		 704#define TSO_IPV4_OK(_ifp, _m) \
705 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
706 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
39236c6e 707
0a7de745
A		 708#define TSO_IPV4_NOTOK(_ifp, _m) \
709 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
710 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
39236c6e 711
0a7de745
A		 712#define TSO_IPV6_OK(_ifp, _m) \
713 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
714 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
39236c6e 715
0a7de745
A		 716#define TSO_IPV6_NOTOK(_ifp, _m) \
717 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
718 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
39236c6e 719
6d2010ae 720#endif /* XNU_KERNEL_PRIVATE */
1c79356b
A		 721
722/* mbuf types */
0a7de745
A		 723#define MT_FREE 0 /* should be on free list */
724#define MT_DATA 1 /* dynamic (data) allocation */
725#define MT_HEADER 2 /* packet header */
726#define MT_SOCKET 3 /* socket structure */
727#define MT_PCB 4 /* protocol control block */
728#define MT_RTABLE 5 /* routing tables */
729#define MT_HTABLE 6 /* IMP host tables */
730#define MT_ATABLE 7 /* address resolution tables */
731#define MT_SONAME 8 /* socket name */
732#define MT_SOOPTS 10 /* socket options */
733#define MT_FTABLE 11 /* fragment reassembly header */
734#define MT_RIGHTS 12 /* access rights */
735#define MT_IFADDR 13 /* interface address */
736#define MT_CONTROL 14 /* extra-data protocol message */
737#define MT_OOBDATA 15 /* expedited data */
738#define MT_TAG 16 /* volatile metadata associated to pkts */
739#define MT_MAX 32 /* enough? */
1c79356b 740
6d2010ae 741#ifdef XNU_KERNEL_PRIVATE
1c79356b
A		 742/*
743 * mbuf allocation/deallocation macros:
744 *
745 * MGET(struct mbuf *m, int how, int type)
746 * allocates an mbuf and initializes it to contain internal data.
747 *
748 * MGETHDR(struct mbuf *m, int how, int type)
749 * allocates an mbuf and initializes it to contain a packet header
750 * and internal data.
751 */
752
e3027f41 753#if 1
0a7de745 754#define MCHECK(m) m_mcheck(m)
1c79356b 755#else
0a7de745 756#define MCHECK(m)
1c79356b
A		 757#endif
758
0a7de745 759#define MGET(m, how, type) ((m) = m_get((how), (type)))
9bccf70c 760
0a7de745 761#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
1c79356b
A		 762
763/*
764 * Mbuf cluster macros.
765 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
766 * MCLGET adds such clusters to a normal mbuf;
767 * the flag M_EXT is set upon success.
768 * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
769 * freeing the cluster if the reference count has reached 0.
770 *
771 * Normal mbuf clusters are normally treated as character arrays
772 * after allocation, but use the first word of the buffer as a free list
773 * pointer while on the free list.
774 */
775union mcluster {
0a7de745
A		 776 union mcluster *mcl_next;
777 char mcl_buf[MCLBYTES];
1c79356b
A		 778};
779
0a7de745 780#define MCLALLOC(p, how) ((p) = m_mclalloc(how))
9bccf70c 781
0a7de745 782#define MCLFREE(p) m_mclfree(p)
9bccf70c 783
0a7de745 784#define MCLGET(m, how) ((m) = m_mclget(m, how))
9bccf70c 785
91447636
A		 786/*
787 * Mbuf big cluster
788 */
91447636 789union mbigcluster {
0a7de745
A		 790 union mbigcluster *mbc_next;
791 char mbc_buf[MBIGCLBYTES];
91447636
A		 792};
793
6d2010ae
A		 794/*
795 * Mbuf jumbo cluster
796 */
2d21ac55 797union m16kcluster {
0a7de745
A		 798 union m16kcluster *m16kcl_next;
799 char m16kcl_buf[M16KCLBYTES];
2d21ac55 800};
91447636 801
0a7de745 802#define MCLHASREFERENCE(m) m_mclhasreference(m)
1c79356b
A		 803
804/*
805 * MFREE(struct mbuf *m, struct mbuf *n)
806 * Free a single mbuf and associated external storage.
807 * Place the successor, if any, in n.
808 */
809
0a7de745 810#define MFREE(m, n) ((n) = m_free(m))
1c79356b
A		 811
812/*
813 * Copy mbuf pkthdr from from to to.
814 * from must have M_PKTHDR set, and to must be empty.
815 * aux pointer will be moved to `to'.
816 */
0a7de745 817#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from)
1c79356b 818
0a7de745 819#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from)
316670eb 820
2a1bd2d3
A		 821#define M_COPY_NECPTAG(to, from) m_copy_necptag(to, from)
822
0a7de745 823#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from)
39236c6e 824
1c79356b 825/*
d9a64523
A		 826 * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
827 * be both the local data payload, or an external buffer area, depending on
828 * whether M_EXT is set).
1c79356b 829 */
0a7de745 830#define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m))
6d2010ae 831
1c79356b 832/*
d9a64523
A		 833 * These macros are mapped to the appropriate KPIs, so that private code
834 * can be simply recompiled in order to be forward-compatible with future
835 * changes toward the struture sizes.
836 */
837#define MLEN mbuf_get_mlen() /* normal mbuf data len */
838#define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */
839#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */
840/*
841 * Return the address of the start of the buffer associated with an mbuf,
842 * handling external storage, packet-header mbufs, and regular data mbufs.
1c79356b 843 */
d9a64523 844#define M_START(m) \
0a7de745
A		 845 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \
846 ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \
847 &(m)->m_dat[0])
1c79356b
A		 848
849/*
d9a64523
A		 850 * Return the size of the buffer associated with an mbuf, handling external
851 * storage, packet-header mbufs, and regular data mbufs.
1c79356b 852 */
d9a64523 853#define M_SIZE(m) \
0a7de745
A		 854 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \
855 ((m)->m_flags & M_PKTHDR) ? MHLEN : \
856 MLEN)
d9a64523 857
0a7de745
A		 858#define M_ALIGN(m, len) m_align(m, len)
859#define MH_ALIGN(m, len) m_align(m, len)
860#define MEXT_ALIGN(m, len) m_align(m, len)
d9a64523
A		 861
862/*
863 * Compute the amount of space available before the current start of data in
864 * an mbuf.
865 *
866 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
867 * of checking writability of the mbuf data area rests solely with the caller.
868 */
0a7de745 869#define M_LEADINGSPACE(m) \
d9a64523 870 (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0)
1c79356b
A		 871
872/*
d9a64523
A		 873 * Compute the amount of space available after the end of data in an mbuf.
874 *
875 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
876 * of checking writability of the mbuf data area rests solely with the caller.
1c79356b 877 */
0a7de745
A		 878#define M_TRAILINGSPACE(m) \
879 (M_WRITABLE(m) ? \
d9a64523 880 ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0)
1c79356b
A		 881
882/*
883 * Arrange to prepend space of size plen to mbuf m.
884 * If a new mbuf must be allocated, how specifies whether to wait.
885 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
886 * is freed and m is set to NULL.
887 */
0a7de745 888#define M_PREPEND(m, plen, how, align) \
3e170ce0 889 ((m) = m_prepend_2((m), (plen), (how), (align)))
1c79356b
A		 890
891/* change mbuf to new type */
0a7de745 892#define MCHTYPE(m, t) m_mchtype(m, t)
1c79356b
A		 893
894/* compatiblity with 4.3 */
0a7de745 895#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
1c79356b 896
0a7de745
A		 897#define MBSHIFT 20 /* 1MB */
898#define MBSIZE (1 << MBSHIFT)
899#define GBSHIFT 30 /* 1GB */
900#define GBSIZE (1 << GBSHIFT)
c910b4d9 901
6d2010ae
A		 902/*
903 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
316670eb 904 * "off+len") is located in single mbuf, on contiguous memory region.
6d2010ae
A		 905 * The pointer to the region will be returned to pointer variable "val",
906 * with type "typ".
907 *
908 * M_STRUCT_GET0 does the same, except that it aligns the structure at
909 * very top of mbuf. GET0 is likely to make memory copy than GET.
910 */
0a7de745
A		 911#define M_STRUCT_GET(val, typ, m, off, len) \
912do { \
913 struct mbuf *t; \
914 int tmp; \
915 \
916 if ((m)->m_len >= (off) + (len)) { \
917 (val) = (typ)(mtod((m), caddr_t) + (off)); \
918 } else { \
919 t = m_pulldown((m), (off), (len), &tmp); \
920 if (t != NULL) { \
921 if (t->m_len < tmp + (len)) \
922 panic("m_pulldown malfunction"); \
923 (val) = (typ)(mtod(t, caddr_t) + tmp); \
924 } else { \
925 (val) = (typ)NULL; \
926 (m) = NULL; \
927 } \
928 } \
6d2010ae
A		 929} while (0)
930
0a7de745
A		 931#define M_STRUCT_GET0(val, typ, m, off, len) \
932do { \
933 struct mbuf *t; \
934 \
935 if ((off) == 0 && ((m)->m_len >= (len))) { \
936 (val) = (typ)(void *)mtod(m, caddr_t); \
937 } else { \
938 t = m_pulldown((m), (off), (len), NULL); \
939 if (t != NULL) { \
940 if (t->m_len < (len)) \
941 panic("m_pulldown malfunction"); \
942 (val) = (typ)(void *)mtod(t, caddr_t); \
943 } else { \
944 (val) = (typ)NULL; \
945 (m) = NULL; \
946 } \
947 } \
6d2010ae
A		 948} while (0)
949
0a7de745
A		 950#define MBUF_INPUT_CHECK(m, rcvif) \
951do { \
952 if (!(m->m_flags & MBUF_PKTHDR) || \
953 m->m_len < 0 || \
954 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \
955 m->m_type == MT_FREE || \
956 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \
957 panic_plain("Failed mbuf validity check: mbuf %p len %d " \
958 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \
959 m, m->m_len, m->m_type, m->m_flags, \
960 ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
961 if_name(rcvif), \
962 (rcvif->if_flags & 0xffff)); \
963 } \
6d2010ae
A		 964} while (0)
965
316670eb
A		 966/*
967 * Simple mbuf queueing system
968 *
969 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
970 * m_nextpkt instead of field.sqe_next).
971 *
972 * m_next is ignored, so queueing chains of mbufs is possible
973 */
0a7de745
A		 974#define MBUFQ_HEAD(name) \
975struct name { \
976 struct mbuf *mq_first; /* first packet */ \
977 struct mbuf **mq_last; /* addr of last next packet */ \
316670eb
A		 978}
979
0a7de745
A		 980#define MBUFQ_INIT(q) do { \
981 MBUFQ_FIRST(q) = NULL; \
982 (q)->mq_last = &MBUFQ_FIRST(q); \
316670eb
A		 983} while (0)
984
0a7de745
A		 985#define MBUFQ_PREPEND(q, m) do { \
986 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \
987 (q)->mq_last = &MBUFQ_NEXT(m); \
988 MBUFQ_FIRST(q) = (m); \
316670eb
A		 989} while (0)
990
0a7de745
A		 991#define MBUFQ_ENQUEUE(q, m) do { \
992 MBUFQ_NEXT(m) = NULL; \
993 *(q)->mq_last = (m); \
994 (q)->mq_last = &MBUFQ_NEXT(m); \
316670eb
A		 995} while (0)
996
0a7de745
A		 997#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \
998 MBUFQ_NEXT(n) = NULL; \
999 *(q)->mq_last = (m); \
1000 (q)->mq_last = &MBUFQ_NEXT(n); \
316670eb
A		 1001} while (0)
1002
0a7de745
A		 1003#define MBUFQ_DEQUEUE(q, m) do { \
1004 if (((m) = MBUFQ_FIRST(q)) != NULL) { \
1005 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \
1006 (q)->mq_last = &MBUFQ_FIRST(q); \
1007 else \
1008 MBUFQ_NEXT(m) = NULL; \
1009 } \
316670eb
A		 1010} while (0)
1011
0a7de745
A		 1012#define MBUFQ_REMOVE(q, m) do { \
1013 if (MBUFQ_FIRST(q) == (m)) { \
1014 MBUFQ_DEQUEUE(q, m); \
1015 } else { \
1016 struct mbuf *_m = MBUFQ_FIRST(q); \
1017 while (MBUFQ_NEXT(_m) != (m)) \
1018 _m = MBUFQ_NEXT(_m); \
1019 if ((MBUFQ_NEXT(_m) = \
1020 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \
1021 (q)->mq_last = &MBUFQ_NEXT(_m); \
1022 } \
316670eb
A		 1023} while (0)
1024
0a7de745
A		 1025#define MBUFQ_DRAIN(q) do { \
1026 struct mbuf *__m0; \
1027 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \
1028 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \
1029 MBUFQ_NEXT(__m0) = NULL; \
1030 m_freem(__m0); \
1031 } \
1032 (q)->mq_last = &MBUFQ_FIRST(q); \
316670eb
A		 1033} while (0)
1034
0a7de745
A		 1035#define MBUFQ_FOREACH(m, q) \
1036 for ((m) = MBUFQ_FIRST(q); \
1037 (m); \
316670eb
A		 1038 (m) = MBUFQ_NEXT(m))
1039
0a7de745
A		 1040#define MBUFQ_FOREACH_SAFE(m, q, tvar) \
1041 for ((m) = MBUFQ_FIRST(q); \
1042 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \
316670eb
A		 1043 (m) = (tvar))
1044
0a7de745
A		 1045#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
1046#define MBUFQ_FIRST(q) ((q)->mq_first)
1047#define MBUFQ_NEXT(m) ((m)->m_nextpkt)
39037602
A		 1048/*
1049 * mq_last is initialized to point to mq_first, so check if they're
1050 * equal and return NULL when the list is empty. Otherwise, we need
1051 * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
1052 * to the base mbuf address to return to caller.
1053 */
0a7de745
A		 1054#define MBUFQ_LAST(head) \
1055 (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \
1056 ((struct mbuf *)(void *)((char *)(head)->mq_last - \
cb323159 1057 __builtin_offsetof(struct mbuf, m_nextpkt))))
316670eb 1058
f427ee49
A		 1059#define max_linkhdr (int)P2ROUNDUP(_max_linkhdr, sizeof (uint32_t))
1060#define max_protohdr (int)P2ROUNDUP(_max_protohdr, sizeof (uint32_t))
6d2010ae 1061#endif /* XNU_KERNEL_PRIVATE */
91447636 1062
1c79356b 1063/*
2d21ac55 1064 * Mbuf statistics (legacy).
1c79356b
A		 1065 */
1066struct mbstat {
0a7de745
A		 1067 u_int32_t m_mbufs; /* mbufs obtained from page pool */
1068 u_int32_t m_clusters; /* clusters obtained from page pool */
1069 u_int32_t m_spare; /* spare field */
1070 u_int32_t m_clfree; /* free clusters */
1071 u_int32_t m_drops; /* times failed to find space */
1072 u_int32_t m_wait; /* times waited for space */
1073 u_int32_t m_drain; /* times drained protocols for space */
1074 u_short m_mtypes[256]; /* type specific mbuf allocations */
1075 u_int32_t m_mcfail; /* times m_copym failed */
1076 u_int32_t m_mpfail; /* times m_pullup failed */
1077 u_int32_t m_msize; /* length of an mbuf */
1078 u_int32_t m_mclbytes; /* length of an mbuf cluster */
1079 u_int32_t m_minclsize; /* min length of data to allocate a cluster */
1080 u_int32_t m_mlen; /* length of data in an mbuf */
1081 u_int32_t m_mhlen; /* length of data in a header mbuf */
1082 u_int32_t m_bigclusters; /* clusters obtained from page pool */
1083 u_int32_t m_bigclfree; /* free clusters */
1084 u_int32_t m_bigmclbytes; /* length of an mbuf cluster */
c3c9b80d 1085 u_int32_t m_forcedefunct; /* times we force defunct'ed an app's sockets */
91447636
A		 1086};
1087
1088/* Compatibillity with 10.3 */
1089struct ombstat {
0a7de745
A		 1090 u_int32_t m_mbufs; /* mbufs obtained from page pool */
1091 u_int32_t m_clusters; /* clusters obtained from page pool */
1092 u_int32_t m_spare; /* spare field */
1093 u_int32_t m_clfree; /* free clusters */
1094 u_int32_t m_drops; /* times failed to find space */
1095 u_int32_t m_wait; /* times waited for space */
1096 u_int32_t m_drain; /* times drained protocols for space */
1097 u_short m_mtypes[256]; /* type specific mbuf allocations */
1098 u_int32_t m_mcfail; /* times m_copym failed */
1099 u_int32_t m_mpfail; /* times m_pullup failed */
1100 u_int32_t m_msize; /* length of an mbuf */
1101 u_int32_t m_mclbytes; /* length of an mbuf cluster */
1102 u_int32_t m_minclsize; /* min length of data to allocate a cluster */
1103 u_int32_t m_mlen; /* length of data in an mbuf */
1104 u_int32_t m_mhlen; /* length of data in a header mbuf */
1c79356b
A		 1105};
1106
1107/*
2d21ac55 1108 * mbuf class statistics.
1c79356b 1109 */
0a7de745 1110#define MAX_MBUF_CNAME 15
2d21ac55 1111
6d2010ae 1112#if defined(XNU_KERNEL_PRIVATE)
b0d623f7
A		 1113/* For backwards compatibility with 32-bit userland process */
1114struct omb_class_stat {
0a7de745
A		 1115 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
1116 u_int32_t mbcl_size; /* buffer size */
1117 u_int32_t mbcl_total; /* # of buffers created */
1118 u_int32_t mbcl_active; /* # of active buffers */
1119 u_int32_t mbcl_infree; /* # of available buffers */
1120 u_int32_t mbcl_slab_cnt; /* # of available slabs */
1121 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
1122 u_int64_t mbcl_free_cnt; /* # of times free is called */
1123 u_int64_t mbcl_notified; /* # of notified wakeups */
1124 u_int64_t mbcl_purge_cnt; /* # of purges so far */
1125 u_int64_t mbcl_fail_cnt; /* # of allocation failures */
1126 u_int32_t mbcl_ctotal; /* total only for this class */
1127 u_int32_t mbcl_release_cnt; /* amount of memory returned */
b0d623f7
A		 1128 /*
1129 * Cache layer statistics
1130 */
0a7de745
A		 1131 u_int32_t mbcl_mc_state; /* cache state (see below) */
1132 u_int32_t mbcl_mc_cached; /* # of cached buffers */
1133 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
1134 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
1135 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
1136 u_int64_t mbcl_reserved[4]; /* for future use */
b0d623f7 1137} __attribute__((__packed__));
6d2010ae 1138#endif /* XNU_KERNEL_PRIVATE */
b0d623f7 1139
2d21ac55 1140typedef struct mb_class_stat {
0a7de745
A		 1141 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
1142 u_int32_t mbcl_size; /* buffer size */
1143 u_int32_t mbcl_total; /* # of buffers created */
1144 u_int32_t mbcl_active; /* # of active buffers */
1145 u_int32_t mbcl_infree; /* # of available buffers */
1146 u_int32_t mbcl_slab_cnt; /* # of available slabs */
b0d623f7 1147#if defined(KERNEL) || defined(__LP64__)
0a7de745 1148 u_int32_t mbcl_pad; /* padding */
b0d623f7 1149#endif /* KERNEL || __LP64__ */
0a7de745
A		 1150 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */
1151 u_int64_t mbcl_free_cnt; /* # of times free is called */
1152 u_int64_t mbcl_notified; /* # of notified wakeups */
1153 u_int64_t mbcl_purge_cnt; /* # of purges so far */
1154 u_int64_t mbcl_fail_cnt; /* # of allocation failures */
1155 u_int32_t mbcl_ctotal; /* total only for this class */
1156 u_int32_t mbcl_release_cnt; /* amount of memory returned */
2d21ac55
A		 1157 /*
1158 * Cache layer statistics
1159 */
0a7de745
A		 1160 u_int32_t mbcl_mc_state; /* cache state (see below) */
1161 u_int32_t mbcl_mc_cached; /* # of cached buffers */
1162 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */
1163 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */
1164 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
1165 u_int32_t mbcl_peak_reported; /* last usage peak reported */
1166 u_int32_t mbcl_reserved[7]; /* for future use */
2d21ac55
A		 1167} mb_class_stat_t;
1168
0a7de745
A		 1169#define MCS_DISABLED 0 /* cache is permanently disabled */
1170#define MCS_ONLINE 1 /* cache is online */
1171#define MCS_PURGING 2 /* cache is being purged */
1172#define MCS_OFFLINE 3 /* cache is offline (resizing) */
2d21ac55 1173
6d2010ae 1174#if defined(XNU_KERNEL_PRIVATE)
b0d623f7
A		 1175/* For backwards compatibility with 32-bit userland process */
1176struct omb_stat {
0a7de745
A		 1177 u_int32_t mbs_cnt; /* number of classes */
1178 struct omb_class_stat mbs_class[1]; /* class array */
b0d623f7 1179} __attribute__((__packed__));
6d2010ae 1180#endif /* XNU_KERNEL_PRIVATE */
b0d623f7 1181
2d21ac55 1182typedef struct mb_stat {
0a7de745 1183 u_int32_t mbs_cnt; /* number of classes */
b0d623f7 1184#if defined(KERNEL) || defined(__LP64__)
0a7de745 1185 u_int32_t mbs_pad; /* padding */
b0d623f7 1186#endif /* KERNEL || __LP64__ */
0a7de745 1187 mb_class_stat_t mbs_class[1]; /* class array */
2d21ac55
A		 1188} mb_stat_t;
1189
6d2010ae 1190#ifdef PRIVATE
0a7de745 1191#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */
6d2010ae
A		 1192
1193typedef struct mleak_trace_stat {
0a7de745
A		 1194 u_int64_t mltr_collisions;
1195 u_int64_t mltr_hitcount;
1196 u_int64_t mltr_allocs;
1197 u_int64_t mltr_depth;
1198 u_int64_t mltr_addr[MLEAK_STACK_DEPTH];
6d2010ae
A		 1199} mleak_trace_stat_t;
1200
1201typedef struct mleak_stat {
0a7de745
A		 1202 u_int32_t ml_isaddr64; /* 64-bit KVA? */
1203 u_int32_t ml_cnt; /* number of traces */
1204 mleak_trace_stat_t ml_trace[1]; /* trace array */
6d2010ae
A		 1205} mleak_stat_t;
1206
1207struct mleak_table {
0a7de745
A		 1208 u_int32_t mleak_capture; /* sampling capture counter */
1209 u_int32_t mleak_sample_factor; /* sample factor */
6d2010ae
A		 1210
1211 /* Times two active records want to occupy the same spot */
1212 u_int64_t alloc_collisions;
1213 u_int64_t trace_collisions;
1214
1215 /* Times new record lands on spot previously occupied by freed alloc */
1216 u_int64_t alloc_overwrites;
1217 u_int64_t trace_overwrites;
1218
1219 /* Times a new alloc or trace is put into the hash table */
1220 u_int64_t alloc_recorded;
1221 u_int64_t trace_recorded;
1222
1223 /* Total number of outstanding allocs */
1224 u_int64_t outstanding_allocs;
1225
1226 /* Times mleak_log returned false because couldn't acquire the lock */
1227 u_int64_t total_conflicts;
1228};
1229#endif /* PRIVATE */
1230
2d21ac55 1231#ifdef KERNEL_PRIVATE
6d2010ae 1232__BEGIN_DECLS
1c79356b 1233
6d2010ae
A		 1234/*
1235 * Exported (private)
1236 */
1237
0a7de745 1238extern struct mbstat mbstat; /* statistics */
6d2010ae
A		 1239
1240__END_DECLS
1241#endif /* KERNEL_PRIVATE */
1c79356b 1242
6d2010ae 1243#ifdef XNU_KERNEL_PRIVATE
91447636 1244__BEGIN_DECLS
6d2010ae
A		 1245
1246/*
1247 * Not exported (xnu private)
1248 */
1249
1250/* flags to m_get/MGET */
1251/* Need to include malloc.h to get right options for malloc */
0a7de745 1252#include <sys/malloc.h>
6d2010ae
A		 1253
1254struct mbuf;
1255
1256/* length to m_copy to copy all */
0a7de745 1257#define M_COPYALL 1000000000
6d2010ae 1258
0a7de745
A		 1259#define M_DONTWAIT M_NOWAIT
1260#define M_WAIT M_WAITOK
6d2010ae 1261
39236c6e 1262/* modes for m_copym and variants */
0a7de745
A		 1263#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
1264#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
1265#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
1266#define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */
1267#define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */
39236c6e 1268
cb323159 1269extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem);
39236c6e 1270extern u_int64_t mcl_to_paddr(char *);
6d2010ae
A		 1271extern void m_adj(struct mbuf *, int);
1272extern void m_cat(struct mbuf *, struct mbuf *);
1273extern void m_copydata(struct mbuf *, int, int, void *);
1274extern struct mbuf *m_copym(struct mbuf *, int, int, int);
39236c6e 1275extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t);
6d2010ae
A		 1276extern struct mbuf *m_get(int, int);
1277extern struct mbuf *m_gethdr(int, int);
1278extern struct mbuf *m_getpacket(void);
1279extern struct mbuf *m_getpackets(int, int, int);
1280extern struct mbuf *m_mclget(struct mbuf *, int);
1281extern void *m_mtod(struct mbuf *);
3e170ce0 1282extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int);
6d2010ae
A		 1283extern struct mbuf *m_pullup(struct mbuf *, int);
1284extern struct mbuf *m_split(struct mbuf *, int, int);
1285extern void m_mclfree(caddr_t p);
2a1bd2d3
A		 1286extern int mbuf_get_class(struct mbuf *m);
1287extern bool mbuf_class_under_pressure(struct mbuf *m);
6d2010ae 1288
316670eb
A		 1289/*
1290 * On platforms which require strict alignment (currently for anything but
1291 * i386 or x86_64), this macro checks whether the data pointer of an mbuf
1292 * is 32-bit aligned (this is the expected minimum alignment for protocol
1293 * headers), and assert otherwise.
1294 */
1295#if defined(__i386__) || defined(__x86_64__)
0a7de745 1296#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
316670eb 1297#else /* !__i386__ && !__x86_64__ */
0a7de745
A		 1298#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \
1299 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \
1300 if (((_m)->m_flags & M_PKTHDR) && \
1301 (_m)->m_pkthdr.rcvif != NULL) { \
1302 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1303 "32-bit aligned [%s: alignerrs=%lld]\n", \
1304 __func__, (_m), (_m)->m_data, \
1305 if_name((_m)->m_pkthdr.rcvif), \
1306 (_m)->m_pkthdr.rcvif->if_alignerrs); \
1307 } else { \
1308 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1309 "32-bit aligned\n", \
1310 __func__, (_m), (_m)->m_data); \
1311 } \
1312 } \
316670eb
A		 1313} while (0)
1314#endif /* !__i386__ && !__x86_64__ */
1315
1316/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
0a7de745 1317#define MBUF_SC_MAX_CLASSES 10
316670eb
A		 1318
1319/*
1320 * These conversion macros rely on the corresponding MBUF_SC and
1321 * MBUF_TC values in order to establish the following mapping:
1322 *
1323 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK
1324 * MBUF_SC_BK ]
1325 *
1326 * MBUF_SC_BE ] ==> MBUF_TC_BE
1327 * MBUF_SC_RD ]
1328 * MBUF_SC_OAM ]
1329 *
1330 * MBUF_SC_AV ] ==> MBUF_TC_VI
1331 * MBUF_SC_RV ]
1332 * MBUF_SC_VI ]
d9a64523 1333 * MBUF_SC_SIG ]
316670eb
A		 1334 *
1335 * MBUF_SC_VO ] ==> MBUF_TC_VO
1336 * MBUF_SC_CTL ]
1337 *
1338 * The values assigned to each service class allows for a fast mapping to
1339 * the corresponding MBUF_TC traffic class values, as well as to retrieve the
1340 * assigned index; therefore care must be taken when comparing against these
1341 * values. Use the corresponding class and index macros to retrieve the
1342 * corresponding portion, and never assume that a higher class corresponds
1343 * to a higher index.
1344 */
0a7de745
A		 1345#define MBUF_SCVAL(x) ((x) & 0xffff)
1346#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3)
1347#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7)
1348#define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
316670eb
A		 1349#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
1350 ((_sc) == MBUF_SC_BK))
0a7de745
A		 1351#define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
1352#define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \
3e170ce0 1353 (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)
316670eb 1354
0a7de745
A		 1355#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
1356#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
1357#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE)
1358#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD)
1359#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM)
1360#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV)
1361#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV)
1362#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI)
1363#define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG)
1364#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO)
1365#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL)
1366
1367#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS)
1368#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK)
1369#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE)
1370#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD)
1371#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM)
1372#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV)
1373#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV)
1374#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI)
1375#define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG)
1376#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO)
1377#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL)
1378
1379#define MBUF_VALID_SC(c) \
1380 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \
1381 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \
1382 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \
d9a64523 1383 c == MBUF_SC_VO || c == MBUF_SC_CTL)
316670eb 1384
0a7de745
A		 1385#define MBUF_VALID_SCIDX(c) \
1386 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \
1387 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \
1388 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \
d9a64523 1389 c == SCIDX_VO || c == SCIDX_CTL)
316670eb 1390
0a7de745
A		 1391#define MBUF_VALID_SCVAL(c) \
1392 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \
1393 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \
1394 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \
d9a64523 1395 c == SCVAL_VO || SCVAL_CTL)
316670eb 1396
0a7de745
A		 1397extern unsigned char *mbutl; /* start VA of mbuf pool */
1398extern unsigned char *embutl; /* end VA of mbuf pool */
1399extern unsigned int nmbclusters; /* number of mapped clusters */
1400extern int njcl; /* # of jumbo clusters */
1401extern int njclbytes; /* size of a jumbo cluster */
1402extern int max_hdr; /* largest link+protocol header */
1403extern int max_datalen; /* MHLEN - max_hdr */
6d2010ae 1404
316670eb 1405/* Use max_linkhdr instead of _max_linkhdr */
0a7de745 1406extern int _max_linkhdr; /* largest link-level header */
316670eb
A		 1407
1408/* Use max_protohdr instead of _max_protohdr */
0a7de745 1409extern int _max_protohdr; /* largest protocol header */
316670eb 1410
f427ee49 1411__private_extern__ unsigned int mbuf_default_ncl(uint64_t);
2d21ac55
A		 1412__private_extern__ void mbinit(void);
1413__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
39037602 1414 void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int, int);
2d21ac55
A		 1415__private_extern__ caddr_t m_bigalloc(int);
1416__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
1417__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
1418__private_extern__ caddr_t m_16kalloc(int);
1419__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
1420__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
39236c6e 1421__private_extern__ int m_reinit(struct mbuf *, int);
cb323159 1422__private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free);
6d2010ae
A		 1423__private_extern__ struct mbuf *m_getclr(int, int);
1424__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *);
1425__private_extern__ unsigned int m_length(struct mbuf *);
316670eb
A		 1426__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **);
1427__private_extern__ unsigned int m_fixhdr(struct mbuf *);
1428__private_extern__ struct mbuf *m_defrag(struct mbuf *, int);
1429__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int);
6d2010ae
A		 1430__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int);
1431__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int);
1432__private_extern__ struct mbuf *m_retry(int, int);
1433__private_extern__ struct mbuf *m_retryhdr(int, int);
1434__private_extern__ int m_freem_list(struct mbuf *);
1435__private_extern__ int m_append(struct mbuf *, int, caddr_t);
1436__private_extern__ struct mbuf *m_last(struct mbuf *);
1437__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *,
1438 void (*)(const void *, void *, size_t));
1439__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *);
1440
1441__private_extern__ struct mbuf *m_getcl(int, int, int);
1442__private_extern__ caddr_t m_mclalloc(int);
1443__private_extern__ int m_mclhasreference(struct mbuf *);
1444__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *);
316670eb 1445__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *);
2a1bd2d3 1446__private_extern__ void m_copy_necptag(struct mbuf *, struct mbuf *);
39236c6e 1447__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *);
6d2010ae
A		 1448
1449__private_extern__ struct mbuf *m_dtom(void *);
1450__private_extern__ int m_mtocl(void *);
1451__private_extern__ union mcluster *m_cltom(int);
1452
d9a64523 1453__private_extern__ void m_align(struct mbuf *, int);
6d2010ae
A		 1454
1455__private_extern__ struct mbuf *m_normalize(struct mbuf *m);
1456__private_extern__ void m_mchtype(struct mbuf *m, int t);
1457__private_extern__ void m_mcheck(struct mbuf *);
1458
1459__private_extern__ void m_copyback(struct mbuf *, int, int, const void *);
1460__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int,
1461 const void *, int);
1462__private_extern__ int m_makewritable(struct mbuf **, int, int, int);
1463__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how);
1464__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int,
39236c6e 1465 struct mbuf **, int *, uint32_t);
6d2010ae
A		 1466__private_extern__ struct mbuf *m_getpackethdrs(int, int);
1467__private_extern__ struct mbuf *m_getpacket_how(int);
1468__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int,
1469 int, int, size_t);
1470__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t,
1471 unsigned int *, int, int, size_t);
9bccf70c 1472
39037602
A		 1473__private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t);
1474__private_extern__ uint32_t m_ext_get_prop(struct mbuf *);
1475__private_extern__ int m_ext_paired_is_active(struct mbuf *);
1476__private_extern__ void m_ext_paired_activate(struct mbuf *);
1477
d9a64523 1478__private_extern__ void mbuf_drain(boolean_t);
fe8ab488 1479
91447636 1480/*
6d2010ae
A		 1481 * Packets may have annotations attached by affixing a list of "packet
1482 * tags" to the pkthdr structure. Packet tags are dynamically allocated
1483 * semi-opaque data structures that have a fixed header (struct m_tag)
1484 * that specifies the size of the memory block and an <id,type> pair that
1485 * identifies it. The id identifies the module and the type identifies the
1486 * type of data for that module. The id of zero is reserved for the kernel.
1487 *
1488 * Note that the packet tag returned by m_tag_allocate has the default
1489 * memory alignment implemented by malloc. To reference private data one
1490 * can use a construct like:
1491 *
1492 * struct m_tag *mtag = m_tag_allocate(...);
1493 * struct foo *p = (struct foo *)(mtag+1);
1494 *
1495 * if the alignment of struct m_tag is sufficient for referencing members
1496 * of struct foo. Otherwise it is necessary to embed struct m_tag within
1497 * the private data structure to insure proper alignment; e.g.
1498 *
1499 * struct foo {
1500 * struct m_tag tag;
1501 * ...
1502 * };
1503 * struct foo *p = (struct foo *) m_tag_allocate(...);
1504 * struct m_tag *mtag = &p->tag;
91447636
A		 1505 */
1506
0a7de745 1507#define KERNEL_MODULE_TAG_ID 0
91447636
A		 1508
1509enum {
0a7de745
A		 1510 KERNEL_TAG_TYPE_NONE = 0,
1511 KERNEL_TAG_TYPE_DUMMYNET = 1,
1512 KERNEL_TAG_TYPE_DIVERT = 2,
1513 KERNEL_TAG_TYPE_IPFORWARD = 3,
1514 KERNEL_TAG_TYPE_IPFILT = 4,
1515 KERNEL_TAG_TYPE_MACLABEL = 5,
1516 KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6,
1517 KERNEL_TAG_TYPE_ENCAP = 8,
1518 KERNEL_TAG_TYPE_INET6 = 9,
1519 KERNEL_TAG_TYPE_IPSEC = 10,
1520 KERNEL_TAG_TYPE_DRVAUX = 11,
1521 KERNEL_TAG_TYPE_CFIL_UDP = 13,
cb323159 1522 KERNEL_TAG_TYPE_PF_REASS = 14,
91447636
A		 1523};
1524
91447636 1525/* Packet tag routines */
6d2010ae
A		 1526__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int);
1527__private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int,
0a7de745 1528 struct mbuf *);
6d2010ae
A		 1529__private_extern__ void m_tag_free(struct m_tag *);
1530__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *);
1531__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *);
1532__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *);
1533__private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *);
1534__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t,
1535 u_int16_t, struct m_tag *);
1536__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int);
1537__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int);
39236c6e 1538__private_extern__ void m_tag_init(struct mbuf *, int);
6d2010ae
A		 1539__private_extern__ struct m_tag *m_tag_first(struct mbuf *);
1540__private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *);
1541
39236c6e
A		 1542__private_extern__ void m_scratch_init(struct mbuf *);
1543__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **);
1544
1545__private_extern__ void m_classifier_init(struct mbuf *, uint32_t);
1546
316670eb
A		 1547__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t);
1548__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *);
1549__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t);
1550__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t);
1551__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t);
1552__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *);
1553
0a7de745
A		 1554#define ADDCARRY(_x) do { \
1555 while (((_x) >> 16) != 0) \
1556 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \
39236c6e
A		 1557} while (0)
1558
1559__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t,
5ba3f43e 1560 u_int32_t, u_int32_t, u_int32_t);
39236c6e
A		 1561__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t);
1562
5ba3f43e
A		 1563__private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *,
1564 m_ext_free_func_t, caddr_t);
813fb2f6
A		 1565__private_extern__ struct ext_ref *m_get_rfa(struct mbuf *);
1566__private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *);
1567__private_extern__ caddr_t m_get_ext_arg(struct mbuf *);
1568
d9a64523
A		 1569__private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *);
1570__private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t);
39037602 1571
316670eb
A		 1572__END_DECLS
1573#endif /* XNU_KERNEL_PRIVATE */
0a7de745 1574#endif /* !_SYS_MBUF_H_ */
mirror of XNU