2 * Copyright (c) 1999-2019 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
29 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
31 * Mach Operating System
32 * Copyright (c) 1987 Carnegie-Mellon University
33 * All rights reserved. The CMU software License Agreement specifies
34 * the terms and conditions for use and redistribution.
37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved.
39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California.
40 * All rights reserved.
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 * may be used to endorse or promote products derived from this software
56 * without specific prior written permission.
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94
73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
74 * support for mandatory and extensible security protections. This notice
75 * is included in support of clause 2.2 (b) of the Apple Public License,
82 #include <sys/appleapiopts.h>
83 #include <sys/cdefs.h>
84 #include <sys/_types/_u_int32_t.h> /* u_int32_t */
85 #include <sys/_types/_u_int64_t.h> /* u_int64_t */
86 #include <sys/_types/_u_short.h> /* u_short */
89 #include <sys/kpi_mbuf.h>
92 #ifdef XNU_KERNEL_PRIVATE
94 #include <sys/queue.h>
95 #include <machine/endian.h>
97 * Mbufs are of a single size, MSIZE (machine/param.h), which
98 * includes overhead. An mbuf may add a single "mbuf cluster" of size
99 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has
100 * no additional overhead and is used instead of the internal data area;
101 * this is done when at least MINCLSIZE of data must be stored.
105 * The following _MLEN and _MHLEN macros are private to xnu. Private code
106 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since
107 * the sizes of the structures are dependent upon specific xnu configs.
109 #define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */
110 #define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */
112 #define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT)
113 #define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */
115 #define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT)
116 #define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */
118 #define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT)
119 #define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */
121 #define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT)
122 #define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */
124 #define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT)
125 #define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */
127 #define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT)
128 #define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */
130 #define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT)
131 #define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */
134 * Macros for type conversion
135 * mtod(m,t) - convert mbuf pointer to data pointer of correct type
136 * mtodo(m, o) -- Same as above but with offset 'o' into data.
137 * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX)
139 #define mtod(m, t) ((t)m_mtod(m))
140 #define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o)))
141 #define dtom(x) m_dtom(x)
143 /* header at beginning of each mbuf: */
145 struct mbuf
*mh_next
; /* next buffer in chain */
146 struct mbuf
*mh_nextpkt
; /* next chain in queue/record */
147 caddr_t mh_data
; /* location of data */
148 int32_t mh_len
; /* amount of data in this mbuf */
149 u_int16_t mh_type
; /* type of data in this mbuf */
150 u_int16_t mh_flags
; /* flags; see below */
151 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4)
152 /* This is needed because of how _MLEN is defined and used. Ideally, _MLEN
153 * should be defined using the offsetof(struct mbuf, M_dat), since there is
154 * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler
155 * may (and does in the armv7k case) insert padding between m_hdr and M_dat in
156 * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced
157 * in the definition of mbuf.
159 } __attribute__((aligned(8)));
165 * Packet tag structure (see below for details).
168 u_int64_t m_tag_cookie
; /* Error checking */
170 u_int32_t pad
; /* For structure alignment */
171 #endif /* !__LP64__ */
172 SLIST_ENTRY(m_tag
) m_tag_link
; /* List of packet tags */
173 u_int16_t m_tag_type
; /* Module specific type */
174 u_int16_t m_tag_len
; /* Length of data */
175 u_int32_t m_tag_id
; /* Module ID */
178 #define M_TAG_ALIGN(len) \
179 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag))
181 #define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL
182 #define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL
185 * Packet tag header structure (at the top of mbuf). Pointers are
186 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded.
190 u_int32_t pad
; /* For structure alignment */
191 #endif /* !__LP64__ */
192 u_int64_t refcnt
; /* Number of tags in this mbuf */
196 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX).
198 struct m_drvaux_tag
{
199 u_int32_t da_family
; /* IFNET_FAMILY values */
200 u_int32_t da_subfamily
; /* IFNET_SUBFAMILY values */
201 u_int32_t da_reserved
; /* for future */
202 u_int32_t da_length
; /* length of following data */
205 /* Values for pftag_flags (16-bit wide) */
206 #define PF_TAG_GENERATED 0x1 /* pkt generated by PF */
207 #define PF_TAG_FRAGCACHE 0x2
208 #define PF_TAG_TRANSLATE_LOCALHOST 0x4
210 #define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */
211 #define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */
213 #define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */
214 #define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */
219 u_int16_t pftag_flags
; /* PF_TAG flags */
220 u_int16_t pftag_rtableid
; /* alternate routing table id */
222 u_int16_t pftag_routed
;
224 void *pftag_hdr
; /* saved hdr pos in mbuf, for ECN */
231 struct pf_fragment_tag
{
232 uint32_t ft_id
; /* fragment id */
233 uint16_t ft_hdrlen
; /* header length of reassembled pkt */
234 uint16_t ft_unfragpartlen
; /* length of the per-fragment headers */
235 uint16_t ft_extoff
; /* last extension header offset or 0 */
236 uint16_t ft_maxlen
; /* maximum fragment payload length */
245 u_int32_t segsz
; /* segment size (actual MSS) */
246 u_int32_t start_seq
; /* start seq of this packet */
251 u_int16_t lro_pktlen
; /* max seg size encountered */
252 u_int8_t lro_npkts
; /* # of coalesced TCP pkts */
253 u_int8_t lro_timediff
; /* time spent in LRO */
257 u_int32_t pri
; /* send msg priority */
258 u_int32_t seq
; /* recv msg sequence # */
260 #define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz
261 #define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq
262 #define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid
263 #define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid
264 #define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen
265 #define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts
266 #define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff
267 #define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri
268 #define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq
274 struct mptcp_pktinfo
{
275 u_int64_t mtpi_dsn
; /* MPTCP Data Sequence Number */
276 u_int32_t mtpi_rel_seq
; /* Relative Seq Number */
277 u_int16_t mtpi_length
; /* Length of mapping */
279 #define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn
280 #define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq
281 #define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length
282 #define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum
286 * TCP specific mbuf tag. Note that the current implementation uses
287 * MPTCP metadata strictly between MPTCP and the TCP subflow layers,
288 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means
289 * that TCP messages functionality is currently incompatible with MPTCP.
293 struct tcp_pktinfo tm_tcp
; /* TCP and below */
294 struct mptcp_pktinfo tm_mptcp
; /* MPTCP-TCP only */
301 #define tx_udp_pid proto_mtag.__pr_u.udp._pid
302 #define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid
308 #define tx_rawip_pid proto_mtag.__pr_u.rawip._pid
309 #define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid
312 struct driver_mtag_
{
313 uintptr_t _drv_tx_compl_arg
;
314 uintptr_t _drv_tx_compl_data
;
315 kern_return_t _drv_tx_status
;
316 uint16_t _drv_flowid
;
317 #define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg
318 #define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data
319 #define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status
320 #define drv_flowid builtin_mtag._drv_mtag._drv_flowid
324 * Protocol specific mbuf tag (at most one protocol metadata per mbuf).
326 * Care must be taken to ensure that they are mutually exclusive, e.g.
327 * IPsec policy ID implies no TCP segment offload (which is fine given
328 * that the former is used on the virtual ipsec interface that does
329 * not advertise the TSO capability.)
333 struct tcp_mtag tcp
; /* TCP specific */
334 struct udp_mtag udp
; /* UDP specific */
335 struct rawip_mtag rawip
; /* raw IPv4/IPv6 specific */
340 * NECP specific mbuf tag.
343 u_int32_t necp_policy_id
;
344 u_int32_t necp_skip_policy_id
;
345 u_int32_t necp_route_rule_id
;
346 u_int16_t necp_last_interface_index
;
347 u_int16_t necp_app_id
;
352 struct proto_mtag_ _proto_mtag
; /* built-in protocol-specific tag */
353 struct pf_mtag _pf_mtag
; /* built-in PF tag */
354 struct necp_mtag_ _necp_mtag
; /* built-in NECP tag */
356 struct driver_mtag_ _drv_mtag
;
357 #define necp_mtag builtin_mtag._net_mtag._necp_mtag
358 #define proto_mtag builtin_mtag._net_mtag._proto_mtag
359 #define driver_mtag builtin_mtag._drv_mtag
363 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set.
366 struct ifnet
*rcvif
; /* rcv interface */
367 /* variables for ip and tcp reassembly */
368 void *pkt_hdr
; /* pointer to packet header */
369 int32_t len
; /* total packet length */
370 /* variables for hardware checksum */
371 /* Note: csum_flags is used for hardware checksum and VLAN */
372 u_int32_t csum_flags
; /* flags regarding checksum */
375 u_int16_t val
; /* checksum value */
376 u_int16_t start
; /* checksum start offset */
378 #define csum_rx_val _csum_rx.val
379 #define csum_rx_start _csum_rx.start
381 u_int16_t start
; /* checksum start offset */
382 u_int16_t stuff
; /* checksum stuff offset */
384 #define csum_tx_start _csum_tx.start
385 #define csum_tx_stuff _csum_tx.stuff
387 * Generic data field used by csum routines.
388 * It gets used differently in different contexts.
392 u_int16_t vlan_tag
; /* VLAN tag, host byte order */
394 * Packet classifier info
396 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value
397 * means the packet has been classified by one of the flow sources.
398 * It is also a prerequisite for flow control advisory, which is
399 * enabled by additionally setting PKTF_FLOW_ADV.
401 * The protocol value is a best-effort representation of the payload.
402 * It is opportunistically updated and used only for optimization.
403 * It is not a substitute for parsing the protocol header(s); use it
406 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the
407 * indices of interfaces which own the source and/or destination
408 * addresses of the packet. For the local/loopback case (PKTF_LOOP),
409 * both should be valid, and thus allows for the receiving end to
410 * quickly determine the actual interfaces used by the the addresses;
411 * they may not necessarily be the same or refer to the loopback
412 * interface. Otherwise, in the non-local/loopback case, the indices
413 * are opportunistically set, and because of that only one may be set
414 * (0 means the index has not been determined.) In addition, the
415 * interface address flags are also recorded. This allows us to avoid
416 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally
417 * this would be a superset of {ia,ia6}_flags, but the namespaces are
418 * overlapping at present, so we'll need a new set of values in future
419 * to achieve this. For now, we will just rely on the address family
420 * related code paths examining this mbuf to interpret the flags.
422 u_int8_t pkt_proto
; /* IPPROTO value */
423 u_int8_t pkt_flowsrc
; /* FLOWSRC values */
424 u_int32_t pkt_flowid
; /* flow ID */
425 u_int32_t pkt_flags
; /* PKTF flags (see below) */
426 u_int32_t pkt_svc
; /* MBUF_SVC value */
428 u_int32_t pkt_compl_context
; /* Packet completion context */
432 u_int16_t src
; /* ifindex of src addr i/f */
433 u_int16_t src_flags
; /* src PKT_IFAIFF flags */
434 u_int16_t dst
; /* ifindex of dst addr i/f */
435 u_int16_t dst_flags
; /* dst PKT_IFAIFF flags */
437 #define src_ifindex _pkt_iaif.src
438 #define src_iff _pkt_iaif.src_flags
439 #define dst_ifindex _pkt_iaif.dst
440 #define dst_iff _pkt_iaif.dst_flags
441 u_int64_t pkt_ifainfo
; /* data field used by ifainfo */
443 u_int32_t if_data
; /* bytes in interface queue */
444 u_int32_t sndbuf_data
; /* bytes in socket buffer */
445 } _pkt_bsr
; /* Buffer status report used by cellular interface */
446 #define bufstatus_if _pkt_bsr.if_data
447 #define bufstatus_sndbuf _pkt_bsr.sndbuf_data
449 u_int64_t pkt_timestamp
; /* TX: enqueue time, RX: receive timestamp */
452 * Tags (external and built-in)
454 SLIST_HEAD(packet_tags
, m_tag
) tags
; /* list of external tags */
455 union builtin_mtag builtin_mtag
;
457 * Module private scratch space (32-bit aligned), currently 16-bytes
458 * large. Anything stored here is not guaranteed to survive across
459 * modules. The AQM layer (outbound) uses all 16-bytes for both
460 * packet scheduling and flow advisory information.
464 u_int8_t __mpriv8
[16];
465 u_int16_t __mpriv16
[8];
469 u_int16_t __val16
[2];
473 u_int64_t __mpriv64
[2];
475 } pkt_mpriv
__attribute__((aligned(4)));
476 #define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32
477 #define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32
478 #define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32
479 #define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32
481 u_int32_t redzone
; /* red zone */
482 u_int32_t pkt_compl_callbacks
; /* Packet completion callbacks */
486 * Flow data source type. A data source module is responsible for generating
487 * a unique flow ID and associating it to each data flow as pkt_flowid.
488 * This is required for flow control/advisory, as it allows the output queue
489 * to identify the data source object and inform that it can resume its
490 * transmission (in the event it was flow controlled.)
492 #define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */
493 #define FLOWSRC_IFNET 2 /* flow ID generated by interface */
494 #define FLOWSRC_PF 3 /* flow ID generated by PF */
495 #define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */
498 * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for
499 * multi-layer packet logging. We're usurping the pkt_flowsrc field because
500 * the mbuf packet header ran out of space and pkt_flowsrc is normally
501 * used on output so we assume we can safely overwrite the normal semantic of
503 * This value is meant to be used on incoming packet from a lower level protocol
504 * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT
505 * is set, the following fields are used:
506 * - pkt_proto: the IP protocol ID of the lower level protocol
507 * - pkt_flowid: the identifier of the packet at the lower protocol.
508 * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI.
512 * Packet flags. Unlike m_flags, all packet flags are copied along when
513 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags
514 * (and other classifier info) will be cleared during DLIL input.
516 * Some notes about M_LOOP and PKTF_LOOP:
518 * - M_LOOP flag is overloaded, and its use is discouraged. Historically,
519 * that flag was used by the KAME implementation for allowing certain
520 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this
521 * was originally meant to be set as the packet is looped back to the
522 * system, and in some circumstances temporarily set in ip6_output().
523 * Over time, this flag was used by the pre-output routines to indicate
524 * to the DLIL frameout and output routines, that the packet may be
525 * looped back to the system under the right conditions. In addition,
526 * this is an mbuf flag rather than an mbuf packet header flag.
528 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only
529 * if the packet was looped back to the system. This flag should be
530 * used instead for newer code.
532 #define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */
533 #define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */
534 #define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */
535 #define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */
536 #define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */
537 #define PKTF_PROXY_DST 0x20 /* processed but not locally destined */
538 #define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */
539 #define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */
540 #define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */
541 #define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */
542 #define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */
543 #define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */
544 #define PKTF_MPSO 0x1000 /* MPTCP socket meta data */
545 #define PKTF_LOOP 0x2000 /* loopbacked packet */
546 #define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */
547 #define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */
548 #define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */
549 #define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */
550 #define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */
551 #define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */
552 #define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */
553 #define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */
554 #define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */
555 #define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */
556 #define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */
557 #define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */
558 #define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */
559 #define PKTF_START_SEQ 0x8000000 /* valid start sequence */
560 #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */
561 #define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */
562 #define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */
563 #define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */
565 /* flags related to flow control/advisory and identification */
566 #define PKTF_FLOW_MASK \
567 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)
570 * Description of external storage mapped into mbuf, valid only if M_EXT set.
572 typedef void (*m_ext_free_func_t
)(caddr_t
, u_int
, caddr_t
);
574 caddr_t ext_buf
; /* start of buffer */
575 m_ext_free_func_t ext_free
; /* free routine if not the usual */
576 u_int ext_size
; /* size of buffer, for ext_free */
577 caddr_t ext_arg
; /* additional ext_free argument */
589 /* define m_ext to a type since it gets redefined below */
590 typedef struct m_ext _m_ext_t
;
599 struct pkthdr MH_pkthdr
; /* M_PKTHDR set */
601 struct m_ext MH_ext
; /* M_EXT set */
602 char MH_databuf
[_MHLEN
];
605 char M_databuf
[_MLEN
]; /* !M_PKTHDR, !M_EXT */
609 #define m_next m_hdr.mh_next
610 #define m_len m_hdr.mh_len
611 #define m_data m_hdr.mh_data
612 #define m_type m_hdr.mh_type
613 #define m_flags m_hdr.mh_flags
614 #define m_nextpkt m_hdr.mh_nextpkt
615 #define m_act m_nextpkt
616 #define m_pkthdr M_dat.MH.MH_pkthdr
617 #define m_ext M_dat.MH.MH_dat.MH_ext
618 #define m_pktdat M_dat.MH.MH_dat.MH_databuf
619 #define m_dat M_dat.M_databuf
620 #define m_pktlen(_m) ((_m)->m_pkthdr.len)
621 #define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag)
623 /* mbuf flags (private) */
624 #define M_EXT 0x0001 /* has associated external storage */
625 #define M_PKTHDR 0x0002 /* start of record */
626 #define M_EOR 0x0004 /* end of record */
627 #define M_PROTO1 0x0008 /* protocol-specific */
628 #define M_PROTO2 0x0010 /* protocol-specific */
629 #define M_PROTO3 0x0020 /* protocol-specific */
630 #define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */
631 #define M_PROTO5 0x0080 /* protocol-specific */
633 /* mbuf pkthdr flags, also in m_flags (private) */
634 #define M_BCAST 0x0100 /* send/received as link-level broadcast */
635 #define M_MCAST 0x0200 /* send/received as link-level multicast */
636 #define M_FRAG 0x0400 /* packet is a fragment of a larger packet */
637 #define M_FIRSTFRAG 0x0800 /* packet is first fragment */
638 #define M_LASTFRAG 0x1000 /* packet is last fragment */
639 #define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */
640 #define M_HASFCS 0x4000 /* packet has FCS */
641 #define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */
644 * Flags to purge when crossing layers.
646 #define M_PROTOFLAGS \
647 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5)
649 /* flags copied when copying m_pkthdr */
650 #define M_COPYFLAGS \
651 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \
652 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \
653 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS)
655 /* flags indicating hw checksum support and sw checksum requirements */
656 #define CSUM_IP 0x0001 /* will csum IP */
657 #define CSUM_TCP 0x0002 /* will csum TCP */
658 #define CSUM_UDP 0x0004 /* will csum UDP */
659 #define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */
660 #define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */
661 #define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */
662 #define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */
663 #define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */
665 #define CSUM_IP_CHECKED 0x0100 /* did csum IP */
666 #define CSUM_IP_VALID 0x0200 /* ... the csum is valid */
667 #define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */
668 #define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */
669 #define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */
670 #define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */
672 #define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP)
673 #define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */
674 #define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6)
675 #define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */
677 #define CSUM_TX_FLAGS \
678 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \
679 CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT)
681 #define CSUM_RX_FLAGS \
682 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \
683 CSUM_DATA_VALID | CSUM_PARTIAL)
686 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h>
689 /* VLAN tag present */
690 #define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */
692 /* checksum start adjustment has been done */
693 #define CSUM_ADJUST_DONE 0x00020000
695 /* TCP Segment Offloading requested on this mbuf */
696 #define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */
697 #define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */
699 #define TSO_IPV4_OK(_ifp, _m) \
700 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
701 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
703 #define TSO_IPV4_NOTOK(_ifp, _m) \
704 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \
705 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \
707 #define TSO_IPV6_OK(_ifp, _m) \
708 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
709 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
711 #define TSO_IPV6_NOTOK(_ifp, _m) \
712 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \
713 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \
715 #endif /* XNU_KERNEL_PRIVATE */
718 #define MT_FREE 0 /* should be on free list */
719 #define MT_DATA 1 /* dynamic (data) allocation */
720 #define MT_HEADER 2 /* packet header */
721 #define MT_SOCKET 3 /* socket structure */
722 #define MT_PCB 4 /* protocol control block */
723 #define MT_RTABLE 5 /* routing tables */
724 #define MT_HTABLE 6 /* IMP host tables */
725 #define MT_ATABLE 7 /* address resolution tables */
726 #define MT_SONAME 8 /* socket name */
727 #define MT_SOOPTS 10 /* socket options */
728 #define MT_FTABLE 11 /* fragment reassembly header */
729 #define MT_RIGHTS 12 /* access rights */
730 #define MT_IFADDR 13 /* interface address */
731 #define MT_CONTROL 14 /* extra-data protocol message */
732 #define MT_OOBDATA 15 /* expedited data */
733 #define MT_TAG 16 /* volatile metadata associated to pkts */
734 #define MT_MAX 32 /* enough? */
736 #ifdef XNU_KERNEL_PRIVATE
738 * mbuf allocation/deallocation macros:
740 * MGET(struct mbuf *m, int how, int type)
741 * allocates an mbuf and initializes it to contain internal data.
743 * MGETHDR(struct mbuf *m, int how, int type)
744 * allocates an mbuf and initializes it to contain a packet header
749 #define MCHECK(m) m_mcheck(m)
754 #define MGET(m, how, type) ((m) = m_get((how), (type)))
756 #define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type)))
759 * Mbuf cluster macros.
760 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster.
761 * MCLGET adds such clusters to a normal mbuf;
762 * the flag M_EXT is set upon success.
763 * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
764 * freeing the cluster if the reference count has reached 0.
766 * Normal mbuf clusters are normally treated as character arrays
767 * after allocation, but use the first word of the buffer as a free list
768 * pointer while on the free list.
771 union mcluster
*mcl_next
;
772 char mcl_buf
[MCLBYTES
];
775 #define MCLALLOC(p, how) ((p) = m_mclalloc(how))
777 #define MCLFREE(p) m_mclfree(p)
779 #define MCLGET(m, how) ((m) = m_mclget(m, how))
785 union mbigcluster
*mbc_next
;
786 char mbc_buf
[MBIGCLBYTES
];
793 union m16kcluster
*m16kcl_next
;
794 char m16kcl_buf
[M16KCLBYTES
];
797 #define MCLHASREFERENCE(m) m_mclhasreference(m)
800 * MFREE(struct mbuf *m, struct mbuf *n)
801 * Free a single mbuf and associated external storage.
802 * Place the successor, if any, in n.
805 #define MFREE(m, n) ((n) = m_free(m))
808 * Copy mbuf pkthdr from from to to.
809 * from must have M_PKTHDR set, and to must be empty.
810 * aux pointer will be moved to `to'.
812 #define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from)
814 #define M_COPY_PFTAG(to, from) m_copy_pftag(to, from)
816 #define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from)
819 * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can
820 * be both the local data payload, or an external buffer area, depending on
821 * whether M_EXT is set).
823 #define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m))
826 * These macros are mapped to the appropriate KPIs, so that private code
827 * can be simply recompiled in order to be forward-compatible with future
828 * changes toward the struture sizes.
830 #define MLEN mbuf_get_mlen() /* normal mbuf data len */
831 #define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */
832 #define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */
834 * Return the address of the start of the buffer associated with an mbuf,
835 * handling external storage, packet-header mbufs, and regular data mbufs.
838 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \
839 ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \
843 * Return the size of the buffer associated with an mbuf, handling external
844 * storage, packet-header mbufs, and regular data mbufs.
847 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \
848 ((m)->m_flags & M_PKTHDR) ? MHLEN : \
851 #define M_ALIGN(m, len) m_align(m, len)
852 #define MH_ALIGN(m, len) m_align(m, len)
853 #define MEXT_ALIGN(m, len) m_align(m, len)
856 * Compute the amount of space available before the current start of data in
859 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
860 * of checking writability of the mbuf data area rests solely with the caller.
862 #define M_LEADINGSPACE(m) \
863 (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0)
866 * Compute the amount of space available after the end of data in an mbuf.
868 * The M_WRITABLE() is a temporary, conservative safety measure: the burden
869 * of checking writability of the mbuf data area rests solely with the caller.
871 #define M_TRAILINGSPACE(m) \
873 ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0)
876 * Arrange to prepend space of size plen to mbuf m.
877 * If a new mbuf must be allocated, how specifies whether to wait.
878 * If how is M_DONTWAIT and allocation fails, the original mbuf chain
879 * is freed and m is set to NULL.
881 #define M_PREPEND(m, plen, how, align) \
882 ((m) = m_prepend_2((m), (plen), (how), (align)))
884 /* change mbuf to new type */
885 #define MCHTYPE(m, t) m_mchtype(m, t)
887 /* compatiblity with 4.3 */
888 #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT)
890 #define MBSHIFT 20 /* 1MB */
891 #define MBSIZE (1 << MBSHIFT)
892 #define GBSHIFT 30 /* 1GB */
893 #define GBSIZE (1 << GBSHIFT)
896 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to
897 * "off+len") is located in single mbuf, on contiguous memory region.
898 * The pointer to the region will be returned to pointer variable "val",
901 * M_STRUCT_GET0 does the same, except that it aligns the structure at
902 * very top of mbuf. GET0 is likely to make memory copy than GET.
904 #define M_STRUCT_GET(val, typ, m, off, len) \
909 if ((m)->m_len >= (off) + (len)) { \
910 (val) = (typ)(mtod((m), caddr_t) + (off)); \
912 t = m_pulldown((m), (off), (len), &tmp); \
914 if (t->m_len < tmp + (len)) \
915 panic("m_pulldown malfunction"); \
916 (val) = (typ)(mtod(t, caddr_t) + tmp); \
924 #define M_STRUCT_GET0(val, typ, m, off, len) \
928 if ((off) == 0 && ((m)->m_len >= (len))) { \
929 (val) = (typ)(void *)mtod(m, caddr_t); \
931 t = m_pulldown((m), (off), (len), NULL); \
933 if (t->m_len < (len)) \
934 panic("m_pulldown malfunction"); \
935 (val) = (typ)(void *)mtod(t, caddr_t); \
943 #define MBUF_INPUT_CHECK(m, rcvif) \
945 if (!(m->m_flags & MBUF_PKTHDR) || \
947 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \
948 m->m_type == MT_FREE || \
949 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \
950 panic_plain("Failed mbuf validity check: mbuf %p len %d " \
951 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \
952 m, m->m_len, m->m_type, m->m_flags, \
953 ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \
955 (rcvif->if_flags & 0xffff)); \
960 * Simple mbuf queueing system
962 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using
963 * m_nextpkt instead of field.sqe_next).
965 * m_next is ignored, so queueing chains of mbufs is possible
967 #define MBUFQ_HEAD(name) \
969 struct mbuf *mq_first; /* first packet */ \
970 struct mbuf **mq_last; /* addr of last next packet */ \
973 #define MBUFQ_INIT(q) do { \
974 MBUFQ_FIRST(q) = NULL; \
975 (q)->mq_last = &MBUFQ_FIRST(q); \
978 #define MBUFQ_PREPEND(q, m) do { \
979 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \
980 (q)->mq_last = &MBUFQ_NEXT(m); \
981 MBUFQ_FIRST(q) = (m); \
984 #define MBUFQ_ENQUEUE(q, m) do { \
985 MBUFQ_NEXT(m) = NULL; \
986 *(q)->mq_last = (m); \
987 (q)->mq_last = &MBUFQ_NEXT(m); \
990 #define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \
991 MBUFQ_NEXT(n) = NULL; \
992 *(q)->mq_last = (m); \
993 (q)->mq_last = &MBUFQ_NEXT(n); \
996 #define MBUFQ_DEQUEUE(q, m) do { \
997 if (((m) = MBUFQ_FIRST(q)) != NULL) { \
998 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \
999 (q)->mq_last = &MBUFQ_FIRST(q); \
1001 MBUFQ_NEXT(m) = NULL; \
1005 #define MBUFQ_REMOVE(q, m) do { \
1006 if (MBUFQ_FIRST(q) == (m)) { \
1007 MBUFQ_DEQUEUE(q, m); \
1009 struct mbuf *_m = MBUFQ_FIRST(q); \
1010 while (MBUFQ_NEXT(_m) != (m)) \
1011 _m = MBUFQ_NEXT(_m); \
1012 if ((MBUFQ_NEXT(_m) = \
1013 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \
1014 (q)->mq_last = &MBUFQ_NEXT(_m); \
1018 #define MBUFQ_DRAIN(q) do { \
1019 struct mbuf *__m0; \
1020 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \
1021 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \
1022 MBUFQ_NEXT(__m0) = NULL; \
1025 (q)->mq_last = &MBUFQ_FIRST(q); \
1028 #define MBUFQ_FOREACH(m, q) \
1029 for ((m) = MBUFQ_FIRST(q); \
1031 (m) = MBUFQ_NEXT(m))
1033 #define MBUFQ_FOREACH_SAFE(m, q, tvar) \
1034 for ((m) = MBUFQ_FIRST(q); \
1035 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \
1038 #define MBUFQ_EMPTY(q) ((q)->mq_first == NULL)
1039 #define MBUFQ_FIRST(q) ((q)->mq_first)
1040 #define MBUFQ_NEXT(m) ((m)->m_nextpkt)
1042 * mq_last is initialized to point to mq_first, so check if they're
1043 * equal and return NULL when the list is empty. Otherwise, we need
1044 * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get
1045 * to the base mbuf address to return to caller.
1047 #define MBUFQ_LAST(head) \
1048 (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \
1049 ((struct mbuf *)(void *)((char *)(head)->mq_last - \
1050 __builtin_offsetof(struct mbuf, m_nextpkt))))
1052 #define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t))
1053 #define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t))
1054 #endif /* XNU_KERNEL_PRIVATE */
1057 * Mbuf statistics (legacy).
1060 u_int32_t m_mbufs
; /* mbufs obtained from page pool */
1061 u_int32_t m_clusters
; /* clusters obtained from page pool */
1062 u_int32_t m_spare
; /* spare field */
1063 u_int32_t m_clfree
; /* free clusters */
1064 u_int32_t m_drops
; /* times failed to find space */
1065 u_int32_t m_wait
; /* times waited for space */
1066 u_int32_t m_drain
; /* times drained protocols for space */
1067 u_short m_mtypes
[256]; /* type specific mbuf allocations */
1068 u_int32_t m_mcfail
; /* times m_copym failed */
1069 u_int32_t m_mpfail
; /* times m_pullup failed */
1070 u_int32_t m_msize
; /* length of an mbuf */
1071 u_int32_t m_mclbytes
; /* length of an mbuf cluster */
1072 u_int32_t m_minclsize
; /* min length of data to allocate a cluster */
1073 u_int32_t m_mlen
; /* length of data in an mbuf */
1074 u_int32_t m_mhlen
; /* length of data in a header mbuf */
1075 u_int32_t m_bigclusters
; /* clusters obtained from page pool */
1076 u_int32_t m_bigclfree
; /* free clusters */
1077 u_int32_t m_bigmclbytes
; /* length of an mbuf cluster */
1080 /* Compatibillity with 10.3 */
1082 u_int32_t m_mbufs
; /* mbufs obtained from page pool */
1083 u_int32_t m_clusters
; /* clusters obtained from page pool */
1084 u_int32_t m_spare
; /* spare field */
1085 u_int32_t m_clfree
; /* free clusters */
1086 u_int32_t m_drops
; /* times failed to find space */
1087 u_int32_t m_wait
; /* times waited for space */
1088 u_int32_t m_drain
; /* times drained protocols for space */
1089 u_short m_mtypes
[256]; /* type specific mbuf allocations */
1090 u_int32_t m_mcfail
; /* times m_copym failed */
1091 u_int32_t m_mpfail
; /* times m_pullup failed */
1092 u_int32_t m_msize
; /* length of an mbuf */
1093 u_int32_t m_mclbytes
; /* length of an mbuf cluster */
1094 u_int32_t m_minclsize
; /* min length of data to allocate a cluster */
1095 u_int32_t m_mlen
; /* length of data in an mbuf */
1096 u_int32_t m_mhlen
; /* length of data in a header mbuf */
1100 * mbuf class statistics.
1102 #define MAX_MBUF_CNAME 15
1104 #if defined(XNU_KERNEL_PRIVATE)
1105 /* For backwards compatibility with 32-bit userland process */
1106 struct omb_class_stat
{
1107 char mbcl_cname
[MAX_MBUF_CNAME
+ 1]; /* class name */
1108 u_int32_t mbcl_size
; /* buffer size */
1109 u_int32_t mbcl_total
; /* # of buffers created */
1110 u_int32_t mbcl_active
; /* # of active buffers */
1111 u_int32_t mbcl_infree
; /* # of available buffers */
1112 u_int32_t mbcl_slab_cnt
; /* # of available slabs */
1113 u_int64_t mbcl_alloc_cnt
; /* # of times alloc is called */
1114 u_int64_t mbcl_free_cnt
; /* # of times free is called */
1115 u_int64_t mbcl_notified
; /* # of notified wakeups */
1116 u_int64_t mbcl_purge_cnt
; /* # of purges so far */
1117 u_int64_t mbcl_fail_cnt
; /* # of allocation failures */
1118 u_int32_t mbcl_ctotal
; /* total only for this class */
1119 u_int32_t mbcl_release_cnt
; /* amount of memory returned */
1121 * Cache layer statistics
1123 u_int32_t mbcl_mc_state
; /* cache state (see below) */
1124 u_int32_t mbcl_mc_cached
; /* # of cached buffers */
1125 u_int32_t mbcl_mc_waiter_cnt
; /* # waiters on the cache */
1126 u_int32_t mbcl_mc_wretry_cnt
; /* # of wait retries */
1127 u_int32_t mbcl_mc_nwretry_cnt
; /* # of no-wait retry attempts */
1128 u_int64_t mbcl_reserved
[4]; /* for future use */
1129 } __attribute__((__packed__
));
1130 #endif /* XNU_KERNEL_PRIVATE */
1132 typedef struct mb_class_stat
{
1133 char mbcl_cname
[MAX_MBUF_CNAME
+ 1]; /* class name */
1134 u_int32_t mbcl_size
; /* buffer size */
1135 u_int32_t mbcl_total
; /* # of buffers created */
1136 u_int32_t mbcl_active
; /* # of active buffers */
1137 u_int32_t mbcl_infree
; /* # of available buffers */
1138 u_int32_t mbcl_slab_cnt
; /* # of available slabs */
1139 #if defined(KERNEL) || defined(__LP64__)
1140 u_int32_t mbcl_pad
; /* padding */
1141 #endif /* KERNEL || __LP64__ */
1142 u_int64_t mbcl_alloc_cnt
; /* # of times alloc is called */
1143 u_int64_t mbcl_free_cnt
; /* # of times free is called */
1144 u_int64_t mbcl_notified
; /* # of notified wakeups */
1145 u_int64_t mbcl_purge_cnt
; /* # of purges so far */
1146 u_int64_t mbcl_fail_cnt
; /* # of allocation failures */
1147 u_int32_t mbcl_ctotal
; /* total only for this class */
1148 u_int32_t mbcl_release_cnt
; /* amount of memory returned */
1150 * Cache layer statistics
1152 u_int32_t mbcl_mc_state
; /* cache state (see below) */
1153 u_int32_t mbcl_mc_cached
; /* # of cached buffers */
1154 u_int32_t mbcl_mc_waiter_cnt
; /* # waiters on the cache */
1155 u_int32_t mbcl_mc_wretry_cnt
; /* # of wait retries */
1156 u_int32_t mbcl_mc_nwretry_cnt
; /* # of no-wait retry attempts */
1157 u_int32_t mbcl_peak_reported
; /* last usage peak reported */
1158 u_int32_t mbcl_reserved
[7]; /* for future use */
1161 #define MCS_DISABLED 0 /* cache is permanently disabled */
1162 #define MCS_ONLINE 1 /* cache is online */
1163 #define MCS_PURGING 2 /* cache is being purged */
1164 #define MCS_OFFLINE 3 /* cache is offline (resizing) */
1166 #if defined(XNU_KERNEL_PRIVATE)
1167 /* For backwards compatibility with 32-bit userland process */
1169 u_int32_t mbs_cnt
; /* number of classes */
1170 struct omb_class_stat mbs_class
[1]; /* class array */
1171 } __attribute__((__packed__
));
1172 #endif /* XNU_KERNEL_PRIVATE */
1174 typedef struct mb_stat
{
1175 u_int32_t mbs_cnt
; /* number of classes */
1176 #if defined(KERNEL) || defined(__LP64__)
1177 u_int32_t mbs_pad
; /* padding */
1178 #endif /* KERNEL || __LP64__ */
1179 mb_class_stat_t mbs_class
[1]; /* class array */
1183 #define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */
1185 typedef struct mleak_trace_stat
{
1186 u_int64_t mltr_collisions
;
1187 u_int64_t mltr_hitcount
;
1188 u_int64_t mltr_allocs
;
1189 u_int64_t mltr_depth
;
1190 u_int64_t mltr_addr
[MLEAK_STACK_DEPTH
];
1191 } mleak_trace_stat_t
;
1193 typedef struct mleak_stat
{
1194 u_int32_t ml_isaddr64
; /* 64-bit KVA? */
1195 u_int32_t ml_cnt
; /* number of traces */
1196 mleak_trace_stat_t ml_trace
[1]; /* trace array */
1199 struct mleak_table
{
1200 u_int32_t mleak_capture
; /* sampling capture counter */
1201 u_int32_t mleak_sample_factor
; /* sample factor */
1203 /* Times two active records want to occupy the same spot */
1204 u_int64_t alloc_collisions
;
1205 u_int64_t trace_collisions
;
1207 /* Times new record lands on spot previously occupied by freed alloc */
1208 u_int64_t alloc_overwrites
;
1209 u_int64_t trace_overwrites
;
1211 /* Times a new alloc or trace is put into the hash table */
1212 u_int64_t alloc_recorded
;
1213 u_int64_t trace_recorded
;
1215 /* Total number of outstanding allocs */
1216 u_int64_t outstanding_allocs
;
1218 /* Times mleak_log returned false because couldn't acquire the lock */
1219 u_int64_t total_conflicts
;
1221 #endif /* PRIVATE */
1223 #ifdef KERNEL_PRIVATE
1227 * Exported (private)
1230 extern struct mbstat mbstat
; /* statistics */
1233 #endif /* KERNEL_PRIVATE */
1235 #ifdef XNU_KERNEL_PRIVATE
1239 * Not exported (xnu private)
1242 /* flags to m_get/MGET */
1243 /* Need to include malloc.h to get right options for malloc */
1244 #include <sys/malloc.h>
1248 /* length to m_copy to copy all */
1249 #define M_COPYALL 1000000000
1251 #define M_DONTWAIT M_NOWAIT
1252 #define M_WAIT M_WAITOK
1254 /* modes for m_copym and variants */
1255 #define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */
1256 #define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */
1257 #define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */
1258 #define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */
1259 #define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */
1261 extern void m_freem(struct mbuf
*) __XNU_INTERNAL(m_freem
);
1262 extern u_int64_t
mcl_to_paddr(char *);
1263 extern void m_adj(struct mbuf
*, int);
1264 extern void m_cat(struct mbuf
*, struct mbuf
*);
1265 extern void m_copydata(struct mbuf
*, int, int, void *);
1266 extern struct mbuf
*m_copym(struct mbuf
*, int, int, int);
1267 extern struct mbuf
*m_copym_mode(struct mbuf
*, int, int, int, uint32_t);
1268 extern struct mbuf
*m_get(int, int);
1269 extern struct mbuf
*m_gethdr(int, int);
1270 extern struct mbuf
*m_getpacket(void);
1271 extern struct mbuf
*m_getpackets(int, int, int);
1272 extern struct mbuf
*m_mclget(struct mbuf
*, int);
1273 extern void *m_mtod(struct mbuf
*);
1274 extern struct mbuf
*m_prepend_2(struct mbuf
*, int, int, int);
1275 extern struct mbuf
*m_pullup(struct mbuf
*, int);
1276 extern struct mbuf
*m_split(struct mbuf
*, int, int);
1277 extern void m_mclfree(caddr_t p
);
1280 * On platforms which require strict alignment (currently for anything but
1281 * i386 or x86_64), this macro checks whether the data pointer of an mbuf
1282 * is 32-bit aligned (this is the expected minimum alignment for protocol
1283 * headers), and assert otherwise.
1285 #if defined(__i386__) || defined(__x86_64__)
1286 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m)
1287 #else /* !__i386__ && !__x86_64__ */
1288 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \
1289 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \
1290 if (((_m)->m_flags & M_PKTHDR) && \
1291 (_m)->m_pkthdr.rcvif != NULL) { \
1292 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1293 "32-bit aligned [%s: alignerrs=%lld]\n", \
1294 __func__, (_m), (_m)->m_data, \
1295 if_name((_m)->m_pkthdr.rcvif), \
1296 (_m)->m_pkthdr.rcvif->if_alignerrs); \
1298 panic_plain("\n%s: mbuf %p data ptr %p is not " \
1299 "32-bit aligned\n", \
1300 __func__, (_m), (_m)->m_data); \
1304 #endif /* !__i386__ && !__x86_64__ */
1306 /* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */
1307 #define MBUF_SC_MAX_CLASSES 10
1310 * These conversion macros rely on the corresponding MBUF_SC and
1311 * MBUF_TC values in order to establish the following mapping:
1313 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK
1316 * MBUF_SC_BE ] ==> MBUF_TC_BE
1320 * MBUF_SC_AV ] ==> MBUF_TC_VI
1325 * MBUF_SC_VO ] ==> MBUF_TC_VO
1328 * The values assigned to each service class allows for a fast mapping to
1329 * the corresponding MBUF_TC traffic class values, as well as to retrieve the
1330 * assigned index; therefore care must be taken when comparing against these
1331 * values. Use the corresponding class and index macros to retrieve the
1332 * corresponding portion, and never assume that a higher class corresponds
1333 * to a higher index.
1335 #define MBUF_SCVAL(x) ((x) & 0xffff)
1336 #define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3)
1337 #define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7)
1338 #define MBUF_TC2SCVAL(_tc) ((_tc) << 7)
1339 #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \
1340 ((_sc) == MBUF_SC_BK))
1341 #define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO)
1342 #define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \
1343 (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM)
1345 #define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS)
1346 #define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK)
1347 #define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE)
1348 #define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD)
1349 #define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM)
1350 #define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV)
1351 #define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV)
1352 #define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI)
1353 #define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG)
1354 #define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO)
1355 #define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL)
1357 #define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS)
1358 #define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK)
1359 #define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE)
1360 #define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD)
1361 #define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM)
1362 #define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV)
1363 #define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV)
1364 #define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI)
1365 #define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG)
1366 #define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO)
1367 #define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL)
1369 #define MBUF_VALID_SC(c) \
1370 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \
1371 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \
1372 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \
1373 c == MBUF_SC_VO || c == MBUF_SC_CTL)
1375 #define MBUF_VALID_SCIDX(c) \
1376 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \
1377 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \
1378 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \
1379 c == SCIDX_VO || c == SCIDX_CTL)
1381 #define MBUF_VALID_SCVAL(c) \
1382 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \
1383 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \
1384 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \
1385 c == SCVAL_VO || SCVAL_CTL)
1387 extern unsigned char *mbutl
; /* start VA of mbuf pool */
1388 extern unsigned char *embutl
; /* end VA of mbuf pool */
1389 extern unsigned int nmbclusters
; /* number of mapped clusters */
1390 extern int njcl
; /* # of jumbo clusters */
1391 extern int njclbytes
; /* size of a jumbo cluster */
1392 extern int max_hdr
; /* largest link+protocol header */
1393 extern int max_datalen
; /* MHLEN - max_hdr */
1395 /* Use max_linkhdr instead of _max_linkhdr */
1396 extern int _max_linkhdr
; /* largest link-level header */
1398 /* Use max_protohdr instead of _max_protohdr */
1399 extern int _max_protohdr
; /* largest protocol header */
1401 __private_extern__
unsigned int mbuf_default_ncl(int, u_int64_t
);
1402 __private_extern__
void mbinit(void);
1403 __private_extern__
struct mbuf
*m_clattach(struct mbuf
*, int, caddr_t
,
1404 void (*)(caddr_t
, u_int
, caddr_t
), u_int
, caddr_t
, int, int);
1405 __private_extern__ caddr_t
m_bigalloc(int);
1406 __private_extern__
void m_bigfree(caddr_t
, u_int
, caddr_t
);
1407 __private_extern__
struct mbuf
*m_mbigget(struct mbuf
*, int);
1408 __private_extern__ caddr_t
m_16kalloc(int);
1409 __private_extern__
void m_16kfree(caddr_t
, u_int
, caddr_t
);
1410 __private_extern__
struct mbuf
*m_m16kget(struct mbuf
*, int);
1411 __private_extern__
int m_reinit(struct mbuf
*, int);
1412 __private_extern__
struct mbuf
*m_free(struct mbuf
*) __XNU_INTERNAL(m_free
);
1413 __private_extern__
struct mbuf
*m_getclr(int, int);
1414 __private_extern__
struct mbuf
*m_getptr(struct mbuf
*, int, int *);
1415 __private_extern__
unsigned int m_length(struct mbuf
*);
1416 __private_extern__
unsigned int m_length2(struct mbuf
*, struct mbuf
**);
1417 __private_extern__
unsigned int m_fixhdr(struct mbuf
*);
1418 __private_extern__
struct mbuf
*m_defrag(struct mbuf
*, int);
1419 __private_extern__
struct mbuf
*m_defrag_offset(struct mbuf
*, u_int32_t
, int);
1420 __private_extern__
struct mbuf
*m_prepend(struct mbuf
*, int, int);
1421 __private_extern__
struct mbuf
*m_copyup(struct mbuf
*, int, int);
1422 __private_extern__
struct mbuf
*m_retry(int, int);
1423 __private_extern__
struct mbuf
*m_retryhdr(int, int);
1424 __private_extern__
int m_freem_list(struct mbuf
*);
1425 __private_extern__
int m_append(struct mbuf
*, int, caddr_t
);
1426 __private_extern__
struct mbuf
*m_last(struct mbuf
*);
1427 __private_extern__
struct mbuf
*m_devget(char *, int, int, struct ifnet
*,
1428 void (*)(const void *, void *, size_t));
1429 __private_extern__
struct mbuf
*m_pulldown(struct mbuf
*, int, int, int *);
1431 __private_extern__
struct mbuf
*m_getcl(int, int, int);
1432 __private_extern__ caddr_t
m_mclalloc(int);
1433 __private_extern__
int m_mclhasreference(struct mbuf
*);
1434 __private_extern__
void m_copy_pkthdr(struct mbuf
*, struct mbuf
*);
1435 __private_extern__
void m_copy_pftag(struct mbuf
*, struct mbuf
*);
1436 __private_extern__
void m_copy_classifier(struct mbuf
*, struct mbuf
*);
1438 __private_extern__
struct mbuf
*m_dtom(void *);
1439 __private_extern__
int m_mtocl(void *);
1440 __private_extern__
union mcluster
*m_cltom(int);
1442 __private_extern__
void m_align(struct mbuf
*, int);
1444 __private_extern__
struct mbuf
*m_normalize(struct mbuf
*m
);
1445 __private_extern__
void m_mchtype(struct mbuf
*m
, int t
);
1446 __private_extern__
void m_mcheck(struct mbuf
*);
1448 __private_extern__
void m_copyback(struct mbuf
*, int, int, const void *);
1449 __private_extern__
struct mbuf
*m_copyback_cow(struct mbuf
*, int, int,
1451 __private_extern__
int m_makewritable(struct mbuf
**, int, int, int);
1452 __private_extern__
struct mbuf
*m_dup(struct mbuf
*m
, int how
);
1453 __private_extern__
struct mbuf
*m_copym_with_hdrs(struct mbuf
*, int, int, int,
1454 struct mbuf
**, int *, uint32_t);
1455 __private_extern__
struct mbuf
*m_getpackethdrs(int, int);
1456 __private_extern__
struct mbuf
*m_getpacket_how(int);
1457 __private_extern__
struct mbuf
*m_getpackets_internal(unsigned int *, int,
1459 __private_extern__
struct mbuf
*m_allocpacket_internal(unsigned int *, size_t,
1460 unsigned int *, int, int, size_t);
1462 __private_extern__
int m_ext_set_prop(struct mbuf
*, uint32_t, uint32_t);
1463 __private_extern__
uint32_t m_ext_get_prop(struct mbuf
*);
1464 __private_extern__
int m_ext_paired_is_active(struct mbuf
*);
1465 __private_extern__
void m_ext_paired_activate(struct mbuf
*);
1467 __private_extern__
void mbuf_drain(boolean_t
);
1470 * Packets may have annotations attached by affixing a list of "packet
1471 * tags" to the pkthdr structure. Packet tags are dynamically allocated
1472 * semi-opaque data structures that have a fixed header (struct m_tag)
1473 * that specifies the size of the memory block and an <id,type> pair that
1474 * identifies it. The id identifies the module and the type identifies the
1475 * type of data for that module. The id of zero is reserved for the kernel.
1477 * Note that the packet tag returned by m_tag_allocate has the default
1478 * memory alignment implemented by malloc. To reference private data one
1479 * can use a construct like:
1481 * struct m_tag *mtag = m_tag_allocate(...);
1482 * struct foo *p = (struct foo *)(mtag+1);
1484 * if the alignment of struct m_tag is sufficient for referencing members
1485 * of struct foo. Otherwise it is necessary to embed struct m_tag within
1486 * the private data structure to insure proper alignment; e.g.
1492 * struct foo *p = (struct foo *) m_tag_allocate(...);
1493 * struct m_tag *mtag = &p->tag;
1496 #define KERNEL_MODULE_TAG_ID 0
1499 KERNEL_TAG_TYPE_NONE
= 0,
1500 KERNEL_TAG_TYPE_DUMMYNET
= 1,
1501 KERNEL_TAG_TYPE_DIVERT
= 2,
1502 KERNEL_TAG_TYPE_IPFORWARD
= 3,
1503 KERNEL_TAG_TYPE_IPFILT
= 4,
1504 KERNEL_TAG_TYPE_MACLABEL
= 5,
1505 KERNEL_TAG_TYPE_MAC_POLICY_LABEL
= 6,
1506 KERNEL_TAG_TYPE_ENCAP
= 8,
1507 KERNEL_TAG_TYPE_INET6
= 9,
1508 KERNEL_TAG_TYPE_IPSEC
= 10,
1509 KERNEL_TAG_TYPE_DRVAUX
= 11,
1510 KERNEL_TAG_TYPE_CFIL_UDP
= 13,
1511 KERNEL_TAG_TYPE_PF_REASS
= 14,
1514 /* Packet tag routines */
1515 __private_extern__
struct m_tag
*m_tag_alloc(u_int32_t
, u_int16_t
, int, int);
1516 __private_extern__
struct m_tag
*m_tag_create(u_int32_t
, u_int16_t
, int, int,
1518 __private_extern__
void m_tag_free(struct m_tag
*);
1519 __private_extern__
void m_tag_prepend(struct mbuf
*, struct m_tag
*);
1520 __private_extern__
void m_tag_unlink(struct mbuf
*, struct m_tag
*);
1521 __private_extern__
void m_tag_delete(struct mbuf
*, struct m_tag
*);
1522 __private_extern__
void m_tag_delete_chain(struct mbuf
*, struct m_tag
*);
1523 __private_extern__
struct m_tag
*m_tag_locate(struct mbuf
*, u_int32_t
,
1524 u_int16_t
, struct m_tag
*);
1525 __private_extern__
struct m_tag
*m_tag_copy(struct m_tag
*, int);
1526 __private_extern__
int m_tag_copy_chain(struct mbuf
*, struct mbuf
*, int);
1527 __private_extern__
void m_tag_init(struct mbuf
*, int);
1528 __private_extern__
struct m_tag
*m_tag_first(struct mbuf
*);
1529 __private_extern__
struct m_tag
*m_tag_next(struct mbuf
*, struct m_tag
*);
1531 __private_extern__
void m_scratch_init(struct mbuf
*);
1532 __private_extern__ u_int32_t
m_scratch_get(struct mbuf
*, u_int8_t
**);
1534 __private_extern__
void m_classifier_init(struct mbuf
*, uint32_t);
1536 __private_extern__
int m_set_service_class(struct mbuf
*, mbuf_svc_class_t
);
1537 __private_extern__ mbuf_svc_class_t
m_get_service_class(struct mbuf
*);
1538 __private_extern__ mbuf_svc_class_t
m_service_class_from_idx(u_int32_t
);
1539 __private_extern__ mbuf_svc_class_t
m_service_class_from_val(u_int32_t
);
1540 __private_extern__
int m_set_traffic_class(struct mbuf
*, mbuf_traffic_class_t
);
1541 __private_extern__ mbuf_traffic_class_t
m_get_traffic_class(struct mbuf
*);
1543 #define ADDCARRY(_x) do { \
1544 while (((_x) >> 16) != 0) \
1545 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \
1548 __private_extern__ u_int16_t
m_adj_sum16(struct mbuf
*, u_int32_t
,
1549 u_int32_t
, u_int32_t
, u_int32_t
);
1550 __private_extern__ u_int16_t
m_sum16(struct mbuf
*, u_int32_t
, u_int32_t
);
1552 __private_extern__
void m_set_ext(struct mbuf
*, struct ext_ref
*,
1553 m_ext_free_func_t
, caddr_t
);
1554 __private_extern__
struct ext_ref
*m_get_rfa(struct mbuf
*);
1555 __private_extern__ m_ext_free_func_t
m_get_ext_free(struct mbuf
*);
1556 __private_extern__ caddr_t
m_get_ext_arg(struct mbuf
*);
1558 __private_extern__
void m_do_tx_compl_callback(struct mbuf
*, struct ifnet
*);
1559 __private_extern__ mbuf_tx_compl_func
m_get_tx_compl_callback(u_int32_t
);
1562 #endif /* XNU_KERNEL_PRIVATE */
1563 #endif /* !_SYS_MBUF_H_ */