[apple/xnu.git] / bsd / net / pf_pbuf.c

/*
 * Copyright (c) 2016-2018 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/cdefs.h>
#include <sys/systm.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/mcache.h>
#include <kern/kern_types.h>
#include <net/pf_pbuf.h>
#include <net/pfvar.h>
#include <netinet/in.h>

void
pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
{
	VERIFY((m->m_flags & M_PKTHDR) != 0);

	pbuf->pb_type = PBUF_TYPE_MBUF;
	pbuf->pb_mbuf = m;
	pbuf->pb_ifp = ifp;
	pbuf->pb_next = NULL;
	pbuf_sync(pbuf);
}

void
pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
{
	pbuf->pb_type = PBUF_TYPE_MEMORY;
	pbuf->pb_memory = *mp;
	pbuf->pb_ifp = ifp;
	pbuf->pb_next = NULL;
	pbuf_sync(pbuf);
}

void
pbuf_destroy(pbuf_t *pbuf)
{
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		if (pbuf->pb_mbuf) {
			m_freem(pbuf->pb_mbuf);
			pbuf->pb_mbuf = NULL;
		}
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		VERIFY(pbuf->pb_memory.pm_buffer != NULL);
		(void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
		    PBUF_ACTION_DESTROY);
	} else {
		VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
	}

	memset(pbuf, 0, sizeof(*pbuf));
}

void
pbuf_sync(pbuf_t *pbuf)
{
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m = pbuf->pb_mbuf;

		VERIFY(m != NULL);
		VERIFY(m->m_flags & M_PKTHDR);

		pbuf->pb_data = mtod(m, void *);
		pbuf->pb_packet_len = m->m_pkthdr.len;
		pbuf->pb_contig_len = m->m_len;
		pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
		pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
		pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
		pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
		pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
		pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
		pbuf->pb_pftag = m_pftag(m);
		pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
		ASSERT((pbuf->pb_pf_fragtag == NULL) ||
		    (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		struct pbuf_memory *nm = &pbuf->pb_memory;

		VERIFY(nm->pm_buffer != NULL);
		VERIFY(nm->pm_buffer_len != 0);
		VERIFY(nm->pm_len != 0);
		VERIFY(nm->pm_len <= nm->pm_buffer_len);
		VERIFY(nm->pm_offset < nm->pm_len);

		pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
		pbuf->pb_packet_len = nm->pm_len;
		pbuf->pb_contig_len = nm->pm_len;
		pbuf->pb_csum_flags = &nm->pm_csum_flags;
		pbuf->pb_csum_data = &nm->pm_csum_data;
		pbuf->pb_proto = &nm->pm_proto;
		pbuf->pb_flowsrc = &nm->pm_flowsrc;
		pbuf->pb_flowid = &nm->pm_flowid;
		pbuf->pb_flags = &nm->pm_flags;
		pbuf->pb_pftag = &nm->pm_pftag;
		pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}
}

struct mbuf *
pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
{
	struct mbuf *m = NULL;

	pbuf_sync(pbuf);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		m = pbuf->pb_mbuf;
		if (release_ptr) {
			pbuf->pb_mbuf = NULL;
		}
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		boolean_t fragtag = FALSE;

		if (pbuf->pb_packet_len > (u_int)MHLEN) {
			if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
				printf("%s: packet too big for cluster (%u)\n",
				    __func__, pbuf->pb_packet_len);
				return NULL;
			}
			m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
		} else {
			m = m_gethdr(M_DONTWAIT, MT_DATA);
		}
		if (m == NULL) {
			goto done;
		}

		m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
		m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
		m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
		m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
		m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
		m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
		m->m_pkthdr.pkt_flags = *pbuf->pb_flags;

		if (pbuf->pb_pftag != NULL) {
			struct pf_mtag *pftag = m_pftag(m);

			ASSERT(pftag != NULL);
			*pftag = *pbuf->pb_pftag;
			fragtag =
			    ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
		}

		if (fragtag && pbuf->pb_pf_fragtag != NULL) {
			if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
			    M_NOWAIT) == NULL) {
				m_freem(m);
				m = NULL;
				goto done;
			}
		}
	}

done:
	if (release_ptr) {
		pbuf_destroy(pbuf);
	}
	return m;
}

struct mbuf *
pbuf_clone_to_mbuf(pbuf_t *pbuf)
{
	struct mbuf *m = NULL;

	pbuf_sync(pbuf);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		m = pbuf_to_mbuf(pbuf, FALSE);
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}

	return m;
}

void *
pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
{
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m = pbuf->pb_mbuf;

		if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
			return NULL;
		}

		if (pbuf->pb_mbuf == NULL) {
			pbuf_destroy(pbuf);
			return NULL;
		}

		if (m != pbuf->pb_mbuf) {
			pbuf_sync(pbuf);
		}
	} else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}

	return pbuf->pb_data;
}

void *
pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
{
	void *rv = NULL;

	VERIFY(off >= 0);
	VERIFY((u_int)off <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m, *n;

		VERIFY(pbuf->pb_mbuf != NULL);

		m = pbuf->pb_mbuf;

		if (off > 0) {
			/* Split the mbuf chain at the specified boundary */
			if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
				return NULL;
			}
		} else {
			n = m;
		}

		/* Trim old length */
		m_adj(n, olen);

		/* Prepend new length */
		if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
			return NULL;
		}

		rv = mtod(n, void *);

		if (off > 0) {
			/* Merge the two chains */
			int mlen;

			mlen = n->m_pkthdr.len;
			m_cat(m, n);
			m->m_pkthdr.len += mlen;
		} else {
			/* The new mbuf becomes the packet header */
			pbuf->pb_mbuf = n;
		}

		pbuf_sync(pbuf);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		struct pbuf_memory *nm = &pbuf->pb_memory;
		u_int true_offset, move_len;
		int delta_len;
		uint8_t *psrc, *pdst;

		delta_len = nlen - olen;
		VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
		    nm->pm_buffer_len);

		true_offset = (u_int)off + nm->pm_offset;
		rv = &nm->pm_buffer[true_offset];
		psrc = &nm->pm_buffer[true_offset + (u_int)olen];
		pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
		move_len = pbuf->pb_packet_len - ((u_int)off + olen);
		memmove(pdst, psrc, move_len);

		nm->pm_len += delta_len;

		VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);

		pbuf_sync(pbuf);
	} else {
		panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
	}
	return rv;
}

void *
pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
{
	void *rv = NULL;

	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	/*
	 * Note: If this fails, then the pbuf is destroyed. This is a
	 * side-effect of m_pulldown().
	 *
	 * PF expects this behaviour so it's not a real problem.
	 */
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *n;
		int moff;

		n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
		if (n == NULL) {
			/* mbuf is freed by m_pulldown() in this case */
			pbuf->pb_mbuf = NULL;
			pbuf_destroy(pbuf);
			return NULL;
		}

		pbuf_sync(pbuf);

		rv = (void *)(mtod(n, uint8_t *) + moff);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		/*
		 * This always succeeds since memory pbufs are fully contig.
		 */
		rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}

	return rv;
}

void
pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
{
	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		m_copyback(pbuf->pb_mbuf, off, len, src);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		if (len) {
			memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
		}
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}
}

void
pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
{
	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		m_copydata(pbuf->pb_mbuf, off, len, dst);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		if (len) {
			memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
		}
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}
}

uint16_t
pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
{
	uint16_t sum = 0;

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}

	return sum;
}

uint16_t
pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
{
	uint16_t sum = 0;

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
	} else {
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	}

	return sum;
}

mbuf_svc_class_t
pbuf_get_service_class(const pbuf_t *pbuf)
{
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		return m_get_service_class(pbuf->pb_mbuf);
	}

	VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);

	return MBUF_SC_BE;
}
Commit	Line	Data
5ba3f43e	1	/*
d9a64523	2	* Copyright (c) 2016-2018 Apple Inc. All rights reserved.
5ba3f43e A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23
	24	#include <sys/cdefs.h>
	25	#include <sys/systm.h>
	26	#include <sys/param.h>
	27	#include <sys/types.h>
	28	#include <sys/mcache.h>
	29	#include <kern/kern_types.h>
	30	#include <net/pf_pbuf.h>
cb323159	31	#include <net/pfvar.h>
5ba3f43e A	32	#include <netinet/in.h>
	33
	34	void
	35	pbuf_init_mbuf(pbuf_t pbuf, struct mbuf m, struct ifnet *ifp)
	36	{
5ba3f43e A	37	VERIFY((m->m_flags & M_PKTHDR) != 0);
	38
	39	pbuf->pb_type = PBUF_TYPE_MBUF;
	40	pbuf->pb_mbuf = m;
	41	pbuf->pb_ifp = ifp;
	42	pbuf->pb_next = NULL;
	43	pbuf_sync(pbuf);
	44	}
	45
	46	void
	47	pbuf_init_memory(pbuf_t pbuf, const struct pbuf_memory mp, struct ifnet *ifp)
	48	{
5ba3f43e A	49	pbuf->pb_type = PBUF_TYPE_MEMORY;
	50	pbuf->pb_memory = *mp;
	51	pbuf->pb_ifp = ifp;
	52	pbuf->pb_next = NULL;
	53	pbuf_sync(pbuf);
	54	}
	55
	56	void
	57	pbuf_destroy(pbuf_t *pbuf)
	58	{
5ba3f43e A	59	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	60	if (pbuf->pb_mbuf) {
	61	m_freem(pbuf->pb_mbuf);
	62	pbuf->pb_mbuf = NULL;
	63	}
0a7de745	64	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e A	65	VERIFY(pbuf->pb_memory.pm_buffer != NULL);
	66	(void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
	67	PBUF_ACTION_DESTROY);
	68	} else {
	69	VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
	70	}
	71
	72	memset(pbuf, 0, sizeof(*pbuf));
	73	}
	74
	75	void
	76	pbuf_sync(pbuf_t *pbuf)
	77	{
5ba3f43e A	78	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	79	struct mbuf *m = pbuf->pb_mbuf;
	80
	81	VERIFY(m != NULL);
	82	VERIFY(m->m_flags & M_PKTHDR);
	83
	84	pbuf->pb_data = mtod(m, void *);
	85	pbuf->pb_packet_len = m->m_pkthdr.len;
	86	pbuf->pb_contig_len = m->m_len;
	87	pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
	88	pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
	89	pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
	90	pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
	91	pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
	92	pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
	93	pbuf->pb_pftag = m_pftag(m);
cb323159 A	94	pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
	95	ASSERT((pbuf->pb_pf_fragtag == NULL) \|\|
	96	(pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
0a7de745	97	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e A	98	struct pbuf_memory *nm = &pbuf->pb_memory;
	99
	100	VERIFY(nm->pm_buffer != NULL);
	101	VERIFY(nm->pm_buffer_len != 0);
	102	VERIFY(nm->pm_len != 0);
	103	VERIFY(nm->pm_len <= nm->pm_buffer_len);
	104	VERIFY(nm->pm_offset < nm->pm_len);
	105
	106	pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
	107	pbuf->pb_packet_len = nm->pm_len;
	108	pbuf->pb_contig_len = nm->pm_len;
	109	pbuf->pb_csum_flags = &nm->pm_csum_flags;
	110	pbuf->pb_csum_data = &nm->pm_csum_data;
	111	pbuf->pb_proto = &nm->pm_proto;
	112	pbuf->pb_flowsrc = &nm->pm_flowsrc;
	113	pbuf->pb_flowid = &nm->pm_flowid;
	114	pbuf->pb_flags = &nm->pm_flags;
	115	pbuf->pb_pftag = &nm->pm_pftag;
cb323159	116	pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
0a7de745	117	} else {
5ba3f43e	118	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	119	}
5ba3f43e A	120	}
	121
	122	struct mbuf *
	123	pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
	124	{
	125	struct mbuf *m = NULL;
	126
	127	pbuf_sync(pbuf);
	128
	129	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	130	m = pbuf->pb_mbuf;
	131	if (release_ptr) {
	132	pbuf->pb_mbuf = NULL;
5ba3f43e	133	}
0a7de745	134	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
cb323159 A	135	boolean_t fragtag = FALSE;
cb323159 A	136
5ba3f43e A	137	if (pbuf->pb_packet_len > (u_int)MHLEN) {
	138	if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
	139	printf("%s: packet too big for cluster (%u)\n",
	140	__func__, pbuf->pb_packet_len);
0a7de745	141	return NULL;
5ba3f43e A	142	}
	143	m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
	144	} else {
	145	m = m_gethdr(M_DONTWAIT, MT_DATA);
	146	}
0a7de745	147	if (m == NULL) {
cb323159	148	goto done;
0a7de745	149	}
5ba3f43e A	150
	151	m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
	152	m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
	153	m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
	154	m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
	155	m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
	156	m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
	157	m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
	158
	159	if (pbuf->pb_pftag != NULL) {
	160	struct pf_mtag *pftag = m_pftag(m);
	161
cb323159 A	162	ASSERT(pftag != NULL);
	163	pftag = pbuf->pb_pftag;
	164	fragtag =
	165	((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
5ba3f43e A	166	}
5ba3f43e A	167
cb323159 A	168	if (fragtag && pbuf->pb_pf_fragtag != NULL) {
	169	if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
	170	M_NOWAIT) == NULL) {
	171	m_freem(m);
	172	m = NULL;
	173	goto done;
	174	}
0a7de745	175	}
5ba3f43e A	176	}
5ba3f43e A	177
cb323159 A	178	done:
	179	if (release_ptr) {
	180	pbuf_destroy(pbuf);
	181	}
0a7de745	182	return m;
5ba3f43e A	183	}
	184
	185	struct mbuf *
	186	pbuf_clone_to_mbuf(pbuf_t *pbuf)
	187	{
	188	struct mbuf *m = NULL;
	189
	190	pbuf_sync(pbuf);
	191
0a7de745	192	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
5ba3f43e	193	m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
0a7de745	194	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e	195	m = pbuf_to_mbuf(pbuf, FALSE);
0a7de745	196	} else {
5ba3f43e	197	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	198	}
5ba3f43e	199
0a7de745	200	return m;
5ba3f43e A	201	}
	202
	203	void *
	204	pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
	205	{
5ba3f43e A	206	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	207	struct mbuf *m = pbuf->pb_mbuf;
	208
0a7de745 A	209	if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
	210	return NULL;
	211	}
5ba3f43e A	212
	213	if (pbuf->pb_mbuf == NULL) {
	214	pbuf_destroy(pbuf);
0a7de745	215	return NULL;
5ba3f43e A	216	}
5ba3f43e A	217
0a7de745	218	if (m != pbuf->pb_mbuf) {
5ba3f43e	219	pbuf_sync(pbuf);
0a7de745 A	220	}
0a7de745 A	221	} else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
5ba3f43e	222	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	223	}
5ba3f43e	224
0a7de745	225	return pbuf->pb_data;
5ba3f43e A	226	}
	227
	228	void *
	229	pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
	230	{
	231	void *rv = NULL;
	232
	233	VERIFY(off >= 0);
	234	VERIFY((u_int)off <= pbuf->pb_packet_len);
	235
	236	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	237	struct mbuf m, n;
	238
	239	VERIFY(pbuf->pb_mbuf != NULL);
	240
	241	m = pbuf->pb_mbuf;
	242
	243	if (off > 0) {
	244	/* Split the mbuf chain at the specified boundary */
0a7de745 A	245	if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
	246	return NULL;
	247	}
5ba3f43e A	248	} else {
	249	n = m;
	250	}
	251
	252	/* Trim old length */
	253	m_adj(n, olen);
	254
	255	/* Prepend new length */
0a7de745 A	256	if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
	257	return NULL;
	258	}
5ba3f43e A	259
	260	rv = mtod(n, void *);
	261
	262	if (off > 0) {
	263	/* Merge the two chains */
	264	int mlen;
	265
	266	mlen = n->m_pkthdr.len;
	267	m_cat(m, n);
	268	m->m_pkthdr.len += mlen;
	269	} else {
	270	/* The new mbuf becomes the packet header */
	271	pbuf->pb_mbuf = n;
	272	}
	273
	274	pbuf_sync(pbuf);
d9a64523	275	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e A	276	struct pbuf_memory *nm = &pbuf->pb_memory;
	277	u_int true_offset, move_len;
	278	int delta_len;
	279	uint8_t psrc, pdst;
	280
	281	delta_len = nlen - olen;
	282	VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
	283	nm->pm_buffer_len);
	284
	285	true_offset = (u_int)off + nm->pm_offset;
	286	rv = &nm->pm_buffer[true_offset];
	287	psrc = &nm->pm_buffer[true_offset + (u_int)olen];
	288	pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
	289	move_len = pbuf->pb_packet_len - ((u_int)off + olen);
	290	memmove(pdst, psrc, move_len);
	291
	292	nm->pm_len += delta_len;
	293
	294	VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
	295
	296	pbuf_sync(pbuf);
d9a64523	297	} else {
5ba3f43e	298	panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
d9a64523	299	}
0a7de745	300	return rv;
5ba3f43e A	301	}
	302
	303	void *
	304	pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
	305	{
	306	void *rv = NULL;
	307
	308	VERIFY(off >= 0);
	309	VERIFY(len >= 0);
d9a64523	310	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
5ba3f43e A	311
	312	/*
	313	* Note: If this fails, then the pbuf is destroyed. This is a
	314	* side-effect of m_pulldown().
	315	*
	316	* PF expects this behaviour so it's not a real problem.
	317	*/
5ba3f43e A	318	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	319	struct mbuf *n;
	320	int moff;
	321
	322	n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
	323	if (n == NULL) {
	324	/* mbuf is freed by m_pulldown() in this case */
	325	pbuf->pb_mbuf = NULL;
	326	pbuf_destroy(pbuf);
0a7de745	327	return NULL;
5ba3f43e A	328	}
	329
	330	pbuf_sync(pbuf);
	331
	332	rv = (void )(mtod(n, uint8_t ) + moff);
0a7de745	333	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e A	334	/*
	335	* This always succeeds since memory pbufs are fully contig.
	336	*/
	337	rv = (void )(uintptr_t)(((uint8_t )pbuf->pb_data)[off]);
0a7de745	338	} else {
5ba3f43e	339	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	340	}
5ba3f43e	341
0a7de745	342	return rv;
5ba3f43e A	343	}
	344
	345	void
	346	pbuf_copy_back(pbuf_t pbuf, int off, int len, void src)
	347	{
5ba3f43e A	348	VERIFY(off >= 0);
	349	VERIFY(len >= 0);
	350	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
	351
5ba3f43e	352	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
0a7de745	353	m_copyback(pbuf->pb_mbuf, off, len, src);
cb323159	354	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
0a7de745	355	if (len) {
5ba3f43e	356	memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
0a7de745 A	357	}
0a7de745 A	358	} else {
5ba3f43e	359	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	360	}
5ba3f43e A	361	}
	362
	363	void
	364	pbuf_copy_data(pbuf_t pbuf, int off, int len, void dst)
	365	{
5ba3f43e A	366	VERIFY(off >= 0);
	367	VERIFY(len >= 0);
	368	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
	369
5ba3f43e	370	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
0a7de745	371	m_copydata(pbuf->pb_mbuf, off, len, dst);
cb323159	372	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
0a7de745	373	if (len) {
5ba3f43e	374	memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
0a7de745 A	375	}
0a7de745 A	376	} else {
5ba3f43e	377	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	378	}
5ba3f43e A	379	}
	380
	381	uint16_t
	382	pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
	383	{
	384	uint16_t sum = 0;
	385
0a7de745	386	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
5ba3f43e	387	sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
0a7de745	388	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e	389	sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
0a7de745	390	} else {
5ba3f43e	391	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	392	}
5ba3f43e	393
0a7de745	394	return sum;
5ba3f43e A	395	}
	396
	397	uint16_t
	398	pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
	399	{
	400	uint16_t sum = 0;
	401
0a7de745	402	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
5ba3f43e	403	sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
0a7de745	404	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e	405	sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
0a7de745	406	} else {
5ba3f43e	407	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
0a7de745	408	}
5ba3f43e	409
0a7de745	410	return sum;
5ba3f43e A	411	}
	412
	413	mbuf_svc_class_t
	414	pbuf_get_service_class(const pbuf_t *pbuf)
	415	{
0a7de745	416	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
5ba3f43e	417	return m_get_service_class(pbuf->pb_mbuf);
0a7de745	418	}
5ba3f43e A	419
	420	VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
	421
0a7de745	422	return MBUF_SC_BE;
5ba3f43e	423	}