[apple/xnu.git] / bsd / net / pf_pbuf.c

/*
 * Copyright (c) 2016-2018 Apple Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/cdefs.h>
#include <sys/systm.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/mcache.h>
#include <kern/kern_types.h>
#include <net/pf_pbuf.h>
#include <netinet/in.h>

void
pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
{

	VERIFY((m->m_flags & M_PKTHDR) != 0);

	pbuf->pb_type = PBUF_TYPE_MBUF;
	pbuf->pb_mbuf = m;
	pbuf->pb_ifp = ifp;
	pbuf->pb_next = NULL;
	pbuf_sync(pbuf);
}

void
pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
{

	pbuf->pb_type = PBUF_TYPE_MEMORY;
	pbuf->pb_memory = *mp;
	pbuf->pb_ifp = ifp;
	pbuf->pb_next = NULL;
	pbuf_sync(pbuf);
}

void
pbuf_destroy(pbuf_t *pbuf)
{

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		if (pbuf->pb_mbuf) {
			m_freem(pbuf->pb_mbuf);
			pbuf->pb_mbuf = NULL;
		}
	} else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		VERIFY(pbuf->pb_memory.pm_buffer != NULL);
		(void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
		    PBUF_ACTION_DESTROY);
	} else {
		VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
	}

	memset(pbuf, 0, sizeof(*pbuf));
}

void
pbuf_sync(pbuf_t *pbuf)
{

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m = pbuf->pb_mbuf;

		VERIFY(m != NULL);
		VERIFY(m->m_flags & M_PKTHDR);

		pbuf->pb_data = mtod(m, void *);
		pbuf->pb_packet_len = m->m_pkthdr.len;
		pbuf->pb_contig_len = m->m_len;
		pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
		pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
		pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
		pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
		pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
		pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
		pbuf->pb_pftag = m_pftag(m);
	} else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		struct pbuf_memory *nm = &pbuf->pb_memory;

		VERIFY(nm->pm_buffer != NULL);
		VERIFY(nm->pm_buffer_len != 0);
		VERIFY(nm->pm_len != 0);
		VERIFY(nm->pm_len <= nm->pm_buffer_len);
		VERIFY(nm->pm_offset < nm->pm_len);

		pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
		pbuf->pb_packet_len = nm->pm_len;
		pbuf->pb_contig_len = nm->pm_len;
		pbuf->pb_csum_flags = &nm->pm_csum_flags;
		pbuf->pb_csum_data = &nm->pm_csum_data;
		pbuf->pb_proto = &nm->pm_proto;
		pbuf->pb_flowsrc = &nm->pm_flowsrc;
		pbuf->pb_flowid = &nm->pm_flowid;
		pbuf->pb_flags = &nm->pm_flags;
		pbuf->pb_pftag = &nm->pm_pftag;
	} else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
}

struct mbuf *
pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
{
	struct mbuf *m = NULL;

	pbuf_sync(pbuf);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		m = pbuf->pb_mbuf;
		if (release_ptr) {
			pbuf->pb_mbuf = NULL;
			pbuf_destroy(pbuf);
		}
	} else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		if (pbuf->pb_packet_len > (u_int)MHLEN) {
			if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
				printf("%s: packet too big for cluster (%u)\n",
				    __func__, pbuf->pb_packet_len);
				return (NULL);
			}
			m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
		} else {
			m = m_gethdr(M_DONTWAIT, MT_DATA);
		}
		if (m == NULL)
			return (NULL);

		m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
		m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
		m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
		m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
		m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
		m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
		m->m_pkthdr.pkt_flags = *pbuf->pb_flags;

		if (pbuf->pb_pftag != NULL) {
			struct pf_mtag *pftag = m_pftag(m);

			if (pftag != NULL)
				*pftag = *pbuf->pb_pftag;
		}

		if (release_ptr)
			pbuf_destroy(pbuf);
	}

	return (m);
}

struct mbuf *
pbuf_clone_to_mbuf(pbuf_t *pbuf)
{
	struct mbuf *m = NULL;

	pbuf_sync(pbuf);

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
	else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
		m = pbuf_to_mbuf(pbuf, FALSE);
	else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);

	return (m);
}

void *
pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
{

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m = pbuf->pb_mbuf;

		if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT))
			return (NULL);

		if (pbuf->pb_mbuf == NULL) {
			pbuf_destroy(pbuf);
			return (NULL);
		}

		if  (m != pbuf->pb_mbuf)
			pbuf_sync(pbuf);

	} else
	if (pbuf->pb_type != PBUF_TYPE_MEMORY)
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);

	return (pbuf->pb_data);
}

void *
pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
{
	void *rv = NULL;

	VERIFY(off >= 0);
	VERIFY((u_int)off <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *m, *n;

		VERIFY(pbuf->pb_mbuf != NULL);

		m = pbuf->pb_mbuf;

		if (off > 0) {
			/* Split the mbuf chain at the specified boundary */
			if ((n = m_split(m, off, M_DONTWAIT)) == NULL)
				return (NULL);
		} else {
			n = m;
		}

		/* Trim old length */
		m_adj(n, olen);

		/* Prepend new length */
		if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL)
			return (NULL);

		rv = mtod(n, void *);

		if (off > 0) {
			/* Merge the two chains */
			int mlen;

			mlen = n->m_pkthdr.len;
			m_cat(m, n);
			m->m_pkthdr.len += mlen;
		} else {
			/* The new mbuf becomes the packet header */
			pbuf->pb_mbuf = n;
		}

		pbuf_sync(pbuf);
	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		struct pbuf_memory *nm = &pbuf->pb_memory;
		u_int true_offset, move_len;
		int delta_len;
		uint8_t *psrc, *pdst;

		delta_len = nlen - olen;
		VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
		    nm->pm_buffer_len);

		true_offset = (u_int)off + nm->pm_offset;
		rv = &nm->pm_buffer[true_offset];
		psrc = &nm->pm_buffer[true_offset + (u_int)olen];
		pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
		move_len = pbuf->pb_packet_len - ((u_int)off + olen);
		memmove(pdst, psrc, move_len);

		nm->pm_len += delta_len;

		VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);

		pbuf_sync(pbuf);
	} else {
		panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
	}
	return (rv);
}

void *
pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
{
	void *rv = NULL;

	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	/*
	 * Note: If this fails, then the pbuf is destroyed. This is a
	 * side-effect of m_pulldown().
	 *
	 * PF expects this behaviour so it's not a real problem.
	 */
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		struct mbuf *n;
		int moff;

		n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
		if (n == NULL) {
			/* mbuf is freed by m_pulldown() in this case */
			pbuf->pb_mbuf = NULL;
			pbuf_destroy(pbuf);
			return (NULL);
		}

		pbuf_sync(pbuf);

		rv = (void *)(mtod(n, uint8_t *) + moff);
	} else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
		/*
		 * This always succeeds since memory pbufs are fully contig.
		 */
		rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
	} else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);

	return (rv);
}

void
pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
{

	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		m_copyback(pbuf->pb_mbuf, off, len, src);
	else
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		if (len)
			memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
	} else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
}

void
pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
{

	VERIFY(off >= 0);
	VERIFY(len >= 0);
	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		m_copydata(pbuf->pb_mbuf, off, len, dst);
	else
	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
		if (len)
			memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
	} else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
}

uint16_t
pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
{
	uint16_t sum = 0;

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
	else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
		sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
	else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);

	return (sum);
}

uint16_t
pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
{
	uint16_t sum = 0;

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
	else
	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
		sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
	else
		panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);

	return (sum);
}

mbuf_svc_class_t
pbuf_get_service_class(const pbuf_t *pbuf)
{

	if (pbuf->pb_type == PBUF_TYPE_MBUF)
		return m_get_service_class(pbuf->pb_mbuf);

	VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);

	return (MBUF_SC_BE);
}
Commit	Line	Data
5ba3f43e	1	/*
d9a64523	2	* Copyright (c) 2016-2018 Apple Inc. All rights reserved.
5ba3f43e A	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23
	24	#include <sys/cdefs.h>
	25	#include <sys/systm.h>
	26	#include <sys/param.h>
	27	#include <sys/types.h>
	28	#include <sys/mcache.h>
	29	#include <kern/kern_types.h>
	30	#include <net/pf_pbuf.h>
	31	#include <netinet/in.h>
	32
	33	void
	34	pbuf_init_mbuf(pbuf_t pbuf, struct mbuf m, struct ifnet *ifp)
	35	{
	36
	37	VERIFY((m->m_flags & M_PKTHDR) != 0);
	38
	39	pbuf->pb_type = PBUF_TYPE_MBUF;
	40	pbuf->pb_mbuf = m;
	41	pbuf->pb_ifp = ifp;
	42	pbuf->pb_next = NULL;
	43	pbuf_sync(pbuf);
	44	}
	45
	46	void
	47	pbuf_init_memory(pbuf_t pbuf, const struct pbuf_memory mp, struct ifnet *ifp)
	48	{
	49
	50	pbuf->pb_type = PBUF_TYPE_MEMORY;
	51	pbuf->pb_memory = *mp;
	52	pbuf->pb_ifp = ifp;
	53	pbuf->pb_next = NULL;
	54	pbuf_sync(pbuf);
	55	}
	56
	57	void
	58	pbuf_destroy(pbuf_t *pbuf)
	59	{
	60
	61	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	62	if (pbuf->pb_mbuf) {
	63	m_freem(pbuf->pb_mbuf);
	64	pbuf->pb_mbuf = NULL;
	65	}
	66	} else
67	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
68	VERIFY(pbuf->pb_memory.pm_buffer != NULL);
69	(void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
70	PBUF_ACTION_DESTROY);
71	} else {
72	VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
73	}
74
75	memset(pbuf, 0, sizeof(*pbuf));
76	}
77
78	void
79	pbuf_sync(pbuf_t *pbuf)
80	{
81
82	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
83	struct mbuf *m = pbuf->pb_mbuf;
84
85	VERIFY(m != NULL);
86	VERIFY(m->m_flags & M_PKTHDR);
87
88	pbuf->pb_data = mtod(m, void *);
89	pbuf->pb_packet_len = m->m_pkthdr.len;
90	pbuf->pb_contig_len = m->m_len;
91	pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
92	pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
93	pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
94	pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
95	pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
96	pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
97	pbuf->pb_pftag = m_pftag(m);
98	} else
99	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
100	struct pbuf_memory *nm = &pbuf->pb_memory;
101
102	VERIFY(nm->pm_buffer != NULL);
103	VERIFY(nm->pm_buffer_len != 0);
104	VERIFY(nm->pm_len != 0);
105	VERIFY(nm->pm_len <= nm->pm_buffer_len);
106	VERIFY(nm->pm_offset < nm->pm_len);
107
108	pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
109	pbuf->pb_packet_len = nm->pm_len;
110	pbuf->pb_contig_len = nm->pm_len;
111	pbuf->pb_csum_flags = &nm->pm_csum_flags;
112	pbuf->pb_csum_data = &nm->pm_csum_data;
113	pbuf->pb_proto = &nm->pm_proto;
114	pbuf->pb_flowsrc = &nm->pm_flowsrc;
115	pbuf->pb_flowid = &nm->pm_flowid;
116	pbuf->pb_flags = &nm->pm_flags;
117	pbuf->pb_pftag = &nm->pm_pftag;
118	} else
119	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
120	}
121
122	struct mbuf *
123	pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
124	{
125	struct mbuf *m = NULL;
126
127	pbuf_sync(pbuf);
128
129	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
130	m = pbuf->pb_mbuf;
131	if (release_ptr) {
132	pbuf->pb_mbuf = NULL;
133	pbuf_destroy(pbuf);
134	}
135	} else
136	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
137	if (pbuf->pb_packet_len > (u_int)MHLEN) {
138	if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
139	printf("%s: packet too big for cluster (%u)\n",
140	__func__, pbuf->pb_packet_len);
141	return (NULL);
142	}
143	m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
144	} else {
145	m = m_gethdr(M_DONTWAIT, MT_DATA);
146	}
147	if (m == NULL)
148	return (NULL);
149
150	m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
151	m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
152	m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
153	m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
154	m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
155	m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
156	m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
157
158	if (pbuf->pb_pftag != NULL) {
159	struct pf_mtag *pftag = m_pftag(m);
160
161	if (pftag != NULL)
162	pftag = pbuf->pb_pftag;
163	}
164
165	if (release_ptr)
166	pbuf_destroy(pbuf);
167	}
168
169	return (m);
170	}
171
172	struct mbuf *
173	pbuf_clone_to_mbuf(pbuf_t *pbuf)
174	{
175	struct mbuf *m = NULL;
176
177	pbuf_sync(pbuf);
178
179	if (pbuf->pb_type == PBUF_TYPE_MBUF)
180	m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
181	else
182	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
183	m = pbuf_to_mbuf(pbuf, FALSE);
184	else
185	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
186
187	return (m);
188	}
189
190	void *
191	pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
192	{
193
194	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
195	struct mbuf *m = pbuf->pb_mbuf;
196
197	if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT))
198	return (NULL);
199
200	if (pbuf->pb_mbuf == NULL) {
201	pbuf_destroy(pbuf);
202	return (NULL);
203	}
204
205	if (m != pbuf->pb_mbuf)
206	pbuf_sync(pbuf);
207
208	} else
209	if (pbuf->pb_type != PBUF_TYPE_MEMORY)
210	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
211
212	return (pbuf->pb_data);
213	}
214
215	void *
216	pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
217	{
218	void *rv = NULL;
219
220	VERIFY(off >= 0);
221	VERIFY((u_int)off <= pbuf->pb_packet_len);
222
223	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
224	struct mbuf m, n;
225
226	VERIFY(pbuf->pb_mbuf != NULL);
227
228	m = pbuf->pb_mbuf;
229
230	if (off > 0) {
231	/* Split the mbuf chain at the specified boundary */
232	if ((n = m_split(m, off, M_DONTWAIT)) == NULL)
233	return (NULL);
234	} else {
235	n = m;
236	}
237
238	/* Trim old length */
239	m_adj(n, olen);
240
241	/* Prepend new length */
242	if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL)
243	return (NULL);
244
245	rv = mtod(n, void *);
246
247	if (off > 0) {
248	/* Merge the two chains */
249	int mlen;
250
251	mlen = n->m_pkthdr.len;
252	m_cat(m, n);
253	m->m_pkthdr.len += mlen;
254	} else {
255	/* The new mbuf becomes the packet header */
256	pbuf->pb_mbuf = n;
257	}
258
259	pbuf_sync(pbuf);
d9a64523	260	} else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
5ba3f43e A	261	struct pbuf_memory *nm = &pbuf->pb_memory;
	262	u_int true_offset, move_len;
	263	int delta_len;
	264	uint8_t psrc, pdst;
	265
	266	delta_len = nlen - olen;
	267	VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
	268	nm->pm_buffer_len);
	269
	270	true_offset = (u_int)off + nm->pm_offset;
	271	rv = &nm->pm_buffer[true_offset];
	272	psrc = &nm->pm_buffer[true_offset + (u_int)olen];
	273	pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
	274	move_len = pbuf->pb_packet_len - ((u_int)off + olen);
	275	memmove(pdst, psrc, move_len);
	276
	277	nm->pm_len += delta_len;
	278
	279	VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
	280
	281	pbuf_sync(pbuf);
d9a64523	282	} else {
5ba3f43e	283	panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
d9a64523	284	}
5ba3f43e A	285	return (rv);
	286	}
	287
	288	void *
	289	pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
	290	{
	291	void *rv = NULL;
	292
	293	VERIFY(off >= 0);
	294	VERIFY(len >= 0);
d9a64523	295	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
5ba3f43e A	296
	297	/*
	298	* Note: If this fails, then the pbuf is destroyed. This is a
	299	* side-effect of m_pulldown().
	300	*
	301	* PF expects this behaviour so it's not a real problem.
	302	*/
5ba3f43e A	303	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	304	struct mbuf *n;
	305	int moff;
	306
	307	n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
	308	if (n == NULL) {
	309	/* mbuf is freed by m_pulldown() in this case */
	310	pbuf->pb_mbuf = NULL;
	311	pbuf_destroy(pbuf);
	312	return (NULL);
	313	}
	314
	315	pbuf_sync(pbuf);
	316
	317	rv = (void )(mtod(n, uint8_t ) + moff);
	318	} else
	319	if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
	320	/*
	321	* This always succeeds since memory pbufs are fully contig.
	322	*/
	323	rv = (void )(uintptr_t)(((uint8_t )pbuf->pb_data)[off]);
	324	} else
	325	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	326
	327	return (rv);
	328	}
	329
	330	void
	331	pbuf_copy_back(pbuf_t pbuf, int off, int len, void src)
	332	{
	333
	334	VERIFY(off >= 0);
	335	VERIFY(len >= 0);
	336	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
	337
	338	if (pbuf->pb_type == PBUF_TYPE_MBUF)
	339	m_copyback(pbuf->pb_mbuf, off, len, src);
	340	else
	341	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	342	if (len)
	343	memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
	344	} else
	345	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	346	}
	347
	348	void
	349	pbuf_copy_data(pbuf_t pbuf, int off, int len, void dst)
	350	{
	351
	352	VERIFY(off >= 0);
	353	VERIFY(len >= 0);
	354	VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
	355
	356	if (pbuf->pb_type == PBUF_TYPE_MBUF)
	357	m_copydata(pbuf->pb_mbuf, off, len, dst);
	358	else
	359	if (pbuf->pb_type == PBUF_TYPE_MBUF) {
	360	if (len)
	361	memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
	362	} else
	363	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
	364	}
	365
	366	uint16_t
367	pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
368	{
369	uint16_t sum = 0;
370
371	if (pbuf->pb_type == PBUF_TYPE_MBUF)
372	sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
373	else
374	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
375	sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
376	else
377	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
378
379	return (sum);
380	}
381
382	uint16_t
383	pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
384	{
385	uint16_t sum = 0;
386
387	if (pbuf->pb_type == PBUF_TYPE_MBUF)
388	sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
389	else
390	if (pbuf->pb_type == PBUF_TYPE_MEMORY)
391	sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
392	else
393	panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
394
395	return (sum);
396	}
397
398	mbuf_svc_class_t
399	pbuf_get_service_class(const pbuf_t *pbuf)
400	{
401
402	if (pbuf->pb_type == PBUF_TYPE_MBUF)
403	return m_get_service_class(pbuf->pb_mbuf);
404
405	VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
406
407	return (MBUF_SC_BE);
408	}