]>
Commit | Line | Data |
---|---|---|
1c79356b | 1 | /* |
a39ff7e2 | 2 | * Copyright (c) 2000-2018 Apple Inc. All rights reserved. |
5d5c5d0d | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
1c79356b | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
8f6c56a5 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
8f6c56a5 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
8f6c56a5 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
1c79356b A |
27 | */ |
28 | /* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */ | |
29 | ||
30 | /* | |
31 | * Copyright (C) 1999 WIDE Project. | |
32 | * All rights reserved. | |
33 | * | |
34 | * Redistribution and use in source and binary forms, with or without | |
35 | * modification, are permitted provided that the following conditions | |
36 | * are met: | |
37 | * 1. Redistributions of source code must retain the above copyright | |
38 | * notice, this list of conditions and the following disclaimer. | |
39 | * 2. Redistributions in binary form must reproduce the above copyright | |
40 | * notice, this list of conditions and the following disclaimer in the | |
41 | * documentation and/or other materials provided with the distribution. | |
42 | * 3. Neither the name of the project nor the names of its contributors | |
43 | * may be used to endorse or promote products derived from this software | |
44 | * without specific prior written permission. | |
45 | * | |
46 | * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND | |
47 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
48 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
49 | * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE | |
50 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
51 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
52 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
53 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
54 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
55 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
56 | * SUCH DAMAGE. | |
57 | */ | |
58 | ||
59 | /* | |
60 | * Copyright (c) 1982, 1986, 1988, 1991, 1993 | |
61 | * The Regents of the University of California. All rights reserved. | |
62 | * | |
63 | * Redistribution and use in source and binary forms, with or without | |
64 | * modification, are permitted provided that the following conditions | |
65 | * are met: | |
66 | * 1. Redistributions of source code must retain the above copyright | |
67 | * notice, this list of conditions and the following disclaimer. | |
68 | * 2. Redistributions in binary form must reproduce the above copyright | |
69 | * notice, this list of conditions and the following disclaimer in the | |
70 | * documentation and/or other materials provided with the distribution. | |
71 | * 3. All advertising materials mentioning features or use of this software | |
72 | * must display the following acknowledgement: | |
73 | * This product includes software developed by the University of | |
74 | * California, Berkeley and its contributors. | |
75 | * 4. Neither the name of the University nor the names of its contributors | |
76 | * may be used to endorse or promote products derived from this software | |
77 | * without specific prior written permission. | |
78 | * | |
79 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
80 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
81 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
82 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
83 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
84 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
85 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
86 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
87 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
88 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
89 | * SUCH DAMAGE. | |
90 | * | |
91 | * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95 | |
92 | */ | |
2d21ac55 A |
93 | /* |
94 | * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce | |
95 | * support for mandatory and extensible security protections. This notice | |
96 | * is included in support of clause 2.2 (b) of the Apple Public License, | |
97 | * Version 2.0. | |
98 | */ | |
1c79356b | 99 | |
1c79356b | 100 | |
9bccf70c | 101 | /*#define PULLDOWN_DEBUG*/ |
1c79356b A |
102 | |
103 | #include <sys/param.h> | |
104 | #include <sys/systm.h> | |
91447636 | 105 | #include <sys/proc_internal.h> |
1c79356b A |
106 | #include <sys/malloc.h> |
107 | #include <sys/mbuf.h> | |
6d2010ae | 108 | #include <sys/mcache.h> |
1c79356b | 109 | #include <netinet/in.h> |
39236c6e A |
110 | #include <netinet/ip_var.h> |
111 | #if INET6 | |
1c79356b A |
112 | #include <netinet/ip6.h> |
113 | #include <netinet6/ip6_var.h> | |
6d2010ae | 114 | #endif /* INET6 */ |
1c79356b | 115 | |
2d21ac55 A |
116 | #if CONFIG_MACF_NET |
117 | #include <security/mac_framework.h> | |
118 | #endif | |
119 | ||
1c79356b A |
120 | /* |
121 | * ensure that [off, off + len) is contiguous on the mbuf chain "m". | |
122 | * packet chain before "off" is kept untouched. | |
123 | * if offp == NULL, the target will start at <retval, 0> on resulting chain. | |
124 | * if offp != NULL, the target will start at <retval, *offp> on resulting chain. | |
125 | * | |
126 | * on error return (NULL return value), original "m" will be freed. | |
127 | * | |
128 | * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster) | |
129 | */ | |
130 | struct mbuf * | |
2d21ac55 | 131 | m_pulldown(struct mbuf *m, int off, int len, int *offp) |
1c79356b | 132 | { |
a39ff7e2 A |
133 | struct mbuf *n = NULL, *o = NULL; |
134 | int hlen = 0, tlen = 0, olen = 0; | |
135 | int sharedcluster = 0; | |
6d2010ae | 136 | #if defined(PULLDOWN_STAT) && INET6 |
1c79356b A |
137 | static struct mbuf *prev = NULL; |
138 | int prevlen = 0, prevmlen = 0; | |
139 | #endif | |
140 | ||
141 | /* check invalid arguments. */ | |
142 | if (m == NULL) | |
143 | panic("m == NULL in m_pulldown()"); | |
144 | if (len > MCLBYTES) { | |
145 | m_freem(m); | |
146 | return NULL; /* impossible */ | |
147 | } | |
148 | ||
6d2010ae | 149 | #if defined(PULLDOWN_STAT) && INET6 |
1c79356b A |
150 | ip6stat.ip6s_pulldown++; |
151 | #endif | |
152 | ||
6d2010ae | 153 | #if defined(PULLDOWN_STAT) && INET6 |
1c79356b A |
154 | /* statistics for m_pullup */ |
155 | ip6stat.ip6s_pullup++; | |
156 | if (off + len > MHLEN) | |
157 | ip6stat.ip6s_pullup_fail++; | |
158 | else { | |
159 | int dlen, mlen; | |
160 | ||
161 | dlen = (prev == m) ? prevlen : m->m_len; | |
162 | mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m); | |
163 | ||
164 | if (dlen >= off + len) | |
165 | ip6stat.ip6s_pullup--; /* call will not be made! */ | |
166 | else if ((m->m_flags & M_EXT) != 0) { | |
167 | ip6stat.ip6s_pullup_alloc++; | |
168 | ip6stat.ip6s_pullup_copy++; | |
169 | } else { | |
170 | if (mlen >= off + len) | |
171 | ip6stat.ip6s_pullup_copy++; | |
172 | else { | |
173 | ip6stat.ip6s_pullup_alloc++; | |
174 | ip6stat.ip6s_pullup_copy++; | |
175 | } | |
176 | } | |
177 | ||
178 | prevlen = off + len; | |
179 | prevmlen = MHLEN; | |
180 | } | |
181 | ||
182 | /* statistics for m_pullup2 */ | |
183 | ip6stat.ip6s_pullup2++; | |
184 | if (off + len > MCLBYTES) | |
185 | ip6stat.ip6s_pullup2_fail++; | |
186 | else { | |
187 | int dlen, mlen; | |
188 | ||
189 | dlen = (prev == m) ? prevlen : m->m_len; | |
190 | mlen = (prev == m) ? prevmlen : m->m_len + M_TRAILINGSPACE(m); | |
191 | prevlen = off + len; | |
192 | prevmlen = mlen; | |
193 | ||
194 | if (dlen >= off + len) | |
195 | ip6stat.ip6s_pullup2--; /* call will not be made! */ | |
196 | else if ((m->m_flags & M_EXT) != 0) { | |
197 | ip6stat.ip6s_pullup2_alloc++; | |
198 | ip6stat.ip6s_pullup2_copy++; | |
199 | prevmlen = (off + len > MHLEN) ? MCLBYTES : MHLEN; | |
200 | } else { | |
201 | if (mlen >= off + len) | |
202 | ip6stat.ip6s_pullup2_copy++; | |
203 | else { | |
204 | ip6stat.ip6s_pullup2_alloc++; | |
205 | ip6stat.ip6s_pullup2_copy++; | |
206 | prevmlen = (off + len > MHLEN) ? MCLBYTES | |
207 | : MHLEN; | |
208 | } | |
209 | } | |
210 | } | |
211 | ||
212 | prev = m; | |
213 | #endif | |
214 | ||
215 | #ifdef PULLDOWN_DEBUG | |
216 | { | |
217 | struct mbuf *t; | |
218 | printf("before:"); | |
219 | for (t = m; t; t = t->m_next) | |
220 | printf(" %d", t->m_len); | |
221 | printf("\n"); | |
222 | } | |
223 | #endif | |
224 | n = m; | |
a39ff7e2 A |
225 | |
226 | /* | |
227 | * Iterate and make n point to the mbuf | |
228 | * within which the first byte at length | |
229 | * offset is contained from the start of | |
230 | * mbuf chain. | |
231 | */ | |
1c79356b A |
232 | while (n != NULL && off > 0) { |
233 | if (n->m_len > off) | |
234 | break; | |
235 | off -= n->m_len; | |
236 | n = n->m_next; | |
237 | } | |
a39ff7e2 | 238 | |
1c79356b A |
239 | /* be sure to point non-empty mbuf */ |
240 | while (n != NULL && n->m_len == 0) | |
241 | n = n->m_next; | |
a39ff7e2 | 242 | |
1c79356b A |
243 | if (!n) { |
244 | m_freem(m); | |
245 | return NULL; /* mbuf chain too short */ | |
246 | } | |
247 | ||
248 | /* | |
249 | * the target data is on <n, off>. | |
250 | * if we got enough data on the mbuf "n", we're done. | |
a39ff7e2 A |
251 | * |
252 | * It should be noted, that we should only do this either | |
253 | * when offset is 0, i.e. data is pointing to the start | |
254 | * or when the caller specifies an out argument to get | |
255 | * the offset value in the mbuf to work with data pointer | |
256 | * correctly. | |
257 | * | |
258 | * If offset is not 0 and caller did not provide out-argument | |
259 | * to get offset, we should split the mbuf even when the length | |
260 | * is contained in current mbuf. | |
1c79356b A |
261 | */ |
262 | if ((off == 0 || offp) && len <= n->m_len - off) | |
263 | goto ok; | |
264 | ||
6d2010ae | 265 | #if defined(PULLDOWN_STAT) && INET6 |
1c79356b A |
266 | ip6stat.ip6s_pulldown_copy++; |
267 | #endif | |
268 | ||
269 | /* | |
a39ff7e2 | 270 | * when len <= n->m_len - off and off != 0, it is a special case. |
1c79356b A |
271 | * len bytes from <n, off> sits in single mbuf, but the caller does |
272 | * not like the starting position (off). | |
273 | * chop the current mbuf into two pieces, set off to 0. | |
274 | */ | |
a39ff7e2 | 275 | if (len <= n->m_len - off) { |
1c79356b A |
276 | o = m_copym(n, off, n->m_len - off, M_DONTWAIT); |
277 | if (o == NULL) { | |
278 | m_freem(m); | |
279 | return NULL; /* ENOBUFS */ | |
280 | } | |
281 | n->m_len = off; | |
282 | o->m_next = n->m_next; | |
283 | n->m_next = o; | |
284 | n = n->m_next; | |
285 | off = 0; | |
286 | goto ok; | |
287 | } | |
288 | ||
289 | /* | |
290 | * we need to take hlen from <n, off> and tlen from <n->m_next, 0>, | |
291 | * and construct contiguous mbuf with m_len == len. | |
292 | * note that hlen + tlen == len, and tlen > 0. | |
a39ff7e2 A |
293 | * |
294 | * Read these variables as head length and tail length | |
1c79356b A |
295 | */ |
296 | hlen = n->m_len - off; | |
297 | tlen = len - hlen; | |
298 | ||
299 | /* | |
300 | * ensure that we have enough trailing data on mbuf chain. | |
301 | * if not, we can do nothing about the chain. | |
302 | */ | |
303 | olen = 0; | |
304 | for (o = n->m_next; o != NULL; o = o->m_next) | |
305 | olen += o->m_len; | |
306 | if (hlen + olen < len) { | |
307 | m_freem(m); | |
308 | return NULL; /* mbuf chain too short */ | |
309 | } | |
310 | ||
311 | /* | |
312 | * easy cases first. | |
313 | * we need to use m_copydata() to get data from <n->m_next, 0>. | |
314 | */ | |
315 | if ((n->m_flags & M_EXT) == 0) | |
316 | sharedcluster = 0; | |
317 | else { | |
813fb2f6 | 318 | if (m_get_ext_free(n) != NULL) |
1c79356b | 319 | sharedcluster = 1; |
91447636 | 320 | else if (m_mclhasreference(n)) |
1c79356b A |
321 | sharedcluster = 1; |
322 | else | |
323 | sharedcluster = 0; | |
324 | } | |
a39ff7e2 A |
325 | |
326 | /* | |
327 | * If we have enough space left in current mbuf to accomodate | |
328 | * tail length, copy tail length worth of data starting with next mbuf | |
329 | * and adjust the length of next one accordingly. | |
330 | */ | |
1c79356b A |
331 | if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen |
332 | && !sharedcluster) { | |
333 | m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len); | |
334 | n->m_len += tlen; | |
335 | m_adj(n->m_next, tlen); | |
336 | goto ok; | |
337 | } | |
a39ff7e2 A |
338 | |
339 | /* | |
340 | * If have enough leading space in next mbuf to accomodate head length | |
341 | * of current mbuf, and total resulting length of next mbuf is greater | |
342 | * than or equal to requested len bytes, then just copy hlen from | |
343 | * current to the next one and adjust sizes accordingly. | |
344 | */ | |
345 | if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen && | |
346 | (n->m_next->m_len + hlen) >= len && !sharedcluster) { | |
1c79356b A |
347 | n->m_next->m_data -= hlen; |
348 | n->m_next->m_len += hlen; | |
349 | bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), hlen); | |
350 | n->m_len -= hlen; | |
351 | n = n->m_next; | |
352 | off = 0; | |
353 | goto ok; | |
354 | } | |
355 | ||
356 | /* | |
357 | * now, we need to do the hard way. don't m_copy as there's no room | |
358 | * on both end. | |
359 | */ | |
6d2010ae | 360 | #if defined(PULLDOWN_STAT) && INET6 |
1c79356b A |
361 | ip6stat.ip6s_pulldown_alloc++; |
362 | #endif | |
363 | MGET(o, M_DONTWAIT, m->m_type); | |
364 | if (o == NULL) { | |
365 | m_freem(m); | |
366 | return NULL; /* ENOBUFS */ | |
367 | } | |
368 | if (len > MHLEN) { /* use MHLEN just for safety */ | |
369 | MCLGET(o, M_DONTWAIT); | |
370 | if ((o->m_flags & M_EXT) == 0) { | |
371 | m_freem(m); | |
372 | m_free(o); | |
373 | return NULL; /* ENOBUFS */ | |
374 | } | |
375 | } | |
376 | /* get hlen from <n, off> into <o, 0> */ | |
377 | o->m_len = hlen; | |
378 | bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), hlen); | |
379 | n->m_len -= hlen; | |
380 | /* get tlen from <n->m_next, 0> into <o, hlen> */ | |
381 | m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len); | |
382 | o->m_len += tlen; | |
383 | m_adj(n->m_next, tlen); | |
384 | o->m_next = n->m_next; | |
385 | n->m_next = o; | |
386 | n = o; | |
387 | off = 0; | |
388 | ||
389 | ok: | |
390 | #ifdef PULLDOWN_DEBUG | |
391 | { | |
392 | struct mbuf *t; | |
393 | printf("after:"); | |
394 | for (t = m; t; t = t->m_next) | |
395 | printf("%c%d", t == n ? '*' : ' ', t->m_len); | |
396 | printf(" (off=%d)\n", off); | |
397 | } | |
398 | #endif | |
399 | if (offp) | |
400 | *offp = off; | |
401 | return n; | |
402 | } | |
403 | ||
6d2010ae A |
404 | /* |
405 | * Create and return an m_tag, either by re-using space in a previous tag | |
406 | * or by allocating a new mbuf/cluster | |
407 | */ | |
408 | struct m_tag * | |
409 | m_tag_create(u_int32_t id, u_int16_t type, int len, int wait, struct mbuf *buf) | |
410 | { | |
411 | struct m_tag *t = NULL; | |
412 | struct m_tag *p; | |
413 | ||
414 | if (len < 0) | |
415 | return (NULL); | |
416 | ||
417 | if (len + sizeof (struct m_tag) + sizeof (struct m_taghdr) > MLEN) | |
418 | return (m_tag_alloc(id, type, len, wait)); | |
419 | ||
420 | /* | |
421 | * We've exhausted all external cases. Now, go through the m_tag | |
422 | * chain and see if we can fit it in any of them. | |
423 | * If not (t == NULL), call m_tag_alloc to store it in a new mbuf. | |
424 | */ | |
425 | p = SLIST_FIRST(&buf->m_pkthdr.tags); | |
426 | while(p != NULL) { | |
427 | /* 2KCL m_tag */ | |
428 | if (M_TAG_ALIGN(p->m_tag_len) + | |
429 | sizeof (struct m_taghdr) > MLEN) { | |
430 | p = SLIST_NEXT(p, m_tag_link); | |
431 | continue; | |
432 | } | |
433 | ||
434 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); | |
435 | ||
436 | struct mbuf *m = m_dtom(p); | |
316670eb | 437 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
6d2010ae | 438 | |
316670eb | 439 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); |
6d2010ae A |
440 | VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT)); |
441 | ||
442 | /* The mbuf can store this m_tag */ | |
443 | if (M_TAG_ALIGN(len) <= MLEN - m->m_len) { | |
316670eb A |
444 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
445 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); | |
6d2010ae A |
446 | hdr->refcnt++; |
447 | m->m_len += M_TAG_ALIGN(len); | |
448 | VERIFY(m->m_len <= MLEN); | |
449 | break; | |
450 | } | |
451 | ||
452 | p = SLIST_NEXT(p, m_tag_link); | |
453 | } | |
454 | ||
455 | if (t == NULL) | |
456 | return (m_tag_alloc(id, type, len, wait)); | |
457 | ||
458 | t->m_tag_cookie = M_TAG_VALID_PATTERN; | |
459 | t->m_tag_type = type; | |
460 | t->m_tag_len = len; | |
461 | t->m_tag_id = id; | |
462 | if (len > 0) | |
463 | bzero(t + 1, len); | |
464 | return (t); | |
465 | } | |
466 | ||
91447636 A |
467 | /* Get a packet tag structure along with specified data following. */ |
468 | struct m_tag * | |
469 | m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait) | |
470 | { | |
471 | struct m_tag *t; | |
472 | ||
473 | if (len < 0) | |
6d2010ae A |
474 | return (NULL); |
475 | ||
476 | if (M_TAG_ALIGN(len) + sizeof (struct m_taghdr) <= MLEN) { | |
91447636 | 477 | struct mbuf *m = m_get(wait, MT_TAG); |
6d2010ae A |
478 | struct m_taghdr *hdr; |
479 | ||
91447636 | 480 | if (m == NULL) |
6d2010ae A |
481 | return (NULL); |
482 | ||
483 | m->m_flags |= M_TAGHDR; | |
484 | ||
316670eb A |
485 | hdr = (struct m_taghdr *)(void *)m->m_data; |
486 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); | |
6d2010ae A |
487 | hdr->refcnt = 1; |
488 | m->m_len += sizeof (struct m_taghdr); | |
316670eb A |
489 | t = (struct m_tag *)(void *)(m->m_data + m->m_len); |
490 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); | |
6d2010ae A |
491 | m->m_len += M_TAG_ALIGN(len); |
492 | VERIFY(m->m_len <= MLEN); | |
493 | } else if (len + sizeof (struct m_tag) <= MCLBYTES) { | |
316670eb | 494 | t = (struct m_tag *)(void *)m_mclalloc(wait); |
6d2010ae | 495 | } else { |
91447636 | 496 | t = NULL; |
6d2010ae A |
497 | } |
498 | ||
91447636 | 499 | if (t == NULL) |
6d2010ae A |
500 | return (NULL); |
501 | ||
316670eb | 502 | VERIFY(IS_P2ALIGNED(t, sizeof (u_int64_t))); |
6d2010ae | 503 | t->m_tag_cookie = M_TAG_VALID_PATTERN; |
91447636 A |
504 | t->m_tag_type = type; |
505 | t->m_tag_len = len; | |
506 | t->m_tag_id = id; | |
6d2010ae A |
507 | if (len > 0) |
508 | bzero(t + 1, len); | |
509 | return (t); | |
91447636 A |
510 | } |
511 | ||
512 | ||
513 | /* Free a packet tag. */ | |
514 | void | |
515 | m_tag_free(struct m_tag *t) | |
516 | { | |
2d21ac55 A |
517 | #if CONFIG_MACF_NET |
518 | if (t != NULL && | |
519 | t->m_tag_id == KERNEL_MODULE_TAG_ID && | |
520 | t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) | |
521 | mac_mbuf_tag_destroy(t); | |
522 | #endif | |
91447636 A |
523 | if (t == NULL) |
524 | return; | |
316670eb A |
525 | |
526 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); | |
527 | ||
6d2010ae | 528 | if (M_TAG_ALIGN(t->m_tag_len) + sizeof (struct m_taghdr) <= MLEN) { |
91447636 | 529 | struct mbuf * m = m_dtom(t); |
6d2010ae | 530 | VERIFY(m->m_flags & M_TAGHDR); |
316670eb A |
531 | struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data; |
532 | ||
533 | VERIFY(IS_P2ALIGNED(hdr + 1, sizeof (u_int64_t))); | |
6d2010ae A |
534 | |
535 | /* No other tags in this mbuf */ | |
536 | if(--hdr->refcnt == 0) { | |
537 | m_free(m); | |
538 | return; | |
539 | } | |
540 | ||
541 | /* Pattern-fill the header */ | |
542 | u_int64_t *fill_ptr = (u_int64_t *)t; | |
543 | u_int64_t *end_ptr = (u_int64_t *)(t + 1); | |
544 | while (fill_ptr < end_ptr) { | |
545 | *fill_ptr = M_TAG_FREE_PATTERN; | |
546 | fill_ptr++; | |
547 | } | |
91447636 | 548 | } else { |
6d2010ae | 549 | m_mclfree((caddr_t)t); |
91447636 | 550 | } |
91447636 A |
551 | } |
552 | ||
553 | /* Prepend a packet tag. */ | |
554 | void | |
555 | m_tag_prepend(struct mbuf *m, struct m_tag *t) | |
556 | { | |
6d2010ae A |
557 | VERIFY(m != NULL && t != NULL); |
558 | ||
91447636 A |
559 | SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link); |
560 | } | |
561 | ||
562 | /* Unlink a packet tag. */ | |
563 | void | |
564 | m_tag_unlink(struct mbuf *m, struct m_tag *t) | |
565 | { | |
39236c6e A |
566 | VERIFY(m->m_flags & M_PKTHDR); |
567 | VERIFY(t != NULL && t->m_tag_cookie == M_TAG_VALID_PATTERN); | |
6d2010ae | 568 | |
91447636 A |
569 | SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link); |
570 | } | |
571 | ||
572 | /* Unlink and free a packet tag. */ | |
573 | void | |
574 | m_tag_delete(struct mbuf *m, struct m_tag *t) | |
575 | { | |
91447636 A |
576 | m_tag_unlink(m, t); |
577 | m_tag_free(t); | |
578 | } | |
579 | ||
580 | /* Unlink and free a packet tag chain, starting from given tag. */ | |
581 | void | |
582 | m_tag_delete_chain(struct mbuf *m, struct m_tag *t) | |
583 | { | |
584 | struct m_tag *p, *q; | |
585 | ||
39236c6e | 586 | VERIFY(m->m_flags & M_PKTHDR); |
6d2010ae A |
587 | |
588 | if (t != NULL) { | |
91447636 | 589 | p = t; |
6d2010ae | 590 | } else { |
91447636 | 591 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
6d2010ae | 592 | } |
91447636 A |
593 | if (p == NULL) |
594 | return; | |
6d2010ae A |
595 | |
596 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); | |
597 | while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) { | |
598 | VERIFY(q->m_tag_cookie == M_TAG_VALID_PATTERN); | |
91447636 | 599 | m_tag_delete(m, q); |
6d2010ae | 600 | } |
91447636 A |
601 | m_tag_delete(m, p); |
602 | } | |
603 | ||
604 | /* Find a tag, starting from a given position. */ | |
605 | struct m_tag * | |
606 | m_tag_locate(struct mbuf *m, u_int32_t id, u_int16_t type, struct m_tag *t) | |
607 | { | |
608 | struct m_tag *p; | |
609 | ||
39236c6e | 610 | VERIFY(m->m_flags & M_PKTHDR); |
6d2010ae A |
611 | |
612 | if (t == NULL) { | |
91447636 | 613 | p = SLIST_FIRST(&m->m_pkthdr.tags); |
6d2010ae A |
614 | } else { |
615 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); | |
91447636 | 616 | p = SLIST_NEXT(t, m_tag_link); |
6d2010ae | 617 | } |
91447636 | 618 | while (p != NULL) { |
6d2010ae | 619 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
91447636 | 620 | if (p->m_tag_id == id && p->m_tag_type == type) |
6d2010ae | 621 | return (p); |
91447636 A |
622 | p = SLIST_NEXT(p, m_tag_link); |
623 | } | |
6d2010ae | 624 | return (NULL); |
91447636 A |
625 | } |
626 | ||
627 | /* Copy a single tag. */ | |
628 | struct m_tag * | |
629 | m_tag_copy(struct m_tag *t, int how) | |
630 | { | |
631 | struct m_tag *p; | |
632 | ||
6d2010ae A |
633 | VERIFY(t != NULL); |
634 | ||
3a60a9f5 | 635 | p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how); |
91447636 A |
636 | if (p == NULL) |
637 | return (NULL); | |
2d21ac55 A |
638 | #if CONFIG_MACF_NET |
639 | /* | |
640 | * XXXMAC: we should probably pass off the initialization, and | |
641 | * copying here? can we hid that KERNEL_TAG_TYPE_MACLABEL is | |
642 | * special from the mbuf code? | |
643 | */ | |
644 | if (t != NULL && | |
645 | t->m_tag_id == KERNEL_MODULE_TAG_ID && | |
646 | t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) { | |
647 | if (mac_mbuf_tag_init(p, how) != 0) { | |
648 | m_tag_free(p); | |
649 | return (NULL); | |
650 | } | |
651 | mac_mbuf_tag_copy(t, p); | |
652 | } else | |
653 | #endif | |
91447636 | 654 | bcopy(t + 1, p + 1, t->m_tag_len); /* Copy the data */ |
6d2010ae | 655 | return (p); |
91447636 A |
656 | } |
657 | ||
658 | /* | |
659 | * Copy two tag chains. The destination mbuf (to) loses any attached | |
660 | * tags even if the operation fails. This should not be a problem, as | |
661 | * m_tag_copy_chain() is typically called with a newly-allocated | |
662 | * destination mbuf. | |
663 | */ | |
664 | int | |
665 | m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how) | |
666 | { | |
667 | struct m_tag *p, *t, *tprev = NULL; | |
668 | ||
39236c6e | 669 | VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR)); |
6d2010ae | 670 | |
91447636 A |
671 | m_tag_delete_chain(to, NULL); |
672 | SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) { | |
6d2010ae | 673 | VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN); |
91447636 A |
674 | t = m_tag_copy(p, how); |
675 | if (t == NULL) { | |
676 | m_tag_delete_chain(to, NULL); | |
6d2010ae | 677 | return (0); |
91447636 | 678 | } |
6d2010ae | 679 | if (tprev == NULL) { |
91447636 | 680 | SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link); |
6d2010ae | 681 | } else { |
91447636 A |
682 | SLIST_INSERT_AFTER(tprev, t, m_tag_link); |
683 | tprev = t; | |
684 | } | |
685 | } | |
6d2010ae | 686 | return (1); |
91447636 A |
687 | } |
688 | ||
39236c6e | 689 | /* Initialize dynamic and static tags on an mbuf. */ |
91447636 | 690 | void |
39236c6e | 691 | m_tag_init(struct mbuf *m, int all) |
91447636 | 692 | { |
39236c6e | 693 | VERIFY(m->m_flags & M_PKTHDR); |
6d2010ae | 694 | |
91447636 | 695 | SLIST_INIT(&m->m_pkthdr.tags); |
39236c6e A |
696 | /* |
697 | * If the caller wants to preserve static mbuf tags | |
698 | * (e.g. m_dup_pkthdr), don't zero them out. | |
699 | */ | |
700 | if (all) { | |
5ba3f43e A |
701 | bzero(&m->m_pkthdr.builtin_mtag._net_mtag, |
702 | sizeof (m->m_pkthdr.builtin_mtag._net_mtag)); | |
39236c6e | 703 | } |
91447636 A |
704 | } |
705 | ||
706 | /* Get first tag in chain. */ | |
707 | struct m_tag * | |
708 | m_tag_first(struct mbuf *m) | |
709 | { | |
39236c6e | 710 | VERIFY(m->m_flags & M_PKTHDR); |
6d2010ae A |
711 | |
712 | return (SLIST_FIRST(&m->m_pkthdr.tags)); | |
91447636 A |
713 | } |
714 | ||
715 | /* Get next tag in chain. */ | |
716 | struct m_tag * | |
6d2010ae | 717 | m_tag_next(struct mbuf *m, struct m_tag *t) |
91447636 | 718 | { |
d1ecb069 | 719 | #pragma unused(m) |
6d2010ae A |
720 | VERIFY(t != NULL); |
721 | VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN); | |
722 | ||
723 | return (SLIST_NEXT(t, m_tag_link)); | |
d1ecb069 A |
724 | } |
725 | ||
316670eb A |
726 | int |
727 | m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc) | |
728 | { | |
729 | u_int32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */ | |
730 | ||
731 | return (m_set_service_class(m, m_service_class_from_val(val))); | |
732 | } | |
733 | ||
734 | mbuf_traffic_class_t | |
735 | m_get_traffic_class(struct mbuf *m) | |
736 | { | |
737 | return (MBUF_SC2TC(m_get_service_class(m))); | |
738 | } | |
739 | ||
316670eb A |
740 | int |
741 | m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc) | |
742 | { | |
743 | int error = 0; | |
744 | ||
745 | VERIFY(m->m_flags & M_PKTHDR); | |
746 | ||
747 | if (MBUF_VALID_SC(sc)) | |
39236c6e | 748 | m->m_pkthdr.pkt_svc = sc; |
316670eb A |
749 | else |
750 | error = EINVAL; | |
751 | ||
752 | return (error); | |
753 | } | |
754 | ||
755 | mbuf_svc_class_t | |
756 | m_get_service_class(struct mbuf *m) | |
757 | { | |
758 | mbuf_svc_class_t sc; | |
759 | ||
760 | VERIFY(m->m_flags & M_PKTHDR); | |
761 | ||
39236c6e A |
762 | if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) |
763 | sc = m->m_pkthdr.pkt_svc; | |
316670eb A |
764 | else |
765 | sc = MBUF_SC_BE; | |
766 | ||
767 | return (sc); | |
768 | } | |
769 | ||
770 | mbuf_svc_class_t | |
771 | m_service_class_from_idx(u_int32_t i) | |
772 | { | |
773 | mbuf_svc_class_t sc = MBUF_SC_BE; | |
774 | ||
775 | switch (i) { | |
776 | case SCIDX_BK_SYS: | |
777 | return (MBUF_SC_BK_SYS); | |
778 | ||
779 | case SCIDX_BK: | |
780 | return (MBUF_SC_BK); | |
781 | ||
782 | case SCIDX_BE: | |
783 | return (MBUF_SC_BE); | |
784 | ||
785 | case SCIDX_RD: | |
786 | return (MBUF_SC_RD); | |
787 | ||
788 | case SCIDX_OAM: | |
789 | return (MBUF_SC_OAM); | |
790 | ||
791 | case SCIDX_AV: | |
792 | return (MBUF_SC_AV); | |
793 | ||
794 | case SCIDX_RV: | |
795 | return (MBUF_SC_RV); | |
796 | ||
797 | case SCIDX_VI: | |
798 | return (MBUF_SC_VI); | |
799 | ||
800 | case SCIDX_VO: | |
801 | return (MBUF_SC_VO); | |
802 | ||
803 | case SCIDX_CTL: | |
804 | return (MBUF_SC_CTL); | |
805 | ||
806 | default: | |
807 | break; | |
808 | } | |
809 | ||
810 | VERIFY(0); | |
811 | /* NOTREACHED */ | |
812 | return (sc); | |
813 | } | |
814 | ||
815 | mbuf_svc_class_t | |
816 | m_service_class_from_val(u_int32_t v) | |
817 | { | |
818 | mbuf_svc_class_t sc = MBUF_SC_BE; | |
819 | ||
820 | switch (v) { | |
821 | case SCVAL_BK_SYS: | |
822 | return (MBUF_SC_BK_SYS); | |
823 | ||
824 | case SCVAL_BK: | |
825 | return (MBUF_SC_BK); | |
826 | ||
827 | case SCVAL_BE: | |
828 | return (MBUF_SC_BE); | |
829 | ||
830 | case SCVAL_RD: | |
831 | return (MBUF_SC_RD); | |
832 | ||
833 | case SCVAL_OAM: | |
834 | return (MBUF_SC_OAM); | |
835 | ||
836 | case SCVAL_AV: | |
837 | return (MBUF_SC_AV); | |
838 | ||
839 | case SCVAL_RV: | |
840 | return (MBUF_SC_RV); | |
841 | ||
842 | case SCVAL_VI: | |
843 | return (MBUF_SC_VI); | |
844 | ||
845 | case SCVAL_VO: | |
846 | return (MBUF_SC_VO); | |
847 | ||
848 | case SCVAL_CTL: | |
849 | return (MBUF_SC_CTL); | |
850 | ||
851 | default: | |
852 | break; | |
853 | } | |
854 | ||
855 | VERIFY(0); | |
856 | /* NOTREACHED */ | |
857 | return (sc); | |
d1ecb069 | 858 | } |
39236c6e A |
859 | |
860 | uint16_t | |
5ba3f43e A |
861 | m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff, |
862 | uint32_t datalen, uint32_t sum) | |
39236c6e | 863 | { |
5ba3f43e A |
864 | uint32_t total_sub = 0; /* total to subtract */ |
865 | uint32_t mlen = m_pktlen(m); /* frame length */ | |
866 | uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */ | |
867 | int len; | |
39236c6e | 868 | |
5ba3f43e A |
869 | ASSERT(bytes <= mlen); |
870 | ||
871 | /* | |
872 | * Take care of excluding (len > 0) or including (len < 0) | |
873 | * extraneous octets at the beginning of the packet, taking | |
874 | * into account the start offset. | |
875 | */ | |
876 | len = (dataoff - start); | |
877 | if (len > 0) | |
878 | total_sub = m_sum16(m, start, len); | |
879 | else if (len < 0) | |
880 | sum += m_sum16(m, dataoff, -len); | |
881 | ||
882 | /* | |
883 | * Take care of excluding any postpended extraneous octets. | |
884 | */ | |
885 | len = (mlen - bytes); | |
39236c6e | 886 | if (len > 0) { |
5ba3f43e A |
887 | struct mbuf *m0 = m; |
888 | uint32_t extra = m_sum16(m, bytes, len); | |
889 | uint32_t off = bytes, off0 = off; | |
890 | ||
891 | while (off > 0) { | |
892 | if (__improbable(m == NULL)) { | |
893 | panic("%s: invalid mbuf chain %p [off %u, " | |
894 | "len %u]", __func__, m0, off0, len); | |
895 | /* NOTREACHED */ | |
896 | } | |
897 | if (off < m->m_len) | |
898 | break; | |
899 | off -= m->m_len; | |
900 | m = m->m_next; | |
901 | } | |
902 | ||
903 | /* if we started on odd-alignment, swap the value */ | |
904 | if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) | |
905 | total_sub += ((extra << 8) & 0xffff) | (extra >> 8); | |
906 | else | |
907 | total_sub += extra; | |
908 | ||
909 | total_sub = (total_sub >> 16) + (total_sub & 0xffff); | |
910 | } | |
911 | ||
912 | /* | |
913 | * 1's complement subtract any extraneous octets. | |
914 | */ | |
915 | if (total_sub != 0) { | |
916 | if (total_sub >= sum) | |
917 | sum = ~(total_sub - sum) & 0xffff; | |
39236c6e | 918 | else |
5ba3f43e | 919 | sum -= total_sub; |
39236c6e A |
920 | } |
921 | ||
5ba3f43e A |
922 | /* fold 32-bit to 16-bit */ |
923 | sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */ | |
924 | sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */ | |
925 | sum = (sum >> 16) + (sum & 0xffff); /* final carry */ | |
39236c6e | 926 | |
5ba3f43e | 927 | return (sum & 0xffff); |
39236c6e A |
928 | } |
929 | ||
39236c6e A |
930 | uint16_t |
931 | m_sum16(struct mbuf *m, uint32_t off, uint32_t len) | |
932 | { | |
933 | int mlen; | |
934 | ||
935 | /* | |
936 | * Sanity check | |
937 | * | |
938 | * Use m_length2() instead of m_length(), as we cannot rely on | |
939 | * the caller setting m_pkthdr.len correctly, if the mbuf is | |
940 | * a M_PKTHDR one. | |
941 | */ | |
942 | if ((mlen = m_length2(m, NULL)) < (off + len)) { | |
5ba3f43e A |
943 | panic("%s: mbuf %p len (%d) < off+len (%d+%d)\n", __func__, |
944 | m, mlen, off, len); | |
945 | /* NOTREACHED */ | |
39236c6e A |
946 | } |
947 | ||
5ba3f43e | 948 | return (os_cpu_in_cksum_mbuf(m, len, off, 0)); |
39236c6e | 949 | } |