]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/kpi_mbuf.c
projects / apple / xnu.git / blob
? search:


587394e4924b96aa1ae03f9aa4bd50cefbd4427c
[apple/xnu.git] / bsd / kern / kpi_mbuf.c
1 /*
2 * Copyright (c) 2004-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #define __KPI__
30
31 #include <sys/param.h>
32 #include <sys/mbuf.h>
33 #include <sys/mcache.h>
34 #include <sys/socket.h>
35 #include <kern/debug.h>
36 #include <libkern/OSAtomic.h>
37 #include <kern/kalloc.h>
38 #include <string.h>
39 #include <net/dlil.h>
40 #include <netinet/in.h>
41 #include <netinet/ip_var.h>
42
43 #include "net/net_str_id.h"
44
45 /* mbuf flags visible to KPI clients; do not add private flags here */
46 static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
47 MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
48 MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS);
49
50 /* Unalterable mbuf flags */
51 static const mbuf_flags_t mbuf_cflags_mask = (MBUF_EXT);
52
53 #define MAX_MBUF_TX_COMPL_FUNC 32
54 mbuf_tx_compl_func
55 mbuf_tx_compl_table[MAX_MBUF_TX_COMPL_FUNC];
56 extern lck_rw_t *mbuf_tx_compl_tbl_lock;
57 u_int32_t mbuf_tx_compl_index = 0;
58
59 #if (DEVELOPMENT || DEBUG)
60 int mbuf_tx_compl_debug = 0;
61 SInt64 mbuf_tx_compl_outstanding __attribute__((aligned(8))) = 0;
62 u_int64_t mbuf_tx_compl_aborted __attribute__((aligned(8))) = 0;
63
64 SYSCTL_DECL(_kern_ipc);
65 SYSCTL_NODE(_kern_ipc, OID_AUTO, mbtxcf,
66 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "");
67 SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, debug,
68 CTLFLAG_RW | CTLFLAG_LOCKED, &mbuf_tx_compl_debug, 0, "");
69 SYSCTL_INT(_kern_ipc_mbtxcf, OID_AUTO, index,
70 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_index, 0, "");
71 SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, oustanding,
72 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_outstanding, "");
73 SYSCTL_QUAD(_kern_ipc_mbtxcf, OID_AUTO, aborted,
74 CTLFLAG_RD | CTLFLAG_LOCKED, &mbuf_tx_compl_aborted, "");
75 #endif /* (DEBUG || DEVELOPMENT) */
76
77 void *
78 mbuf_data(mbuf_t mbuf)
79 {
80 return (mbuf->m_data);
81 }
82
83 void *
84 mbuf_datastart(mbuf_t mbuf)
85 {
86 if (mbuf->m_flags & M_EXT)
87 return (mbuf->m_ext.ext_buf);
88 if (mbuf->m_flags & M_PKTHDR)
89 return (mbuf->m_pktdat);
90 return (mbuf->m_dat);
91 }
92
93 errno_t
94 mbuf_setdata(mbuf_t mbuf, void *data, size_t len)
95 {
96 size_t start = (size_t)((char *)mbuf_datastart(mbuf));
97 size_t maxlen = mbuf_maxlen(mbuf);
98
99 if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
100 return (EINVAL);
101 mbuf->m_data = data;
102 mbuf->m_len = len;
103
104 return (0);
105 }
106
107 errno_t
108 mbuf_align_32(mbuf_t mbuf, size_t len)
109 {
110 if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
111 return (ENOTSUP);
112 mbuf->m_data = mbuf_datastart(mbuf);
113 mbuf->m_data +=
114 ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
115
116 return (0);
117 }
118
119 /*
120 * This function is used to provide mcl_to_paddr via symbol indirection,
121 * please avoid any change in behavior or remove the indirection in
122 * config/Unsupported*
123 */
124 addr64_t
125 mbuf_data_to_physical(void *ptr)
126 {
127 return ((addr64_t)mcl_to_paddr(ptr));
128 }
129
130 errno_t
131 mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
132 {
133 /* Must set *mbuf to NULL in failure case */
134 *mbuf = m_get(how, type);
135
136 return (*mbuf == NULL ? ENOMEM : 0);
137 }
138
139 errno_t
140 mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
141 {
142 /* Must set *mbuf to NULL in failure case */
143 *mbuf = m_gethdr(how, type);
144
145 return (*mbuf == NULL ? ENOMEM : 0);
146 }
147
148 errno_t
149 mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
150 caddr_t extbuf, void (*extfree)(caddr_t, u_int, caddr_t),
151 size_t extsize, caddr_t extarg)
152 {
153 if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0)
154 return (EINVAL);
155
156 if ((*mbuf = m_clattach(*mbuf, type, extbuf,
157 extfree, extsize, extarg, how, 0)) == NULL)
158 return (ENOMEM);
159
160 return (0);
161 }
162
163 errno_t
164 mbuf_ring_cluster_alloc(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
165 void (*extfree)(caddr_t, u_int, caddr_t), size_t *size)
166 {
167 caddr_t extbuf = NULL;
168 errno_t err;
169
170 if (mbuf == NULL || extfree == NULL || size == NULL || *size == 0)
171 return (EINVAL);
172
173 if ((err = mbuf_alloccluster(how, size, &extbuf)) != 0)
174 return (err);
175
176 if ((*mbuf = m_clattach(*mbuf, type, extbuf,
177 extfree, *size, NULL, how, 1)) == NULL) {
178 mbuf_freecluster(extbuf, *size);
179 return (ENOMEM);
180 }
181
182 return (0);
183 }
184
185 int
186 mbuf_ring_cluster_is_active(mbuf_t mbuf)
187 {
188 return (m_ext_paired_is_active(mbuf));
189 }
190
191 errno_t
192 mbuf_ring_cluster_activate(mbuf_t mbuf)
193 {
194 if (mbuf_ring_cluster_is_active(mbuf))
195 return (EBUSY);
196
197 m_ext_paired_activate(mbuf);
198 return (0);
199 }
200
201 errno_t
202 mbuf_cluster_set_prop(mbuf_t mbuf, u_int32_t oldprop, u_int32_t newprop)
203 {
204 if (mbuf == NULL || !(mbuf->m_flags & M_EXT))
205 return (EINVAL);
206
207 return (m_ext_set_prop(mbuf, oldprop, newprop) ? 0 : EBUSY);
208 }
209
210 errno_t
211 mbuf_cluster_get_prop(mbuf_t mbuf, u_int32_t *prop)
212 {
213 if (mbuf == NULL || prop == NULL || !(mbuf->m_flags & M_EXT))
214 return (EINVAL);
215
216 *prop = m_ext_get_prop(mbuf);
217 return (0);
218 }
219
220 errno_t
221 mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
222 {
223 if (size == NULL || *size == 0 || addr == NULL)
224 return (EINVAL);
225
226 *addr = NULL;
227
228 /* Jumbo cluster pool not available? */
229 if (*size > MBIGCLBYTES && njcl == 0)
230 return (ENOTSUP);
231
232 if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
233 *size = MCLBYTES;
234 else if (*size > MCLBYTES && *size <= MBIGCLBYTES &&
235 (*addr = m_bigalloc(how)) != NULL)
236 *size = MBIGCLBYTES;
237 else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES &&
238 (*addr = m_16kalloc(how)) != NULL)
239 *size = M16KCLBYTES;
240 else
241 *size = 0;
242
243 if (*addr == NULL)
244 return (ENOMEM);
245
246 return (0);
247 }
248
249 void
250 mbuf_freecluster(caddr_t addr, size_t size)
251 {
252 if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES)
253 panic("%s: invalid size (%ld) for cluster %p", __func__,
254 size, (void *)addr);
255
256 if (size == MCLBYTES)
257 m_mclfree(addr);
258 else if (size == MBIGCLBYTES)
259 m_bigfree(addr, MBIGCLBYTES, NULL);
260 else if (njcl > 0)
261 m_16kfree(addr, M16KCLBYTES, NULL);
262 else
263 panic("%s: freeing jumbo cluster to an empty pool", __func__);
264 }
265
266 errno_t
267 mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t *mbuf)
268 {
269 /* Must set *mbuf to NULL in failure case */
270 errno_t error = 0;
271 int created = 0;
272
273 if (mbuf == NULL)
274 return (EINVAL);
275 if (*mbuf == NULL) {
276 *mbuf = m_get(how, type);
277 if (*mbuf == NULL)
278 return (ENOMEM);
279 created = 1;
280 }
281 /*
282 * At the time this code was written, m_{mclget,mbigget,m16kget}
283 * would always return the same value that was passed in to it.
284 */
285 if (size == MCLBYTES) {
286 *mbuf = m_mclget(*mbuf, how);
287 } else if (size == MBIGCLBYTES) {
288 *mbuf = m_mbigget(*mbuf, how);
289 } else if (size == M16KCLBYTES) {
290 if (njcl > 0) {
291 *mbuf = m_m16kget(*mbuf, how);
292 } else {
293 /* Jumbo cluster pool not available? */
294 error = ENOTSUP;
295 goto out;
296 }
297 } else {
298 error = EINVAL;
299 goto out;
300 }
301 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
302 error = ENOMEM;
303 out:
304 if (created && error != 0) {
305 mbuf_free(*mbuf);
306 *mbuf = NULL;
307 }
308 return (error);
309 }
310
311 errno_t
312 mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
313 {
314 /* Must set *mbuf to NULL in failure case */
315 errno_t error = 0;
316 int created = 0;
317 if (mbuf == NULL)
318 return (EINVAL);
319 if (*mbuf == NULL) {
320 error = mbuf_get(how, type, mbuf);
321 if (error)
322 return (error);
323 created = 1;
324 }
325
326 /*
327 * At the time this code was written, m_mclget would always
328 * return the same value that was passed in to it.
329 */
330 *mbuf = m_mclget(*mbuf, how);
331
332 if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
333 mbuf_free(*mbuf);
334 *mbuf = NULL;
335 }
336 if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
337 error = ENOMEM;
338 return (error);
339 }
340
341
342 errno_t
343 mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
344 {
345 /* Must set *mbuf to NULL in failure case */
346 errno_t error = 0;
347
348 *mbuf = m_getpacket_how(how);
349
350 if (*mbuf == NULL) {
351 if (how == MBUF_WAITOK)
352 error = ENOMEM;
353 else
354 error = EWOULDBLOCK;
355 }
356
357 return (error);
358 }
359
360 /*
361 * This function is used to provide m_free via symbol indirection, please avoid
362 * any change in behavior or remove the indirection in config/Unsupported*
363 */
364 mbuf_t
365 mbuf_free(mbuf_t mbuf)
366 {
367 return (m_free(mbuf));
368 }
369
370 /*
371 * This function is used to provide m_freem via symbol indirection, please avoid
372 * any change in behavior or remove the indirection in config/Unsupported*
373 */
374 void
375 mbuf_freem(mbuf_t mbuf)
376 {
377 m_freem(mbuf);
378 }
379
380 int
381 mbuf_freem_list(mbuf_t mbuf)
382 {
383 return (m_freem_list(mbuf));
384 }
385
386 size_t
387 mbuf_leadingspace(const mbuf_t mbuf)
388 {
389 return (m_leadingspace(mbuf));
390 }
391
392 /*
393 * This function is used to provide m_trailingspace via symbol indirection,
394 * please avoid any change in behavior or remove the indirection in
395 * config/Unsupported*
396 */
397 size_t
398 mbuf_trailingspace(const mbuf_t mbuf)
399 {
400 return (m_trailingspace(mbuf));
401 }
402
403 /* Manipulation */
404 errno_t
405 mbuf_copym(const mbuf_t src, size_t offset, size_t len,
406 mbuf_how_t how, mbuf_t *new_mbuf)
407 {
408 /* Must set *mbuf to NULL in failure case */
409 *new_mbuf = m_copym(src, offset, len, how);
410
411 return (*new_mbuf == NULL ? ENOMEM : 0);
412 }
413
414 errno_t
415 mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
416 {
417 /* Must set *new_mbuf to NULL in failure case */
418 *new_mbuf = m_dup(src, how);
419
420 return (*new_mbuf == NULL ? ENOMEM : 0);
421 }
422
423 errno_t
424 mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
425 {
426 /* Must set *orig to NULL in failure case */
427 *orig = m_prepend_2(*orig, len, how, 0);
428
429 return (*orig == NULL ? ENOMEM : 0);
430 }
431
432 errno_t
433 mbuf_split(mbuf_t src, size_t offset,
434 mbuf_how_t how, mbuf_t *new_mbuf)
435 {
436 /* Must set *new_mbuf to NULL in failure case */
437 *new_mbuf = m_split(src, offset, how);
438
439 return (*new_mbuf == NULL ? ENOMEM : 0);
440 }
441
442 errno_t
443 mbuf_pullup(mbuf_t *mbuf, size_t len)
444 {
445 /* Must set *mbuf to NULL in failure case */
446 *mbuf = m_pullup(*mbuf, len);
447
448 return (*mbuf == NULL ? ENOMEM : 0);
449 }
450
451 errno_t
452 mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
453 {
454 /* Must set *location to NULL in failure case */
455 int new_offset;
456 *location = m_pulldown(src, *offset, len, &new_offset);
457 *offset = new_offset;
458
459 return (*location == NULL ? ENOMEM : 0);
460 }
461
462 /*
463 * This function is used to provide m_adj via symbol indirection, please avoid
464 * any change in behavior or remove the indirection in config/Unsupported*
465 */
466 void
467 mbuf_adj(mbuf_t mbuf, int len)
468 {
469 m_adj(mbuf, len);
470 }
471
472 errno_t
473 mbuf_adjustlen(mbuf_t m, int amount)
474 {
475 /* Verify m_len will be valid after adding amount */
476 if (amount > 0) {
477 int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
478 m->m_len;
479
480 if ((size_t)(amount + used) > mbuf_maxlen(m))
481 return (EINVAL);
482 } else if (-amount > m->m_len) {
483 return (EINVAL);
484 }
485
486 m->m_len += amount;
487 return (0);
488 }
489
490 mbuf_t
491 mbuf_concatenate(mbuf_t dst, mbuf_t src)
492 {
493 if (dst == NULL)
494 return (NULL);
495
496 m_cat(dst, src);
497
498 /* return dst as is in the current implementation */
499 return (dst);
500 }
501 errno_t
502 mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void *out_data)
503 {
504 /* Copied m_copydata, added error handling (don't just panic) */
505 int count;
506 mbuf_t m = m0;
507
508 while (off > 0) {
509 if (m == 0)
510 return (EINVAL);
511 if (off < (size_t)m->m_len)
512 break;
513 off -= m->m_len;
514 m = m->m_next;
515 }
516 while (len > 0) {
517 if (m == 0)
518 return (EINVAL);
519 count = m->m_len - off > len ? len : m->m_len - off;
520 bcopy(mtod(m, caddr_t) + off, out_data, count);
521 len -= count;
522 out_data = ((char *)out_data) + count;
523 off = 0;
524 m = m->m_next;
525 }
526
527 return (0);
528 }
529
530 int
531 mbuf_mclhasreference(mbuf_t mbuf)
532 {
533 if ((mbuf->m_flags & M_EXT))
534 return (m_mclhasreference(mbuf));
535 else
536 return (0);
537 }
538
539
540 /* mbuf header */
541 mbuf_t
542 mbuf_next(const mbuf_t mbuf)
543 {
544 return (mbuf->m_next);
545 }
546
547 errno_t
548 mbuf_setnext(mbuf_t mbuf, mbuf_t next)
549 {
550 if (next && ((next)->m_nextpkt != NULL ||
551 (next)->m_type == MT_FREE))
552 return (EINVAL);
553 mbuf->m_next = next;
554
555 return (0);
556 }
557
558 mbuf_t
559 mbuf_nextpkt(const mbuf_t mbuf)
560 {
561 return (mbuf->m_nextpkt);
562 }
563
564 void
565 mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
566 {
567 mbuf->m_nextpkt = nextpkt;
568 }
569
570 size_t
571 mbuf_len(const mbuf_t mbuf)
572 {
573 return (mbuf->m_len);
574 }
575
576 void
577 mbuf_setlen(mbuf_t mbuf, size_t len)
578 {
579 mbuf->m_len = len;
580 }
581
582 size_t
583 mbuf_maxlen(const mbuf_t mbuf)
584 {
585 if (mbuf->m_flags & M_EXT)
586 return (mbuf->m_ext.ext_size);
587 return (&mbuf->m_dat[MLEN] - ((char *)mbuf_datastart(mbuf)));
588 }
589
590 mbuf_type_t
591 mbuf_type(const mbuf_t mbuf)
592 {
593 return (mbuf->m_type);
594 }
595
596 errno_t
597 mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
598 {
599 if (new_type == MBUF_TYPE_FREE)
600 return (EINVAL);
601
602 m_mchtype(mbuf, new_type);
603
604 return (0);
605 }
606
607 mbuf_flags_t
608 mbuf_flags(const mbuf_t mbuf)
609 {
610 return (mbuf->m_flags & mbuf_flags_mask);
611 }
612
613 errno_t
614 mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
615 {
616 errno_t ret = 0;
617 mbuf_flags_t oflags = mbuf->m_flags;
618
619 /*
620 * 1. Return error if public but un-alterable flags are changed
621 * in flags argument.
622 * 2. Return error if bits other than public flags are set in passed
623 * flags argument.
624 * Please note that private flag bits must be passed as reset by
625 * kexts, as they must use mbuf_flags KPI to get current set of
626 * mbuf flags and mbuf_flags KPI does not expose private flags.
627 */
628 if ((flags ^ oflags) & mbuf_cflags_mask) {
629 ret = EINVAL;
630 } else if (flags & ~mbuf_flags_mask) {
631 ret = EINVAL;
632 } else {
633 mbuf->m_flags = flags | (mbuf->m_flags & ~mbuf_flags_mask);
634 /*
635 * If M_PKTHDR bit has changed, we have work to do;
636 * m_reinit() will take care of setting/clearing the
637 * bit, as well as the rest of bookkeeping.
638 */
639 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) {
640 mbuf->m_flags ^= M_PKTHDR; /* restore */
641 ret = m_reinit(mbuf,
642 (mbuf->m_flags & M_PKTHDR) ? 0 : 1);
643 }
644 }
645
646 return (ret);
647 }
648
649 errno_t
650 mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
651 {
652 errno_t ret = 0;
653
654 if (mask & (~mbuf_flags_mask | mbuf_cflags_mask)) {
655 ret = EINVAL;
656 } else {
657 mbuf_flags_t oflags = mbuf->m_flags;
658 mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
659 /*
660 * If M_PKTHDR bit has changed, we have work to do;
661 * m_reinit() will take care of setting/clearing the
662 * bit, as well as the rest of bookkeeping.
663 */
664 if ((oflags ^ mbuf->m_flags) & M_PKTHDR) {
665 mbuf->m_flags ^= M_PKTHDR; /* restore */
666 ret = m_reinit(mbuf,
667 (mbuf->m_flags & M_PKTHDR) ? 0 : 1);
668 }
669 }
670
671 return (ret);
672 }
673
674 errno_t
675 mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
676 {
677 if (((src)->m_flags & M_PKTHDR) == 0)
678 return (EINVAL);
679
680 m_copy_pkthdr(dest, src);
681
682 return (0);
683 }
684
685 size_t
686 mbuf_pkthdr_len(const mbuf_t mbuf)
687 {
688 return (mbuf->m_pkthdr.len);
689 }
690
691 __private_extern__ size_t
692 mbuf_pkthdr_maxlen(mbuf_t m)
693 {
694 size_t maxlen = 0;
695 mbuf_t n = m;
696
697 while (n) {
698 maxlen += mbuf_maxlen(n);
699 n = mbuf_next(n);
700 }
701 return (maxlen);
702 }
703
704 void
705 mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
706 {
707 mbuf->m_pkthdr.len = len;
708 }
709
710 void
711 mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
712 {
713 mbuf->m_pkthdr.len += amount;
714 }
715
716 ifnet_t
717 mbuf_pkthdr_rcvif(const mbuf_t mbuf)
718 {
719 /*
720 * If we reference count ifnets, we should take a reference here
721 * before returning
722 */
723 return (mbuf->m_pkthdr.rcvif);
724 }
725
726 errno_t
727 mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
728 {
729 /* May want to walk ifnet list to determine if interface is valid */
730 mbuf->m_pkthdr.rcvif = (struct ifnet *)ifnet;
731 return (0);
732 }
733
734 void*
735 mbuf_pkthdr_header(const mbuf_t mbuf)
736 {
737 return (mbuf->m_pkthdr.pkt_hdr);
738 }
739
740 void
741 mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
742 {
743 mbuf->m_pkthdr.pkt_hdr = (void*)header;
744 }
745
746 void
747 mbuf_inbound_modified(mbuf_t mbuf)
748 {
749 /* Invalidate hardware generated checksum flags */
750 mbuf->m_pkthdr.csum_flags = 0;
751 }
752
753 void
754 mbuf_outbound_finalize(struct mbuf *m, u_int32_t pf, size_t o)
755 {
756 /* Generate the packet in software, client needs it */
757 switch (pf) {
758 case PF_INET:
759 (void) in_finalize_cksum(m, o, m->m_pkthdr.csum_flags);
760 break;
761
762 case PF_INET6:
763 #if INET6
764 /*
765 * Checksum offload should not have been enabled when
766 * extension headers exist; indicate that the callee
767 * should skip such case by setting optlen to -1.
768 */
769 (void) in6_finalize_cksum(m, o, -1, -1, m->m_pkthdr.csum_flags);
770 #endif /* INET6 */
771 break;
772
773 default:
774 break;
775 }
776 }
777
778 errno_t
779 mbuf_set_vlan_tag(
780 mbuf_t mbuf,
781 u_int16_t vlan)
782 {
783 mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
784 mbuf->m_pkthdr.vlan_tag = vlan;
785
786 return (0);
787 }
788
789 errno_t
790 mbuf_get_vlan_tag(
791 mbuf_t mbuf,
792 u_int16_t *vlan)
793 {
794 if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
795 return (ENXIO); // No vlan tag set
796
797 *vlan = mbuf->m_pkthdr.vlan_tag;
798
799 return (0);
800 }
801
802 errno_t
803 mbuf_clear_vlan_tag(
804 mbuf_t mbuf)
805 {
806 mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
807 mbuf->m_pkthdr.vlan_tag = 0;
808
809 return (0);
810 }
811
812 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
813 MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP |
814 MBUF_CSUM_PARTIAL | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6;
815
816 errno_t
817 mbuf_set_csum_requested(
818 mbuf_t mbuf,
819 mbuf_csum_request_flags_t request,
820 u_int32_t value)
821 {
822 request &= mbuf_valid_csum_request_flags;
823 mbuf->m_pkthdr.csum_flags =
824 (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
825 mbuf->m_pkthdr.csum_data = value;
826
827 return (0);
828 }
829
830 static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags =
831 MBUF_TSO_IPV4 | MBUF_TSO_IPV6;
832
833 errno_t
834 mbuf_get_tso_requested(
835 mbuf_t mbuf,
836 mbuf_tso_request_flags_t *request,
837 u_int32_t *value)
838 {
839 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
840 request == NULL || value == NULL)
841 return (EINVAL);
842
843 *request = mbuf->m_pkthdr.csum_flags;
844 *request &= mbuf_valid_tso_request_flags;
845 if (*request && value != NULL)
846 *value = mbuf->m_pkthdr.tso_segsz;
847
848 return (0);
849 }
850
851 errno_t
852 mbuf_get_csum_requested(
853 mbuf_t mbuf,
854 mbuf_csum_request_flags_t *request,
855 u_int32_t *value)
856 {
857 *request = mbuf->m_pkthdr.csum_flags;
858 *request &= mbuf_valid_csum_request_flags;
859 if (value != NULL) {
860 *value = mbuf->m_pkthdr.csum_data;
861 }
862
863 return (0);
864 }
865
866 errno_t
867 mbuf_clear_csum_requested(
868 mbuf_t mbuf)
869 {
870 mbuf->m_pkthdr.csum_flags &= 0xffff0000;
871 mbuf->m_pkthdr.csum_data = 0;
872
873 return (0);
874 }
875
876 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
877 MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
878 MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_PARTIAL;
879
880 errno_t
881 mbuf_set_csum_performed(
882 mbuf_t mbuf,
883 mbuf_csum_performed_flags_t performed,
884 u_int32_t value)
885 {
886 performed &= mbuf_valid_csum_performed_flags;
887 mbuf->m_pkthdr.csum_flags =
888 (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
889 mbuf->m_pkthdr.csum_data = value;
890
891 return (0);
892 }
893
894 errno_t
895 mbuf_get_csum_performed(
896 mbuf_t mbuf,
897 mbuf_csum_performed_flags_t *performed,
898 u_int32_t *value)
899 {
900 *performed =
901 mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
902 *value = mbuf->m_pkthdr.csum_data;
903
904 return (0);
905 }
906
907 errno_t
908 mbuf_clear_csum_performed(
909 mbuf_t mbuf)
910 {
911 mbuf->m_pkthdr.csum_flags &= 0xffff0000;
912 mbuf->m_pkthdr.csum_data = 0;
913
914 return (0);
915 }
916
917 errno_t
918 mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
919 u_int16_t *csum)
920 {
921 if (mbuf == NULL || length == 0 || csum == NULL ||
922 (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
923 return (EINVAL);
924
925 *csum = inet_cksum(mbuf, protocol, offset, length);
926 return (0);
927 }
928
929 #if INET6
930 errno_t
931 mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
932 u_int16_t *csum)
933 {
934 if (mbuf == NULL || length == 0 || csum == NULL ||
935 (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
936 return (EINVAL);
937
938 *csum = inet6_cksum(mbuf, protocol, offset, length);
939 return (0);
940 }
941 #else /* INET6 */
942 errno_t
943 mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
944 __unused u_int32_t offset, __unused u_int32_t length,
945 __unused u_int16_t *csum)
946 {
947 panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
948 return (0);
949 }
950
951 u_int16_t
952 inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
953 __unused unsigned int off, __unused unsigned int len)
954 {
955 panic("inet6_cksum() doesn't exist on this platform\n");
956 return (0);
957 }
958
959 void nd6_lookup_ipv6(void);
960 void
961 nd6_lookup_ipv6(void)
962 {
963 panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
964 }
965
966 int
967 in6addr_local(__unused struct in6_addr *a)
968 {
969 panic("in6addr_local() doesn't exist on this platform\n");
970 return (0);
971 }
972
973 void nd6_storelladdr(void);
974 void
975 nd6_storelladdr(void)
976 {
977 panic("nd6_storelladdr() doesn't exist on this platform\n");
978 }
979 #endif /* INET6 */
980
981 /*
982 * Mbuf tag KPIs
983 */
984
985 #define MTAG_FIRST_ID FIRST_KPI_STR_ID
986
987 errno_t
988 mbuf_tag_id_find(
989 const char *string,
990 mbuf_tag_id_t *out_id)
991 {
992 return (net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1));
993 }
994
995 errno_t
996 mbuf_tag_allocate(
997 mbuf_t mbuf,
998 mbuf_tag_id_t id,
999 mbuf_tag_type_t type,
1000 size_t length,
1001 mbuf_how_t how,
1002 void** data_p)
1003 {
1004 struct m_tag *tag;
1005 u_int32_t mtag_id_first, mtag_id_last;
1006
1007 if (data_p != NULL)
1008 *data_p = NULL;
1009
1010 /* Sanity check parameters */
1011 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1012 NSI_MBUF_TAG);
1013 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1014 id < mtag_id_first || id > mtag_id_last || length < 1 ||
1015 (length & 0xffff0000) != 0 || data_p == NULL) {
1016 return (EINVAL);
1017 }
1018
1019 /* Make sure this mtag hasn't already been allocated */
1020 tag = m_tag_locate(mbuf, id, type, NULL);
1021 if (tag != NULL) {
1022 return (EEXIST);
1023 }
1024
1025 /* Allocate an mtag */
1026 tag = m_tag_create(id, type, length, how, mbuf);
1027 if (tag == NULL) {
1028 return (how == M_WAITOK ? ENOMEM : EWOULDBLOCK);
1029 }
1030
1031 /* Attach the mtag and set *data_p */
1032 m_tag_prepend(mbuf, tag);
1033 *data_p = tag + 1;
1034
1035 return (0);
1036 }
1037
1038 errno_t
1039 mbuf_tag_find(
1040 mbuf_t mbuf,
1041 mbuf_tag_id_t id,
1042 mbuf_tag_type_t type,
1043 size_t *length,
1044 void **data_p)
1045 {
1046 struct m_tag *tag;
1047 u_int32_t mtag_id_first, mtag_id_last;
1048
1049 if (length != NULL)
1050 *length = 0;
1051 if (data_p != NULL)
1052 *data_p = NULL;
1053
1054 /* Sanity check parameters */
1055 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1056 NSI_MBUF_TAG);
1057 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1058 id < mtag_id_first || id > mtag_id_last || length == NULL ||
1059 data_p == NULL) {
1060 return (EINVAL);
1061 }
1062
1063 /* Locate an mtag */
1064 tag = m_tag_locate(mbuf, id, type, NULL);
1065 if (tag == NULL) {
1066 return (ENOENT);
1067 }
1068
1069 /* Copy out the pointer to the data and the lenght value */
1070 *length = tag->m_tag_len;
1071 *data_p = tag + 1;
1072
1073 return (0);
1074 }
1075
1076 void
1077 mbuf_tag_free(
1078 mbuf_t mbuf,
1079 mbuf_tag_id_t id,
1080 mbuf_tag_type_t type)
1081 {
1082 struct m_tag *tag;
1083 u_int32_t mtag_id_first, mtag_id_last;
1084
1085 /* Sanity check parameters */
1086 (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last,
1087 NSI_MBUF_TAG);
1088 if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
1089 id < mtag_id_first || id > mtag_id_last)
1090 return;
1091
1092 tag = m_tag_locate(mbuf, id, type, NULL);
1093 if (tag == NULL) {
1094 return;
1095 }
1096
1097 m_tag_delete(mbuf, tag);
1098 }
1099
1100 /*
1101 * Maximum length of driver auxiliary data; keep this small to
1102 * fit in a single mbuf to avoid wasting memory, rounded down to
1103 * the nearest 64-bit boundary. This takes into account mbuf
1104 * tag-related (m_taghdr + m_tag) as well m_drvaux_tag structs.
1105 */
1106 #define MBUF_DRVAUX_MAXLEN \
1107 P2ROUNDDOWN(MLEN - sizeof (struct m_taghdr) - \
1108 M_TAG_ALIGN(sizeof (struct m_drvaux_tag)), sizeof (uint64_t))
1109
1110 errno_t
1111 mbuf_add_drvaux(mbuf_t mbuf, mbuf_how_t how, u_int32_t family,
1112 u_int32_t subfamily, size_t length, void **data_p)
1113 {
1114 struct m_drvaux_tag *p;
1115 struct m_tag *tag;
1116
1117 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) ||
1118 length == 0 || length > MBUF_DRVAUX_MAXLEN)
1119 return (EINVAL);
1120
1121 if (data_p != NULL)
1122 *data_p = NULL;
1123
1124 /* Check if one is already associated */
1125 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1126 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL)
1127 return (EEXIST);
1128
1129 /* Tag is (m_drvaux_tag + module specific data) */
1130 if ((tag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_DRVAUX,
1131 sizeof (*p) + length, how, mbuf)) == NULL)
1132 return ((how == MBUF_WAITOK) ? ENOMEM : EWOULDBLOCK);
1133
1134 p = (struct m_drvaux_tag *)(tag + 1);
1135 p->da_family = family;
1136 p->da_subfamily = subfamily;
1137 p->da_length = length;
1138
1139 /* Associate the tag */
1140 m_tag_prepend(mbuf, tag);
1141
1142 if (data_p != NULL)
1143 *data_p = (p + 1);
1144
1145 return (0);
1146 }
1147
1148 errno_t
1149 mbuf_find_drvaux(mbuf_t mbuf, u_int32_t *family_p, u_int32_t *subfamily_p,
1150 u_int32_t *length_p, void **data_p)
1151 {
1152 struct m_drvaux_tag *p;
1153 struct m_tag *tag;
1154
1155 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR) || data_p == NULL)
1156 return (EINVAL);
1157
1158 *data_p = NULL;
1159
1160 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1161 KERNEL_TAG_TYPE_DRVAUX, NULL)) == NULL)
1162 return (ENOENT);
1163
1164 /* Must be at least size of m_drvaux_tag */
1165 VERIFY(tag->m_tag_len >= sizeof (*p));
1166
1167 p = (struct m_drvaux_tag *)(tag + 1);
1168 VERIFY(p->da_length > 0 && p->da_length <= MBUF_DRVAUX_MAXLEN);
1169
1170 if (family_p != NULL)
1171 *family_p = p->da_family;
1172 if (subfamily_p != NULL)
1173 *subfamily_p = p->da_subfamily;
1174 if (length_p != NULL)
1175 *length_p = p->da_length;
1176
1177 *data_p = (p + 1);
1178
1179 return (0);
1180 }
1181
1182 void
1183 mbuf_del_drvaux(mbuf_t mbuf)
1184 {
1185 struct m_tag *tag;
1186
1187 if (mbuf == NULL || !(mbuf->m_flags & M_PKTHDR))
1188 return;
1189
1190 if ((tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID,
1191 KERNEL_TAG_TYPE_DRVAUX, NULL)) != NULL)
1192 m_tag_delete(mbuf, tag);
1193 }
1194
1195 /* mbuf stats */
1196 void
1197 mbuf_stats(struct mbuf_stat *stats)
1198 {
1199 stats->mbufs = mbstat.m_mbufs;
1200 stats->clusters = mbstat.m_clusters;
1201 stats->clfree = mbstat.m_clfree;
1202 stats->drops = mbstat.m_drops;
1203 stats->wait = mbstat.m_wait;
1204 stats->drain = mbstat.m_drain;
1205 __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
1206 stats->mcfail = mbstat.m_mcfail;
1207 stats->mpfail = mbstat.m_mpfail;
1208 stats->msize = mbstat.m_msize;
1209 stats->mclbytes = mbstat.m_mclbytes;
1210 stats->minclsize = mbstat.m_minclsize;
1211 stats->mlen = mbstat.m_mlen;
1212 stats->mhlen = mbstat.m_mhlen;
1213 stats->bigclusters = mbstat.m_bigclusters;
1214 stats->bigclfree = mbstat.m_bigclfree;
1215 stats->bigmclbytes = mbstat.m_bigmclbytes;
1216 }
1217
1218 errno_t
1219 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks,
1220 mbuf_t *mbuf)
1221 {
1222 errno_t error;
1223 struct mbuf *m;
1224 unsigned int numpkts = 1;
1225 unsigned int numchunks = maxchunks ? *maxchunks : 0;
1226
1227 if (packetlen == 0) {
1228 error = EINVAL;
1229 goto out;
1230 }
1231 m = m_allocpacket_internal(&numpkts, packetlen,
1232 maxchunks ? &numchunks : NULL, how, 1, 0);
1233 if (m == 0) {
1234 if (maxchunks && *maxchunks && numchunks > *maxchunks)
1235 error = ENOBUFS;
1236 else
1237 error = ENOMEM;
1238 } else {
1239 if (maxchunks)
1240 *maxchunks = numchunks;
1241 error = 0;
1242 *mbuf = m;
1243 }
1244 out:
1245 return (error);
1246 }
1247
1248 errno_t
1249 mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen,
1250 unsigned int *maxchunks, mbuf_t *mbuf)
1251 {
1252 errno_t error;
1253 struct mbuf *m;
1254 unsigned int numchunks = maxchunks ? *maxchunks : 0;
1255
1256 if (numpkts == 0) {
1257 error = EINVAL;
1258 goto out;
1259 }
1260 if (packetlen == 0) {
1261 error = EINVAL;
1262 goto out;
1263 }
1264 m = m_allocpacket_internal(&numpkts, packetlen,
1265 maxchunks ? &numchunks : NULL, how, 1, 0);
1266 if (m == 0) {
1267 if (maxchunks && *maxchunks && numchunks > *maxchunks)
1268 error = ENOBUFS;
1269 else
1270 error = ENOMEM;
1271 } else {
1272 if (maxchunks)
1273 *maxchunks = numchunks;
1274 error = 0;
1275 *mbuf = m;
1276 }
1277 out:
1278 return (error);
1279 }
1280
1281 __private_extern__ size_t
1282 mbuf_pkt_list_len(mbuf_t m)
1283 {
1284 size_t len = 0;
1285 mbuf_t n = m;
1286
1287 while (n) {
1288 len += mbuf_pkthdr_len(n);
1289 n = mbuf_nextpkt(n);
1290 }
1291 return (len);
1292 }
1293
1294 __private_extern__ size_t
1295 mbuf_pkt_list_maxlen(mbuf_t m)
1296 {
1297 size_t maxlen = 0;
1298 mbuf_t n = m;
1299
1300 while (n) {
1301 maxlen += mbuf_pkthdr_maxlen(n);
1302 n = mbuf_nextpkt(n);
1303 }
1304 return (maxlen);
1305 }
1306
1307 /*
1308 * mbuf_copyback differs from m_copyback in a few ways:
1309 * 1) mbuf_copyback will allocate clusters for new mbufs we append
1310 * 2) mbuf_copyback will grow the last mbuf in the chain if possible
1311 * 3) mbuf_copyback reports whether or not the operation succeeded
1312 * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
1313 */
1314 errno_t
1315 mbuf_copyback(
1316 mbuf_t m,
1317 size_t off,
1318 size_t len,
1319 const void *data,
1320 mbuf_how_t how)
1321 {
1322 size_t mlen;
1323 mbuf_t m_start = m;
1324 mbuf_t n;
1325 int totlen = 0;
1326 errno_t result = 0;
1327 const char *cp = data;
1328
1329 if (m == NULL || len == 0 || data == NULL)
1330 return (EINVAL);
1331
1332 while (off > (mlen = m->m_len)) {
1333 off -= mlen;
1334 totlen += mlen;
1335 if (m->m_next == 0) {
1336 n = m_getclr(how, m->m_type);
1337 if (n == 0) {
1338 result = ENOBUFS;
1339 goto out;
1340 }
1341 n->m_len = MIN(MLEN, len + off);
1342 m->m_next = n;
1343 }
1344 m = m->m_next;
1345 }
1346
1347 while (len > 0) {
1348 mlen = MIN(m->m_len - off, len);
1349 if (mlen < len && m->m_next == NULL &&
1350 mbuf_trailingspace(m) > 0) {
1351 size_t grow = MIN(mbuf_trailingspace(m), len - mlen);
1352 mlen += grow;
1353 m->m_len += grow;
1354 }
1355 bcopy(cp, off + (char *)mbuf_data(m), (unsigned)mlen);
1356 cp += mlen;
1357 len -= mlen;
1358 mlen += off;
1359 off = 0;
1360 totlen += mlen;
1361 if (len == 0)
1362 break;
1363 if (m->m_next == 0) {
1364 n = m_get(how, m->m_type);
1365 if (n == NULL) {
1366 result = ENOBUFS;
1367 goto out;
1368 }
1369 if (len > MINCLSIZE) {
1370 /*
1371 * cluster allocation failure is okay,
1372 * we can grow chain
1373 */
1374 mbuf_mclget(how, m->m_type, &n);
1375 }
1376 n->m_len = MIN(mbuf_maxlen(n), len);
1377 m->m_next = n;
1378 }
1379 m = m->m_next;
1380 }
1381
1382 out:
1383 if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
1384 m_start->m_pkthdr.len = totlen;
1385
1386 return (result);
1387 }
1388
1389 u_int32_t
1390 mbuf_get_mlen(void)
1391 {
1392 return (_MLEN);
1393 }
1394
1395 u_int32_t
1396 mbuf_get_mhlen(void)
1397 {
1398 return (_MHLEN);
1399 }
1400
1401 u_int32_t
1402 mbuf_get_minclsize(void)
1403 {
1404 return (MHLEN + MLEN);
1405 }
1406
1407 u_int32_t
1408 mbuf_get_traffic_class_max_count(void)
1409 {
1410 return (MBUF_TC_MAX);
1411 }
1412
1413 errno_t
1414 mbuf_get_traffic_class_index(mbuf_traffic_class_t tc, u_int32_t *index)
1415 {
1416 if (index == NULL || (u_int32_t)tc >= MBUF_TC_MAX)
1417 return (EINVAL);
1418
1419 *index = MBUF_SCIDX(m_service_class_from_val(MBUF_TC2SCVAL(tc)));
1420 return (0);
1421 }
1422
1423 mbuf_traffic_class_t
1424 mbuf_get_traffic_class(mbuf_t m)
1425 {
1426 if (m == NULL || !(m->m_flags & M_PKTHDR))
1427 return (MBUF_TC_BE);
1428
1429 return (m_get_traffic_class(m));
1430 }
1431
1432 errno_t
1433 mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc)
1434 {
1435 if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1436 ((u_int32_t)tc >= MBUF_TC_MAX))
1437 return (EINVAL);
1438
1439 return (m_set_traffic_class(m, tc));
1440 }
1441
1442 int
1443 mbuf_is_traffic_class_privileged(mbuf_t m)
1444 {
1445 if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1446 !MBUF_VALID_SC(m->m_pkthdr.pkt_svc))
1447 return (0);
1448
1449 return ((m->m_pkthdr.pkt_flags & PKTF_PRIO_PRIVILEGED) ? 1 : 0);
1450 }
1451
1452 u_int32_t
1453 mbuf_get_service_class_max_count(void)
1454 {
1455 return (MBUF_SC_MAX_CLASSES);
1456 }
1457
1458 errno_t
1459 mbuf_get_service_class_index(mbuf_svc_class_t sc, u_int32_t *index)
1460 {
1461 if (index == NULL || !MBUF_VALID_SC(sc))
1462 return (EINVAL);
1463
1464 *index = MBUF_SCIDX(sc);
1465 return (0);
1466 }
1467
1468 mbuf_svc_class_t
1469 mbuf_get_service_class(mbuf_t m)
1470 {
1471 if (m == NULL || !(m->m_flags & M_PKTHDR))
1472 return (MBUF_SC_BE);
1473
1474 return (m_get_service_class(m));
1475 }
1476
1477 errno_t
1478 mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc)
1479 {
1480 if (m == NULL || !(m->m_flags & M_PKTHDR))
1481 return (EINVAL);
1482
1483 return (m_set_service_class(m, sc));
1484 }
1485
1486 errno_t
1487 mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp)
1488 {
1489 u_int32_t flags;
1490
1491 if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL)
1492 return (EINVAL);
1493
1494 *flagsp = 0;
1495 flags = m->m_pkthdr.pkt_flags;
1496 if ((flags & (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR)) ==
1497 (PKTF_INET_RESOLVE|PKTF_RESOLVE_RTR))
1498 *flagsp |= MBUF_PKTAUXF_INET_RESOLVE_RTR;
1499 if ((flags & (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR)) ==
1500 (PKTF_INET6_RESOLVE|PKTF_RESOLVE_RTR))
1501 *flagsp |= MBUF_PKTAUXF_INET6_RESOLVE_RTR;
1502
1503 /* These 2 flags are mutually exclusive */
1504 VERIFY((*flagsp &
1505 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) !=
1506 (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR));
1507
1508 return (0);
1509 }
1510
1511 errno_t
1512 mbuf_get_driver_scratch(mbuf_t m, u_int8_t **area, size_t *area_len)
1513 {
1514 if (m == NULL || area == NULL || area_len == NULL ||
1515 !(m->m_flags & M_PKTHDR))
1516 return (EINVAL);
1517
1518 *area_len = m_scratch_get(m, area);
1519 return (0);
1520 }
1521
1522 errno_t
1523 mbuf_get_unsent_data_bytes(const mbuf_t m, u_int32_t *unsent_data)
1524 {
1525 if (m == NULL || unsent_data == NULL || !(m->m_flags & M_PKTHDR))
1526 return (EINVAL);
1527
1528 if (!(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA))
1529 return (EINVAL);
1530
1531 *unsent_data = m->m_pkthdr.bufstatus_if +
1532 m->m_pkthdr.bufstatus_sndbuf;
1533 return (0);
1534 }
1535
1536 errno_t
1537 mbuf_get_buffer_status(const mbuf_t m, mbuf_buffer_status_t *buf_status)
1538 {
1539 if (m == NULL || buf_status == NULL || !(m->m_flags & M_PKTHDR) ||
1540 !(m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA))
1541 return (EINVAL);
1542
1543 buf_status->buf_interface = m->m_pkthdr.bufstatus_if;
1544 buf_status->buf_sndbuf = m->m_pkthdr.bufstatus_sndbuf;
1545 return (0);
1546 }
1547
1548 errno_t
1549 mbuf_pkt_new_flow(const mbuf_t m, u_int32_t *retval)
1550 {
1551 if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR))
1552 return (EINVAL);
1553 if (m->m_pkthdr.pkt_flags & PKTF_NEW_FLOW)
1554 *retval = 1;
1555 else
1556 *retval = 0;
1557 return (0);
1558 }
1559
1560 errno_t
1561 mbuf_last_pkt(const mbuf_t m, u_int32_t *retval)
1562 {
1563 if (m == NULL || retval == NULL || !(m->m_flags & M_PKTHDR))
1564 return (EINVAL);
1565 if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT)
1566 *retval = 1;
1567 else
1568 *retval = 0;
1569 return (0);
1570 }
1571
1572 errno_t
1573 mbuf_get_timestamp(mbuf_t m, u_int64_t *ts, boolean_t *valid)
1574 {
1575 if (m == NULL || !(m->m_flags & M_PKTHDR) || ts == NULL ||
1576 valid == NULL)
1577 return (EINVAL);
1578
1579 if ((m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) {
1580 *valid = FALSE;
1581 *ts = 0;
1582 } else {
1583 *valid = TRUE;
1584 *ts = m->m_pkthdr.pkt_timestamp;
1585 }
1586 return (0);
1587 }
1588
1589 errno_t
1590 mbuf_set_timestamp(mbuf_t m, u_int64_t ts, boolean_t valid)
1591 {
1592 if (m == NULL || !(m->m_flags & M_PKTHDR))
1593 return (EINVAL);
1594
1595 if (valid == FALSE) {
1596 m->m_pkthdr.pkt_flags &= ~PKTF_DRV_TS_VALID;
1597 m->m_pkthdr.pkt_timestamp = 0;
1598 } else {
1599 m->m_pkthdr.pkt_flags |= PKTF_DRV_TS_VALID;
1600 m->m_pkthdr.pkt_timestamp = ts;
1601 }
1602 return (0);
1603 }
1604
1605 errno_t
1606 mbuf_get_status(mbuf_t m, kern_return_t *status)
1607 {
1608 if (m == NULL || !(m->m_flags & M_PKTHDR) || status == NULL)
1609 return (EINVAL);
1610
1611 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1612 *status = 0;
1613 } else {
1614 *status = m->m_pkthdr.drv_tx_status;
1615 }
1616 return (0);
1617 }
1618
1619 static void
1620 driver_mtag_init(mbuf_t m)
1621 {
1622 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1623 m->m_pkthdr.pkt_flags |= PKTF_DRIVER_MTAG;
1624 bzero(&m->m_pkthdr.driver_mtag,
1625 sizeof(m->m_pkthdr.driver_mtag));
1626 }
1627 }
1628
1629 errno_t
1630 mbuf_set_status(mbuf_t m, kern_return_t status)
1631 {
1632 if (m == NULL || !(m->m_flags & M_PKTHDR))
1633 return (EINVAL);
1634
1635 driver_mtag_init(m);
1636
1637 m->m_pkthdr.drv_tx_status = status;
1638
1639 return (0);
1640 }
1641
1642 errno_t
1643 mbuf_get_flowid(mbuf_t m, u_int16_t *flowid)
1644 {
1645 if (m == NULL || !(m->m_flags & M_PKTHDR) || flowid == NULL)
1646 return (EINVAL);
1647
1648 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1649 *flowid = 0;
1650 } else {
1651 *flowid = m->m_pkthdr.drv_flowid;
1652 }
1653 return (0);
1654 }
1655
1656 errno_t
1657 mbuf_set_flowid(mbuf_t m, u_int16_t flowid)
1658 {
1659 if (m == NULL || !(m->m_flags & M_PKTHDR))
1660 return (EINVAL);
1661
1662 driver_mtag_init(m);
1663
1664 m->m_pkthdr.drv_flowid = flowid;
1665
1666 return (0);
1667 }
1668
1669 errno_t
1670 mbuf_get_tx_compl_data(mbuf_t m, uintptr_t *arg, uintptr_t *data)
1671 {
1672 if (m == NULL || !(m->m_flags & M_PKTHDR) || arg == NULL ||
1673 data == NULL)
1674 return (EINVAL);
1675
1676 if ((m->m_pkthdr.pkt_flags & PKTF_DRIVER_MTAG) == 0) {
1677 *arg = 0;
1678 *data = 0;
1679 } else {
1680 *arg = m->m_pkthdr.drv_tx_compl_arg;
1681 *data = m->m_pkthdr.drv_tx_compl_data;
1682 }
1683 return (0);
1684 }
1685
1686 errno_t
1687 mbuf_set_tx_compl_data(mbuf_t m, uintptr_t arg, uintptr_t data)
1688 {
1689 if (m == NULL || !(m->m_flags & M_PKTHDR))
1690 return (EINVAL);
1691
1692 driver_mtag_init(m);
1693
1694 m->m_pkthdr.drv_tx_compl_arg = arg;
1695 m->m_pkthdr.drv_tx_compl_data = data;
1696
1697 return (0);
1698 }
1699
1700 static u_int32_t
1701 get_tx_compl_callback_index_locked(mbuf_tx_compl_func callback)
1702 {
1703 u_int32_t i;
1704
1705 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1706 if (mbuf_tx_compl_table[i] == callback) {
1707 return (i);
1708 }
1709 }
1710 return (UINT32_MAX);
1711 }
1712
1713 static u_int32_t
1714 get_tx_compl_callback_index(mbuf_tx_compl_func callback)
1715 {
1716 u_int32_t i;
1717
1718 lck_rw_lock_shared(mbuf_tx_compl_tbl_lock);
1719
1720 i = get_tx_compl_callback_index_locked(callback);
1721
1722 lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock);
1723
1724 return (i);
1725 }
1726
1727 errno_t
1728 mbuf_register_tx_compl_callback(mbuf_tx_compl_func callback)
1729 {
1730 int i;
1731 errno_t error;
1732
1733 if (callback == NULL)
1734 return (EINVAL);
1735
1736 lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock);
1737
1738 i = get_tx_compl_callback_index_locked(callback);
1739 if (i != -1) {
1740 error = EEXIST;
1741 goto unlock;
1742 }
1743
1744 /* assume the worst */
1745 error = ENOSPC;
1746 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1747 if (mbuf_tx_compl_table[i] == NULL) {
1748 mbuf_tx_compl_table[i] = callback;
1749 error = 0;
1750 goto unlock;
1751 }
1752 }
1753 unlock:
1754 lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock);
1755
1756 return (error);
1757 }
1758
1759 errno_t
1760 mbuf_unregister_tx_compl_callback(mbuf_tx_compl_func callback)
1761 {
1762 int i;
1763 errno_t error;
1764
1765 if (callback == NULL)
1766 return (EINVAL);
1767
1768 lck_rw_lock_exclusive(mbuf_tx_compl_tbl_lock);
1769
1770 /* assume the worst */
1771 error = ENOENT;
1772 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1773 if (mbuf_tx_compl_table[i] == callback) {
1774 mbuf_tx_compl_table[i] = NULL;
1775 error = 0;
1776 goto unlock;
1777 }
1778 }
1779 unlock:
1780 lck_rw_unlock_exclusive(mbuf_tx_compl_tbl_lock);
1781
1782 return (error);
1783 }
1784
1785 errno_t
1786 mbuf_get_timestamp_requested(mbuf_t m, boolean_t *requested)
1787 {
1788 if (m == NULL || !(m->m_flags & M_PKTHDR))
1789 return (EINVAL);
1790
1791 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) {
1792 *requested = FALSE;
1793 } else {
1794 *requested = TRUE;
1795 }
1796 return (0);
1797 }
1798
1799 errno_t
1800 mbuf_set_timestamp_requested(mbuf_t m, uintptr_t *pktid,
1801 mbuf_tx_compl_func callback)
1802 {
1803 size_t i;
1804
1805 if (m == NULL || !(m->m_flags & M_PKTHDR) || callback == NULL ||
1806 pktid == NULL)
1807 return (EINVAL);
1808
1809 i = get_tx_compl_callback_index(callback);
1810 if (i == UINT32_MAX)
1811 return (ENOENT);
1812
1813 #if (DEBUG || DEVELOPMENT)
1814 VERIFY(i < sizeof(m->m_pkthdr.pkt_compl_callbacks));
1815 #endif /* (DEBUG || DEVELOPMENT) */
1816
1817 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0) {
1818 m->m_pkthdr.pkt_compl_callbacks = 0;
1819 m->m_pkthdr.pkt_flags |= PKTF_TX_COMPL_TS_REQ;
1820 m->m_pkthdr.pkt_compl_context =
1821 atomic_add_32_ov(&mbuf_tx_compl_index, 1);
1822
1823 #if (DEBUG || DEVELOPMENT)
1824 if (mbuf_tx_compl_debug != 0) {
1825 OSIncrementAtomic64(&mbuf_tx_compl_outstanding);
1826 }
1827 #endif /* (DEBUG || DEVELOPMENT) */
1828 }
1829 m->m_pkthdr.pkt_compl_callbacks |= (1 << i);
1830 *pktid = m->m_pkthdr.pkt_compl_context;
1831
1832 return (0);
1833 }
1834
1835 void
1836 m_do_tx_compl_callback(struct mbuf *m, struct ifnet *ifp)
1837 {
1838 int i;
1839
1840 if (m == NULL)
1841 return;
1842
1843 if ((m->m_pkthdr.pkt_flags & PKTF_TX_COMPL_TS_REQ) == 0)
1844 return;
1845
1846 #if (DEBUG || DEVELOPMENT)
1847 if (mbuf_tx_compl_debug != 0 && ifp != NULL &&
1848 (ifp->if_xflags & IFXF_TIMESTAMP_ENABLED) != 0 &&
1849 (m->m_pkthdr.pkt_flags & PKTF_DRV_TS_VALID) == 0) {
1850 struct timespec now;
1851
1852 nanouptime(&now);
1853 net_timernsec(&now, &m->m_pkthdr.pkt_timestamp);
1854 }
1855 #endif /* (DEBUG || DEVELOPMENT) */
1856
1857 for (i = 0; i < MAX_MBUF_TX_COMPL_FUNC; i++) {
1858 mbuf_tx_compl_func callback;
1859
1860 if ((m->m_pkthdr.pkt_compl_callbacks & (1 << i)) == 0)
1861 continue;
1862
1863 lck_rw_lock_shared(mbuf_tx_compl_tbl_lock);
1864 callback = mbuf_tx_compl_table[i];
1865 lck_rw_unlock_shared(mbuf_tx_compl_tbl_lock);
1866
1867 if (callback != NULL) {
1868 callback(m->m_pkthdr.pkt_compl_context,
1869 ifp, m->m_pkthdr.pkt_timestamp,
1870 m->m_pkthdr.drv_tx_compl_arg,
1871 m->m_pkthdr.drv_tx_compl_data,
1872 m->m_pkthdr.drv_tx_status);
1873 }
1874 }
1875 m->m_pkthdr.pkt_compl_callbacks = 0;
1876
1877 #if (DEBUG || DEVELOPMENT)
1878 if (mbuf_tx_compl_debug != 0) {
1879 OSDecrementAtomic64(&mbuf_tx_compl_outstanding);
1880 if (ifp == NULL)
1881 atomic_add_64(&mbuf_tx_compl_aborted, 1);
1882 }
1883 #endif /* (DEBUG || DEVELOPMENT) */
1884 }
mirror of XNU