bsd/kern/kpi_mbuf.c

   1 /*
   2  * Copyright (c) 2004-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define __KPI__
  30 //#include <sys/kpi_interface.h>
  31
  32 #include <sys/param.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/mcache.h>
  35 #include <sys/socket.h>
  36 #include <kern/debug.h>
  37 #include <libkern/OSAtomic.h>
  38 #include <kern/kalloc.h>
  39 #include <string.h>
  40 #include <netinet/in.h>
  41
  42 #include "net/net_str_id.h"
  43
  44 static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
  45     MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
  46     MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS);
  47
  48 #define MBUF_PKTAUXF_MASK       \
  49         (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)
  50
  51 void* mbuf_data(mbuf_t mbuf)
  52 {
  53         return mbuf->m_data;
  54 }
  55
  56 void* mbuf_datastart(mbuf_t mbuf)
  57 {
  58         if (mbuf->m_flags & M_EXT)
  59                 return mbuf->m_ext.ext_buf;
  60         if (mbuf->m_flags & M_PKTHDR)
  61                 return mbuf->m_pktdat;
  62         return mbuf->m_dat;
  63 }
  64
  65 errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len)
  66 {
  67         size_t  start = (size_t)((char*)mbuf_datastart(mbuf));
  68         size_t  maxlen = mbuf_maxlen(mbuf);
  69
  70         if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
  71                 return EINVAL;
  72         mbuf->m_data = data;
  73         mbuf->m_len = len;
  74
  75         return 0;
  76 }
  77
  78 errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
  79 {
  80         if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
  81                 return ENOTSUP;
  82         mbuf->m_data = mbuf_datastart(mbuf);
  83         mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
  84
  85         return 0;
  86 }
  87
  88 /* This function is used to provide mcl_to_paddr via symbol indirection,
  89  * please avoid any change in behavior or remove the indirection in
  90  * config/Unsupported*
  91  */
  92 addr64_t mbuf_data_to_physical(void* ptr)
  93 {
  94         return (addr64_t)(uintptr_t)mcl_to_paddr(ptr);
  95 }
  96
  97 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  98 {
  99         /* Must set *mbuf to NULL in failure case */
 100         *mbuf = m_get(how, type);
 101
 102         return (*mbuf == NULL) ? ENOMEM : 0;
 103 }
 104
 105 errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 106 {
 107         /* Must set *mbuf to NULL in failure case */
 108         *mbuf = m_gethdr(how, type);
 109
 110         return (*mbuf == NULL) ? ENOMEM : 0;
 111 }
 112
 113 errno_t
 114 mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
 115     caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t),
 116     size_t extsize, caddr_t extarg)
 117 {
 118         if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0)
 119                 return (EINVAL);
 120
 121         if ((*mbuf = m_clattach(*mbuf, type, extbuf,
 122             extfree, extsize, extarg, how)) == NULL)
 123                 return (ENOMEM);
 124
 125         return (0);
 126 }
 127
 128 errno_t
 129 mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
 130 {
 131         if (size == NULL || *size == 0 || addr == NULL)
 132                 return (EINVAL);
 133
 134         *addr = NULL;
 135
 136         /* Jumbo cluster pool not available? */
 137         if (*size > MBIGCLBYTES && njcl == 0)
 138                 return (ENOTSUP);
 139
 140         if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
 141                 *size = MCLBYTES;
 142         else if (*size > MCLBYTES && *size <= MBIGCLBYTES &&
 143             (*addr = m_bigalloc(how)) != NULL)
 144                 *size = MBIGCLBYTES;
 145         else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES &&
 146             (*addr = m_16kalloc(how)) != NULL)
 147                 *size = M16KCLBYTES;
 148         else
 149                 *size = 0;
 150
 151         if (*addr == NULL)
 152                 return (ENOMEM);
 153
 154         return (0);
 155 }
 156
 157 void
 158 mbuf_freecluster(caddr_t addr, size_t size)
 159 {
 160         if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES)
 161                 panic("%s: invalid size (%ld) for cluster %p", __func__,
 162                     size, (void *)addr);
 163
 164         if (size == MCLBYTES)
 165                 m_mclfree(addr);
 166         else if (size == MBIGCLBYTES)
 167                 m_bigfree(addr, MBIGCLBYTES, NULL);
 168         else if (njcl > 0)
 169                 m_16kfree(addr, M16KCLBYTES, NULL);
 170         else
 171                 panic("%s: freeing jumbo cluster to an empty pool", __func__);
 172 }
 173
 174 errno_t
 175 mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 176 {
 177         /* Must set *mbuf to NULL in failure case */
 178         errno_t error = 0;
 179         int     created = 0;
 180
 181         if (mbuf == NULL)
 182                 return EINVAL;
 183         if (*mbuf == NULL) {
 184                 *mbuf = m_get(how, type);
 185                 if (*mbuf == NULL)
 186                         return ENOMEM;
 187                 created = 1;
 188         }
 189         /*
 190          * At the time this code was written, m_{mclget,mbigget,m16kget}
 191          * would always return the same value that was passed in to it.
 192          */
 193         if (size == MCLBYTES) {
 194                 *mbuf = m_mclget(*mbuf, how);
 195         } else if (size == MBIGCLBYTES) {
 196                 *mbuf = m_mbigget(*mbuf, how);
 197         } else if (size == M16KCLBYTES) {
 198                 if (njcl > 0) {
 199                         *mbuf = m_m16kget(*mbuf, how);
 200                 } else {
 201                         /* Jumbo cluster pool not available? */
 202                         error = ENOTSUP;
 203                         goto out;
 204                 }
 205         } else {
 206                 error = EINVAL;
 207                 goto out;
 208         }
 209         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 210                 error = ENOMEM;
 211 out:
 212         if (created && error != 0) {
 213                 mbuf_free(*mbuf);
 214                 *mbuf = NULL;
 215         }
 216         return error;
 217 }
 218
 219 errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 220 {
 221         /* Must set *mbuf to NULL in failure case */
 222         errno_t error = 0;
 223         int             created = 0;
 224         if (mbuf == NULL) return EINVAL;
 225         if (*mbuf == NULL) {
 226                 error = mbuf_get(how, type, mbuf);
 227                 if (error)
 228                         return error;
 229                 created = 1;
 230         }
 231
 232         /*
 233          * At the time this code was written, m_mclget would always
 234          * return the same value that was passed in to it.
 235          */
 236         *mbuf = m_mclget(*mbuf, how);
 237
 238         if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
 239                 mbuf_free(*mbuf);
 240                 *mbuf = NULL;
 241         }
 242         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 243                 error = ENOMEM;
 244         return error;
 245 }
 246
 247
 248 errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
 249 {
 250         /* Must set *mbuf to NULL in failure case */
 251         errno_t error = 0;
 252
 253         *mbuf = m_getpacket_how(how);
 254
 255         if (*mbuf == NULL) {
 256                 if (how == MBUF_WAITOK)
 257                         error = ENOMEM;
 258                 else
 259                         error = EWOULDBLOCK;
 260         }
 261
 262         return error;
 263 }
 264
 265 /* This function is used to provide m_free via symbol indirection, please avoid
 266  * any change in behavior or remove the indirection in config/Unsupported*
 267  */
 268 mbuf_t mbuf_free(mbuf_t mbuf)
 269 {
 270         return m_free(mbuf);
 271 }
 272
 273 /* This function is used to provide m_freem via symbol indirection, please avoid
 274  * any change in behavior or remove the indirection in config/Unsupported*
 275  */
 276 void mbuf_freem(mbuf_t mbuf)
 277 {
 278         m_freem(mbuf);
 279 }
 280
 281 int     mbuf_freem_list(mbuf_t mbuf)
 282 {
 283         return m_freem_list(mbuf);
 284 }
 285
 286 size_t mbuf_leadingspace(const mbuf_t mbuf)
 287 {
 288         return m_leadingspace(mbuf);
 289 }
 290
 291 /* This function is used to provide m_trailingspace via symbol indirection,
 292  * please avoid any change in behavior or remove the indirection in
 293  * config/Unsupported*
 294  */
 295 size_t mbuf_trailingspace(const mbuf_t mbuf)
 296 {
 297         return m_trailingspace(mbuf);
 298 }
 299
 300 /* Manipulation */
 301 errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len,
 302                                    mbuf_how_t how, mbuf_t *new_mbuf)
 303 {
 304         /* Must set *mbuf to NULL in failure case */
 305         *new_mbuf = m_copym(src, offset, len, how);
 306
 307         return (*new_mbuf == NULL) ? ENOMEM : 0;
 308 }
 309
 310 errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 311 {
 312         /* Must set *new_mbuf to NULL in failure case */
 313         *new_mbuf = m_dup(src, how);
 314
 315         return (*new_mbuf == NULL) ? ENOMEM : 0;
 316 }
 317
 318 errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
 319 {
 320         /* Must set *orig to NULL in failure case */
 321         *orig = m_prepend_2(*orig, len, how);
 322
 323         return (*orig == NULL) ? ENOMEM : 0;
 324 }
 325
 326 errno_t mbuf_split(mbuf_t src, size_t offset,
 327                                         mbuf_how_t how, mbuf_t *new_mbuf)
 328 {
 329         /* Must set *new_mbuf to NULL in failure case */
 330         *new_mbuf = m_split(src, offset, how);
 331
 332         return (*new_mbuf == NULL) ? ENOMEM : 0;
 333 }
 334
 335 errno_t mbuf_pullup(mbuf_t *mbuf, size_t len)
 336 {
 337         /* Must set *mbuf to NULL in failure case */
 338         *mbuf = m_pullup(*mbuf, len);
 339
 340         return (*mbuf == NULL) ? ENOMEM : 0;
 341 }
 342
 343 errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
 344 {
 345         /* Must set *location to NULL in failure case */
 346         int new_offset;
 347         *location = m_pulldown(src, *offset, len, &new_offset);
 348         *offset = new_offset;
 349
 350         return (*location == NULL) ? ENOMEM : 0;
 351 }
 352
 353 /* This function is used to provide m_adj via symbol indirection, please avoid
 354  * any change in behavior or remove the indirection in config/Unsupported*
 355  */
 356 void mbuf_adj(mbuf_t mbuf, int len)
 357 {
 358         m_adj(mbuf, len);
 359 }
 360
 361 errno_t mbuf_adjustlen(mbuf_t m, int amount)
 362 {
 363         /* Verify m_len will be valid after adding amount */
 364         if (amount > 0) {
 365                 int             used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
 366                                            m->m_len;
 367
 368                 if ((size_t)(amount + used) > mbuf_maxlen(m))
 369                         return EINVAL;
 370         }
 371         else if (-amount > m->m_len) {
 372                 return EINVAL;
 373         }
 374
 375         m->m_len += amount;
 376         return 0;
 377 }
 378
 379 mbuf_t
 380 mbuf_concatenate(mbuf_t dst, mbuf_t src)
 381 {
 382         if (dst == NULL)
 383                 return (NULL);
 384
 385         m_cat(dst, src);
 386
 387         /* return dst as is in the current implementation */
 388         return (dst);
 389 }
 390 errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data)
 391 {
 392         /* Copied m_copydata, added error handling (don't just panic) */
 393         int count;
 394         mbuf_t  m = m0;
 395
 396         while (off > 0) {
 397                 if (m == 0)
 398                         return EINVAL;
 399                 if (off < (size_t)m->m_len)
 400                         break;
 401                 off -= m->m_len;
 402                 m = m->m_next;
 403         }
 404         while (len > 0) {
 405                 if (m == 0)
 406                         return EINVAL;
 407                 count = m->m_len - off > len ? len : m->m_len - off;
 408                 bcopy(mtod(m, caddr_t) + off, out_data, count);
 409                 len -= count;
 410                 out_data = ((char*)out_data) + count;
 411                 off = 0;
 412                 m = m->m_next;
 413         }
 414
 415         return 0;
 416 }
 417
 418 int mbuf_mclhasreference(mbuf_t mbuf)
 419 {
 420         if ((mbuf->m_flags & M_EXT))
 421                 return m_mclhasreference(mbuf);
 422         else
 423                 return 0;
 424 }
 425
 426
 427 /* mbuf header */
 428 mbuf_t mbuf_next(const mbuf_t mbuf)
 429 {
 430         return mbuf->m_next;
 431 }
 432
 433 errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 434 {
 435         if (next && ((next)->m_nextpkt != NULL ||
 436                 (next)->m_type == MT_FREE)) return EINVAL;
 437         mbuf->m_next = next;
 438
 439         return 0;
 440 }
 441
 442 mbuf_t mbuf_nextpkt(const mbuf_t mbuf)
 443 {
 444         return mbuf->m_nextpkt;
 445 }
 446
 447 void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 448 {
 449         mbuf->m_nextpkt = nextpkt;
 450 }
 451
 452 size_t mbuf_len(const mbuf_t mbuf)
 453 {
 454         return mbuf->m_len;
 455 }
 456
 457 void mbuf_setlen(mbuf_t mbuf, size_t len)
 458 {
 459         mbuf->m_len = len;
 460 }
 461
 462 size_t mbuf_maxlen(const mbuf_t mbuf)
 463 {
 464         if (mbuf->m_flags & M_EXT)
 465                 return mbuf->m_ext.ext_size;
 466         return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 467 }
 468
 469 mbuf_type_t mbuf_type(const mbuf_t mbuf)
 470 {
 471         return mbuf->m_type;
 472 }
 473
 474 errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 475 {
 476         if (new_type == MBUF_TYPE_FREE) return EINVAL;
 477
 478         m_mchtype(mbuf, new_type);
 479
 480         return 0;
 481 }
 482
 483 mbuf_flags_t mbuf_flags(const mbuf_t mbuf)
 484 {
 485         return mbuf->m_flags & mbuf_flags_mask;
 486 }
 487
 488 errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
 489 {
 490         if ((flags & ~mbuf_flags_mask) != 0) return EINVAL;
 491         mbuf->m_flags = flags |
 492                 (mbuf->m_flags & ~mbuf_flags_mask);
 493
 494         return 0;
 495 }
 496
 497 errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 498 {
 499         if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL;
 500
 501         mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
 502
 503         return 0;
 504 }
 505
 506 errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
 507 {
 508         if (((src)->m_flags & M_PKTHDR) == 0)
 509                 return EINVAL;
 510
 511         m_copy_pkthdr(dest, src);
 512
 513         return 0;
 514 }
 515
 516 size_t mbuf_pkthdr_len(const mbuf_t mbuf)
 517 {
 518         return mbuf->m_pkthdr.len;
 519 }
 520
 521 void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 522 {
 523         mbuf->m_pkthdr.len = len;
 524 }
 525
 526 void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
 527 {
 528         mbuf->m_pkthdr.len += amount;
 529 }
 530
 531 ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf)
 532 {
 533         // If we reference count ifnets, we should take a reference here before returning
 534         return mbuf->m_pkthdr.rcvif;
 535 }
 536
 537 errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 538 {
 539         /* May want to walk ifnet list to determine if interface is valid */
 540         mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet;
 541         return 0;
 542 }
 543
 544 void* mbuf_pkthdr_header(const mbuf_t mbuf)
 545 {
 546         return mbuf->m_pkthdr.header;
 547 }
 548
 549 void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 550 {
 551         mbuf->m_pkthdr.header = (void*)header;
 552 }
 553
 554 void
 555 mbuf_inbound_modified(mbuf_t mbuf)
 556 {
 557         /* Invalidate hardware generated checksum flags */
 558         mbuf->m_pkthdr.csum_flags = 0;
 559 }
 560
 561 extern void in_cksum_offset(struct mbuf* m, size_t ip_offset);
 562 extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset);
 563
 564 void
 565 mbuf_outbound_finalize(mbuf_t mbuf, u_int32_t protocol_family, size_t protocol_offset)
 566 {
 567         if ((mbuf->m_pkthdr.csum_flags &
 568                  (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16 | CSUM_DELAY_IPV6_DATA)) == 0)
 569                 return;
 570
 571         /* Generate the packet in software, client needs it */
 572         switch (protocol_family) {
 573                 case PF_INET:
 574                         if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 575                                 /*
 576                                  * If you're wondering where this lovely code comes
 577                                  * from, we're trying to undo what happens in ip_output.
 578                                  * Look for CSUM_TCP_SUM16 in ip_output.
 579                                  */
 580                                 u_int16_t       first, second;
 581                                 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16;
 582                                 mbuf->m_pkthdr.csum_flags |= CSUM_TCP;
 583                                 first = mbuf->m_pkthdr.csum_data >> 16;
 584                                 second = mbuf->m_pkthdr.csum_data & 0xffff;
 585                                 mbuf->m_pkthdr.csum_data = first - second;
 586                         }
 587                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
 588                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 589                         }
 590
 591                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
 592                                 in_cksum_offset(mbuf, protocol_offset);
 593                         }
 594
 595                         mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP);
 596                         break;
 597
 598                 case PF_INET6:
 599
 600                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IPV6_DATA) {
 601                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 602                         }
 603                         mbuf->m_pkthdr.csum_flags &= ~CSUM_DELAY_IPV6_DATA;
 604                         break;
 605
 606
 607                 default:
 608                         /*
 609                          * Not sure what to do here if anything.
 610                          * Hardware checksum code looked pretty IPv4/IPv6 specific.
 611                          */
 612                         if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_DELAY_IPV6_DATA)) != 0)
 613                                 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 or IPv6 packet (%u)!\n", protocol_family);
 614         }
 615 }
 616
 617 errno_t
 618 mbuf_set_vlan_tag(
 619         mbuf_t mbuf,
 620         u_int16_t vlan)
 621 {
 622         mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
 623         mbuf->m_pkthdr.vlan_tag = vlan;
 624
 625         return 0;
 626 }
 627
 628 errno_t
 629 mbuf_get_vlan_tag(
 630         mbuf_t mbuf,
 631         u_int16_t *vlan)
 632 {
 633         if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
 634                 return ENXIO; // No vlan tag set
 635
 636         *vlan = mbuf->m_pkthdr.vlan_tag;
 637
 638         return 0;
 639 }
 640
 641 errno_t
 642 mbuf_clear_vlan_tag(
 643         mbuf_t mbuf)
 644 {
 645         mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
 646         mbuf->m_pkthdr.vlan_tag = 0;
 647
 648         return 0;
 649 }
 650
 651 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
 652         MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP |
 653         MBUF_CSUM_REQ_SUM16 | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6;
 654
 655 errno_t
 656 mbuf_set_csum_requested(
 657         mbuf_t mbuf,
 658         mbuf_csum_request_flags_t request,
 659         u_int32_t value)
 660 {
 661         request &= mbuf_valid_csum_request_flags;
 662         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
 663         mbuf->m_pkthdr.csum_data = value;
 664
 665         return 0;
 666 }
 667
 668 static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags =
 669         MBUF_TSO_IPV4 | MBUF_TSO_IPV6;
 670
 671 errno_t
 672 mbuf_get_tso_requested(
 673         mbuf_t mbuf,
 674         mbuf_tso_request_flags_t *request,
 675         u_int32_t *value)
 676 {
 677         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
 678                         request == NULL || value == NULL)
 679                 return EINVAL;
 680
 681         *request = mbuf->m_pkthdr.csum_flags;
 682         *request &= mbuf_valid_tso_request_flags;
 683         if (*request && value != NULL)
 684                 *value = mbuf->m_pkthdr.tso_segsz;
 685
 686         return 0;
 687 }
 688
 689 errno_t
 690 mbuf_get_csum_requested(
 691         mbuf_t mbuf,
 692         mbuf_csum_request_flags_t *request,
 693         u_int32_t *value)
 694 {
 695         *request = mbuf->m_pkthdr.csum_flags;
 696         *request &= mbuf_valid_csum_request_flags;
 697         if (value != NULL) {
 698                 *value = mbuf->m_pkthdr.csum_data;
 699         }
 700
 701         return 0;
 702 }
 703
 704 errno_t
 705 mbuf_clear_csum_requested(
 706         mbuf_t mbuf)
 707 {
 708         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 709         mbuf->m_pkthdr.csum_data = 0;
 710
 711         return 0;
 712 }
 713
 714 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
 715         MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
 716         MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16;
 717
 718 errno_t
 719 mbuf_set_csum_performed(
 720         mbuf_t mbuf,
 721         mbuf_csum_performed_flags_t performed,
 722         u_int32_t value)
 723 {
 724         performed &= mbuf_valid_csum_performed_flags;
 725         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
 726         mbuf->m_pkthdr.csum_data = value;
 727
 728         return 0;
 729 }
 730
 731 errno_t
 732 mbuf_get_csum_performed(
 733         mbuf_t mbuf,
 734         mbuf_csum_performed_flags_t *performed,
 735         u_int32_t *value)
 736 {
 737         *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
 738         *value = mbuf->m_pkthdr.csum_data;
 739
 740         return 0;
 741 }
 742
 743 errno_t
 744 mbuf_clear_csum_performed(
 745         mbuf_t mbuf)
 746 {
 747         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 748         mbuf->m_pkthdr.csum_data = 0;
 749
 750         return 0;
 751 }
 752
 753 errno_t
 754 mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 755     u_int16_t *csum)
 756 {
 757         if (mbuf == NULL || length == 0 || csum == NULL ||
 758            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 759                 return (EINVAL);
 760
 761         *csum = inet_cksum(mbuf, protocol, offset, length);
 762         return (0);
 763 }
 764
 765 #if INET6
 766 errno_t
 767 mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 768     u_int16_t *csum)
 769 {
 770         if (mbuf == NULL || length == 0 || csum == NULL ||
 771            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 772                 return (EINVAL);
 773
 774         *csum = inet6_cksum(mbuf, protocol, offset, length);
 775         return (0);
 776 }
 777 #else /* INET6 */
 778 errno_t
 779 mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
 780                 __unused u_int32_t offset, __unused u_int32_t length,
 781                 __unused u_int16_t *csum)
 782 {
 783         panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
 784         return (0);
 785 }
 786
 787 u_int16_t
 788 inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
 789                 __unused unsigned int off, __unused unsigned int len)
 790 {
 791         panic("inet6_cksum() doesn't exist on this platform\n");
 792         return (0);
 793 }
 794
 795 void nd6_lookup_ipv6(void);
 796 void
 797 nd6_lookup_ipv6(void)
 798 {
 799         panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
 800 }
 801
 802 int
 803 in6addr_local(__unused struct in6_addr *a)
 804 {
 805         panic("in6addr_local() doesn't exist on this platform\n");
 806         return (0);
 807 }
 808
 809 void nd6_storelladdr(void);
 810 void
 811 nd6_storelladdr(void)
 812 {
 813         panic("nd6_storelladdr() doesn't exist on this platform\n");
 814 }
 815 #endif /* INET6 */
 816
 817 /*
 818  * Mbuf tag KPIs
 819  */
 820
 821 #define MTAG_FIRST_ID FIRST_KPI_STR_ID
 822
 823 errno_t
 824 mbuf_tag_id_find(
 825         const char              *string,
 826         mbuf_tag_id_t   *out_id)
 827 {
 828         return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1);
 829 }
 830
 831 errno_t
 832 mbuf_tag_allocate(
 833         mbuf_t                  mbuf,
 834         mbuf_tag_id_t   id,
 835         mbuf_tag_type_t type,
 836         size_t                  length,
 837         mbuf_how_t              how,
 838         void**                  data_p)
 839 {
 840         struct m_tag *tag;
 841         u_int32_t mtag_id_first, mtag_id_last;
 842
 843         if (data_p != NULL)
 844                 *data_p = NULL;
 845
 846         /* Sanity check parameters */
 847         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 848         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 849                 id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 ||
 850                 data_p == NULL) {
 851                 return EINVAL;
 852         }
 853
 854         /* Make sure this mtag hasn't already been allocated */
 855         tag = m_tag_locate(mbuf, id, type, NULL);
 856         if (tag != NULL) {
 857                 return EEXIST;
 858         }
 859
 860         /* Allocate an mtag */
 861         tag = m_tag_create(id, type, length, how, mbuf);
 862         if (tag == NULL) {
 863                 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK;
 864         }
 865
 866         /* Attach the mtag and set *data_p */
 867         m_tag_prepend(mbuf, tag);
 868         *data_p = tag + 1;
 869
 870         return 0;
 871 }
 872
 873 errno_t
 874 mbuf_tag_find(
 875         mbuf_t                  mbuf,
 876         mbuf_tag_id_t   id,
 877         mbuf_tag_type_t type,
 878         size_t*                 length,
 879         void**                  data_p)
 880 {
 881         struct m_tag *tag;
 882         u_int32_t mtag_id_first, mtag_id_last;
 883
 884         if (length != NULL)
 885                 *length = 0;
 886         if (data_p != NULL)
 887                 *data_p = NULL;
 888
 889         /* Sanity check parameters */
 890         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 891         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 892                 id > mtag_id_last || length == NULL || data_p == NULL) {
 893                 return EINVAL;
 894         }
 895
 896         /* Locate an mtag */
 897         tag = m_tag_locate(mbuf, id, type, NULL);
 898         if (tag == NULL) {
 899                 return ENOENT;
 900         }
 901
 902         /* Copy out the pointer to the data and the lenght value */
 903         *length = tag->m_tag_len;
 904         *data_p = tag + 1;
 905
 906         return 0;
 907 }
 908
 909 void
 910 mbuf_tag_free(
 911         mbuf_t                  mbuf,
 912         mbuf_tag_id_t   id,
 913         mbuf_tag_type_t type)
 914 {
 915         struct m_tag *tag;
 916         u_int32_t mtag_id_first, mtag_id_last;
 917
 918         /* Sanity check parameters */
 919         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 920         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 921                 id > mtag_id_last)
 922                 return;
 923
 924         tag = m_tag_locate(mbuf, id, type, NULL);
 925         if (tag == NULL) {
 926                 return;
 927         }
 928
 929         m_tag_delete(mbuf, tag);
 930         return;
 931 }
 932
 933 /* mbuf stats */
 934 void mbuf_stats(struct mbuf_stat *stats)
 935 {
 936         stats->mbufs = mbstat.m_mbufs;
 937         stats->clusters = mbstat.m_clusters;
 938         stats->clfree = mbstat.m_clfree;
 939         stats->drops = mbstat.m_drops;
 940         stats->wait = mbstat.m_wait;
 941         stats->drain = mbstat.m_drain;
 942         __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
 943         stats->mcfail = mbstat.m_mcfail;
 944         stats->mpfail = mbstat.m_mpfail;
 945         stats->msize = mbstat.m_msize;
 946         stats->mclbytes = mbstat.m_mclbytes;
 947         stats->minclsize = mbstat.m_minclsize;
 948         stats->mlen = mbstat.m_mlen;
 949         stats->mhlen = mbstat.m_mhlen;
 950         stats->bigclusters = mbstat.m_bigclusters;
 951         stats->bigclfree = mbstat.m_bigclfree;
 952         stats->bigmclbytes = mbstat.m_bigmclbytes;
 953 }
 954
 955 errno_t
 956 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 957 {
 958         errno_t error;
 959         struct mbuf *m;
 960         unsigned int numpkts = 1;
 961         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 962
 963         if (packetlen == 0) {
 964                 error = EINVAL;
 965                 goto out;
 966         }
 967         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 968         if (m == 0) {
 969                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 970                         error = ENOBUFS;
 971                 else
 972                         error = ENOMEM;
 973         } else {
 974                 if (maxchunks)
 975                         *maxchunks = numchunks;
 976                 error = 0;
 977                 *mbuf = m;
 978         }
 979 out:
 980         return error;
 981 }
 982
 983 errno_t
 984 mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 985 {
 986         errno_t error;
 987         struct mbuf *m;
 988         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 989
 990         if (numpkts == 0) {
 991                 error = EINVAL;
 992                 goto out;
 993         }
 994         if (packetlen == 0) {
 995                 error = EINVAL;
 996                 goto out;
 997         }
 998         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 999         if (m == 0) {
1000                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
1001                         error = ENOBUFS;
1002                 else
1003                         error = ENOMEM;
1004         } else {
1005                 if (maxchunks)
1006                         *maxchunks = numchunks;
1007                 error = 0;
1008                 *mbuf = m;
1009         }
1010 out:
1011         return error;
1012 }
1013
1014
1015
1016 /*
1017  * mbuf_copyback differs from m_copyback in a few ways:
1018  * 1) mbuf_copyback will allocate clusters for new mbufs we append
1019  * 2) mbuf_copyback will grow the last mbuf in the chain if possible
1020  * 3) mbuf_copyback reports whether or not the operation succeeded
1021  * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
1022  */
1023 errno_t
1024 mbuf_copyback(
1025         mbuf_t          m,
1026         size_t          off,
1027         size_t          len,
1028         const void      *data,
1029         mbuf_how_t      how)
1030 {
1031         size_t  mlen;
1032         mbuf_t  m_start = m;
1033         mbuf_t  n;
1034         int             totlen = 0;
1035         errno_t         result = 0;
1036         const char      *cp = data;
1037
1038         if (m == NULL || len == 0 || data == NULL)
1039                 return EINVAL;
1040
1041         while (off > (mlen = m->m_len)) {
1042                 off -= mlen;
1043                 totlen += mlen;
1044                 if (m->m_next == 0) {
1045                         n = m_getclr(how, m->m_type);
1046                         if (n == 0) {
1047                                 result = ENOBUFS;
1048                                 goto out;
1049                         }
1050                         n->m_len = MIN(MLEN, len + off);
1051                         m->m_next = n;
1052                 }
1053                 m = m->m_next;
1054         }
1055
1056         while (len > 0) {
1057                 mlen = MIN(m->m_len - off, len);
1058                 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) {
1059                         size_t  grow = MIN(mbuf_trailingspace(m), len - mlen);
1060                         mlen += grow;
1061                         m->m_len += grow;
1062                 }
1063                 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen);
1064                 cp += mlen;
1065                 len -= mlen;
1066                 mlen += off;
1067                 off = 0;
1068                 totlen += mlen;
1069                 if (len == 0)
1070                         break;
1071                 if (m->m_next == 0) {
1072                         n = m_get(how, m->m_type);
1073                         if (n == NULL) {
1074                                 result = ENOBUFS;
1075                                 goto out;
1076                         }
1077                         if (len > MINCLSIZE) {
1078                                 /* cluter allocation failure is okay, we can grow chain */
1079                                 mbuf_mclget(how, m->m_type, &n);
1080                         }
1081                         n->m_len = MIN(mbuf_maxlen(n), len);
1082                         m->m_next = n;
1083                 }
1084                 m = m->m_next;
1085         }
1086
1087 out:
1088         if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
1089                 m_start->m_pkthdr.len = totlen;
1090
1091         return result;
1092 }
1093
1094 u_int32_t
1095 mbuf_get_mlen(void)
1096 {
1097         return (_MLEN);
1098 }
1099
1100 u_int32_t
1101 mbuf_get_mhlen(void)
1102 {
1103         return (_MHLEN);
1104 }
1105
1106 u_int32_t
1107 mbuf_get_minclsize(void)
1108 {
1109         return (MHLEN + MLEN);
1110 }
1111
1112 mbuf_traffic_class_t
1113 mbuf_get_traffic_class(mbuf_t m)
1114 {
1115         if (m == NULL || !(m->m_flags & M_PKTHDR))
1116                 return (MBUF_TC_BE);
1117
1118         return (m_get_traffic_class(m));
1119 }
1120
1121 errno_t
1122 mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc)
1123 {
1124         if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1125             ((u_int32_t)tc >= MBUF_TC_MAX))
1126                 return (EINVAL);
1127
1128         return (m_set_traffic_class(m, tc));
1129 }
1130
1131 int
1132 mbuf_is_traffic_class_privileged(mbuf_t m)
1133 {
1134         if (m == NULL || !(m->m_flags & M_PKTHDR) ||
1135             !MBUF_VALID_SC(m->m_pkthdr.svc))
1136                 return (0);
1137
1138         return (m->m_pkthdr.aux_flags & MAUXF_PRIO_PRIVILEGED);
1139 }
1140
1141 mbuf_svc_class_t
1142 mbuf_get_service_class(mbuf_t m)
1143 {
1144         if (m == NULL || !(m->m_flags & M_PKTHDR))
1145                 return (MBUF_SC_BE);
1146
1147         return (m_get_service_class(m));
1148 }
1149
1150 errno_t
1151 mbuf_set_service_class(mbuf_t m, mbuf_svc_class_t sc)
1152 {
1153         if (m == NULL || !(m->m_flags & M_PKTHDR))
1154                 return (EINVAL);
1155
1156         return (m_set_service_class(m, sc));
1157 }
1158
1159 errno_t
1160 mbuf_pkthdr_aux_flags(mbuf_t m, mbuf_pkthdr_aux_flags_t *flagsp)
1161 {
1162         u_int32_t flags;
1163         if (m == NULL || !(m->m_flags & M_PKTHDR) || flagsp == NULL)
1164                 return (EINVAL);
1165
1166         flags = m->m_pkthdr.aux_flags & MBUF_PKTAUXF_MASK;
1167
1168         /* These 2 flags are mutually exclusive */
1169         VERIFY((flags &
1170             (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR)) !=
1171             (MBUF_PKTAUXF_INET_RESOLVE_RTR | MBUF_PKTAUXF_INET6_RESOLVE_RTR));
1172
1173         *flagsp = flags;
1174         return (0);
1175 }