bsd/kern/kpi_mbuf.c

   1 /*
   2  * Copyright (c) 2004-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define __KPI__
  30 //#include <sys/kpi_interface.h>
  31
  32 #include <sys/param.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/socket.h>
  35 #include <kern/debug.h>
  36 #include <libkern/OSAtomic.h>
  37 #include <kern/kalloc.h>
  38 #include <string.h>
  39 #include <netinet/in.h>
  40
  41 #include "net/net_str_id.h"
  42
  43 static const mbuf_flags_t mbuf_flags_mask = (MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
  44     MBUF_LOOP | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
  45     MBUF_LASTFRAG | MBUF_PROMISC | MBUF_HASFCS);
  46
  47 void* mbuf_data(mbuf_t mbuf)
  48 {
  49         return mbuf->m_data;
  50 }
  51
  52 void* mbuf_datastart(mbuf_t mbuf)
  53 {
  54         if (mbuf->m_flags & M_EXT)
  55                 return mbuf->m_ext.ext_buf;
  56         if (mbuf->m_flags & M_PKTHDR)
  57                 return mbuf->m_pktdat;
  58         return mbuf->m_dat;
  59 }
  60
  61 errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len)
  62 {
  63         size_t  start = (size_t)((char*)mbuf_datastart(mbuf));
  64         size_t  maxlen = mbuf_maxlen(mbuf);
  65
  66         if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
  67                 return EINVAL;
  68         mbuf->m_data = data;
  69         mbuf->m_len = len;
  70
  71         return 0;
  72 }
  73
  74 errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
  75 {
  76         if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
  77                 return ENOTSUP;
  78         mbuf->m_data = mbuf_datastart(mbuf);
  79         mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
  80
  81         return 0;
  82 }
  83
  84 /* This function is used to provide mcl_to_paddr via symbol indirection,
  85  * please avoid any change in behavior or remove the indirection in
  86  * config/Unsupported*
  87  */
  88 addr64_t mbuf_data_to_physical(void* ptr)
  89 {
  90         return (addr64_t)(uintptr_t)mcl_to_paddr(ptr);
  91 }
  92
  93 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  94 {
  95         /* Must set *mbuf to NULL in failure case */
  96         *mbuf = m_get(how, type);
  97
  98         return (*mbuf == NULL) ? ENOMEM : 0;
  99 }
 100
 101 errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 102 {
 103         /* Must set *mbuf to NULL in failure case */
 104         *mbuf = m_gethdr(how, type);
 105
 106         return (*mbuf == NULL) ? ENOMEM : 0;
 107 }
 108
 109 errno_t
 110 mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
 111     caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t),
 112     size_t extsize, caddr_t extarg)
 113 {
 114         if (mbuf == NULL || extbuf == NULL || extfree == NULL || extsize == 0)
 115                 return (EINVAL);
 116
 117         if ((*mbuf = m_clattach(*mbuf, type, extbuf,
 118             extfree, extsize, extarg, how)) == NULL)
 119                 return (ENOMEM);
 120
 121         return (0);
 122 }
 123
 124 errno_t
 125 mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
 126 {
 127         if (size == NULL || *size == 0 || addr == NULL)
 128                 return (EINVAL);
 129
 130         *addr = NULL;
 131
 132         /* Jumbo cluster pool not available? */
 133         if (*size > MBIGCLBYTES && njcl == 0)
 134                 return (ENOTSUP);
 135
 136         if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
 137                 *size = MCLBYTES;
 138         else if (*size > MCLBYTES && *size <= MBIGCLBYTES &&
 139             (*addr = m_bigalloc(how)) != NULL)
 140                 *size = MBIGCLBYTES;
 141         else if (*size > MBIGCLBYTES && *size <= M16KCLBYTES &&
 142             (*addr = m_16kalloc(how)) != NULL)
 143                 *size = M16KCLBYTES;
 144         else
 145                 *size = 0;
 146
 147         if (*addr == NULL)
 148                 return (ENOMEM);
 149
 150         return (0);
 151 }
 152
 153 void
 154 mbuf_freecluster(caddr_t addr, size_t size)
 155 {
 156         if (size != MCLBYTES && size != MBIGCLBYTES && size != M16KCLBYTES)
 157                 panic("%s: invalid size (%ld) for cluster %p", __func__,
 158                     size, (void *)addr);
 159
 160         if (size == MCLBYTES)
 161                 m_mclfree(addr);
 162         else if (size == MBIGCLBYTES)
 163                 m_bigfree(addr, MBIGCLBYTES, NULL);
 164         else if (njcl > 0)
 165                 m_16kfree(addr, M16KCLBYTES, NULL);
 166         else
 167                 panic("%s: freeing jumbo cluster to an empty pool", __func__);
 168 }
 169
 170 errno_t
 171 mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 172 {
 173         /* Must set *mbuf to NULL in failure case */
 174         errno_t error = 0;
 175         int     created = 0;
 176
 177         if (mbuf == NULL)
 178                 return EINVAL;
 179         if (*mbuf == NULL) {
 180                 *mbuf = m_get(how, type);
 181                 if (*mbuf == NULL)
 182                         return ENOMEM;
 183                 created = 1;
 184         }
 185         /*
 186          * At the time this code was written, m_{mclget,mbigget,m16kget}
 187          * would always return the same value that was passed in to it.
 188          */
 189         if (size == MCLBYTES) {
 190                 *mbuf = m_mclget(*mbuf, how);
 191         } else if (size == MBIGCLBYTES) {
 192                 *mbuf = m_mbigget(*mbuf, how);
 193         } else if (size == M16KCLBYTES) {
 194                 if (njcl > 0) {
 195                         *mbuf = m_m16kget(*mbuf, how);
 196                 } else {
 197                         /* Jumbo cluster pool not available? */
 198                         error = ENOTSUP;
 199                         goto out;
 200                 }
 201         } else {
 202                 error = EINVAL;
 203                 goto out;
 204         }
 205         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 206                 error = ENOMEM;
 207 out:
 208         if (created && error != 0) {
 209                 mbuf_free(*mbuf);
 210                 *mbuf = NULL;
 211         }
 212         return error;
 213 }
 214
 215 errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 216 {
 217         /* Must set *mbuf to NULL in failure case */
 218         errno_t error = 0;
 219         int             created = 0;
 220         if (mbuf == NULL) return EINVAL;
 221         if (*mbuf == NULL) {
 222                 error = mbuf_get(how, type, mbuf);
 223                 if (error)
 224                         return error;
 225                 created = 1;
 226         }
 227
 228         /*
 229          * At the time this code was written, m_mclget would always
 230          * return the same value that was passed in to it.
 231          */
 232         *mbuf = m_mclget(*mbuf, how);
 233
 234         if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
 235                 mbuf_free(*mbuf);
 236                 *mbuf = NULL;
 237         }
 238         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 239                 error = ENOMEM;
 240         return error;
 241 }
 242
 243
 244 errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
 245 {
 246         /* Must set *mbuf to NULL in failure case */
 247         errno_t error = 0;
 248
 249         *mbuf = m_getpacket_how(how);
 250
 251         if (*mbuf == NULL) {
 252                 if (how == MBUF_WAITOK)
 253                         error = ENOMEM;
 254                 else
 255                         error = EWOULDBLOCK;
 256         }
 257
 258         return error;
 259 }
 260
 261 /* This function is used to provide m_free via symbol indirection, please avoid
 262  * any change in behavior or remove the indirection in config/Unsupported*
 263  */
 264 mbuf_t mbuf_free(mbuf_t mbuf)
 265 {
 266         return m_free(mbuf);
 267 }
 268
 269 /* This function is used to provide m_freem via symbol indirection, please avoid
 270  * any change in behavior or remove the indirection in config/Unsupported*
 271  */
 272 void mbuf_freem(mbuf_t mbuf)
 273 {
 274         m_freem(mbuf);
 275 }
 276
 277 int     mbuf_freem_list(mbuf_t mbuf)
 278 {
 279         return m_freem_list(mbuf);
 280 }
 281
 282 size_t mbuf_leadingspace(const mbuf_t mbuf)
 283 {
 284         return m_leadingspace(mbuf);
 285 }
 286
 287 /* This function is used to provide m_trailingspace via symbol indirection,
 288  * please avoid any change in behavior or remove the indirection in
 289  * config/Unsupported*
 290  */
 291 size_t mbuf_trailingspace(const mbuf_t mbuf)
 292 {
 293         return m_trailingspace(mbuf);
 294 }
 295
 296 /* Manipulation */
 297 errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len,
 298                                    mbuf_how_t how, mbuf_t *new_mbuf)
 299 {
 300         /* Must set *mbuf to NULL in failure case */
 301         *new_mbuf = m_copym(src, offset, len, how);
 302
 303         return (*new_mbuf == NULL) ? ENOMEM : 0;
 304 }
 305
 306 errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 307 {
 308         /* Must set *new_mbuf to NULL in failure case */
 309         *new_mbuf = m_dup(src, how);
 310
 311         return (*new_mbuf == NULL) ? ENOMEM : 0;
 312 }
 313
 314 errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
 315 {
 316         /* Must set *orig to NULL in failure case */
 317         *orig = m_prepend_2(*orig, len, how);
 318
 319         return (*orig == NULL) ? ENOMEM : 0;
 320 }
 321
 322 errno_t mbuf_split(mbuf_t src, size_t offset,
 323                                         mbuf_how_t how, mbuf_t *new_mbuf)
 324 {
 325         /* Must set *new_mbuf to NULL in failure case */
 326         *new_mbuf = m_split(src, offset, how);
 327
 328         return (*new_mbuf == NULL) ? ENOMEM : 0;
 329 }
 330
 331 errno_t mbuf_pullup(mbuf_t *mbuf, size_t len)
 332 {
 333         /* Must set *mbuf to NULL in failure case */
 334         *mbuf = m_pullup(*mbuf, len);
 335
 336         return (*mbuf == NULL) ? ENOMEM : 0;
 337 }
 338
 339 errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
 340 {
 341         /* Must set *location to NULL in failure case */
 342         int new_offset;
 343         *location = m_pulldown(src, *offset, len, &new_offset);
 344         *offset = new_offset;
 345
 346         return (*location == NULL) ? ENOMEM : 0;
 347 }
 348
 349 /* This function is used to provide m_adj via symbol indirection, please avoid
 350  * any change in behavior or remove the indirection in config/Unsupported*
 351  */
 352 void mbuf_adj(mbuf_t mbuf, int len)
 353 {
 354         m_adj(mbuf, len);
 355 }
 356
 357 errno_t mbuf_adjustlen(mbuf_t m, int amount)
 358 {
 359         /* Verify m_len will be valid after adding amount */
 360         if (amount > 0) {
 361                 int             used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
 362                                            m->m_len;
 363
 364                 if ((size_t)(amount + used) > mbuf_maxlen(m))
 365                         return EINVAL;
 366         }
 367         else if (-amount > m->m_len) {
 368                 return EINVAL;
 369         }
 370
 371         m->m_len += amount;
 372         return 0;
 373 }
 374
 375 mbuf_t
 376 mbuf_concatenate(mbuf_t dst, mbuf_t src)
 377 {
 378         if (dst == NULL)
 379                 return (NULL);
 380
 381         m_cat(dst, src);
 382
 383         /* return dst as is in the current implementation */
 384         return (dst);
 385 }
 386 errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data)
 387 {
 388         /* Copied m_copydata, added error handling (don't just panic) */
 389         int count;
 390         mbuf_t  m = m0;
 391
 392         while (off > 0) {
 393                 if (m == 0)
 394                         return EINVAL;
 395                 if (off < (size_t)m->m_len)
 396                         break;
 397                 off -= m->m_len;
 398                 m = m->m_next;
 399         }
 400         while (len > 0) {
 401                 if (m == 0)
 402                         return EINVAL;
 403                 count = m->m_len - off > len ? len : m->m_len - off;
 404                 bcopy(mtod(m, caddr_t) + off, out_data, count);
 405                 len -= count;
 406                 out_data = ((char*)out_data) + count;
 407                 off = 0;
 408                 m = m->m_next;
 409         }
 410
 411         return 0;
 412 }
 413
 414 int mbuf_mclhasreference(mbuf_t mbuf)
 415 {
 416         if ((mbuf->m_flags & M_EXT))
 417                 return m_mclhasreference(mbuf);
 418         else
 419                 return 0;
 420 }
 421
 422
 423 /* mbuf header */
 424 mbuf_t mbuf_next(const mbuf_t mbuf)
 425 {
 426         return mbuf->m_next;
 427 }
 428
 429 errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 430 {
 431         if (next && ((next)->m_nextpkt != NULL ||
 432                 (next)->m_type == MT_FREE)) return EINVAL;
 433         mbuf->m_next = next;
 434
 435         return 0;
 436 }
 437
 438 mbuf_t mbuf_nextpkt(const mbuf_t mbuf)
 439 {
 440         return mbuf->m_nextpkt;
 441 }
 442
 443 void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 444 {
 445         mbuf->m_nextpkt = nextpkt;
 446 }
 447
 448 size_t mbuf_len(const mbuf_t mbuf)
 449 {
 450         return mbuf->m_len;
 451 }
 452
 453 void mbuf_setlen(mbuf_t mbuf, size_t len)
 454 {
 455         mbuf->m_len = len;
 456 }
 457
 458 size_t mbuf_maxlen(const mbuf_t mbuf)
 459 {
 460         if (mbuf->m_flags & M_EXT)
 461                 return mbuf->m_ext.ext_size;
 462         return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 463 }
 464
 465 mbuf_type_t mbuf_type(const mbuf_t mbuf)
 466 {
 467         return mbuf->m_type;
 468 }
 469
 470 errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 471 {
 472         if (new_type == MBUF_TYPE_FREE) return EINVAL;
 473
 474         m_mchtype(mbuf, new_type);
 475
 476         return 0;
 477 }
 478
 479 mbuf_flags_t mbuf_flags(const mbuf_t mbuf)
 480 {
 481         return mbuf->m_flags & mbuf_flags_mask;
 482 }
 483
 484 errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
 485 {
 486         if ((flags & ~mbuf_flags_mask) != 0) return EINVAL;
 487         mbuf->m_flags = flags |
 488                 (mbuf->m_flags & ~mbuf_flags_mask);
 489
 490         return 0;
 491 }
 492
 493 errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 494 {
 495         if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL;
 496
 497         mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
 498
 499         return 0;
 500 }
 501
 502 errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
 503 {
 504         if (((src)->m_flags & M_PKTHDR) == 0)
 505                 return EINVAL;
 506
 507         m_copy_pkthdr(dest, src);
 508
 509         return 0;
 510 }
 511
 512 size_t mbuf_pkthdr_len(const mbuf_t mbuf)
 513 {
 514         return mbuf->m_pkthdr.len;
 515 }
 516
 517 void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 518 {
 519         mbuf->m_pkthdr.len = len;
 520 }
 521
 522 void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
 523 {
 524         mbuf->m_pkthdr.len += amount;
 525 }
 526
 527 ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf)
 528 {
 529         // If we reference count ifnets, we should take a reference here before returning
 530         return mbuf->m_pkthdr.rcvif;
 531 }
 532
 533 errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 534 {
 535         /* May want to walk ifnet list to determine if interface is valid */
 536         mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet;
 537         return 0;
 538 }
 539
 540 void* mbuf_pkthdr_header(const mbuf_t mbuf)
 541 {
 542         return mbuf->m_pkthdr.header;
 543 }
 544
 545 void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 546 {
 547         mbuf->m_pkthdr.header = (void*)header;
 548 }
 549
 550 void
 551 mbuf_inbound_modified(mbuf_t mbuf)
 552 {
 553         /* Invalidate hardware generated checksum flags */
 554         mbuf->m_pkthdr.csum_flags = 0;
 555 }
 556
 557 extern void in_cksum_offset(struct mbuf* m, size_t ip_offset);
 558 extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset);
 559
 560 void
 561 mbuf_outbound_finalize(mbuf_t mbuf, u_int32_t protocol_family, size_t protocol_offset)
 562 {
 563         if ((mbuf->m_pkthdr.csum_flags &
 564                  (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16 | CSUM_DELAY_IPV6_DATA)) == 0)
 565                 return;
 566
 567         /* Generate the packet in software, client needs it */
 568         switch (protocol_family) {
 569                 case PF_INET:
 570                         if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 571                                 /*
 572                                  * If you're wondering where this lovely code comes
 573                                  * from, we're trying to undo what happens in ip_output.
 574                                  * Look for CSUM_TCP_SUM16 in ip_output.
 575                                  */
 576                                 u_int16_t       first, second;
 577                                 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16;
 578                                 mbuf->m_pkthdr.csum_flags |= CSUM_TCP;
 579                                 first = mbuf->m_pkthdr.csum_data >> 16;
 580                                 second = mbuf->m_pkthdr.csum_data & 0xffff;
 581                                 mbuf->m_pkthdr.csum_data = first - second;
 582                         }
 583                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
 584                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 585                         }
 586
 587                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
 588                                 in_cksum_offset(mbuf, protocol_offset);
 589                         }
 590
 591                         mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP);
 592                         break;
 593
 594                 case PF_INET6:
 595
 596                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IPV6_DATA) {
 597                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 598                         }
 599                         mbuf->m_pkthdr.csum_flags &= ~CSUM_DELAY_IPV6_DATA;
 600                         break;
 601
 602
 603                 default:
 604                         /*
 605                          * Not sure what to do here if anything.
 606                          * Hardware checksum code looked pretty IPv4/IPv6 specific.
 607                          */
 608                         if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_DELAY_IPV6_DATA)) != 0)
 609                                 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 or IPv6 packet (%u)!\n", protocol_family);
 610         }
 611 }
 612
 613 errno_t
 614 mbuf_set_vlan_tag(
 615         mbuf_t mbuf,
 616         u_int16_t vlan)
 617 {
 618         mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
 619         mbuf->m_pkthdr.vlan_tag = vlan;
 620
 621         return 0;
 622 }
 623
 624 errno_t
 625 mbuf_get_vlan_tag(
 626         mbuf_t mbuf,
 627         u_int16_t *vlan)
 628 {
 629         if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
 630                 return ENXIO; // No vlan tag set
 631
 632         *vlan = mbuf->m_pkthdr.vlan_tag;
 633
 634         return 0;
 635 }
 636
 637 errno_t
 638 mbuf_clear_vlan_tag(
 639         mbuf_t mbuf)
 640 {
 641         mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
 642         mbuf->m_pkthdr.vlan_tag = 0;
 643
 644         return 0;
 645 }
 646
 647 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
 648         MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP |
 649         MBUF_CSUM_REQ_SUM16 | MBUF_CSUM_REQ_TCPIPV6 | MBUF_CSUM_REQ_UDPIPV6;
 650
 651 errno_t
 652 mbuf_set_csum_requested(
 653         mbuf_t mbuf,
 654         mbuf_csum_request_flags_t request,
 655         u_int32_t value)
 656 {
 657         request &= mbuf_valid_csum_request_flags;
 658         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
 659         mbuf->m_pkthdr.csum_data = value;
 660
 661         return 0;
 662 }
 663
 664 static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags =
 665         MBUF_TSO_IPV4 | MBUF_TSO_IPV6;
 666
 667 errno_t
 668 mbuf_get_tso_requested(
 669         mbuf_t mbuf,
 670         mbuf_tso_request_flags_t *request,
 671         u_int32_t *value)
 672 {
 673         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
 674                         request == NULL || value == NULL)
 675                 return EINVAL;
 676
 677         *request = mbuf->m_pkthdr.csum_flags;
 678         *request &= mbuf_valid_tso_request_flags;
 679         if (*request && value != NULL)
 680                 *value = mbuf->m_pkthdr.tso_segsz;
 681
 682         return 0;
 683 }
 684
 685 errno_t
 686 mbuf_get_csum_requested(
 687         mbuf_t mbuf,
 688         mbuf_csum_request_flags_t *request,
 689         u_int32_t *value)
 690 {
 691         *request = mbuf->m_pkthdr.csum_flags;
 692         *request &= mbuf_valid_csum_request_flags;
 693         if (value != NULL) {
 694                 *value = mbuf->m_pkthdr.csum_data;
 695         }
 696
 697         return 0;
 698 }
 699
 700 errno_t
 701 mbuf_clear_csum_requested(
 702         mbuf_t mbuf)
 703 {
 704         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 705         mbuf->m_pkthdr.csum_data = 0;
 706
 707         return 0;
 708 }
 709
 710 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
 711         MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
 712         MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16;
 713
 714 errno_t
 715 mbuf_set_csum_performed(
 716         mbuf_t mbuf,
 717         mbuf_csum_performed_flags_t performed,
 718         u_int32_t value)
 719 {
 720         performed &= mbuf_valid_csum_performed_flags;
 721         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
 722         mbuf->m_pkthdr.csum_data = value;
 723
 724         return 0;
 725 }
 726
 727 errno_t
 728 mbuf_get_csum_performed(
 729         mbuf_t mbuf,
 730         mbuf_csum_performed_flags_t *performed,
 731         u_int32_t *value)
 732 {
 733         *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
 734         *value = mbuf->m_pkthdr.csum_data;
 735
 736         return 0;
 737 }
 738
 739 errno_t
 740 mbuf_clear_csum_performed(
 741         mbuf_t mbuf)
 742 {
 743         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 744         mbuf->m_pkthdr.csum_data = 0;
 745
 746         return 0;
 747 }
 748
 749 errno_t
 750 mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 751     u_int16_t *csum)
 752 {
 753         if (mbuf == NULL || length == 0 || csum == NULL ||
 754            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 755                 return (EINVAL);
 756
 757         *csum = inet_cksum(mbuf, protocol, offset, length);
 758         return (0);
 759 }
 760
 761 #if INET6
 762 errno_t
 763 mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 764     u_int16_t *csum)
 765 {
 766         if (mbuf == NULL || length == 0 || csum == NULL ||
 767            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 768                 return (EINVAL);
 769
 770         *csum = inet6_cksum(mbuf, protocol, offset, length);
 771         return (0);
 772 }
 773 #else /* INET6 */
 774 errno_t
 775 mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
 776                 __unused u_int32_t offset, __unused u_int32_t length,
 777                 __unused u_int16_t *csum)
 778 {
 779         panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
 780         return (0);
 781 }
 782
 783 u_int16_t
 784 inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
 785                 __unused unsigned int off, __unused unsigned int len)
 786 {
 787         panic("inet6_cksum() doesn't exist on this platform\n");
 788         return (0);
 789 }
 790
 791 void nd6_lookup_ipv6(void);
 792 void
 793 nd6_lookup_ipv6(void)
 794 {
 795         panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
 796 }
 797
 798 int
 799 in6addr_local(__unused struct in6_addr *a)
 800 {
 801         panic("in6addr_local() doesn't exist on this platform\n");
 802         return (0);
 803 }
 804
 805 void nd6_storelladdr(void);
 806 void
 807 nd6_storelladdr(void)
 808 {
 809         panic("nd6_storelladdr() doesn't exist on this platform\n");
 810 }
 811 #endif /* INET6 */
 812
 813 /*
 814  * Mbuf tag KPIs
 815  */
 816
 817 #define MTAG_FIRST_ID FIRST_KPI_STR_ID
 818
 819 errno_t
 820 mbuf_tag_id_find(
 821         const char              *string,
 822         mbuf_tag_id_t   *out_id)
 823 {
 824         return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1);
 825 }
 826
 827 errno_t
 828 mbuf_tag_allocate(
 829         mbuf_t                  mbuf,
 830         mbuf_tag_id_t   id,
 831         mbuf_tag_type_t type,
 832         size_t                  length,
 833         mbuf_how_t              how,
 834         void**                  data_p)
 835 {
 836         struct m_tag *tag;
 837         u_int32_t mtag_id_first, mtag_id_last;
 838
 839         if (data_p != NULL)
 840                 *data_p = NULL;
 841
 842         /* Sanity check parameters */
 843         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 844         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 845                 id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 ||
 846                 data_p == NULL) {
 847                 return EINVAL;
 848         }
 849
 850         /* Make sure this mtag hasn't already been allocated */
 851         tag = m_tag_locate(mbuf, id, type, NULL);
 852         if (tag != NULL) {
 853                 return EEXIST;
 854         }
 855
 856         /* Allocate an mtag */
 857         tag = m_tag_create(id, type, length, how, mbuf);
 858         if (tag == NULL) {
 859                 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK;
 860         }
 861
 862         /* Attach the mtag and set *data_p */
 863         m_tag_prepend(mbuf, tag);
 864         *data_p = tag + 1;
 865
 866         return 0;
 867 }
 868
 869 errno_t
 870 mbuf_tag_find(
 871         mbuf_t                  mbuf,
 872         mbuf_tag_id_t   id,
 873         mbuf_tag_type_t type,
 874         size_t*                 length,
 875         void**                  data_p)
 876 {
 877         struct m_tag *tag;
 878         u_int32_t mtag_id_first, mtag_id_last;
 879
 880         if (length != NULL)
 881                 *length = 0;
 882         if (data_p != NULL)
 883                 *data_p = NULL;
 884
 885         /* Sanity check parameters */
 886         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 887         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 888                 id > mtag_id_last || length == NULL || data_p == NULL) {
 889                 return EINVAL;
 890         }
 891
 892         /* Locate an mtag */
 893         tag = m_tag_locate(mbuf, id, type, NULL);
 894         if (tag == NULL) {
 895                 return ENOENT;
 896         }
 897
 898         /* Copy out the pointer to the data and the lenght value */
 899         *length = tag->m_tag_len;
 900         *data_p = tag + 1;
 901
 902         return 0;
 903 }
 904
 905 void
 906 mbuf_tag_free(
 907         mbuf_t                  mbuf,
 908         mbuf_tag_id_t   id,
 909         mbuf_tag_type_t type)
 910 {
 911         struct m_tag *tag;
 912         u_int32_t mtag_id_first, mtag_id_last;
 913
 914         /* Sanity check parameters */
 915         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 916         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 917                 id > mtag_id_last)
 918                 return;
 919
 920         tag = m_tag_locate(mbuf, id, type, NULL);
 921         if (tag == NULL) {
 922                 return;
 923         }
 924
 925         m_tag_delete(mbuf, tag);
 926         return;
 927 }
 928
 929 /* mbuf stats */
 930 void mbuf_stats(struct mbuf_stat *stats)
 931 {
 932         stats->mbufs = mbstat.m_mbufs;
 933         stats->clusters = mbstat.m_clusters;
 934         stats->clfree = mbstat.m_clfree;
 935         stats->drops = mbstat.m_drops;
 936         stats->wait = mbstat.m_wait;
 937         stats->drain = mbstat.m_drain;
 938         __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
 939         stats->mcfail = mbstat.m_mcfail;
 940         stats->mpfail = mbstat.m_mpfail;
 941         stats->msize = mbstat.m_msize;
 942         stats->mclbytes = mbstat.m_mclbytes;
 943         stats->minclsize = mbstat.m_minclsize;
 944         stats->mlen = mbstat.m_mlen;
 945         stats->mhlen = mbstat.m_mhlen;
 946         stats->bigclusters = mbstat.m_bigclusters;
 947         stats->bigclfree = mbstat.m_bigclfree;
 948         stats->bigmclbytes = mbstat.m_bigmclbytes;
 949 }
 950
 951 errno_t
 952 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 953 {
 954         errno_t error;
 955         struct mbuf *m;
 956         unsigned int numpkts = 1;
 957         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 958
 959         if (packetlen == 0) {
 960                 error = EINVAL;
 961                 goto out;
 962         }
 963         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 964         if (m == 0) {
 965                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 966                         error = ENOBUFS;
 967                 else
 968                         error = ENOMEM;
 969         } else {
 970                 if (maxchunks)
 971                         *maxchunks = numchunks;
 972                 error = 0;
 973                 *mbuf = m;
 974         }
 975 out:
 976         return error;
 977 }
 978
 979 errno_t
 980 mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 981 {
 982         errno_t error;
 983         struct mbuf *m;
 984         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 985
 986         if (numpkts == 0) {
 987                 error = EINVAL;
 988                 goto out;
 989         }
 990         if (packetlen == 0) {
 991                 error = EINVAL;
 992                 goto out;
 993         }
 994         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 995         if (m == 0) {
 996                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 997                         error = ENOBUFS;
 998                 else
 999                         error = ENOMEM;
1000         } else {
1001                 if (maxchunks)
1002                         *maxchunks = numchunks;
1003                 error = 0;
1004                 *mbuf = m;
1005         }
1006 out:
1007         return error;
1008 }
1009
1010
1011
1012 /*
1013  * mbuf_copyback differs from m_copyback in a few ways:
1014  * 1) mbuf_copyback will allocate clusters for new mbufs we append
1015  * 2) mbuf_copyback will grow the last mbuf in the chain if possible
1016  * 3) mbuf_copyback reports whether or not the operation succeeded
1017  * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
1018  */
1019 errno_t
1020 mbuf_copyback(
1021         mbuf_t          m,
1022         size_t          off,
1023         size_t          len,
1024         const void      *data,
1025         mbuf_how_t      how)
1026 {
1027         size_t  mlen;
1028         mbuf_t  m_start = m;
1029         mbuf_t  n;
1030         int             totlen = 0;
1031         errno_t         result = 0;
1032         const char      *cp = data;
1033
1034         if (m == NULL || len == 0 || data == NULL)
1035                 return EINVAL;
1036
1037         while (off > (mlen = m->m_len)) {
1038                 off -= mlen;
1039                 totlen += mlen;
1040                 if (m->m_next == 0) {
1041                         n = m_getclr(how, m->m_type);
1042                         if (n == 0) {
1043                                 result = ENOBUFS;
1044                                 goto out;
1045                         }
1046                         n->m_len = MIN(MLEN, len + off);
1047                         m->m_next = n;
1048                 }
1049                 m = m->m_next;
1050         }
1051
1052         while (len > 0) {
1053                 mlen = MIN(m->m_len - off, len);
1054                 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) {
1055                         size_t  grow = MIN(mbuf_trailingspace(m), len - mlen);
1056                         mlen += grow;
1057                         m->m_len += grow;
1058                 }
1059                 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen);
1060                 cp += mlen;
1061                 len -= mlen;
1062                 mlen += off;
1063                 off = 0;
1064                 totlen += mlen;
1065                 if (len == 0)
1066                         break;
1067                 if (m->m_next == 0) {
1068                         n = m_get(how, m->m_type);
1069                         if (n == NULL) {
1070                                 result = ENOBUFS;
1071                                 goto out;
1072                         }
1073                         if (len > MINCLSIZE) {
1074                                 /* cluter allocation failure is okay, we can grow chain */
1075                                 mbuf_mclget(how, m->m_type, &n);
1076                         }
1077                         n->m_len = MIN(mbuf_maxlen(n), len);
1078                         m->m_next = n;
1079                 }
1080                 m = m->m_next;
1081         }
1082
1083 out:
1084         if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
1085                 m_start->m_pkthdr.len = totlen;
1086
1087         return result;
1088 }
1089
1090 u_int32_t
1091 mbuf_get_mlen(void)
1092 {
1093         return (_MLEN);
1094 }
1095
1096 u_int32_t
1097 mbuf_get_mhlen(void)
1098 {
1099         return (_MHLEN);
1100 }
1101
1102 u_int32_t
1103 mbuf_get_minclsize(void)
1104 {
1105         return (MHLEN + MLEN);
1106 }
1107
1108 mbuf_traffic_class_t
1109 mbuf_get_traffic_class(mbuf_t m)
1110 {
1111         mbuf_traffic_class_t prio = MBUF_TC_BE;
1112
1113         if (m == NULL || !(m->m_flags & M_PKTHDR))
1114                 return (prio);
1115
1116         if (m->m_pkthdr.prio <= MBUF_TC_VO)
1117                 prio = m->m_pkthdr.prio;
1118
1119         return (prio);
1120 }
1121
1122 errno_t
1123 mbuf_set_traffic_class(mbuf_t m, mbuf_traffic_class_t tc)
1124 {
1125         errno_t error = 0;
1126
1127         if (m == NULL || !(m->m_flags & M_PKTHDR))
1128                 return EINVAL;
1129
1130         switch (tc) {
1131                 case MBUF_TC_BE:
1132                 case MBUF_TC_BK:
1133                 case MBUF_TC_VI:
1134                 case MBUF_TC_VO:
1135                         m->m_pkthdr.prio = tc;
1136                         break;
1137                 default:
1138                         error = EINVAL;
1139                         break;
1140         }
1141         return error;
1142 }