bsd/kern/kpi_mbuf.c

   1 /*
   2  * Copyright (c) 2004-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define __KPI__
  30 //#include <sys/kpi_interface.h>
  31
  32 #include <sys/param.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/socket.h>
  35 #include <kern/debug.h>
  36 #include <libkern/OSAtomic.h>
  37 #include <kern/kalloc.h>
  38 #include <string.h>
  39 #include <netinet/in.h>
  40
  41 #include "net/net_str_id.h"
  42
  43 static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
  44                                 MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
  45                                 MBUF_LASTFRAG | MBUF_PROMISC;
  46
  47 void* mbuf_data(mbuf_t mbuf)
  48 {
  49         return mbuf->m_data;
  50 }
  51
  52 void* mbuf_datastart(mbuf_t mbuf)
  53 {
  54         if (mbuf->m_flags & M_EXT)
  55                 return mbuf->m_ext.ext_buf;
  56         if (mbuf->m_flags & M_PKTHDR)
  57                 return mbuf->m_pktdat;
  58         return mbuf->m_dat;
  59 }
  60
  61 errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len)
  62 {
  63         size_t  start = (size_t)((char*)mbuf_datastart(mbuf));
  64         size_t  maxlen = mbuf_maxlen(mbuf);
  65
  66         if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
  67                 return EINVAL;
  68         mbuf->m_data = data;
  69         mbuf->m_len = len;
  70
  71         return 0;
  72 }
  73
  74 errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
  75 {
  76         if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
  77                 return ENOTSUP;
  78         mbuf->m_data = mbuf_datastart(mbuf);
  79         mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
  80
  81         return 0;
  82 }
  83
  84 addr64_t mbuf_data_to_physical(void* ptr)
  85 {
  86         return (addr64_t)(intptr_t)mcl_to_paddr(ptr);
  87 }
  88
  89 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  90 {
  91         /* Must set *mbuf to NULL in failure case */
  92         *mbuf = m_get(how, type);
  93
  94         return (*mbuf == NULL) ? ENOMEM : 0;
  95 }
  96
  97 errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  98 {
  99         /* Must set *mbuf to NULL in failure case */
 100         *mbuf = m_gethdr(how, type);
 101
 102         return (*mbuf == NULL) ? ENOMEM : 0;
 103 }
 104
 105 errno_t
 106 mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
 107     caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t),
 108     size_t extsize, caddr_t extarg)
 109 {
 110         if (extbuf == NULL || extfree == NULL || extsize == 0)
 111                 return (EINVAL);
 112
 113         if ((*mbuf = m_clattach(mbuf != NULL ? *mbuf : NULL, type, extbuf,
 114             extfree, extsize, extarg, how)) == NULL)
 115                 return (ENOMEM);
 116
 117         return (0);
 118 }
 119
 120 errno_t
 121 mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
 122 {
 123         if (size == NULL || *size == 0 || addr == NULL)
 124                 return (EINVAL);
 125
 126         *addr = NULL;
 127
 128         /* Jumbo cluster pool not available? */
 129         if (*size > NBPG && njcl == 0)
 130                 return (ENOTSUP);
 131
 132         if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
 133                 *size = MCLBYTES;
 134         else if (*size > MCLBYTES && *size <= NBPG &&
 135             (*addr = m_bigalloc(how)) != NULL)
 136                 *size = NBPG;
 137         else if (*size > NBPG && *size <= M16KCLBYTES &&
 138             (*addr = m_16kalloc(how)) != NULL)
 139                 *size = M16KCLBYTES;
 140         else
 141                 *size = 0;
 142
 143         if (*addr == NULL)
 144                 return (ENOMEM);
 145
 146         return (0);
 147 }
 148
 149 void
 150 mbuf_freecluster(caddr_t addr, size_t size)
 151 {
 152         if (size != MCLBYTES && size != NBPG && size != M16KCLBYTES)
 153                 panic("%s: invalid size (%ld) for cluster %p", __func__,
 154                     size, (void *)addr);
 155
 156         if (size == MCLBYTES)
 157                 m_mclfree(addr);
 158         else if (size == NBPG)
 159                 m_bigfree(addr, NBPG, NULL);
 160         else if (njcl > 0)
 161                 m_16kfree(addr, M16KCLBYTES, NULL);
 162         else
 163                 panic("%s: freeing jumbo cluster to an empty pool", __func__);
 164 }
 165
 166 errno_t
 167 mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 168 {
 169         /* Must set *mbuf to NULL in failure case */
 170         errno_t error = 0;
 171         int     created = 0;
 172
 173         if (mbuf == NULL)
 174                 return EINVAL;
 175         if (*mbuf == NULL) {
 176                 *mbuf = m_get(how, type);
 177                 if (*mbuf == NULL)
 178                         return ENOMEM;
 179                 created = 1;
 180         }
 181         /*
 182          * At the time this code was written, m_{mclget,mbigget,m16kget}
 183          * would always return the same value that was passed in to it.
 184          */
 185         if (size == MCLBYTES) {
 186                 *mbuf = m_mclget(*mbuf, how);
 187         } else if (size == NBPG) {
 188                 *mbuf = m_mbigget(*mbuf, how);
 189         } else if (size == M16KCLBYTES) {
 190                 if (njcl > 0) {
 191                         *mbuf = m_m16kget(*mbuf, how);
 192                 } else {
 193                         /* Jumbo cluster pool not available? */
 194                         error = ENOTSUP;
 195                         goto out;
 196                 }
 197         } else {
 198                 error = EINVAL;
 199                 goto out;
 200         }
 201         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 202                 error = ENOMEM;
 203 out:
 204         if (created && error != 0) {
 205                 mbuf_free(*mbuf);
 206                 *mbuf = NULL;
 207         }
 208         return error;
 209 }
 210
 211 errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 212 {
 213         /* Must set *mbuf to NULL in failure case */
 214         errno_t error = 0;
 215         int             created = 0;
 216         if (mbuf == NULL) return EINVAL;
 217         if (*mbuf == NULL) {
 218                 error = mbuf_get(how, type, mbuf);
 219                 if (error)
 220                         return error;
 221                 created = 1;
 222         }
 223
 224         /*
 225          * At the time this code was written, m_mclget would always
 226          * return the same value that was passed in to it.
 227          */
 228         *mbuf = m_mclget(*mbuf, how);
 229
 230         if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
 231                 mbuf_free(*mbuf);
 232                 *mbuf = NULL;
 233         }
 234         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 235                 error = ENOMEM;
 236         return error;
 237 }
 238
 239
 240 errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
 241 {
 242         /* Must set *mbuf to NULL in failure case */
 243         errno_t error = 0;
 244
 245         *mbuf = m_getpacket_how(how);
 246
 247         if (*mbuf == NULL) {
 248                 if (how == MBUF_WAITOK)
 249                         error = ENOMEM;
 250                 else
 251                         error = EWOULDBLOCK;
 252         }
 253
 254         return error;
 255 }
 256
 257 mbuf_t mbuf_free(mbuf_t mbuf)
 258 {
 259         return m_free(mbuf);
 260 }
 261
 262 void mbuf_freem(mbuf_t mbuf)
 263 {
 264         m_freem(mbuf);
 265 }
 266
 267 int     mbuf_freem_list(mbuf_t mbuf)
 268 {
 269         return m_freem_list(mbuf);
 270 }
 271
 272 size_t mbuf_leadingspace(const mbuf_t mbuf)
 273 {
 274         return m_leadingspace(mbuf);
 275 }
 276
 277 size_t mbuf_trailingspace(const mbuf_t mbuf)
 278 {
 279         return m_trailingspace(mbuf);
 280 }
 281
 282 /* Manipulation */
 283 errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len,
 284                                    mbuf_how_t how, mbuf_t *new_mbuf)
 285 {
 286         /* Must set *mbuf to NULL in failure case */
 287         *new_mbuf = m_copym(src, offset, len, how);
 288
 289         return (*new_mbuf == NULL) ? ENOMEM : 0;
 290 }
 291
 292 errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 293 {
 294         /* Must set *new_mbuf to NULL in failure case */
 295         *new_mbuf = m_dup(src, how);
 296
 297         return (*new_mbuf == NULL) ? ENOMEM : 0;
 298 }
 299
 300 errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
 301 {
 302         /* Must set *orig to NULL in failure case */
 303         *orig = m_prepend_2(*orig, len, how);
 304
 305         return (*orig == NULL) ? ENOMEM : 0;
 306 }
 307
 308 errno_t mbuf_split(mbuf_t src, size_t offset,
 309                                         mbuf_how_t how, mbuf_t *new_mbuf)
 310 {
 311         /* Must set *new_mbuf to NULL in failure case */
 312         *new_mbuf = m_split(src, offset, how);
 313
 314         return (*new_mbuf == NULL) ? ENOMEM : 0;
 315 }
 316
 317 errno_t mbuf_pullup(mbuf_t *mbuf, size_t len)
 318 {
 319         /* Must set *mbuf to NULL in failure case */
 320         *mbuf = m_pullup(*mbuf, len);
 321
 322         return (*mbuf == NULL) ? ENOMEM : 0;
 323 }
 324
 325 errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
 326 {
 327         /* Must set *location to NULL in failure case */
 328         int new_offset;
 329         *location = m_pulldown(src, *offset, len, &new_offset);
 330         *offset = new_offset;
 331
 332         return (*location == NULL) ? ENOMEM : 0;
 333 }
 334
 335 void mbuf_adj(mbuf_t mbuf, int len)
 336 {
 337         m_adj(mbuf, len);
 338 }
 339
 340 errno_t mbuf_adjustlen(mbuf_t m, int amount)
 341 {
 342         /* Verify m_len will be valid after adding amount */
 343         if (amount > 0) {
 344                 int             used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
 345                                            m->m_len;
 346
 347                 if ((size_t)(amount + used) > mbuf_maxlen(m))
 348                         return EINVAL;
 349         }
 350         else if (-amount > m->m_len) {
 351                 return EINVAL;
 352         }
 353
 354         m->m_len += amount;
 355         return 0;
 356 }
 357
 358 mbuf_t
 359 mbuf_concatenate(mbuf_t dst, mbuf_t src)
 360 {
 361         if (dst == NULL)
 362                 return (NULL);
 363
 364         m_cat(dst, src);
 365
 366         /* return dst as is in the current implementation */
 367         return (dst);
 368 }
 369 errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data)
 370 {
 371         /* Copied m_copydata, added error handling (don't just panic) */
 372         int count;
 373         mbuf_t  m = m0;
 374
 375         while (off > 0) {
 376                 if (m == 0)
 377                         return EINVAL;
 378                 if (off < (size_t)m->m_len)
 379                         break;
 380                 off -= m->m_len;
 381                 m = m->m_next;
 382         }
 383         while (len > 0) {
 384                 if (m == 0)
 385                         return EINVAL;
 386                 count = m->m_len - off > len ? len : m->m_len - off;
 387                 bcopy(mtod(m, caddr_t) + off, out_data, count);
 388                 len -= count;
 389                 out_data = ((char*)out_data) + count;
 390                 off = 0;
 391                 m = m->m_next;
 392         }
 393
 394         return 0;
 395 }
 396
 397 int mbuf_mclhasreference(mbuf_t mbuf)
 398 {
 399         if ((mbuf->m_flags & M_EXT))
 400                 return m_mclhasreference(mbuf);
 401         else
 402                 return 0;
 403 }
 404
 405
 406 /* mbuf header */
 407 mbuf_t mbuf_next(const mbuf_t mbuf)
 408 {
 409         return mbuf->m_next;
 410 }
 411
 412 errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 413 {
 414         if (next && ((next)->m_nextpkt != NULL ||
 415                 (next)->m_type == MT_FREE)) return EINVAL;
 416         mbuf->m_next = next;
 417
 418         return 0;
 419 }
 420
 421 mbuf_t mbuf_nextpkt(const mbuf_t mbuf)
 422 {
 423         return mbuf->m_nextpkt;
 424 }
 425
 426 void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 427 {
 428         mbuf->m_nextpkt = nextpkt;
 429 }
 430
 431 size_t mbuf_len(const mbuf_t mbuf)
 432 {
 433         return mbuf->m_len;
 434 }
 435
 436 void mbuf_setlen(mbuf_t mbuf, size_t len)
 437 {
 438         mbuf->m_len = len;
 439 }
 440
 441 size_t mbuf_maxlen(const mbuf_t mbuf)
 442 {
 443         if (mbuf->m_flags & M_EXT)
 444                 return mbuf->m_ext.ext_size;
 445         return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 446 }
 447
 448 mbuf_type_t mbuf_type(const mbuf_t mbuf)
 449 {
 450         return mbuf->m_type;
 451 }
 452
 453 errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 454 {
 455         if (new_type == MBUF_TYPE_FREE) return EINVAL;
 456
 457         m_mchtype(mbuf, new_type);
 458
 459         return 0;
 460 }
 461
 462 mbuf_flags_t mbuf_flags(const mbuf_t mbuf)
 463 {
 464         return mbuf->m_flags & mbuf_flags_mask;
 465 }
 466
 467 errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
 468 {
 469         if ((flags & ~mbuf_flags_mask) != 0) return EINVAL;
 470         mbuf->m_flags = flags |
 471                 (mbuf->m_flags & ~mbuf_flags_mask);
 472
 473         return 0;
 474 }
 475
 476 errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 477 {
 478         if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL;
 479
 480         mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
 481
 482         return 0;
 483 }
 484
 485 errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
 486 {
 487         if (((src)->m_flags & M_PKTHDR) == 0)
 488                 return EINVAL;
 489
 490         m_copy_pkthdr(dest, src);
 491
 492         return 0;
 493 }
 494
 495 size_t mbuf_pkthdr_len(const mbuf_t mbuf)
 496 {
 497         return mbuf->m_pkthdr.len;
 498 }
 499
 500 void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 501 {
 502         mbuf->m_pkthdr.len = len;
 503 }
 504
 505 void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
 506 {
 507         mbuf->m_pkthdr.len += amount;
 508 }
 509
 510 ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf)
 511 {
 512         // If we reference count ifnets, we should take a reference here before returning
 513         return mbuf->m_pkthdr.rcvif;
 514 }
 515
 516 errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 517 {
 518         /* May want to walk ifnet list to determine if interface is valid */
 519         mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet;
 520         return 0;
 521 }
 522
 523 void* mbuf_pkthdr_header(const mbuf_t mbuf)
 524 {
 525         return mbuf->m_pkthdr.header;
 526 }
 527
 528 void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 529 {
 530         mbuf->m_pkthdr.header = (void*)header;
 531 }
 532
 533 void
 534 mbuf_inbound_modified(mbuf_t mbuf)
 535 {
 536         /* Invalidate hardware generated checksum flags */
 537         mbuf->m_pkthdr.csum_flags = 0;
 538 }
 539
 540 extern void in_cksum_offset(struct mbuf* m, size_t ip_offset);
 541 extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset);
 542
 543 void
 544 mbuf_outbound_finalize(mbuf_t mbuf, u_int32_t protocol_family, size_t protocol_offset)
 545 {
 546         if ((mbuf->m_pkthdr.csum_flags &
 547                  (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16)) == 0)
 548                 return;
 549
 550         /* Generate the packet in software, client needs it */
 551         switch (protocol_family) {
 552                 case PF_INET:
 553                         if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 554                                 /*
 555                                  * If you're wondering where this lovely code comes
 556                                  * from, we're trying to undo what happens in ip_output.
 557                                  * Look for CSUM_TCP_SUM16 in ip_output.
 558                                  */
 559                                 u_int16_t       first, second;
 560                                 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16;
 561                                 mbuf->m_pkthdr.csum_flags |= CSUM_TCP;
 562                                 first = mbuf->m_pkthdr.csum_data >> 16;
 563                                 second = mbuf->m_pkthdr.csum_data & 0xffff;
 564                                 mbuf->m_pkthdr.csum_data = first - second;
 565                         }
 566                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
 567                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 568                         }
 569
 570                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
 571                                 in_cksum_offset(mbuf, protocol_offset);
 572                         }
 573
 574                         mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP);
 575                         break;
 576
 577                 default:
 578                         /*
 579                          * Not sure what to do here if anything.
 580                          * Hardware checksum code looked pretty IPv4 specific.
 581                          */
 582                         if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP)) != 0)
 583                                 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%u)!\n", protocol_family);
 584         }
 585 }
 586
 587 errno_t
 588 mbuf_set_vlan_tag(
 589         mbuf_t mbuf,
 590         u_int16_t vlan)
 591 {
 592         mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
 593         mbuf->m_pkthdr.vlan_tag = vlan;
 594
 595         return 0;
 596 }
 597
 598 errno_t
 599 mbuf_get_vlan_tag(
 600         mbuf_t mbuf,
 601         u_int16_t *vlan)
 602 {
 603         if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
 604                 return ENXIO; // No vlan tag set
 605
 606         *vlan = mbuf->m_pkthdr.vlan_tag;
 607
 608         return 0;
 609 }
 610
 611 errno_t
 612 mbuf_clear_vlan_tag(
 613         mbuf_t mbuf)
 614 {
 615         mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
 616         mbuf->m_pkthdr.vlan_tag = 0;
 617
 618         return 0;
 619 }
 620
 621 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
 622         MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | MBUF_CSUM_REQ_SUM16;
 623
 624 errno_t
 625 mbuf_set_csum_requested(
 626         mbuf_t mbuf,
 627         mbuf_csum_request_flags_t request,
 628         u_int32_t value)
 629 {
 630         request &= mbuf_valid_csum_request_flags;
 631         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
 632         mbuf->m_pkthdr.csum_data = value;
 633
 634         return 0;
 635 }
 636
 637 static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags =
 638         MBUF_TSO_IPV4 | MBUF_TSO_IPV6;
 639
 640 errno_t
 641 mbuf_get_tso_requested(
 642         mbuf_t mbuf,
 643         mbuf_tso_request_flags_t *request,
 644         u_int32_t *value)
 645 {
 646         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 ||
 647                         request == NULL || value == NULL)
 648                 return EINVAL;
 649
 650         *request = mbuf->m_pkthdr.csum_flags;
 651         *request &= mbuf_valid_tso_request_flags;
 652         if (*request && value != NULL)
 653                 *value = mbuf->m_pkthdr.tso_segsz;
 654
 655         return 0;
 656 }
 657
 658 errno_t
 659 mbuf_get_csum_requested(
 660         mbuf_t mbuf,
 661         mbuf_csum_request_flags_t *request,
 662         u_int32_t *value)
 663 {
 664         *request = mbuf->m_pkthdr.csum_flags;
 665         *request &= mbuf_valid_csum_request_flags;
 666         if (value != NULL) {
 667                 *value = mbuf->m_pkthdr.csum_data;
 668         }
 669
 670         return 0;
 671 }
 672
 673 errno_t
 674 mbuf_clear_csum_requested(
 675         mbuf_t mbuf)
 676 {
 677         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 678         mbuf->m_pkthdr.csum_data = 0;
 679
 680         return 0;
 681 }
 682
 683 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
 684         MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
 685         MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16;
 686
 687 errno_t
 688 mbuf_set_csum_performed(
 689         mbuf_t mbuf,
 690         mbuf_csum_performed_flags_t performed,
 691         u_int32_t value)
 692 {
 693         performed &= mbuf_valid_csum_performed_flags;
 694         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
 695         mbuf->m_pkthdr.csum_data = value;
 696
 697         return 0;
 698 }
 699
 700 errno_t
 701 mbuf_get_csum_performed(
 702         mbuf_t mbuf,
 703         mbuf_csum_performed_flags_t *performed,
 704         u_int32_t *value)
 705 {
 706         *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
 707         *value = mbuf->m_pkthdr.csum_data;
 708
 709         return 0;
 710 }
 711
 712 errno_t
 713 mbuf_clear_csum_performed(
 714         mbuf_t mbuf)
 715 {
 716         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 717         mbuf->m_pkthdr.csum_data = 0;
 718
 719         return 0;
 720 }
 721
 722 errno_t
 723 mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 724     u_int16_t *csum)
 725 {
 726         if (mbuf == NULL || length == 0 || csum == NULL ||
 727            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 728                 return (EINVAL);
 729
 730         *csum = inet_cksum(mbuf, protocol, offset, length);
 731         return (0);
 732 }
 733
 734 #if INET6
 735 errno_t
 736 mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
 737     u_int16_t *csum)
 738 {
 739         if (mbuf == NULL || length == 0 || csum == NULL ||
 740            (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
 741                 return (EINVAL);
 742
 743         *csum = inet6_cksum(mbuf, protocol, offset, length);
 744         return (0);
 745 }
 746 #else /* INET6 */
 747 errno_t
 748 mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
 749                 __unused u_int32_t offset, __unused u_int32_t length,
 750                 __unused u_int16_t *csum)
 751 {
 752         panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
 753         return (0);
 754 }
 755
 756 u_int16_t
 757 inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
 758                 __unused unsigned int off, __unused unsigned int len)
 759 {
 760         panic("inet6_cksum() doesn't exist on this platform\n");
 761         return (0);
 762 }
 763
 764 void nd6_lookup_ipv6(void);
 765 void
 766 nd6_lookup_ipv6(void)
 767 {
 768         panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
 769 }
 770
 771 int
 772 in6addr_local(__unused struct in6_addr *a)
 773 {
 774         panic("in6addr_local() doesn't exist on this platform\n");
 775         return (0);
 776 }
 777
 778 void nd6_storelladdr(void);
 779 void
 780 nd6_storelladdr(void)
 781 {
 782         panic("nd6_storelladdr() doesn't exist on this platform\n");
 783 }
 784 #endif /* INET6 */
 785
 786 /*
 787  * Mbuf tag KPIs
 788  */
 789
 790 #define MTAG_FIRST_ID FIRST_KPI_STR_ID
 791
 792 errno_t
 793 mbuf_tag_id_find(
 794         const char              *string,
 795         mbuf_tag_id_t   *out_id)
 796 {
 797         return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1);
 798 }
 799
 800 errno_t
 801 mbuf_tag_allocate(
 802         mbuf_t                  mbuf,
 803         mbuf_tag_id_t   id,
 804         mbuf_tag_type_t type,
 805         size_t                  length,
 806         mbuf_how_t              how,
 807         void**                  data_p)
 808 {
 809         struct m_tag *tag;
 810         u_int32_t mtag_id_first, mtag_id_last;
 811
 812         if (data_p != NULL)
 813                 *data_p = NULL;
 814
 815         /* Sanity check parameters */
 816         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 817         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 818                 id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 ||
 819                 data_p == NULL) {
 820                 return EINVAL;
 821         }
 822
 823         /* Make sure this mtag hasn't already been allocated */
 824         tag = m_tag_locate(mbuf, id, type, NULL);
 825         if (tag != NULL) {
 826                 return EEXIST;
 827         }
 828
 829         /* Allocate an mtag */
 830         tag = m_tag_alloc(id, type, length, how);
 831         if (tag == NULL) {
 832                 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK;
 833         }
 834
 835         /* Attach the mtag and set *data_p */
 836         m_tag_prepend(mbuf, tag);
 837         *data_p = tag + 1;
 838
 839         return 0;
 840 }
 841
 842 errno_t
 843 mbuf_tag_find(
 844         mbuf_t                  mbuf,
 845         mbuf_tag_id_t   id,
 846         mbuf_tag_type_t type,
 847         size_t*                 length,
 848         void**                  data_p)
 849 {
 850         struct m_tag *tag;
 851         u_int32_t mtag_id_first, mtag_id_last;
 852
 853         if (length != NULL)
 854                 *length = 0;
 855         if (data_p != NULL)
 856                 *data_p = NULL;
 857
 858         /* Sanity check parameters */
 859         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 860         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 861                 id > mtag_id_last || length == NULL || data_p == NULL) {
 862                 return EINVAL;
 863         }
 864
 865         /* Locate an mtag */
 866         tag = m_tag_locate(mbuf, id, type, NULL);
 867         if (tag == NULL) {
 868                 return ENOENT;
 869         }
 870
 871         /* Copy out the pointer to the data and the lenght value */
 872         *length = tag->m_tag_len;
 873         *data_p = tag + 1;
 874
 875         return 0;
 876 }
 877
 878 void
 879 mbuf_tag_free(
 880         mbuf_t                  mbuf,
 881         mbuf_tag_id_t   id,
 882         mbuf_tag_type_t type)
 883 {
 884         struct m_tag *tag;
 885         u_int32_t mtag_id_first, mtag_id_last;
 886
 887         /* Sanity check parameters */
 888         (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG);
 889         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first ||
 890                 id > mtag_id_last)
 891                 return;
 892
 893         tag = m_tag_locate(mbuf, id, type, NULL);
 894         if (tag == NULL) {
 895                 return;
 896         }
 897
 898         m_tag_delete(mbuf, tag);
 899         return;
 900 }
 901
 902 /* mbuf stats */
 903 void mbuf_stats(struct mbuf_stat *stats)
 904 {
 905         stats->mbufs = mbstat.m_mbufs;
 906         stats->clusters = mbstat.m_clusters;
 907         stats->clfree = mbstat.m_clfree;
 908         stats->drops = mbstat.m_drops;
 909         stats->wait = mbstat.m_wait;
 910         stats->drain = mbstat.m_drain;
 911         __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
 912         stats->mcfail = mbstat.m_mcfail;
 913         stats->mpfail = mbstat.m_mpfail;
 914         stats->msize = mbstat.m_msize;
 915         stats->mclbytes = mbstat.m_mclbytes;
 916         stats->minclsize = mbstat.m_minclsize;
 917         stats->mlen = mbstat.m_mlen;
 918         stats->mhlen = mbstat.m_mhlen;
 919         stats->bigclusters = mbstat.m_bigclusters;
 920         stats->bigclfree = mbstat.m_bigclfree;
 921         stats->bigmclbytes = mbstat.m_bigmclbytes;
 922 }
 923
 924 errno_t
 925 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 926 {
 927         errno_t error;
 928         struct mbuf *m;
 929         unsigned int numpkts = 1;
 930         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 931
 932         if (packetlen == 0) {
 933                 error = EINVAL;
 934                 goto out;
 935         }
 936         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 937         if (m == 0) {
 938                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 939                         error = ENOBUFS;
 940                 else
 941                         error = ENOMEM;
 942         } else {
 943                 if (maxchunks)
 944                         *maxchunks = numchunks;
 945                 error = 0;
 946                 *mbuf = m;
 947         }
 948 out:
 949         return error;
 950 }
 951
 952 errno_t
 953 mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 954 {
 955         errno_t error;
 956         struct mbuf *m;
 957         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 958
 959         if (numpkts == 0) {
 960                 error = EINVAL;
 961                 goto out;
 962         }
 963         if (packetlen == 0) {
 964                 error = EINVAL;
 965                 goto out;
 966         }
 967         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 968         if (m == 0) {
 969                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 970                         error = ENOBUFS;
 971                 else
 972                         error = ENOMEM;
 973         } else {
 974                 if (maxchunks)
 975                         *maxchunks = numchunks;
 976                 error = 0;
 977                 *mbuf = m;
 978         }
 979 out:
 980         return error;
 981 }
 982
 983
 984
 985 /*
 986  * mbuf_copyback differs from m_copyback in a few ways:
 987  * 1) mbuf_copyback will allocate clusters for new mbufs we append
 988  * 2) mbuf_copyback will grow the last mbuf in the chain if possible
 989  * 3) mbuf_copyback reports whether or not the operation succeeded
 990  * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
 991  */
 992 errno_t
 993 mbuf_copyback(
 994         mbuf_t          m,
 995         size_t          off,
 996         size_t          len,
 997         const void      *data,
 998         mbuf_how_t      how)
 999 {
1000         size_t  mlen;
1001         mbuf_t  m_start = m;
1002         mbuf_t  n;
1003         int             totlen = 0;
1004         errno_t         result = 0;
1005         const char      *cp = data;
1006
1007         if (m == NULL || len == 0 || data == NULL)
1008                 return EINVAL;
1009
1010         while (off > (mlen = m->m_len)) {
1011                 off -= mlen;
1012                 totlen += mlen;
1013                 if (m->m_next == 0) {
1014                         n = m_getclr(how, m->m_type);
1015                         if (n == 0) {
1016                                 result = ENOBUFS;
1017                                 goto out;
1018                         }
1019                         n->m_len = MIN(MLEN, len + off);
1020                         m->m_next = n;
1021                 }
1022                 m = m->m_next;
1023         }
1024
1025         while (len > 0) {
1026                 mlen = MIN(m->m_len - off, len);
1027                 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) {
1028                         size_t  grow = MIN(mbuf_trailingspace(m), len - mlen);
1029                         mlen += grow;
1030                         m->m_len += grow;
1031                 }
1032                 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen);
1033                 cp += mlen;
1034                 len -= mlen;
1035                 mlen += off;
1036                 off = 0;
1037                 totlen += mlen;
1038                 if (len == 0)
1039                         break;
1040                 if (m->m_next == 0) {
1041                         n = m_get(how, m->m_type);
1042                         if (n == NULL) {
1043                                 result = ENOBUFS;
1044                                 goto out;
1045                         }
1046                         if (len > MINCLSIZE) {
1047                                 /* cluter allocation failure is okay, we can grow chain */
1048                                 mbuf_mclget(how, m->m_type, &n);
1049                         }
1050                         n->m_len = MIN(mbuf_maxlen(n), len);
1051                         m->m_next = n;
1052                 }
1053                 m = m->m_next;
1054         }
1055
1056 out:
1057         if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
1058                 m_start->m_pkthdr.len = totlen;
1059
1060         return result;
1061 }
1062
1063 u_int32_t
1064 mbuf_get_mlen(void)
1065 {
1066         return (_MLEN);
1067 }
1068
1069 u_int32_t
1070 mbuf_get_mhlen(void)
1071 {
1072         return (_MHLEN);
1073 }