bsd/kern/kpi_mbuf.c

   1 /*
   2  * Copyright (c) 2006 Apple Computer, Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 #define __KPI__
  32 //#include <sys/kpi_interface.h>
  33
  34 #include <sys/param.h>
  35 #include <sys/mbuf.h>
  36 #include <sys/socket.h>
  37 #include <kern/debug.h>
  38 #include <libkern/OSAtomic.h>
  39 #include <kern/kalloc.h>
  40 #include <string.h>
  41
  42 void mbuf_tag_id_first_last(u_long *first, u_long *last);
  43 errno_t mbuf_tag_id_find_internal(const char *string, u_long *out_id, int create);
  44
  45 static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
  46                                 MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
  47                                 MBUF_LASTFRAG | MBUF_PROMISC;
  48
  49 void* mbuf_data(mbuf_t mbuf)
  50 {
  51         return m_mtod(mbuf);
  52 }
  53
  54 void* mbuf_datastart(mbuf_t mbuf)
  55 {
  56         if (mbuf->m_flags & M_EXT)
  57                 return mbuf->m_ext.ext_buf;
  58         if (mbuf->m_flags & M_PKTHDR)
  59                 return mbuf->m_pktdat;
  60         return mbuf->m_dat;
  61 }
  62
  63 errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len)
  64 {
  65         size_t  start = (size_t)((char*)mbuf_datastart(mbuf));
  66         size_t  maxlen = mbuf_maxlen(mbuf);
  67
  68         if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
  69                 return EINVAL;
  70         mbuf->m_data = data;
  71         mbuf->m_len = len;
  72
  73         return 0;
  74 }
  75
  76 errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
  77 {
  78         if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
  79                 return ENOTSUP;
  80         mbuf->m_data = mbuf_datastart(mbuf);
  81         mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
  82
  83         return 0;
  84 }
  85
  86 addr64_t mbuf_data_to_physical(void* ptr)
  87 {
  88         return (addr64_t)mcl_to_paddr(ptr);
  89 }
  90
  91 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  92 {
  93         /* Must set *mbuf to NULL in failure case */
  94         *mbuf = m_get(how, type);
  95
  96         return (*mbuf == NULL) ? ENOMEM : 0;
  97 }
  98
  99 errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 100 {
 101         /* Must set *mbuf to NULL in failure case */
 102         *mbuf = m_gethdr(how, type);
 103
 104         return (*mbuf == NULL) ? ENOMEM : 0;
 105 }
 106
 107 extern struct mbuf * m_mbigget(struct mbuf *m, int nowait);
 108
 109 errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 110 {
 111         /* Must set *mbuf to NULL in failure case */
 112         errno_t error = 0;
 113         int             created = 0;
 114
 115         if (mbuf == NULL)
 116                 return EINVAL;
 117         if (*mbuf == NULL) {
 118                 *mbuf = m_get(how, type);
 119                 if (*mbuf == NULL)
 120                         return ENOMEM;
 121                 created = 1;
 122         }
 123         /*
 124          * At the time this code was written, m_mclget and m_mbigget would always
 125          * return the same value that was passed in to it.
 126          */
 127         if (size == MCLBYTES) {
 128                 *mbuf = m_mclget(*mbuf, how);
 129         } else if (size == NBPG) {
 130                 *mbuf = m_mbigget(*mbuf, how);
 131         } else {
 132                 error = EINVAL;
 133                 goto out;
 134         }
 135         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 136                 error = ENOMEM;
 137 out:
 138         if (created && error != 0) {
 139                 error = ENOMEM;
 140                 mbuf_free(*mbuf);
 141                 *mbuf = NULL;
 142         }
 143         return error;
 144 }
 145
 146 errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 147 {
 148         /* Must set *mbuf to NULL in failure case */
 149         errno_t error = 0;
 150         int             created = 0;
 151         if (mbuf == NULL) return EINVAL;
 152         if (*mbuf == NULL) {
 153                 error = mbuf_get(how, type, mbuf);
 154                 if (error)
 155                         return error;
 156                 created = 1;
 157         }
 158
 159         /*
 160          * At the time this code was written, m_mclget would always
 161          * return the same value that was passed in to it.
 162          */
 163         *mbuf = m_mclget(*mbuf, how);
 164
 165         if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
 166                 mbuf_free(*mbuf);
 167                 *mbuf = NULL;
 168         }
 169         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 170                 error = ENOMEM;
 171         return error;
 172 }
 173
 174
 175 errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
 176 {
 177         /* Must set *mbuf to NULL in failure case */
 178         errno_t error = 0;
 179
 180         *mbuf = m_getpacket_how(how);
 181
 182         if (*mbuf == NULL) {
 183                 if (how == MBUF_WAITOK)
 184                         error = ENOMEM;
 185                 else
 186                         error = EWOULDBLOCK;
 187         }
 188
 189         return error;
 190 }
 191
 192 mbuf_t mbuf_free(mbuf_t mbuf)
 193 {
 194         return m_free(mbuf);
 195 }
 196
 197 void mbuf_freem(mbuf_t mbuf)
 198 {
 199         m_freem(mbuf);
 200 }
 201
 202 int     mbuf_freem_list(mbuf_t mbuf)
 203 {
 204         return m_freem_list(mbuf);
 205 }
 206
 207 size_t mbuf_leadingspace(mbuf_t mbuf)
 208 {
 209         return m_leadingspace(mbuf);
 210 }
 211
 212 size_t mbuf_trailingspace(mbuf_t mbuf)
 213 {
 214         return m_trailingspace(mbuf);
 215 }
 216
 217 /* Manipulation */
 218 errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len,
 219                                    mbuf_how_t how, mbuf_t *new_mbuf)
 220 {
 221         /* Must set *mbuf to NULL in failure case */
 222         *new_mbuf = m_copym(src, offset, len, how);
 223
 224         return (*new_mbuf == NULL) ? ENOMEM : 0;
 225 }
 226
 227 errno_t mbuf_dup(mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 228 {
 229         /* Must set *new_mbuf to NULL in failure case */
 230         *new_mbuf = m_dup(src, how);
 231
 232         return (*new_mbuf == NULL) ? ENOMEM : 0;
 233 }
 234
 235 errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
 236 {
 237         /* Must set *orig to NULL in failure case */
 238         *orig = m_prepend_2(*orig, len, how);
 239
 240         return (*orig == NULL) ? ENOMEM : 0;
 241 }
 242
 243 errno_t mbuf_split(mbuf_t src, size_t offset,
 244                                         mbuf_how_t how, mbuf_t *new_mbuf)
 245 {
 246         /* Must set *new_mbuf to NULL in failure case */
 247         *new_mbuf = m_split(src, offset, how);
 248
 249         return (*new_mbuf == NULL) ? ENOMEM : 0;
 250 }
 251
 252 errno_t mbuf_pullup(mbuf_t *mbuf, size_t len)
 253 {
 254         /* Must set *mbuf to NULL in failure case */
 255         *mbuf = m_pullup(*mbuf, len);
 256
 257         return (*mbuf == NULL) ? ENOMEM : 0;
 258 }
 259
 260 errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
 261 {
 262         /* Must set *location to NULL in failure case */
 263         int new_offset;
 264         *location = m_pulldown(src, *offset, len, &new_offset);
 265         *offset = new_offset;
 266
 267         return (*location == NULL) ? ENOMEM : 0;
 268 }
 269
 270 void mbuf_adj(mbuf_t mbuf, int len)
 271 {
 272         m_adj(mbuf, len);
 273 }
 274
 275 errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data)
 276 {
 277         /* Copied m_copydata, added error handling (don't just panic) */
 278         int count;
 279
 280         while (off > 0) {
 281                 if (m == 0)
 282                         return EINVAL;
 283                 if (off < (size_t)m->m_len)
 284                         break;
 285                 off -= m->m_len;
 286                 m = m->m_next;
 287         }
 288         while (len > 0) {
 289                 if (m == 0)
 290                         return EINVAL;
 291                 count = m->m_len - off > len ? len : m->m_len - off;
 292                 bcopy(mtod(m, caddr_t) + off, out_data, count);
 293                 len -= count;
 294                 out_data = ((char*)out_data) + count;
 295                 off = 0;
 296                 m = m->m_next;
 297         }
 298
 299         return 0;
 300 }
 301
 302 int mbuf_mclref(mbuf_t mbuf)
 303 {
 304         return m_mclref(mbuf);
 305 }
 306
 307 int mbuf_mclunref(mbuf_t mbuf)
 308 {
 309         return m_mclunref(mbuf);
 310 }
 311
 312 int mbuf_mclhasreference(mbuf_t mbuf)
 313 {
 314         if ((mbuf->m_flags & M_EXT))
 315                 return m_mclhasreference(mbuf);
 316         else
 317                 return 0;
 318 }
 319
 320
 321 /* mbuf header */
 322 mbuf_t mbuf_next(mbuf_t mbuf)
 323 {
 324         return mbuf->m_next;
 325 }
 326
 327 errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 328 {
 329         if (next && ((next)->m_nextpkt != NULL ||
 330                 (next)->m_type == MT_FREE)) return EINVAL;
 331         mbuf->m_next = next;
 332
 333         return 0;
 334 }
 335
 336 mbuf_t mbuf_nextpkt(mbuf_t mbuf)
 337 {
 338         return mbuf->m_nextpkt;
 339 }
 340
 341 void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 342 {
 343         mbuf->m_nextpkt = nextpkt;
 344 }
 345
 346 size_t mbuf_len(mbuf_t mbuf)
 347 {
 348         return mbuf->m_len;
 349 }
 350
 351 void mbuf_setlen(mbuf_t mbuf, size_t len)
 352 {
 353         mbuf->m_len = len;
 354 }
 355
 356 size_t mbuf_maxlen(mbuf_t mbuf)
 357 {
 358         if (mbuf->m_flags & M_EXT)
 359                 return mbuf->m_ext.ext_size;
 360         return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 361 }
 362
 363 mbuf_type_t mbuf_type(mbuf_t mbuf)
 364 {
 365         return mbuf->m_type;
 366 }
 367
 368 errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 369 {
 370         if (new_type == MBUF_TYPE_FREE) return EINVAL;
 371
 372         m_mchtype(mbuf, new_type);
 373
 374         return 0;
 375 }
 376
 377 mbuf_flags_t mbuf_flags(mbuf_t mbuf)
 378 {
 379         return mbuf->m_flags & mbuf_flags_mask;
 380 }
 381
 382 errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
 383 {
 384         if ((flags & ~mbuf_flags_mask) != 0) return EINVAL;
 385         mbuf->m_flags = flags |
 386                 (mbuf->m_flags & ~mbuf_flags_mask);
 387
 388         return 0;
 389 }
 390
 391 errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 392 {
 393         if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL;
 394
 395         mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
 396
 397         return 0;
 398 }
 399
 400 errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src)
 401 {
 402         if (((src)->m_flags & M_PKTHDR) == 0)
 403                 return EINVAL;
 404
 405         m_copy_pkthdr(dest, src);
 406
 407         return 0;
 408 }
 409
 410 size_t mbuf_pkthdr_len(mbuf_t mbuf)
 411 {
 412         return mbuf->m_pkthdr.len;
 413 }
 414
 415 void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 416 {
 417         mbuf->m_pkthdr.len = len;
 418 }
 419
 420 ifnet_t mbuf_pkthdr_rcvif(mbuf_t mbuf)
 421 {
 422         // If we reference count ifnets, we should take a reference here before returning
 423         return mbuf->m_pkthdr.rcvif;
 424 }
 425
 426 errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 427 {
 428         /* May want to walk ifnet list to determine if interface is valid */
 429         mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet;
 430         return 0;
 431 }
 432
 433 void* mbuf_pkthdr_header(mbuf_t mbuf)
 434 {
 435         return mbuf->m_pkthdr.header;
 436 }
 437
 438 void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 439 {
 440         mbuf->m_pkthdr.header = (void*)header;
 441 }
 442
 443 /* mbuf aux data */
 444 errno_t mbuf_aux_add(mbuf_t mbuf, int family, mbuf_type_t type, mbuf_t *aux_mbuf)
 445 {
 446         *aux_mbuf = m_aux_add(mbuf, family, type);
 447         return (*aux_mbuf == NULL) ? ENOMEM : 0;
 448 }
 449
 450 mbuf_t mbuf_aux_find(mbuf_t mbuf, int family, mbuf_type_t type)
 451 {
 452         return m_aux_find(mbuf, family, type);
 453 }
 454
 455 void mbuf_aux_delete(mbuf_t mbuf, mbuf_t aux)
 456 {
 457         m_aux_delete(mbuf, aux);
 458 }
 459
 460 void
 461 mbuf_inbound_modified(mbuf_t mbuf)
 462 {
 463         /* Invalidate hardware generated checksum flags */
 464         mbuf->m_pkthdr.csum_flags = 0;
 465 }
 466
 467 extern void in_cksum_offset(struct mbuf* m, size_t ip_offset);
 468 extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset);
 469
 470 void
 471 mbuf_outbound_finalize(mbuf_t mbuf, u_long protocol_family, size_t protocol_offset)
 472 {
 473         if ((mbuf->m_pkthdr.csum_flags &
 474                  (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16)) == 0)
 475                 return;
 476
 477         /* Generate the packet in software, client needs it */
 478         switch (protocol_family) {
 479                 case PF_INET:
 480                         if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 481                                 /*
 482                                  * If you're wondering where this lovely code comes
 483                                  * from, we're trying to undo what happens in ip_output.
 484                                  * Look for CSUM_TCP_SUM16 in ip_output.
 485                                  */
 486                                 u_int16_t       first, second;
 487                                 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16;
 488                                 mbuf->m_pkthdr.csum_flags |= CSUM_TCP;
 489                                 first = mbuf->m_pkthdr.csum_data >> 16;
 490                                 second = mbuf->m_pkthdr.csum_data & 0xffff;
 491                                 mbuf->m_pkthdr.csum_data = first - second;
 492                         }
 493                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
 494                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 495                         }
 496
 497                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
 498                                 in_cksum_offset(mbuf, protocol_offset);
 499                         }
 500
 501                         mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP);
 502                         break;
 503
 504                 default:
 505                         /*
 506                          * Not sure what to do here if anything.
 507                          * Hardware checksum code looked pretty IPv4 specific.
 508                          */
 509                         if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP)) != 0)
 510                                 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%d)!\n", protocol_family);
 511         }
 512 }
 513
 514 errno_t
 515 mbuf_set_vlan_tag(
 516         mbuf_t mbuf,
 517         u_int16_t vlan)
 518 {
 519         mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
 520         mbuf->m_pkthdr.vlan_tag = vlan;
 521
 522         return 0;
 523 }
 524
 525 errno_t
 526 mbuf_get_vlan_tag(
 527         mbuf_t mbuf,
 528         u_int16_t *vlan)
 529 {
 530         if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
 531                 return ENXIO; // No vlan tag set
 532
 533         *vlan = mbuf->m_pkthdr.vlan_tag;
 534
 535         return 0;
 536 }
 537
 538 errno_t
 539 mbuf_clear_vlan_tag(
 540         mbuf_t mbuf)
 541 {
 542         mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
 543         mbuf->m_pkthdr.vlan_tag = 0;
 544
 545         return 0;
 546 }
 547
 548 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
 549         MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | MBUF_CSUM_REQ_SUM16;
 550
 551 errno_t
 552 mbuf_set_csum_requested(
 553         mbuf_t mbuf,
 554         mbuf_csum_request_flags_t request,
 555         u_int32_t value)
 556 {
 557         request &= mbuf_valid_csum_request_flags;
 558         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
 559         mbuf->m_pkthdr.csum_data = value;
 560
 561         return 0;
 562 }
 563
 564 errno_t
 565 mbuf_get_csum_requested(
 566         mbuf_t mbuf,
 567         mbuf_csum_request_flags_t *request,
 568         u_int32_t *value)
 569 {
 570         *request = mbuf->m_pkthdr.csum_flags;
 571         *request &= mbuf_valid_csum_request_flags;
 572         if (value != NULL) {
 573                 *value = mbuf->m_pkthdr.csum_data;
 574         }
 575
 576         return 0;
 577 }
 578
 579 errno_t
 580 mbuf_clear_csum_requested(
 581         mbuf_t mbuf)
 582 {
 583         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 584         mbuf->m_pkthdr.csum_data = 0;
 585
 586         return 0;
 587 }
 588
 589 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
 590         MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
 591         MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16;
 592
 593 errno_t
 594 mbuf_set_csum_performed(
 595         mbuf_t mbuf,
 596         mbuf_csum_performed_flags_t performed,
 597         u_int32_t value)
 598 {
 599         performed &= mbuf_valid_csum_performed_flags;
 600         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
 601         mbuf->m_pkthdr.csum_data = value;
 602
 603         return 0;
 604 }
 605
 606 errno_t
 607 mbuf_get_csum_performed(
 608         mbuf_t mbuf,
 609         mbuf_csum_performed_flags_t *performed,
 610         u_int32_t *value)
 611 {
 612         *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
 613         *value = mbuf->m_pkthdr.csum_data;
 614
 615         return 0;
 616 }
 617
 618 errno_t
 619 mbuf_clear_csum_performed(
 620         mbuf_t mbuf)
 621 {
 622         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 623         mbuf->m_pkthdr.csum_data = 0;
 624
 625         return 0;
 626 }
 627
 628 /*
 629  * Mbuf tag KPIs
 630  */
 631
 632 struct mbuf_tag_id_entry {
 633         SLIST_ENTRY(mbuf_tag_id_entry)  next;
 634         mbuf_tag_id_t                                   id;
 635         char                                                    string[];
 636 };
 637
 638 #define MBUF_TAG_ID_ENTRY_SIZE(__str) \
 639         ((size_t)&(((struct mbuf_tag_id_entry*)0)->string[0]) + \
 640          strlen(__str) + 1)
 641
 642 #define MTAG_FIRST_ID                                   1000
 643 static u_long                                                   mtag_id_next = MTAG_FIRST_ID;
 644 static SLIST_HEAD(,mbuf_tag_id_entry)   mtag_id_list = {NULL};
 645 static lck_mtx_t                                                *mtag_id_lock = NULL;
 646
 647 __private_extern__ void
 648 mbuf_tag_id_first_last(
 649         u_long  *first,
 650         u_long  *last)
 651 {
 652         *first = MTAG_FIRST_ID;
 653         *last = mtag_id_next - 1;
 654 }
 655
 656 __private_extern__ errno_t
 657 mbuf_tag_id_find_internal(
 658         const char      *string,
 659         u_long          *out_id,
 660         int                     create)
 661 {
 662         struct mbuf_tag_id_entry                                *entry = NULL;
 663
 664
 665         *out_id = 0;
 666
 667         if (string == NULL || out_id == NULL) {
 668                 return EINVAL;
 669         }
 670
 671         /* Don't bother allocating the lock if we're only doing a lookup */
 672         if (create == 0 && mtag_id_lock == NULL)
 673                 return ENOENT;
 674
 675         /* Allocate lock if necessary */
 676         if (mtag_id_lock == NULL) {
 677                 lck_grp_attr_t  *grp_attrib = NULL;
 678                 lck_attr_t              *lck_attrb = NULL;
 679                 lck_grp_t               *lck_group = NULL;
 680                 lck_mtx_t               *new_lock = NULL;
 681
 682                 grp_attrib = lck_grp_attr_alloc_init();
 683                 lck_grp_attr_setdefault(grp_attrib);
 684                 lck_group = lck_grp_alloc_init("mbuf_tag_allocate_id", grp_attrib);
 685                 lck_grp_attr_free(grp_attrib);
 686                 lck_attrb = lck_attr_alloc_init();
 687                 lck_attr_setdefault(lck_attrb);
 688                 lck_attr_setdebug(lck_attrb);
 689                 new_lock = lck_mtx_alloc_init(lck_group, lck_attrb);
 690                 if (!OSCompareAndSwap((UInt32)0, (UInt32)new_lock, (UInt32*)&mtag_id_lock)) {
 691                         /*
 692                          * If the atomic swap fails, someone else has already
 693                          * done this work. We can free the stuff we allocated.
 694                          */
 695                         lck_mtx_free(new_lock, lck_group);
 696                         lck_grp_free(lck_group);
 697                 }
 698                 lck_attr_free(lck_attrb);
 699         }
 700
 701         /* Look for an existing entry */
 702         lck_mtx_lock(mtag_id_lock);
 703         SLIST_FOREACH(entry, &mtag_id_list, next) {
 704                 if (strcmp(string, entry->string) == 0) {
 705                         break;
 706                 }
 707         }
 708
 709         if (entry == NULL) {
 710                 if (create == 0) {
 711                         lck_mtx_unlock(mtag_id_lock);
 712                         return ENOENT;
 713                 }
 714
 715                 entry = kalloc(MBUF_TAG_ID_ENTRY_SIZE(string));
 716                 if (entry == NULL) {
 717                         lck_mtx_unlock(mtag_id_lock);
 718                         return ENOMEM;
 719                 }
 720
 721                 strcpy(entry->string, string);
 722                 entry->id = mtag_id_next;
 723                 mtag_id_next++;
 724                 SLIST_INSERT_HEAD(&mtag_id_list, entry, next);
 725         }
 726         lck_mtx_unlock(mtag_id_lock);
 727
 728         *out_id = entry->id;
 729
 730         return 0;
 731 }
 732
 733 errno_t
 734 mbuf_tag_id_find(
 735         const char              *string,
 736         mbuf_tag_id_t   *out_id)
 737 {
 738         return mbuf_tag_id_find_internal(string, (u_long*)out_id, 1);
 739 }
 740
 741 errno_t
 742 mbuf_tag_allocate(
 743         mbuf_t                  mbuf,
 744         mbuf_tag_id_t   id,
 745         mbuf_tag_type_t type,
 746         size_t                  length,
 747         mbuf_how_t              how,
 748         void**                  data_p)
 749 {
 750         struct m_tag *tag;
 751
 752         if (data_p != NULL)
 753                 *data_p = NULL;
 754
 755         /* Sanity check parameters */
 756         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 757                 id >= mtag_id_next || length < 1 || (length & 0xffff0000) != 0 ||
 758                 data_p == NULL) {
 759                 return EINVAL;
 760         }
 761
 762         /* Make sure this mtag hasn't already been allocated */
 763         tag = m_tag_locate(mbuf, id, type, NULL);
 764         if (tag != NULL) {
 765                 return EEXIST;
 766         }
 767
 768         /* Allocate an mtag */
 769         tag = m_tag_alloc(id, type, length, how);
 770         if (tag == NULL) {
 771                 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK;
 772         }
 773
 774         /* Attach the mtag and set *data_p */
 775         m_tag_prepend(mbuf, tag);
 776         *data_p = tag + 1;
 777
 778         return 0;
 779 }
 780
 781 errno_t
 782 mbuf_tag_find(
 783         mbuf_t                  mbuf,
 784         mbuf_tag_id_t   id,
 785         mbuf_tag_type_t type,
 786         size_t*                 length,
 787         void**                  data_p)
 788 {
 789         struct m_tag *tag;
 790
 791         if (length != NULL)
 792                 *length = 0;
 793         if (data_p != NULL)
 794                 *data_p = NULL;
 795
 796         /* Sanity check parameters */
 797         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 798                 id >= mtag_id_next || length == NULL || data_p == NULL) {
 799                 return EINVAL;
 800         }
 801
 802         /* Locate an mtag */
 803         tag = m_tag_locate(mbuf, id, type, NULL);
 804         if (tag == NULL) {
 805                 return ENOENT;
 806         }
 807
 808         /* Copy out the pointer to the data and the lenght value */
 809         *length = tag->m_tag_len;
 810         *data_p = tag + 1;
 811
 812         return 0;
 813 }
 814
 815 void
 816 mbuf_tag_free(
 817         mbuf_t                  mbuf,
 818         mbuf_tag_id_t   id,
 819         mbuf_tag_type_t type)
 820 {
 821         struct m_tag *tag;
 822
 823         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 824                 id >= mtag_id_next)
 825                 return;
 826
 827         tag = m_tag_locate(mbuf, id, type, NULL);
 828         if (tag == NULL) {
 829                 return;
 830         }
 831
 832         m_tag_delete(mbuf, tag);
 833         return;
 834 }
 835
 836 /* mbuf stats */
 837 void mbuf_stats(struct mbuf_stat *stats)
 838 {
 839         stats->mbufs = mbstat.m_mbufs;
 840         stats->clusters = mbstat.m_clusters;
 841         stats->clfree = mbstat.m_clfree;
 842         stats->drops = mbstat.m_drops;
 843         stats->wait = mbstat.m_wait;
 844         stats->drain = mbstat.m_drain;
 845         __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
 846         stats->mcfail = mbstat.m_mcfail;
 847         stats->mpfail = mbstat.m_mpfail;
 848         stats->msize = mbstat.m_msize;
 849         stats->mclbytes = mbstat.m_mclbytes;
 850         stats->minclsize = mbstat.m_minclsize;
 851         stats->mlen = mbstat.m_mlen;
 852         stats->mhlen = mbstat.m_mhlen;
 853         stats->bigclusters = mbstat.m_bigclusters;
 854         stats->bigclfree = mbstat.m_bigclfree;
 855         stats->bigmclbytes = mbstat.m_bigmclbytes;
 856 }
 857
 858 errno_t
 859 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 860 {
 861         errno_t error;
 862         struct mbuf *m;
 863         unsigned int numpkts = 1;
 864         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 865
 866         if (packetlen == 0) {
 867                 error = EINVAL;
 868                 goto out;
 869         }
 870         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 871         if (m == 0) {
 872                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 873                         error = ENOBUFS;
 874                 else
 875                         error = ENOMEM;
 876         } else {
 877                 error = 0;
 878                 *mbuf = m;
 879         }
 880 out:
 881         return error;
 882 }
 883
 884
 885 /*
 886  * mbuf_copyback differs from m_copyback in a few ways:
 887  * 1) mbuf_copyback will allocate clusters for new mbufs we append
 888  * 2) mbuf_copyback will grow the last mbuf in the chain if possible
 889  * 3) mbuf_copyback reports whether or not the operation succeeded
 890  * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
 891  */
 892 errno_t
 893 mbuf_copyback(
 894         mbuf_t          m,
 895         size_t          off,
 896         size_t          len,
 897         const void      *data,
 898         mbuf_how_t      how)
 899 {
 900         size_t  mlen;
 901         mbuf_t  m_start = m;
 902         mbuf_t  n;
 903         int             totlen = 0;
 904         errno_t         result = 0;
 905         const char      *cp = data;
 906
 907         if (m == NULL || len == 0 || data == NULL)
 908                 return EINVAL;
 909
 910         while (off > (mlen = m->m_len)) {
 911                 off -= mlen;
 912                 totlen += mlen;
 913                 if (m->m_next == 0) {
 914                         n = m_getclr(how, m->m_type);
 915                         if (n == 0) {
 916                                 result = ENOBUFS;
 917                                 goto out;
 918                         }
 919                         n->m_len = MIN(MLEN, len + off);
 920                         m->m_next = n;
 921                 }
 922                 m = m->m_next;
 923         }
 924
 925         while (len > 0) {
 926                 mlen = MIN(m->m_len - off, len);
 927                 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) {
 928                         size_t  grow = MIN(mbuf_trailingspace(m), len - mlen);
 929                         mlen += grow;
 930                         m->m_len += grow;
 931                 }
 932                 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen);
 933                 cp += mlen;
 934                 len -= mlen;
 935                 mlen += off;
 936                 off = 0;
 937                 totlen += mlen;
 938                 if (len == 0)
 939                         break;
 940                 if (m->m_next == 0) {
 941                         n = m_get(how, m->m_type);
 942                         if (n == NULL) {
 943                                 result = ENOBUFS;
 944                                 goto out;
 945                         }
 946                         if (len > MINCLSIZE) {
 947                                 /* cluter allocation failure is okay, we can grow chain */
 948                                 mbuf_mclget(how, m->m_type, &n);
 949                         }
 950                         n->m_len = MIN(mbuf_maxlen(n), len);
 951                         m->m_next = n;
 952                 }
 953                 m = m->m_next;
 954         }
 955
 956 out:
 957         if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
 958                 m_start->m_pkthdr.len = totlen;
 959
 960         return result;
 961 }