bsd/kern/kpi_mbuf.c

   1 /*
   2  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define __KPI__
  30 //#include <sys/kpi_interface.h>
  31
  32 #include <sys/param.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/socket.h>
  35 #include <kern/debug.h>
  36 #include <libkern/OSAtomic.h>
  37 #include <kern/kalloc.h>
  38 #include <string.h>
  39
  40 void mbuf_tag_id_first_last(u_long *first, u_long *last);
  41 errno_t mbuf_tag_id_find_internal(const char *string, u_long *out_id, int create);
  42
  43 static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
  44                                 MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
  45                                 MBUF_LASTFRAG | MBUF_PROMISC;
  46
  47 void* mbuf_data(mbuf_t mbuf)
  48 {
  49         return m_mtod(mbuf);
  50 }
  51
  52 void* mbuf_datastart(mbuf_t mbuf)
  53 {
  54         if (mbuf->m_flags & M_EXT)
  55                 return mbuf->m_ext.ext_buf;
  56         if (mbuf->m_flags & M_PKTHDR)
  57                 return mbuf->m_pktdat;
  58         return mbuf->m_dat;
  59 }
  60
  61 errno_t mbuf_setdata(mbuf_t mbuf, void* data, size_t len)
  62 {
  63         size_t  start = (size_t)((char*)mbuf_datastart(mbuf));
  64         size_t  maxlen = mbuf_maxlen(mbuf);
  65
  66         if ((size_t)data < start || ((size_t)data) + len > start + maxlen)
  67                 return EINVAL;
  68         mbuf->m_data = data;
  69         mbuf->m_len = len;
  70
  71         return 0;
  72 }
  73
  74 errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
  75 {
  76         if ((mbuf->m_flags & M_EXT) != 0 && m_mclhasreference(mbuf))
  77                 return ENOTSUP;
  78         mbuf->m_data = mbuf_datastart(mbuf);
  79         mbuf->m_data += ((mbuf_trailingspace(mbuf) - len) &~ (sizeof(u_int32_t) - 1));
  80
  81         return 0;
  82 }
  83
  84 addr64_t mbuf_data_to_physical(void* ptr)
  85 {
  86         return (addr64_t)mcl_to_paddr(ptr);
  87 }
  88
  89 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  90 {
  91         /* Must set *mbuf to NULL in failure case */
  92         *mbuf = m_get(how, type);
  93
  94         return (*mbuf == NULL) ? ENOMEM : 0;
  95 }
  96
  97 errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
  98 {
  99         /* Must set *mbuf to NULL in failure case */
 100         *mbuf = m_gethdr(how, type);
 101
 102         return (*mbuf == NULL) ? ENOMEM : 0;
 103 }
 104
 105 extern struct mbuf * m_mbigget(struct mbuf *m, int nowait);
 106
 107 errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 108 {
 109         /* Must set *mbuf to NULL in failure case */
 110         errno_t error = 0;
 111         int             created = 0;
 112
 113         if (mbuf == NULL)
 114                 return EINVAL;
 115         if (*mbuf == NULL) {
 116                 *mbuf = m_get(how, type);
 117                 if (*mbuf == NULL)
 118                         return ENOMEM;
 119                 created = 1;
 120         }
 121         /*
 122          * At the time this code was written, m_mclget and m_mbigget would always
 123          * return the same value that was passed in to it.
 124          */
 125         if (size == MCLBYTES) {
 126                 *mbuf = m_mclget(*mbuf, how);
 127         } else if (size == NBPG) {
 128                 *mbuf = m_mbigget(*mbuf, how);
 129         } else {
 130                 error = EINVAL;
 131                 goto out;
 132         }
 133         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 134                 error = ENOMEM;
 135 out:
 136         if (created && error != 0) {
 137                 error = ENOMEM;
 138                 mbuf_free(*mbuf);
 139                 *mbuf = NULL;
 140         }
 141         return error;
 142 }
 143
 144 errno_t mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 145 {
 146         /* Must set *mbuf to NULL in failure case */
 147         errno_t error = 0;
 148         int             created = 0;
 149         if (mbuf == NULL) return EINVAL;
 150         if (*mbuf == NULL) {
 151                 error = mbuf_get(how, type, mbuf);
 152                 if (error)
 153                         return error;
 154                 created = 1;
 155         }
 156
 157         /*
 158          * At the time this code was written, m_mclget would always
 159          * return the same value that was passed in to it.
 160          */
 161         *mbuf = m_mclget(*mbuf, how);
 162
 163         if (created && ((*mbuf)->m_flags & M_EXT) == 0) {
 164                 mbuf_free(*mbuf);
 165                 *mbuf = NULL;
 166         }
 167         if (*mbuf == NULL || ((*mbuf)->m_flags & M_EXT) == 0)
 168                 error = ENOMEM;
 169         return error;
 170 }
 171
 172
 173 errno_t mbuf_getpacket(mbuf_how_t how, mbuf_t *mbuf)
 174 {
 175         /* Must set *mbuf to NULL in failure case */
 176         errno_t error = 0;
 177
 178         *mbuf = m_getpacket_how(how);
 179
 180         if (*mbuf == NULL) {
 181                 if (how == MBUF_WAITOK)
 182                         error = ENOMEM;
 183                 else
 184                         error = EWOULDBLOCK;
 185         }
 186
 187         return error;
 188 }
 189
 190 mbuf_t mbuf_free(mbuf_t mbuf)
 191 {
 192         return m_free(mbuf);
 193 }
 194
 195 void mbuf_freem(mbuf_t mbuf)
 196 {
 197         m_freem(mbuf);
 198 }
 199
 200 int     mbuf_freem_list(mbuf_t mbuf)
 201 {
 202         return m_freem_list(mbuf);
 203 }
 204
 205 size_t mbuf_leadingspace(mbuf_t mbuf)
 206 {
 207         return m_leadingspace(mbuf);
 208 }
 209
 210 size_t mbuf_trailingspace(mbuf_t mbuf)
 211 {
 212         return m_trailingspace(mbuf);
 213 }
 214
 215 /* Manipulation */
 216 errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len,
 217                                    mbuf_how_t how, mbuf_t *new_mbuf)
 218 {
 219         /* Must set *mbuf to NULL in failure case */
 220         *new_mbuf = m_copym(src, offset, len, how);
 221
 222         return (*new_mbuf == NULL) ? ENOMEM : 0;
 223 }
 224
 225 errno_t mbuf_dup(mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 226 {
 227         /* Must set *new_mbuf to NULL in failure case */
 228         *new_mbuf = m_dup(src, how);
 229
 230         return (*new_mbuf == NULL) ? ENOMEM : 0;
 231 }
 232
 233 errno_t mbuf_prepend(mbuf_t *orig, size_t len, mbuf_how_t how)
 234 {
 235         /* Must set *orig to NULL in failure case */
 236         *orig = m_prepend_2(*orig, len, how);
 237
 238         return (*orig == NULL) ? ENOMEM : 0;
 239 }
 240
 241 errno_t mbuf_split(mbuf_t src, size_t offset,
 242                                         mbuf_how_t how, mbuf_t *new_mbuf)
 243 {
 244         /* Must set *new_mbuf to NULL in failure case */
 245         *new_mbuf = m_split(src, offset, how);
 246
 247         return (*new_mbuf == NULL) ? ENOMEM : 0;
 248 }
 249
 250 errno_t mbuf_pullup(mbuf_t *mbuf, size_t len)
 251 {
 252         /* Must set *mbuf to NULL in failure case */
 253         *mbuf = m_pullup(*mbuf, len);
 254
 255         return (*mbuf == NULL) ? ENOMEM : 0;
 256 }
 257
 258 errno_t mbuf_pulldown(mbuf_t src, size_t *offset, size_t len, mbuf_t *location)
 259 {
 260         /* Must set *location to NULL in failure case */
 261         int new_offset;
 262         *location = m_pulldown(src, *offset, len, &new_offset);
 263         *offset = new_offset;
 264
 265         return (*location == NULL) ? ENOMEM : 0;
 266 }
 267
 268 void mbuf_adj(mbuf_t mbuf, int len)
 269 {
 270         m_adj(mbuf, len);
 271 }
 272
 273 errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data)
 274 {
 275         /* Copied m_copydata, added error handling (don't just panic) */
 276         int count;
 277
 278         while (off > 0) {
 279                 if (m == 0)
 280                         return EINVAL;
 281                 if (off < (size_t)m->m_len)
 282                         break;
 283                 off -= m->m_len;
 284                 m = m->m_next;
 285         }
 286         while (len > 0) {
 287                 if (m == 0)
 288                         return EINVAL;
 289                 count = m->m_len - off > len ? len : m->m_len - off;
 290                 bcopy(mtod(m, caddr_t) + off, out_data, count);
 291                 len -= count;
 292                 out_data = ((char*)out_data) + count;
 293                 off = 0;
 294                 m = m->m_next;
 295         }
 296
 297         return 0;
 298 }
 299
 300 int mbuf_mclref(mbuf_t mbuf)
 301 {
 302         return m_mclref(mbuf);
 303 }
 304
 305 int mbuf_mclunref(mbuf_t mbuf)
 306 {
 307         return m_mclunref(mbuf);
 308 }
 309
 310 int mbuf_mclhasreference(mbuf_t mbuf)
 311 {
 312         if ((mbuf->m_flags & M_EXT))
 313                 return m_mclhasreference(mbuf);
 314         else
 315                 return 0;
 316 }
 317
 318
 319 /* mbuf header */
 320 mbuf_t mbuf_next(mbuf_t mbuf)
 321 {
 322         return mbuf->m_next;
 323 }
 324
 325 errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 326 {
 327         if (next && ((next)->m_nextpkt != NULL ||
 328                 (next)->m_type == MT_FREE)) return EINVAL;
 329         mbuf->m_next = next;
 330
 331         return 0;
 332 }
 333
 334 mbuf_t mbuf_nextpkt(mbuf_t mbuf)
 335 {
 336         return mbuf->m_nextpkt;
 337 }
 338
 339 void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 340 {
 341         mbuf->m_nextpkt = nextpkt;
 342 }
 343
 344 size_t mbuf_len(mbuf_t mbuf)
 345 {
 346         return mbuf->m_len;
 347 }
 348
 349 void mbuf_setlen(mbuf_t mbuf, size_t len)
 350 {
 351         mbuf->m_len = len;
 352 }
 353
 354 size_t mbuf_maxlen(mbuf_t mbuf)
 355 {
 356         if (mbuf->m_flags & M_EXT)
 357                 return mbuf->m_ext.ext_size;
 358         return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 359 }
 360
 361 mbuf_type_t mbuf_type(mbuf_t mbuf)
 362 {
 363         return mbuf->m_type;
 364 }
 365
 366 errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 367 {
 368         if (new_type == MBUF_TYPE_FREE) return EINVAL;
 369
 370         m_mchtype(mbuf, new_type);
 371
 372         return 0;
 373 }
 374
 375 mbuf_flags_t mbuf_flags(mbuf_t mbuf)
 376 {
 377         return mbuf->m_flags & mbuf_flags_mask;
 378 }
 379
 380 errno_t mbuf_setflags(mbuf_t mbuf, mbuf_flags_t flags)
 381 {
 382         if ((flags & ~mbuf_flags_mask) != 0) return EINVAL;
 383         mbuf->m_flags = flags |
 384                 (mbuf->m_flags & ~mbuf_flags_mask);
 385
 386         return 0;
 387 }
 388
 389 errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 390 {
 391         if (((flags | mask) & ~mbuf_flags_mask) != 0) return EINVAL;
 392
 393         mbuf->m_flags = (flags & mask) | (mbuf->m_flags & ~mask);
 394
 395         return 0;
 396 }
 397
 398 errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src)
 399 {
 400         if (((src)->m_flags & M_PKTHDR) == 0)
 401                 return EINVAL;
 402
 403         m_copy_pkthdr(dest, src);
 404
 405         return 0;
 406 }
 407
 408 size_t mbuf_pkthdr_len(mbuf_t mbuf)
 409 {
 410         return mbuf->m_pkthdr.len;
 411 }
 412
 413 void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 414 {
 415         mbuf->m_pkthdr.len = len;
 416 }
 417
 418 ifnet_t mbuf_pkthdr_rcvif(mbuf_t mbuf)
 419 {
 420         // If we reference count ifnets, we should take a reference here before returning
 421         return mbuf->m_pkthdr.rcvif;
 422 }
 423
 424 errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 425 {
 426         /* May want to walk ifnet list to determine if interface is valid */
 427         mbuf->m_pkthdr.rcvif = (struct ifnet*)ifnet;
 428         return 0;
 429 }
 430
 431 void* mbuf_pkthdr_header(mbuf_t mbuf)
 432 {
 433         return mbuf->m_pkthdr.header;
 434 }
 435
 436 void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 437 {
 438         mbuf->m_pkthdr.header = (void*)header;
 439 }
 440
 441 /* mbuf aux data */
 442 errno_t mbuf_aux_add(mbuf_t mbuf, int family, mbuf_type_t type, mbuf_t *aux_mbuf)
 443 {
 444         *aux_mbuf = m_aux_add(mbuf, family, type);
 445         return (*aux_mbuf == NULL) ? ENOMEM : 0;
 446 }
 447
 448 mbuf_t mbuf_aux_find(mbuf_t mbuf, int family, mbuf_type_t type)
 449 {
 450         return m_aux_find(mbuf, family, type);
 451 }
 452
 453 void mbuf_aux_delete(mbuf_t mbuf, mbuf_t aux)
 454 {
 455         m_aux_delete(mbuf, aux);
 456 }
 457
 458 void
 459 mbuf_inbound_modified(mbuf_t mbuf)
 460 {
 461         /* Invalidate hardware generated checksum flags */
 462         mbuf->m_pkthdr.csum_flags = 0;
 463 }
 464
 465 extern void in_cksum_offset(struct mbuf* m, size_t ip_offset);
 466 extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset);
 467
 468 void
 469 mbuf_outbound_finalize(mbuf_t mbuf, u_long protocol_family, size_t protocol_offset)
 470 {
 471         if ((mbuf->m_pkthdr.csum_flags &
 472                  (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16)) == 0)
 473                 return;
 474
 475         /* Generate the packet in software, client needs it */
 476         switch (protocol_family) {
 477                 case PF_INET:
 478                         if (mbuf->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 479                                 /*
 480                                  * If you're wondering where this lovely code comes
 481                                  * from, we're trying to undo what happens in ip_output.
 482                                  * Look for CSUM_TCP_SUM16 in ip_output.
 483                                  */
 484                                 u_int16_t       first, second;
 485                                 mbuf->m_pkthdr.csum_flags &= ~CSUM_TCP_SUM16;
 486                                 mbuf->m_pkthdr.csum_flags |= CSUM_TCP;
 487                                 first = mbuf->m_pkthdr.csum_data >> 16;
 488                                 second = mbuf->m_pkthdr.csum_data & 0xffff;
 489                                 mbuf->m_pkthdr.csum_data = first - second;
 490                         }
 491                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
 492                                 in_delayed_cksum_offset(mbuf, protocol_offset);
 493                         }
 494
 495                         if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) {
 496                                 in_cksum_offset(mbuf, protocol_offset);
 497                         }
 498
 499                         mbuf->m_pkthdr.csum_flags &= ~(CSUM_DELAY_DATA | CSUM_DELAY_IP);
 500                         break;
 501
 502                 default:
 503                         /*
 504                          * Not sure what to do here if anything.
 505                          * Hardware checksum code looked pretty IPv4 specific.
 506                          */
 507                         if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP)) != 0)
 508                                 panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%d)!\n", protocol_family);
 509         }
 510 }
 511
 512 errno_t
 513 mbuf_set_vlan_tag(
 514         mbuf_t mbuf,
 515         u_int16_t vlan)
 516 {
 517         mbuf->m_pkthdr.csum_flags |= CSUM_VLAN_TAG_VALID;
 518         mbuf->m_pkthdr.vlan_tag = vlan;
 519
 520         return 0;
 521 }
 522
 523 errno_t
 524 mbuf_get_vlan_tag(
 525         mbuf_t mbuf,
 526         u_int16_t *vlan)
 527 {
 528         if ((mbuf->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0)
 529                 return ENXIO; // No vlan tag set
 530
 531         *vlan = mbuf->m_pkthdr.vlan_tag;
 532
 533         return 0;
 534 }
 535
 536 errno_t
 537 mbuf_clear_vlan_tag(
 538         mbuf_t mbuf)
 539 {
 540         mbuf->m_pkthdr.csum_flags &= ~CSUM_VLAN_TAG_VALID;
 541         mbuf->m_pkthdr.vlan_tag = 0;
 542
 543         return 0;
 544 }
 545
 546 static const mbuf_csum_request_flags_t mbuf_valid_csum_request_flags =
 547         MBUF_CSUM_REQ_IP | MBUF_CSUM_REQ_TCP | MBUF_CSUM_REQ_UDP | MBUF_CSUM_REQ_SUM16;
 548
 549 errno_t
 550 mbuf_set_csum_requested(
 551         mbuf_t mbuf,
 552         mbuf_csum_request_flags_t request,
 553         u_int32_t value)
 554 {
 555         request &= mbuf_valid_csum_request_flags;
 556         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | request;
 557         mbuf->m_pkthdr.csum_data = value;
 558
 559         return 0;
 560 }
 561
 562 errno_t
 563 mbuf_get_csum_requested(
 564         mbuf_t mbuf,
 565         mbuf_csum_request_flags_t *request,
 566         u_int32_t *value)
 567 {
 568         *request = mbuf->m_pkthdr.csum_flags;
 569         *request &= mbuf_valid_csum_request_flags;
 570         if (value != NULL) {
 571                 *value = mbuf->m_pkthdr.csum_data;
 572         }
 573
 574         return 0;
 575 }
 576
 577 errno_t
 578 mbuf_clear_csum_requested(
 579         mbuf_t mbuf)
 580 {
 581         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 582         mbuf->m_pkthdr.csum_data = 0;
 583
 584         return 0;
 585 }
 586
 587 static const mbuf_csum_performed_flags_t mbuf_valid_csum_performed_flags =
 588         MBUF_CSUM_DID_IP | MBUF_CSUM_IP_GOOD | MBUF_CSUM_DID_DATA |
 589         MBUF_CSUM_PSEUDO_HDR | MBUF_CSUM_TCP_SUM16;
 590
 591 errno_t
 592 mbuf_set_csum_performed(
 593         mbuf_t mbuf,
 594         mbuf_csum_performed_flags_t performed,
 595         u_int32_t value)
 596 {
 597         performed &= mbuf_valid_csum_performed_flags;
 598         mbuf->m_pkthdr.csum_flags = (mbuf->m_pkthdr.csum_flags & 0xffff0000) | performed;
 599         mbuf->m_pkthdr.csum_data = value;
 600
 601         return 0;
 602 }
 603
 604 errno_t
 605 mbuf_get_csum_performed(
 606         mbuf_t mbuf,
 607         mbuf_csum_performed_flags_t *performed,
 608         u_int32_t *value)
 609 {
 610         *performed = mbuf->m_pkthdr.csum_flags & mbuf_valid_csum_performed_flags;
 611         *value = mbuf->m_pkthdr.csum_data;
 612
 613         return 0;
 614 }
 615
 616 errno_t
 617 mbuf_clear_csum_performed(
 618         mbuf_t mbuf)
 619 {
 620         mbuf->m_pkthdr.csum_flags &= 0xffff0000;
 621         mbuf->m_pkthdr.csum_data = 0;
 622
 623         return 0;
 624 }
 625
 626 /*
 627  * Mbuf tag KPIs
 628  */
 629
 630 struct mbuf_tag_id_entry {
 631         SLIST_ENTRY(mbuf_tag_id_entry)  next;
 632         mbuf_tag_id_t                                   id;
 633         char                                                    string[];
 634 };
 635
 636 #define MBUF_TAG_ID_ENTRY_SIZE(__str) \
 637         ((size_t)&(((struct mbuf_tag_id_entry*)0)->string[0]) + \
 638          strlen(__str) + 1)
 639
 640 #define MTAG_FIRST_ID                                   1000
 641 static u_long                                                   mtag_id_next = MTAG_FIRST_ID;
 642 static SLIST_HEAD(,mbuf_tag_id_entry)   mtag_id_list = {NULL};
 643 static lck_mtx_t                                                *mtag_id_lock = NULL;
 644
 645 __private_extern__ void
 646 mbuf_tag_id_first_last(
 647         u_long  *first,
 648         u_long  *last)
 649 {
 650         *first = MTAG_FIRST_ID;
 651         *last = mtag_id_next - 1;
 652 }
 653
 654 __private_extern__ errno_t
 655 mbuf_tag_id_find_internal(
 656         const char      *string,
 657         u_long          *out_id,
 658         int                     create)
 659 {
 660         struct mbuf_tag_id_entry                                *entry = NULL;
 661
 662
 663         *out_id = 0;
 664
 665         if (string == NULL || out_id == NULL) {
 666                 return EINVAL;
 667         }
 668
 669         /* Don't bother allocating the lock if we're only doing a lookup */
 670         if (create == 0 && mtag_id_lock == NULL)
 671                 return ENOENT;
 672
 673         /* Allocate lock if necessary */
 674         if (mtag_id_lock == NULL) {
 675                 lck_grp_attr_t  *grp_attrib = NULL;
 676                 lck_attr_t              *lck_attrb = NULL;
 677                 lck_grp_t               *lck_group = NULL;
 678                 lck_mtx_t               *new_lock = NULL;
 679
 680                 grp_attrib = lck_grp_attr_alloc_init();
 681                 lck_grp_attr_setdefault(grp_attrib);
 682                 lck_group = lck_grp_alloc_init("mbuf_tag_allocate_id", grp_attrib);
 683                 lck_grp_attr_free(grp_attrib);
 684                 lck_attrb = lck_attr_alloc_init();
 685                 lck_attr_setdefault(lck_attrb);
 686                 lck_attr_setdebug(lck_attrb);
 687                 new_lock = lck_mtx_alloc_init(lck_group, lck_attrb);
 688                 if (!OSCompareAndSwap((UInt32)0, (UInt32)new_lock, (UInt32*)&mtag_id_lock)) {
 689                         /*
 690                          * If the atomic swap fails, someone else has already
 691                          * done this work. We can free the stuff we allocated.
 692                          */
 693                         lck_mtx_free(new_lock, lck_group);
 694                         lck_grp_free(lck_group);
 695                 }
 696                 lck_attr_free(lck_attrb);
 697         }
 698
 699         /* Look for an existing entry */
 700         lck_mtx_lock(mtag_id_lock);
 701         SLIST_FOREACH(entry, &mtag_id_list, next) {
 702                 if (strcmp(string, entry->string) == 0) {
 703                         break;
 704                 }
 705         }
 706
 707         if (entry == NULL) {
 708                 if (create == 0) {
 709                         lck_mtx_unlock(mtag_id_lock);
 710                         return ENOENT;
 711                 }
 712
 713                 entry = kalloc(MBUF_TAG_ID_ENTRY_SIZE(string));
 714                 if (entry == NULL) {
 715                         lck_mtx_unlock(mtag_id_lock);
 716                         return ENOMEM;
 717                 }
 718
 719                 strcpy(entry->string, string);
 720                 entry->id = mtag_id_next;
 721                 mtag_id_next++;
 722                 SLIST_INSERT_HEAD(&mtag_id_list, entry, next);
 723         }
 724         lck_mtx_unlock(mtag_id_lock);
 725
 726         *out_id = entry->id;
 727
 728         return 0;
 729 }
 730
 731 errno_t
 732 mbuf_tag_id_find(
 733         const char              *string,
 734         mbuf_tag_id_t   *out_id)
 735 {
 736         return mbuf_tag_id_find_internal(string, (u_long*)out_id, 1);
 737 }
 738
 739 errno_t
 740 mbuf_tag_allocate(
 741         mbuf_t                  mbuf,
 742         mbuf_tag_id_t   id,
 743         mbuf_tag_type_t type,
 744         size_t                  length,
 745         mbuf_how_t              how,
 746         void**                  data_p)
 747 {
 748         struct m_tag *tag;
 749
 750         if (data_p != NULL)
 751                 *data_p = NULL;
 752
 753         /* Sanity check parameters */
 754         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 755                 id >= mtag_id_next || length < 1 || (length & 0xffff0000) != 0 ||
 756                 data_p == NULL) {
 757                 return EINVAL;
 758         }
 759
 760         /* Make sure this mtag hasn't already been allocated */
 761         tag = m_tag_locate(mbuf, id, type, NULL);
 762         if (tag != NULL) {
 763                 return EEXIST;
 764         }
 765
 766         /* Allocate an mtag */
 767         tag = m_tag_alloc(id, type, length, how);
 768         if (tag == NULL) {
 769                 return how == M_WAITOK ? ENOMEM : EWOULDBLOCK;
 770         }
 771
 772         /* Attach the mtag and set *data_p */
 773         m_tag_prepend(mbuf, tag);
 774         *data_p = tag + 1;
 775
 776         return 0;
 777 }
 778
 779 errno_t
 780 mbuf_tag_find(
 781         mbuf_t                  mbuf,
 782         mbuf_tag_id_t   id,
 783         mbuf_tag_type_t type,
 784         size_t*                 length,
 785         void**                  data_p)
 786 {
 787         struct m_tag *tag;
 788
 789         if (length != NULL)
 790                 *length = 0;
 791         if (data_p != NULL)
 792                 *data_p = NULL;
 793
 794         /* Sanity check parameters */
 795         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 796                 id >= mtag_id_next || length == NULL || data_p == NULL) {
 797                 return EINVAL;
 798         }
 799
 800         /* Locate an mtag */
 801         tag = m_tag_locate(mbuf, id, type, NULL);
 802         if (tag == NULL) {
 803                 return ENOENT;
 804         }
 805
 806         /* Copy out the pointer to the data and the lenght value */
 807         *length = tag->m_tag_len;
 808         *data_p = tag + 1;
 809
 810         return 0;
 811 }
 812
 813 void
 814 mbuf_tag_free(
 815         mbuf_t                  mbuf,
 816         mbuf_tag_id_t   id,
 817         mbuf_tag_type_t type)
 818 {
 819         struct m_tag *tag;
 820
 821         if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID ||
 822                 id >= mtag_id_next)
 823                 return;
 824
 825         tag = m_tag_locate(mbuf, id, type, NULL);
 826         if (tag == NULL) {
 827                 return;
 828         }
 829
 830         m_tag_delete(mbuf, tag);
 831         return;
 832 }
 833
 834 /* mbuf stats */
 835 void mbuf_stats(struct mbuf_stat *stats)
 836 {
 837         stats->mbufs = mbstat.m_mbufs;
 838         stats->clusters = mbstat.m_clusters;
 839         stats->clfree = mbstat.m_clfree;
 840         stats->drops = mbstat.m_drops;
 841         stats->wait = mbstat.m_wait;
 842         stats->drain = mbstat.m_drain;
 843         __builtin_memcpy(stats->mtypes, mbstat.m_mtypes, sizeof(stats->mtypes));
 844         stats->mcfail = mbstat.m_mcfail;
 845         stats->mpfail = mbstat.m_mpfail;
 846         stats->msize = mbstat.m_msize;
 847         stats->mclbytes = mbstat.m_mclbytes;
 848         stats->minclsize = mbstat.m_minclsize;
 849         stats->mlen = mbstat.m_mlen;
 850         stats->mhlen = mbstat.m_mhlen;
 851         stats->bigclusters = mbstat.m_bigclusters;
 852         stats->bigclfree = mbstat.m_bigclfree;
 853         stats->bigmclbytes = mbstat.m_bigmclbytes;
 854 }
 855
 856 errno_t
 857 mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
 858 {
 859         errno_t error;
 860         struct mbuf *m;
 861         unsigned int numpkts = 1;
 862         unsigned int numchunks = maxchunks ? *maxchunks : 0;
 863
 864         if (packetlen == 0) {
 865                 error = EINVAL;
 866                 goto out;
 867         }
 868         m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
 869         if (m == 0) {
 870                 if (maxchunks && *maxchunks && numchunks > *maxchunks)
 871                         error = ENOBUFS;
 872                 else
 873                         error = ENOMEM;
 874         } else {
 875                 error = 0;
 876                 *mbuf = m;
 877         }
 878 out:
 879         return error;
 880 }
 881
 882
 883 /*
 884  * mbuf_copyback differs from m_copyback in a few ways:
 885  * 1) mbuf_copyback will allocate clusters for new mbufs we append
 886  * 2) mbuf_copyback will grow the last mbuf in the chain if possible
 887  * 3) mbuf_copyback reports whether or not the operation succeeded
 888  * 4) mbuf_copyback allows the caller to specify M_WAITOK or M_NOWAIT
 889  */
 890 errno_t
 891 mbuf_copyback(
 892         mbuf_t          m,
 893         size_t          off,
 894         size_t          len,
 895         const void      *data,
 896         mbuf_how_t      how)
 897 {
 898         size_t  mlen;
 899         mbuf_t  m_start = m;
 900         mbuf_t  n;
 901         int             totlen = 0;
 902         errno_t         result = 0;
 903         const char      *cp = data;
 904
 905         if (m == NULL || len == 0 || data == NULL)
 906                 return EINVAL;
 907
 908         while (off > (mlen = m->m_len)) {
 909                 off -= mlen;
 910                 totlen += mlen;
 911                 if (m->m_next == 0) {
 912                         n = m_getclr(how, m->m_type);
 913                         if (n == 0) {
 914                                 result = ENOBUFS;
 915                                 goto out;
 916                         }
 917                         n->m_len = MIN(MLEN, len + off);
 918                         m->m_next = n;
 919                 }
 920                 m = m->m_next;
 921         }
 922
 923         while (len > 0) {
 924                 mlen = MIN(m->m_len - off, len);
 925                 if (mlen < len && m->m_next == NULL && mbuf_trailingspace(m) > 0) {
 926                         size_t  grow = MIN(mbuf_trailingspace(m), len - mlen);
 927                         mlen += grow;
 928                         m->m_len += grow;
 929                 }
 930                 bcopy(cp, off + (char*)mbuf_data(m), (unsigned)mlen);
 931                 cp += mlen;
 932                 len -= mlen;
 933                 mlen += off;
 934                 off = 0;
 935                 totlen += mlen;
 936                 if (len == 0)
 937                         break;
 938                 if (m->m_next == 0) {
 939                         n = m_get(how, m->m_type);
 940                         if (n == NULL) {
 941                                 result = ENOBUFS;
 942                                 goto out;
 943                         }
 944                         if (len > MINCLSIZE) {
 945                                 /* cluter allocation failure is okay, we can grow chain */
 946                                 mbuf_mclget(how, m->m_type, &n);
 947                         }
 948                         n->m_len = MIN(mbuf_maxlen(n), len);
 949                         m->m_next = n;
 950                 }
 951                 m = m->m_next;
 952         }
 953
 954 out:
 955         if ((m_start->m_flags & M_PKTHDR) && (m_start->m_pkthdr.len < totlen))
 956                 m_start->m_pkthdr.len = totlen;
 957
 958         return result;
 959 }