bsd/net/pf_pbuf.c

   1 /*
   2  * Copyright (c) 2016-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <sys/cdefs.h>
  25 #include <sys/systm.h>
  26 #include <sys/param.h>
  27 #include <sys/types.h>
  28 #include <sys/mcache.h>
  29 #include <kern/kern_types.h>
  30 #include <net/pf_pbuf.h>
  31 #include <netinet/in.h>
  32
  33 void
  34 pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
  35 {
  36
  37         VERIFY((m->m_flags & M_PKTHDR) != 0);
  38
  39         pbuf->pb_type = PBUF_TYPE_MBUF;
  40         pbuf->pb_mbuf = m;
  41         pbuf->pb_ifp = ifp;
  42         pbuf->pb_next = NULL;
  43         pbuf_sync(pbuf);
  44 }
  45
  46 void
  47 pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
  48 {
  49
  50         pbuf->pb_type = PBUF_TYPE_MEMORY;
  51         pbuf->pb_memory = *mp;
  52         pbuf->pb_ifp = ifp;
  53         pbuf->pb_next = NULL;
  54         pbuf_sync(pbuf);
  55 }
  56
  57 void
  58 pbuf_destroy(pbuf_t *pbuf)
  59 {
  60
  61         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
  62                 if (pbuf->pb_mbuf) {
  63                         m_freem(pbuf->pb_mbuf);
  64                         pbuf->pb_mbuf = NULL;
  65                 }
  66         } else
  67         if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
  68                 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
  69                 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
  70                     PBUF_ACTION_DESTROY);
  71         } else {
  72                 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
  73         }
  74
  75         memset(pbuf, 0, sizeof(*pbuf));
  76 }
  77
  78 void
  79 pbuf_sync(pbuf_t *pbuf)
  80 {
  81
  82         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
  83                 struct mbuf *m = pbuf->pb_mbuf;
  84
  85                 VERIFY(m != NULL);
  86                 VERIFY(m->m_flags & M_PKTHDR);
  87
  88                 pbuf->pb_data = mtod(m, void *);
  89                 pbuf->pb_packet_len = m->m_pkthdr.len;
  90                 pbuf->pb_contig_len = m->m_len;
  91                 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
  92                 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
  93                 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
  94                 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
  95                 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
  96                 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
  97                 pbuf->pb_pftag = m_pftag(m);
  98         } else
  99         if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
 100                 struct pbuf_memory *nm = &pbuf->pb_memory;
 101
 102                 VERIFY(nm->pm_buffer != NULL);
 103                 VERIFY(nm->pm_buffer_len != 0);
 104                 VERIFY(nm->pm_len != 0);
 105                 VERIFY(nm->pm_len <= nm->pm_buffer_len);
 106                 VERIFY(nm->pm_offset < nm->pm_len);
 107
 108                 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
 109                 pbuf->pb_packet_len = nm->pm_len;
 110                 pbuf->pb_contig_len = nm->pm_len;
 111                 pbuf->pb_csum_flags = &nm->pm_csum_flags;
 112                 pbuf->pb_csum_data = &nm->pm_csum_data;
 113                 pbuf->pb_proto = &nm->pm_proto;
 114                 pbuf->pb_flowsrc = &nm->pm_flowsrc;
 115                 pbuf->pb_flowid = &nm->pm_flowid;
 116                 pbuf->pb_flags = &nm->pm_flags;
 117                 pbuf->pb_pftag = &nm->pm_pftag;
 118         } else
 119                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 120 }
 121
 122 struct mbuf *
 123 pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
 124 {
 125         struct mbuf *m = NULL;
 126
 127         pbuf_sync(pbuf);
 128
 129         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 130                 m = pbuf->pb_mbuf;
 131                 if (release_ptr) {
 132                         pbuf->pb_mbuf = NULL;
 133                         pbuf_destroy(pbuf);
 134                 }
 135         } else
 136         if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
 137                 if (pbuf->pb_packet_len > (u_int)MHLEN) {
 138                         if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
 139                                 printf("%s: packet too big for cluster (%u)\n",
 140                                     __func__, pbuf->pb_packet_len);
 141                                 return (NULL);
 142                         }
 143                         m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
 144                 } else {
 145                         m = m_gethdr(M_DONTWAIT, MT_DATA);
 146                 }
 147                 if (m == NULL)
 148                         return (NULL);
 149
 150                 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
 151                 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
 152                 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
 153                 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
 154                 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
 155                 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
 156                 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
 157
 158                 if (pbuf->pb_pftag != NULL) {
 159                         struct pf_mtag *pftag = m_pftag(m);
 160
 161                         if (pftag != NULL)
 162                                 *pftag = *pbuf->pb_pftag;
 163                 }
 164
 165                 if (release_ptr)
 166                         pbuf_destroy(pbuf);
 167         }
 168
 169         return (m);
 170 }
 171
 172 struct mbuf *
 173 pbuf_clone_to_mbuf(pbuf_t *pbuf)
 174 {
 175         struct mbuf *m = NULL;
 176
 177         pbuf_sync(pbuf);
 178
 179         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 180                 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
 181         else
 182         if (pbuf->pb_type == PBUF_TYPE_MEMORY)
 183                 m = pbuf_to_mbuf(pbuf, FALSE);
 184         else
 185                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 186
 187         return (m);
 188 }
 189
 190 void *
 191 pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
 192 {
 193
 194         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 195                 struct mbuf *m = pbuf->pb_mbuf;
 196
 197                 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT))
 198                         return (NULL);
 199
 200                 if (pbuf->pb_mbuf == NULL) {
 201                         pbuf_destroy(pbuf);
 202                         return (NULL);
 203                 }
 204
 205                 if  (m != pbuf->pb_mbuf)
 206                         pbuf_sync(pbuf);
 207
 208         } else
 209         if (pbuf->pb_type != PBUF_TYPE_MEMORY)
 210                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 211
 212         return (pbuf->pb_data);
 213 }
 214
 215 void *
 216 pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
 217 {
 218         void *rv = NULL;
 219
 220         VERIFY(off >= 0);
 221         VERIFY((u_int)off <= pbuf->pb_packet_len);
 222
 223         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 224                 struct mbuf *m, *n;
 225
 226                 VERIFY(pbuf->pb_mbuf != NULL);
 227
 228                 m = pbuf->pb_mbuf;
 229
 230                 if (off > 0) {
 231                         /* Split the mbuf chain at the specified boundary */
 232                         if ((n = m_split(m, off, M_DONTWAIT)) == NULL)
 233                                 return (NULL);
 234                 } else {
 235                         n = m;
 236                 }
 237
 238                 /* Trim old length */
 239                 m_adj(n, olen);
 240
 241                 /* Prepend new length */
 242                 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL)
 243                         return (NULL);
 244
 245                 rv = mtod(n, void *);
 246
 247                 if (off > 0) {
 248                         /* Merge the two chains */
 249                         int mlen;
 250
 251                         mlen = n->m_pkthdr.len;
 252                         m_cat(m, n);
 253                         m->m_pkthdr.len += mlen;
 254                 } else {
 255                         /* The new mbuf becomes the packet header */
 256                         pbuf->pb_mbuf = n;
 257                 }
 258
 259                 pbuf_sync(pbuf);
 260         } else
 261         if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
 262                 struct pbuf_memory *nm = &pbuf->pb_memory;
 263                 u_int true_offset, move_len;
 264                 int delta_len;
 265                 uint8_t *psrc, *pdst;
 266
 267                 delta_len = nlen - olen;
 268                 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
 269                     nm->pm_buffer_len);
 270
 271                 true_offset = (u_int)off + nm->pm_offset;
 272                 rv = &nm->pm_buffer[true_offset];
 273                 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
 274                 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
 275                 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
 276                 memmove(pdst, psrc, move_len);
 277
 278                 nm->pm_len += delta_len;
 279
 280                 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
 281
 282                 pbuf_sync(pbuf);
 283         } else
 284                 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
 285
 286         return (rv);
 287 }
 288
 289 void *
 290 pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
 291 {
 292         void *rv = NULL;
 293
 294         VERIFY(off >= 0);
 295         VERIFY(len >= 0);
 296         VERIFY((u_int)(off + len) < pbuf->pb_packet_len);
 297
 298         /*
 299          * Note: If this fails, then the pbuf is destroyed. This is a
 300          * side-effect of m_pulldown().
 301          *
 302          * PF expects this behaviour so it's not a real problem.
 303          */
 304
 305         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 306                 struct mbuf *n;
 307                 int moff;
 308
 309                 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
 310                 if (n == NULL) {
 311                         /* mbuf is freed by m_pulldown() in this case */
 312                         pbuf->pb_mbuf = NULL;
 313                         pbuf_destroy(pbuf);
 314                         return (NULL);
 315                 }
 316
 317                 pbuf_sync(pbuf);
 318
 319                 rv = (void *)(mtod(n, uint8_t *) + moff);
 320         } else
 321         if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
 322                 /*
 323                  * This always succeeds since memory pbufs are fully contig.
 324                  */
 325                 rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
 326         } else
 327                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 328
 329         return (rv);
 330 }
 331
 332 void
 333 pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
 334 {
 335
 336         VERIFY(off >= 0);
 337         VERIFY(len >= 0);
 338         VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
 339
 340         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 341                 m_copyback(pbuf->pb_mbuf, off, len, src);
 342         else
 343         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 344                 if (len)
 345                         memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
 346         } else
 347                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 348 }
 349
 350 void
 351 pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
 352 {
 353
 354         VERIFY(off >= 0);
 355         VERIFY(len >= 0);
 356         VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
 357
 358         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 359                 m_copydata(pbuf->pb_mbuf, off, len, dst);
 360         else
 361         if (pbuf->pb_type == PBUF_TYPE_MBUF) {
 362                 if (len)
 363                         memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
 364         } else
 365                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 366 }
 367
 368 uint16_t
 369 pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
 370 {
 371         uint16_t sum = 0;
 372
 373         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 374                 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
 375         else
 376         if (pbuf->pb_type == PBUF_TYPE_MEMORY)
 377                 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
 378         else
 379                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 380
 381         return (sum);
 382 }
 383
 384 uint16_t
 385 pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
 386 {
 387         uint16_t sum = 0;
 388
 389         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 390                 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
 391         else
 392         if (pbuf->pb_type == PBUF_TYPE_MEMORY)
 393                 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
 394         else
 395                 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
 396
 397         return (sum);
 398 }
 399
 400 mbuf_svc_class_t
 401 pbuf_get_service_class(const pbuf_t *pbuf)
 402 {
 403
 404         if (pbuf->pb_type == PBUF_TYPE_MBUF)
 405                 return m_get_service_class(pbuf->pb_mbuf);
 406
 407         VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
 408
 409         return (MBUF_SC_BE);
 410 }