bsd/net/pktsched/pktsched.c

   1 /*
   2  * Copyright (c) 2011-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/malloc.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/errno.h>
  37 #include <sys/mcache.h>
  38 #include <sys/sysctl.h>
  39
  40 #include <dev/random/randomdev.h>
  41 #include <net/if.h>
  42 #include <net/if_var.h>
  43 #include <net/if_dl.h>
  44 #include <net/if_types.h>
  45 #include <net/net_osdep.h>
  46 #include <net/pktsched/pktsched.h>
  47 #include <net/pktsched/pktsched_fq_codel.h>
  48 #include <net/pktsched/pktsched_netem.h>
  49
  50 #include <pexpert/pexpert.h>
  51
  52
  53 u_int32_t machclk_freq = 0;
  54 u_int64_t machclk_per_sec = 0;
  55 u_int32_t pktsched_verbose = 0; /* more noise if greater than 1 */
  56
  57 static void init_machclk(void);
  58
  59 SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "pktsched");
  60
  61 SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
  62     &pktsched_verbose, 0, "Packet scheduler verbosity level");
  63
  64 void
  65 pktsched_init(void)
  66 {
  67         init_machclk();
  68         if (machclk_freq == 0) {
  69                 panic("%s: no CPU clock available!\n", __func__);
  70                 /* NOTREACHED */
  71         }
  72
  73         netem_init();
  74 }
  75
  76 static void
  77 init_machclk(void)
  78 {
  79         /*
  80          * Initialize machclk_freq using the timerbase frequency
  81          * value from device specific info.
  82          */
  83         machclk_freq = (uint32_t)gPEClockFrequencyInfo.timebase_frequency_hz;
  84
  85         clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
  86             &machclk_per_sec);
  87 }
  88
  89 u_int64_t
  90 pktsched_abs_to_nsecs(u_int64_t abstime)
  91 {
  92         u_int64_t nsecs;
  93
  94         absolutetime_to_nanoseconds(abstime, &nsecs);
  95         return nsecs;
  96 }
  97
  98 u_int64_t
  99 pktsched_nsecs_to_abstime(u_int64_t nsecs)
 100 {
 101         u_int64_t abstime;
 102
 103         nanoseconds_to_absolutetime(nsecs, &abstime);
 104         return abstime;
 105 }
 106
 107 int
 108 pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
 109     classq_pkt_type_t ptype)
 110 {
 111         int error = 0;
 112         u_int32_t rflags;
 113
 114         IFCQ_LOCK_ASSERT_HELD(ifq);
 115
 116         VERIFY(machclk_freq != 0);
 117
 118         /* Nothing to do unless the scheduler type changes */
 119         if (ifq->ifcq_type == scheduler) {
 120                 return 0;
 121         }
 122
 123         /*
 124          * Remember the flags that need to be restored upon success, as
 125          * they may be cleared when we tear down existing scheduler.
 126          */
 127         rflags = (ifq->ifcq_flags & IFCQF_ENABLED);
 128
 129         if (ifq->ifcq_type != PKTSCHEDT_NONE) {
 130                 pktsched_teardown(ifq);
 131
 132                 /* Teardown should have succeeded */
 133                 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 134                 VERIFY(ifq->ifcq_disc == NULL);
 135         }
 136
 137         error = fq_if_setup_ifclassq(ifq, sflags, ptype);
 138         if (error == 0) {
 139                 ifq->ifcq_flags |= rflags;
 140         }
 141
 142         return error;
 143 }
 144
 145 void
 146 pktsched_teardown(struct ifclassq *ifq)
 147 {
 148         IFCQ_LOCK_ASSERT_HELD(ifq);
 149
 150         if_qflush(ifq->ifcq_ifp, 1);
 151         VERIFY(IFCQ_IS_EMPTY(ifq));
 152
 153         ifq->ifcq_flags &= ~IFCQF_ENABLED;
 154
 155         if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
 156                 /* Could be PKTSCHEDT_NONE */
 157                 fq_if_teardown_ifclassq(ifq);
 158         }
 159
 160         return;
 161 }
 162
 163 int
 164 pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
 165     struct if_ifclassq_stats *ifqs)
 166 {
 167         int error = 0;
 168
 169         IFCQ_LOCK_ASSERT_HELD(ifq);
 170
 171         if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
 172                 /* Could be PKTSCHEDT_NONE */
 173                 error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
 174         }
 175
 176         return error;
 177 }
 178
 179 void
 180 pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_t *cpkt)
 181 {
 182         pkt->pktsched_pkt = *cpkt;
 183         pkt->pktsched_tail = *cpkt;
 184         pkt->pktsched_pcnt = 1;
 185
 186         switch (cpkt->cp_ptype) {
 187         case QP_MBUF:
 188                 pkt->pktsched_plen =
 189                     (uint32_t)m_pktlen(pkt->pktsched_pkt_mbuf);
 190                 break;
 191
 192
 193         default:
 194                 VERIFY(0);
 195                 /* NOTREACHED */
 196                 __builtin_unreachable();
 197         }
 198 }
 199
 200 void
 201 pktsched_pkt_encap_chain(pktsched_pkt_t *pkt, classq_pkt_t *cpkt,
 202     classq_pkt_t *tail, uint32_t cnt, uint32_t bytes)
 203 {
 204         pkt->pktsched_pkt = *cpkt;
 205         pkt->pktsched_tail = *tail;
 206         pkt->pktsched_pcnt = cnt;
 207         pkt->pktsched_plen = bytes;
 208
 209         switch (cpkt->cp_ptype) {
 210         case QP_MBUF:
 211                 break;
 212
 213
 214         default:
 215                 VERIFY(0);
 216                 /* NOTREACHED */
 217                 __builtin_unreachable();
 218         }
 219 }
 220
 221 int
 222 pktsched_clone_pkt(pktsched_pkt_t *pkt1, pktsched_pkt_t *pkt2)
 223 {
 224         struct mbuf *m1, *m2;
 225
 226         ASSERT(pkt1 != NULL);
 227         ASSERT(pkt1->pktsched_pkt_mbuf != NULL);
 228         ASSERT(pkt1->pktsched_pcnt == 1);
 229
 230         /* allow in place clone, but make sure pkt2->pktsched_pkt won't leak */
 231         ASSERT((pkt1 == pkt2 && pkt1->pktsched_pkt_mbuf ==
 232             pkt2->pktsched_pkt_mbuf) || (pkt1 != pkt2 &&
 233             pkt2->pktsched_pkt_mbuf == NULL));
 234
 235         switch (pkt1->pktsched_ptype) {
 236         case QP_MBUF:
 237                 m1 = (struct mbuf *)pkt1->pktsched_pkt_mbuf;
 238                 m2 = m_dup(m1, M_NOWAIT);
 239                 if (__improbable(m2 == NULL)) {
 240                         return ENOBUFS;
 241                 }
 242                 pkt2->pktsched_pkt_mbuf = m2;
 243                 break;
 244
 245
 246         default:
 247                 VERIFY(0);
 248                 /* NOTREACHED */
 249                 __builtin_unreachable();
 250         }
 251
 252         pkt2->pktsched_plen = pkt1->pktsched_plen;
 253         pkt2->pktsched_ptype = pkt1->pktsched_ptype;
 254         pkt2->pktsched_tail = pkt2->pktsched_pkt;
 255         pkt2->pktsched_pcnt = 1;
 256         return 0;
 257 }
 258
 259 void
 260 pktsched_corrupt_packet(pktsched_pkt_t *pkt)
 261 {
 262         struct mbuf *m = NULL;
 263         uint8_t *data = NULL;
 264         uint32_t data_len = 0;
 265         uint32_t rand32, rand_off, rand_bit;
 266
 267         switch (pkt->pktsched_ptype) {
 268         case QP_MBUF:
 269                 m = pkt->pktsched_pkt_mbuf;
 270                 data = mtod(m, uint8_t *);
 271                 data_len = m->m_pkthdr.len;
 272                 break;
 273
 274         default:
 275                 /* NOTREACHED */
 276                 VERIFY(0);
 277                 __builtin_unreachable();
 278         }
 279
 280         read_frandom(&rand32, sizeof(rand32));
 281         rand_bit = rand32 & 0x8;
 282         rand_off = (rand32 >> 3) % data_len;
 283         data[rand_off] ^= 1 << rand_bit;
 284 }
 285
 286 void
 287 pktsched_free_pkt(pktsched_pkt_t *pkt)
 288 {
 289         uint32_t cnt = pkt->pktsched_pcnt;
 290         ASSERT(cnt != 0);
 291
 292         switch (pkt->pktsched_ptype) {
 293         case QP_MBUF: {
 294                 struct mbuf *m;
 295
 296                 m = pkt->pktsched_pkt_mbuf;
 297                 if (cnt == 1) {
 298                         VERIFY(m->m_nextpkt == NULL);
 299                 } else {
 300                         VERIFY(m->m_nextpkt != NULL);
 301                 }
 302                 m_freem_list(m);
 303                 break;
 304         }
 305
 306         default:
 307                 VERIFY(0);
 308                 /* NOTREACHED */
 309                 __builtin_unreachable();
 310         }
 311         pkt->pktsched_pkt = CLASSQ_PKT_INITIALIZER(pkt->pktsched_pkt);
 312         pkt->pktsched_tail = CLASSQ_PKT_INITIALIZER(pkt->pktsched_tail);
 313         pkt->pktsched_plen = 0;
 314         pkt->pktsched_pcnt = 0;
 315 }
 316
 317 mbuf_svc_class_t
 318 pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
 319 {
 320         mbuf_svc_class_t svc = MBUF_SC_UNSPEC;
 321
 322         switch (pkt->pktsched_ptype) {
 323         case QP_MBUF:
 324                 svc = m_get_service_class(pkt->pktsched_pkt_mbuf);
 325                 break;
 326
 327
 328         default:
 329                 VERIFY(0);
 330                 /* NOTREACHED */
 331                 __builtin_unreachable();
 332         }
 333
 334         return svc;
 335 }
 336
 337 void
 338 pktsched_get_pkt_vars(pktsched_pkt_t *pkt, volatile uint32_t **flags,
 339     uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto,
 340     uint32_t *comp_gencnt)
 341 {
 342         switch (pkt->pktsched_ptype) {
 343         case QP_MBUF: {
 344                 struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
 345
 346                 if (flags != NULL) {
 347                         *flags = &pkth->pkt_flags;
 348                 }
 349                 if (timestamp != NULL) {
 350                         *timestamp = &pkth->pkt_timestamp;
 351                 }
 352                 if (flowid != NULL) {
 353                         *flowid = pkth->pkt_flowid;
 354                 }
 355                 if (flowsrc != NULL) {
 356                         *flowsrc = pkth->pkt_flowsrc;
 357                 }
 358                 if (proto != NULL) {
 359                         *proto = pkth->pkt_proto;
 360                 }
 361                 if (comp_gencnt != NULL) {
 362                         *comp_gencnt = pkth->comp_gencnt;
 363                 }
 364
 365                 break;
 366         }
 367
 368
 369         default:
 370                 VERIFY(0);
 371                 /* NOTREACHED */
 372                 __builtin_unreachable();
 373         }
 374 }
 375
 376 struct flowadv_fcentry *
 377 pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how)
 378 {
 379 #pragma unused(ifp)
 380         struct flowadv_fcentry *fce = NULL;
 381
 382         switch (pkt->pktsched_ptype) {
 383         case QP_MBUF: {
 384                 struct mbuf *m = pkt->pktsched_pkt_mbuf;
 385
 386                 fce = flowadv_alloc_entry(how);
 387                 if (fce == NULL) {
 388                         break;
 389                 }
 390
 391                 _CASSERT(sizeof(m->m_pkthdr.pkt_flowid) ==
 392                     sizeof(fce->fce_flowid));
 393
 394                 fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
 395                 fce->fce_flowid = m->m_pkthdr.pkt_flowid;
 396                 break;
 397         }
 398
 399
 400         default:
 401                 VERIFY(0);
 402                 /* NOTREACHED */
 403                 __builtin_unreachable();
 404         }
 405
 406         return fce;
 407 }
 408
 409 uint32_t *
 410 pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags)
 411 {
 412         uint32_t *hashp = NULL;
 413
 414         switch (pkt->pktsched_ptype) {
 415         case QP_MBUF: {
 416                 struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
 417
 418                 _CASSERT(sizeof(pkth->pkt_mpriv_hash) == sizeof(uint32_t));
 419                 _CASSERT(sizeof(pkth->pkt_mpriv_flags) == sizeof(uint32_t));
 420                 *sfb_flags = &pkth->pkt_mpriv_flags;
 421                 hashp = &pkth->pkt_mpriv_hash;
 422                 break;
 423         }
 424
 425
 426         default:
 427                 VERIFY(0);
 428                 /* NOTREACHED */
 429                 __builtin_unreachable();
 430         }
 431
 432         return hashp;
 433 }