bsd/net/pktsched/pktsched.c

   1 /*
   2  * Copyright (c) 2011-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/malloc.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/errno.h>
  37 #include <sys/mcache.h>
  38 #include <sys/sysctl.h>
  39
  40 #include <dev/random/randomdev.h>
  41 #include <net/if.h>
  42 #include <net/if_var.h>
  43 #include <net/if_dl.h>
  44 #include <net/if_types.h>
  45 #include <net/net_osdep.h>
  46 #include <net/pktsched/pktsched.h>
  47 #include <net/pktsched/pktsched_tcq.h>
  48 #include <net/pktsched/pktsched_qfq.h>
  49 #include <net/pktsched/pktsched_fq_codel.h>
  50 #include <net/pktsched/pktsched_netem.h>
  51
  52 #include <pexpert/pexpert.h>
  53
  54
  55 u_int32_t machclk_freq = 0;
  56 u_int64_t machclk_per_sec = 0;
  57 u_int32_t pktsched_verbose = 0; /* more noise if greater than 1 */
  58
  59 static void init_machclk(void);
  60
  61 SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "pktsched");
  62
  63 SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
  64     &pktsched_verbose, 0, "Packet scheduler verbosity level");
  65
  66 void
  67 pktsched_init(void)
  68 {
  69         init_machclk();
  70         if (machclk_freq == 0) {
  71                 panic("%s: no CPU clock available!\n", __func__);
  72                 /* NOTREACHED */
  73         }
  74
  75         tcq_init();
  76         qfq_init();
  77         netem_init();
  78 }
  79
  80 static void
  81 init_machclk(void)
  82 {
  83         /*
  84          * Initialize machclk_freq using the timerbase frequency
  85          * value from device specific info.
  86          */
  87         machclk_freq = gPEClockFrequencyInfo.timebase_frequency_hz;
  88
  89         clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
  90             &machclk_per_sec);
  91 }
  92
  93 u_int64_t
  94 pktsched_abs_to_nsecs(u_int64_t abstime)
  95 {
  96         u_int64_t nsecs;
  97
  98         absolutetime_to_nanoseconds(abstime, &nsecs);
  99         return nsecs;
 100 }
 101
 102 u_int64_t
 103 pktsched_nsecs_to_abstime(u_int64_t nsecs)
 104 {
 105         u_int64_t abstime;
 106
 107         nanoseconds_to_absolutetime(nsecs, &abstime);
 108         return abstime;
 109 }
 110
 111 int
 112 pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
 113     classq_pkt_type_t ptype)
 114 {
 115         int error = 0;
 116         u_int32_t rflags;
 117
 118         IFCQ_LOCK_ASSERT_HELD(ifq);
 119
 120         VERIFY(machclk_freq != 0);
 121
 122         /* Nothing to do unless the scheduler type changes */
 123         if (ifq->ifcq_type == scheduler) {
 124                 return 0;
 125         }
 126
 127         /*
 128          * Remember the flags that need to be restored upon success, as
 129          * they may be cleared when we tear down existing scheduler.
 130          */
 131         rflags = (ifq->ifcq_flags & IFCQF_ENABLED);
 132
 133         if (ifq->ifcq_type != PKTSCHEDT_NONE) {
 134                 (void) pktsched_teardown(ifq);
 135
 136                 /* Teardown should have succeeded */
 137                 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 138                 VERIFY(ifq->ifcq_disc == NULL);
 139                 VERIFY(ifq->ifcq_enqueue == NULL);
 140                 VERIFY(ifq->ifcq_dequeue == NULL);
 141                 VERIFY(ifq->ifcq_dequeue_sc == NULL);
 142                 VERIFY(ifq->ifcq_request == NULL);
 143         }
 144
 145         switch (scheduler) {
 146         case PKTSCHEDT_TCQ:
 147                 error = tcq_setup_ifclassq(ifq, sflags, ptype);
 148                 break;
 149
 150         case PKTSCHEDT_QFQ:
 151                 error = qfq_setup_ifclassq(ifq, sflags, ptype);
 152                 break;
 153         case PKTSCHEDT_FQ_CODEL:
 154                 error = fq_if_setup_ifclassq(ifq, sflags, ptype);
 155                 break;
 156         default:
 157                 error = ENXIO;
 158                 break;
 159         }
 160
 161         if (error == 0) {
 162                 ifq->ifcq_flags |= rflags;
 163         }
 164
 165         return error;
 166 }
 167
 168 int
 169 pktsched_teardown(struct ifclassq *ifq)
 170 {
 171         int error = 0;
 172
 173         IFCQ_LOCK_ASSERT_HELD(ifq);
 174
 175         if_qflush(ifq->ifcq_ifp, 1);
 176         VERIFY(IFCQ_IS_EMPTY(ifq));
 177
 178         ifq->ifcq_flags &= ~IFCQF_ENABLED;
 179
 180         switch (ifq->ifcq_type) {
 181         case PKTSCHEDT_NONE:
 182                 break;
 183
 184         case PKTSCHEDT_TCQ:
 185                 error = tcq_teardown_ifclassq(ifq);
 186                 break;
 187
 188         case PKTSCHEDT_QFQ:
 189                 error = qfq_teardown_ifclassq(ifq);
 190                 break;
 191
 192         case PKTSCHEDT_FQ_CODEL:
 193                 error = fq_if_teardown_ifclassq(ifq);
 194                 break;
 195         default:
 196                 error = ENXIO;
 197                 break;
 198         }
 199         return error;
 200 }
 201
 202 int
 203 pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
 204     struct if_ifclassq_stats *ifqs)
 205 {
 206         int error;
 207
 208         IFCQ_LOCK_ASSERT_HELD(ifq);
 209
 210         switch (ifq->ifcq_type) {
 211         case PKTSCHEDT_TCQ:
 212                 error = tcq_getqstats_ifclassq(ifq, qid, ifqs);
 213                 break;
 214
 215         case PKTSCHEDT_QFQ:
 216                 error = qfq_getqstats_ifclassq(ifq, qid, ifqs);
 217                 break;
 218
 219         case PKTSCHEDT_FQ_CODEL:
 220                 error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
 221                 break;
 222         default:
 223                 error = ENXIO;
 224                 break;
 225         }
 226
 227         return error;
 228 }
 229
 230 void
 231 pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_t *cpkt)
 232 {
 233         pkt->pktsched_pkt = *cpkt;
 234
 235         switch (cpkt->cp_ptype) {
 236         case QP_MBUF:
 237                 pkt->pktsched_plen =
 238                     (uint32_t)m_pktlen(pkt->pktsched_pkt_mbuf);
 239                 break;
 240
 241
 242         default:
 243                 VERIFY(0);
 244                 /* NOTREACHED */
 245                 __builtin_unreachable();
 246         }
 247 }
 248
 249 int
 250 pktsched_clone_pkt(pktsched_pkt_t *pkt1, pktsched_pkt_t *pkt2)
 251 {
 252         struct mbuf *m1, *m2;
 253
 254         ASSERT(pkt1 != NULL);
 255         ASSERT(pkt1->pktsched_pkt_mbuf != NULL);
 256         /* allow in place clone, but make sure pkt2->pktsched_pkt won't leak */
 257         ASSERT((pkt1 == pkt2 && pkt1->pktsched_pkt_mbuf ==
 258             pkt2->pktsched_pkt_mbuf) || (pkt1 != pkt2 &&
 259             pkt2->pktsched_pkt_mbuf == NULL));
 260
 261         switch (pkt1->pktsched_ptype) {
 262         case QP_MBUF:
 263                 m1 = (struct mbuf *)pkt1->pktsched_pkt_mbuf;
 264                 m2 = m_dup(m1, M_NOWAIT);
 265                 if (__improbable(m2 == NULL)) {
 266                         return ENOBUFS;
 267                 }
 268                 pkt2->pktsched_pkt_mbuf = m2;
 269                 break;
 270
 271
 272         default:
 273                 VERIFY(0);
 274                 /* NOTREACHED */
 275                 __builtin_unreachable();
 276         }
 277
 278         pkt2->pktsched_plen = pkt1->pktsched_plen;
 279         pkt2->pktsched_ptype = pkt1->pktsched_ptype;
 280         return 0;
 281 }
 282
 283 void
 284 pktsched_corrupt_packet(pktsched_pkt_t *pkt)
 285 {
 286         struct mbuf *m = NULL;
 287         uint8_t *data = NULL;
 288         uint32_t data_len = 0;
 289         uint32_t rand32, rand_off, rand_bit;
 290
 291         switch (pkt->pktsched_ptype) {
 292         case QP_MBUF:
 293                 m = pkt->pktsched_pkt_mbuf;
 294                 data = mtod(m, uint8_t *);
 295                 data_len = m->m_pkthdr.len;
 296                 break;
 297
 298         default:
 299                 /* NOTREACHED */
 300                 VERIFY(0);
 301                 __builtin_unreachable();
 302         }
 303
 304         read_frandom(&rand32, sizeof(rand32));
 305         rand_bit = rand32 & 0x8;
 306         rand_off = (rand32 >> 3) % data_len;
 307         data[rand_off] ^= 1 << rand_bit;
 308 }
 309
 310 void
 311 pktsched_free_pkt(pktsched_pkt_t *pkt)
 312 {
 313         switch (pkt->pktsched_ptype) {
 314         case QP_MBUF:
 315                 m_freem(pkt->pktsched_pkt_mbuf);
 316                 break;
 317
 318
 319         default:
 320                 VERIFY(0);
 321                 /* NOTREACHED */
 322                 __builtin_unreachable();
 323         }
 324
 325         pkt->pktsched_pkt = CLASSQ_PKT_INITIALIZER(pkt->pktsched_pkt);
 326         pkt->pktsched_plen = 0;
 327 }
 328
 329 mbuf_svc_class_t
 330 pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
 331 {
 332         mbuf_svc_class_t svc = MBUF_SC_UNSPEC;
 333
 334         switch (pkt->pktsched_ptype) {
 335         case QP_MBUF:
 336                 svc = m_get_service_class(pkt->pktsched_pkt_mbuf);
 337                 break;
 338
 339
 340         default:
 341                 VERIFY(0);
 342                 /* NOTREACHED */
 343                 __builtin_unreachable();
 344         }
 345
 346         return svc;
 347 }
 348
 349 void
 350 pktsched_get_pkt_vars(pktsched_pkt_t *pkt, volatile uint32_t **flags,
 351     uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto,
 352     uint32_t *tcp_start_seq)
 353 {
 354         switch (pkt->pktsched_ptype) {
 355         case QP_MBUF: {
 356                 struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
 357
 358                 if (flags != NULL) {
 359                         *flags = &pkth->pkt_flags;
 360                 }
 361                 if (timestamp != NULL) {
 362                         *timestamp = &pkth->pkt_timestamp;
 363                 }
 364                 if (flowid != NULL) {
 365                         *flowid = pkth->pkt_flowid;
 366                 }
 367                 if (flowsrc != NULL) {
 368                         *flowsrc = pkth->pkt_flowsrc;
 369                 }
 370                 if (proto != NULL) {
 371                         *proto = pkth->pkt_proto;
 372                 }
 373                 /*
 374                  * caller should use this value only if PKTF_START_SEQ
 375                  * is set in the mbuf packet flags
 376                  */
 377                 if (tcp_start_seq != NULL) {
 378                         *tcp_start_seq = pkth->tx_start_seq;
 379                 }
 380
 381                 break;
 382         }
 383
 384
 385         default:
 386                 VERIFY(0);
 387                 /* NOTREACHED */
 388                 __builtin_unreachable();
 389         }
 390 }
 391
 392 struct flowadv_fcentry *
 393 pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how)
 394 {
 395 #pragma unused(ifp)
 396         struct flowadv_fcentry *fce = NULL;
 397
 398         switch (pkt->pktsched_ptype) {
 399         case QP_MBUF: {
 400                 struct mbuf *m = pkt->pktsched_pkt_mbuf;
 401
 402                 fce = flowadv_alloc_entry(how);
 403                 if (fce == NULL) {
 404                         break;
 405                 }
 406
 407                 _CASSERT(sizeof(m->m_pkthdr.pkt_flowid) ==
 408                     sizeof(fce->fce_flowid));
 409
 410                 fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
 411                 fce->fce_flowid = m->m_pkthdr.pkt_flowid;
 412                 break;
 413         }
 414
 415
 416         default:
 417                 VERIFY(0);
 418                 /* NOTREACHED */
 419                 __builtin_unreachable();
 420         }
 421
 422         return fce;
 423 }
 424
 425 uint32_t *
 426 pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags)
 427 {
 428         uint32_t *hashp = NULL;
 429
 430         switch (pkt->pktsched_ptype) {
 431         case QP_MBUF: {
 432                 struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
 433
 434                 _CASSERT(sizeof(pkth->pkt_mpriv_hash) == sizeof(uint32_t));
 435                 _CASSERT(sizeof(pkth->pkt_mpriv_flags) == sizeof(uint32_t));
 436                 *sfb_flags = &pkth->pkt_mpriv_flags;
 437                 hashp = &pkth->pkt_mpriv_hash;
 438                 break;
 439         }
 440
 441
 442         default:
 443                 VERIFY(0);
 444                 /* NOTREACHED */
 445                 __builtin_unreachable();
 446         }
 447
 448         return hashp;
 449 }