bsd/net/pktsched/pktsched.c

   1 /*
   2  * Copyright (c) 2011-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30
  31 #include <sys/param.h>
  32 #include <sys/malloc.h>
  33 #include <sys/mbuf.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/errno.h>
  37 #include <sys/mcache.h>
  38 #include <sys/sysctl.h>
  39
  40 #include <net/if.h>
  41 #include <net/if_var.h>
  42 #include <net/if_dl.h>
  43 #include <net/if_types.h>
  44 #include <net/net_osdep.h>
  45 #include <net/pktsched/pktsched.h>
  46 #include <net/pktsched/pktsched_tcq.h>
  47 #include <net/pktsched/pktsched_qfq.h>
  48 #include <net/pktsched/pktsched_fq_codel.h>
  49
  50 #include <pexpert/pexpert.h>
  51
  52
  53 u_int32_t machclk_freq = 0;
  54 u_int64_t machclk_per_sec = 0;
  55 u_int32_t pktsched_verbose;     /* more noise if greater than 1 */
  56
  57 static void init_machclk(void);
  58
  59 SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "pktsched");
  60
  61 SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED,
  62         &pktsched_verbose, 0, "Packet scheduler verbosity level");
  63
  64 void
  65 pktsched_init(void)
  66 {
  67         init_machclk();
  68         if (machclk_freq == 0) {
  69                 panic("%s: no CPU clock available!\n", __func__);
  70                 /* NOTREACHED */
  71         }
  72
  73         tcq_init();
  74         qfq_init();
  75 }
  76
  77 static void
  78 init_machclk(void)
  79 {
  80         /*
  81          * Initialize machclk_freq using the timerbase frequency
  82          * value from device specific info.
  83          */
  84         machclk_freq = gPEClockFrequencyInfo.timebase_frequency_hz;
  85
  86         clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
  87             &machclk_per_sec);
  88 }
  89
  90 u_int64_t
  91 pktsched_abs_to_nsecs(u_int64_t abstime)
  92 {
  93         u_int64_t nsecs;
  94
  95         absolutetime_to_nanoseconds(abstime, &nsecs);
  96         return (nsecs);
  97 }
  98
  99 u_int64_t
 100 pktsched_nsecs_to_abstime(u_int64_t nsecs)
 101 {
 102         u_int64_t abstime;
 103
 104         nanoseconds_to_absolutetime(nsecs, &abstime);
 105         return (abstime);
 106 }
 107
 108 int
 109 pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
 110     classq_pkt_type_t ptype)
 111 {
 112         int error = 0;
 113         u_int32_t rflags;
 114
 115         IFCQ_LOCK_ASSERT_HELD(ifq);
 116
 117         VERIFY(machclk_freq != 0);
 118
 119         /* Nothing to do unless the scheduler type changes */
 120         if (ifq->ifcq_type == scheduler)
 121                 return (0);
 122
 123         /*
 124          * Remember the flags that need to be restored upon success, as
 125          * they may be cleared when we tear down existing scheduler.
 126          */
 127         rflags = (ifq->ifcq_flags & IFCQF_ENABLED);
 128
 129         if (ifq->ifcq_type != PKTSCHEDT_NONE) {
 130                 (void) pktsched_teardown(ifq);
 131
 132                 /* Teardown should have succeeded */
 133                 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 134                 VERIFY(ifq->ifcq_disc == NULL);
 135                 VERIFY(ifq->ifcq_enqueue == NULL);
 136                 VERIFY(ifq->ifcq_dequeue == NULL);
 137                 VERIFY(ifq->ifcq_dequeue_sc == NULL);
 138                 VERIFY(ifq->ifcq_request == NULL);
 139         }
 140
 141         switch (scheduler) {
 142         case PKTSCHEDT_TCQ:
 143                 error = tcq_setup_ifclassq(ifq, sflags, ptype);
 144                 break;
 145
 146         case PKTSCHEDT_QFQ:
 147                 error = qfq_setup_ifclassq(ifq, sflags, ptype);
 148                 break;
 149         case PKTSCHEDT_FQ_CODEL:
 150                 error = fq_if_setup_ifclassq(ifq, sflags, ptype);
 151                 break;
 152         default:
 153                 error = ENXIO;
 154                 break;
 155         }
 156
 157         if (error == 0)
 158                 ifq->ifcq_flags |= rflags;
 159
 160         return (error);
 161 }
 162
 163 int
 164 pktsched_teardown(struct ifclassq *ifq)
 165 {
 166         int error = 0;
 167
 168         IFCQ_LOCK_ASSERT_HELD(ifq);
 169
 170         if_qflush(ifq->ifcq_ifp, 1);
 171         VERIFY(IFCQ_IS_EMPTY(ifq));
 172
 173         ifq->ifcq_flags &= ~IFCQF_ENABLED;
 174
 175         switch (ifq->ifcq_type) {
 176         case PKTSCHEDT_NONE:
 177                 break;
 178
 179         case PKTSCHEDT_TCQ:
 180                 error = tcq_teardown_ifclassq(ifq);
 181                 break;
 182
 183         case PKTSCHEDT_QFQ:
 184                 error = qfq_teardown_ifclassq(ifq);
 185                 break;
 186
 187         case PKTSCHEDT_FQ_CODEL:
 188                 error = fq_if_teardown_ifclassq(ifq);
 189                 break;
 190         default:
 191                 error = ENXIO;
 192                 break;
 193         }
 194         return (error);
 195 }
 196
 197 int
 198 pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
 199     struct if_ifclassq_stats *ifqs)
 200 {
 201         int error;
 202
 203         IFCQ_LOCK_ASSERT_HELD(ifq);
 204
 205         switch (ifq->ifcq_type) {
 206         case PKTSCHEDT_TCQ:
 207                 error = tcq_getqstats_ifclassq(ifq, qid, ifqs);
 208                 break;
 209
 210         case PKTSCHEDT_QFQ:
 211                 error = qfq_getqstats_ifclassq(ifq, qid, ifqs);
 212                 break;
 213
 214         case PKTSCHEDT_FQ_CODEL:
 215                 error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
 216                 break;
 217         default:
 218                 error = ENXIO;
 219                 break;
 220         }
 221
 222         return (error);
 223 }
 224
 225 void
 226 pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_type_t ptype, void *pp)
 227 {
 228         pkt->pktsched_ptype = ptype;
 229         pkt->pktsched_pkt = pp;
 230
 231         switch (ptype) {
 232         case QP_MBUF:
 233                 pkt->pktsched_plen =
 234                     (uint32_t)m_pktlen((struct mbuf *)pkt->pktsched_pkt);
 235                 break;
 236
 237
 238         default:
 239                 VERIFY(0);
 240                 /* NOTREACHED */
 241         }
 242 }
 243
 244 void
 245 pktsched_free_pkt(pktsched_pkt_t *pkt)
 246 {
 247         switch (pkt->pktsched_ptype) {
 248         case QP_MBUF:
 249                 m_freem(pkt->pktsched_pkt);
 250                 break;
 251
 252
 253         default:
 254                 VERIFY(0);
 255                 /* NOTREACHED */
 256         }
 257
 258         pkt->pktsched_pkt = NULL;
 259         pkt->pktsched_plen = 0;
 260         pkt->pktsched_ptype = 0;
 261 }
 262
 263 uint32_t
 264 pktsched_get_pkt_len(pktsched_pkt_t *pkt)
 265 {
 266         return (pkt->pktsched_plen);
 267 }
 268
 269 mbuf_svc_class_t
 270 pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
 271 {
 272         mbuf_svc_class_t svc = MBUF_SC_UNSPEC;
 273
 274         switch (pkt->pktsched_ptype) {
 275         case QP_MBUF:
 276                 svc = m_get_service_class((mbuf_t)pkt->pktsched_pkt);
 277                 break;
 278
 279
 280         default:
 281                 VERIFY(0);
 282                 /* NOTREACHED */
 283         }
 284
 285         return (svc);
 286 }
 287
 288 void
 289 pktsched_get_pkt_vars(pktsched_pkt_t *pkt, uint32_t **flags,
 290     uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto,
 291     uint32_t *tcp_start_seq)
 292 {
 293         switch (pkt->pktsched_ptype) {
 294         case QP_MBUF: {
 295                 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
 296                 struct pkthdr *pkth = &m->m_pkthdr;
 297
 298                 if (flags != NULL)
 299                         *flags = &pkth->pkt_flags;
 300                 if (timestamp != NULL)
 301                         *timestamp = &pkth->pkt_timestamp;
 302                 if (flowid != NULL)
 303                         *flowid = pkth->pkt_flowid;
 304                 if (flowsrc != NULL)
 305                         *flowsrc = pkth->pkt_flowsrc;
 306                 if (proto != NULL)
 307                         *proto = pkth->pkt_proto;
 308                 /*
 309                  * caller should use this value only if PKTF_START_SEQ
 310                  * is set in the mbuf packet flags
 311                  */
 312                 if (tcp_start_seq != NULL)
 313                         *tcp_start_seq = pkth->tx_start_seq;
 314
 315                 break;
 316         }
 317
 318
 319         default:
 320                 VERIFY(0);
 321                 /* NOTREACHED */
 322         }
 323 }
 324
 325 struct flowadv_fcentry *
 326 pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how)
 327 {
 328 #pragma unused(ifp)
 329         struct flowadv_fcentry *fce = NULL;
 330
 331         switch (pkt->pktsched_ptype) {
 332         case QP_MBUF: {
 333                 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
 334
 335                 fce = flowadv_alloc_entry(how);
 336                 if (fce == NULL)
 337                         break;
 338
 339                 _CASSERT(sizeof (m->m_pkthdr.pkt_flowid) ==
 340                     sizeof (fce->fce_flowid));
 341
 342                 fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
 343                 fce->fce_flowid = m->m_pkthdr.pkt_flowid;
 344                 break;
 345         }
 346
 347
 348         default:
 349                 VERIFY(0);
 350                 /* NOTREACHED */
 351         }
 352
 353         return (fce);
 354 }
 355
 356 uint32_t *
 357 pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags)
 358 {
 359         uint32_t *hashp = NULL;
 360
 361         switch (pkt->pktsched_ptype) {
 362         case QP_MBUF: {
 363                 struct mbuf *m = (struct mbuf *)pkt->pktsched_pkt;
 364                 struct pkthdr *pkth = &m->m_pkthdr;
 365
 366                 _CASSERT(sizeof (pkth->pkt_mpriv_hash) == sizeof (uint32_t));
 367                 _CASSERT(sizeof (pkth->pkt_mpriv_flags) == sizeof (uint32_t));
 368
 369                 *sfb_flags = &pkth->pkt_mpriv_flags;
 370                 hashp = &pkth->pkt_mpriv_hash;
 371                 break;
 372         }
 373
 374
 375         default:
 376                 VERIFY(0);
 377                 /* NOTREACHED */
 378         }
 379
 380         return (hashp);
 381 }