bsd/net/classq/classq_subr.c

   1 /*
   2  * Copyright (c) 2011-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include <sys/param.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/errno.h>
  33 #include <sys/random.h>
  34 #include <sys/kernel_types.h>
  35 #include <sys/sysctl.h>
  36
  37 #include <kern/zalloc.h>
  38
  39 #include <net/if.h>
  40 #include <net/net_osdep.h>
  41 #include <net/classq/classq.h>
  42 #include <pexpert/pexpert.h>
  43 #include <net/classq/classq_sfb.h>
  44 #include <net/classq/classq_fq_codel.h>
  45 #include <net/pktsched/pktsched.h>
  46 #include <net/pktsched/pktsched_fq_codel.h>
  47
  48 #include <libkern/libkern.h>
  49
  50
  51 static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  52     u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *,
  53     boolean_t, classq_pkt_type_t *);
  54 static void *ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  55     boolean_t, classq_pkt_type_t *);
  56
  57 static u_int64_t ifclassq_target_qdelay = 0;
  58 SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED,
  59     &ifclassq_target_qdelay, "target queue delay in nanoseconds");
  60
  61 static u_int64_t ifclassq_update_interval = 0;
  62 SYSCTL_QUAD(_net_classq, OID_AUTO, update_interval,
  63     CTLFLAG_RW | CTLFLAG_LOCKED, &ifclassq_update_interval,
  64     "update interval in nanoseconds");
  65
  66 static int32_t ifclassq_sched_fq_codel;
  67
  68 void
  69 classq_init(void)
  70 {
  71         _CASSERT(MBUF_TC_BE == 0);
  72         _CASSERT(MBUF_SC_BE == 0);
  73         _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES);
  74
  75         sfb_init();
  76         fq_codel_scheduler_init();
  77
  78         if (!PE_parse_boot_argn("fq_codel", &ifclassq_sched_fq_codel,
  79             sizeof(ifclassq_sched_fq_codel))) {
  80                 ifclassq_sched_fq_codel = 1;
  81         }
  82 }
  83
  84 int
  85 ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse)
  86 {
  87 #pragma unused(reuse)
  88         struct ifclassq *ifq = &ifp->if_snd;
  89         int err = 0;
  90
  91         IFCQ_LOCK(ifq);
  92         VERIFY(IFCQ_IS_EMPTY(ifq));
  93         ifq->ifcq_ifp = ifp;
  94         IFCQ_LEN(ifq) = 0;
  95         IFCQ_BYTES(ifq) = 0;
  96         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
  97         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
  98
  99         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 100         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 101         VERIFY(ifq->ifcq_flags == 0);
 102         VERIFY(ifq->ifcq_sflags == 0);
 103         VERIFY(ifq->ifcq_disc == NULL);
 104         VERIFY(ifq->ifcq_enqueue == NULL);
 105         VERIFY(ifq->ifcq_dequeue == NULL);
 106         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 107         VERIFY(ifq->ifcq_request == NULL);
 108
 109         if (ifp->if_eflags & IFEF_TXSTART) {
 110                 u_int32_t maxlen = 0;
 111
 112                 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) {
 113                         maxlen = if_sndq_maxlen;
 114                 }
 115                 IFCQ_SET_MAXLEN(ifq, maxlen);
 116
 117                 if (IFCQ_MAXLEN(ifq) != if_sndq_maxlen &&
 118                     IFCQ_TARGET_QDELAY(ifq) == 0) {
 119                         /*
 120                          * Choose static queues because the interface has
 121                          * maximum queue size set
 122                          */
 123                         sflags &= ~PKTSCHEDF_QALG_DELAYBASED;
 124                 }
 125                 ifq->ifcq_sflags = sflags;
 126                 err = ifclassq_pktsched_setup(ifq);
 127                 if (err == 0) {
 128                         ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED);
 129                 }
 130         }
 131         IFCQ_UNLOCK(ifq);
 132         return err;
 133 }
 134
 135 void
 136 ifclassq_teardown(struct ifnet *ifp)
 137 {
 138         struct ifclassq *ifq = &ifp->if_snd;
 139
 140         IFCQ_LOCK(ifq);
 141
 142         if (IFCQ_IS_READY(ifq)) {
 143                 if (IFCQ_TBR_IS_ENABLED(ifq)) {
 144                         struct tb_profile tb = { 0, 0, 0 };
 145                         (void) ifclassq_tbr_set(ifq, &tb, FALSE);
 146                 }
 147                 (void) pktsched_teardown(ifq);
 148                 ifq->ifcq_flags = 0;
 149         }
 150         ifq->ifcq_sflags = 0;
 151
 152         VERIFY(IFCQ_IS_EMPTY(ifq));
 153         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 154         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 155         VERIFY(ifq->ifcq_flags == 0);
 156         VERIFY(ifq->ifcq_sflags == 0);
 157         VERIFY(ifq->ifcq_disc == NULL);
 158         VERIFY(ifq->ifcq_enqueue == NULL);
 159         VERIFY(ifq->ifcq_dequeue == NULL);
 160         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 161         VERIFY(ifq->ifcq_request == NULL);
 162         IFCQ_LEN(ifq) = 0;
 163         IFCQ_BYTES(ifq) = 0;
 164         IFCQ_MAXLEN(ifq) = 0;
 165         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
 166         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
 167
 168         IFCQ_UNLOCK(ifq);
 169 }
 170
 171 int
 172 ifclassq_pktsched_setup(struct ifclassq *ifq)
 173 {
 174         struct ifnet *ifp = ifq->ifcq_ifp;
 175         classq_pkt_type_t ptype = QP_MBUF;
 176         int err = 0;
 177
 178         IFCQ_LOCK_ASSERT_HELD(ifq);
 179         VERIFY(ifp->if_eflags & IFEF_TXSTART);
 180
 181         switch (ifp->if_output_sched_model) {
 182         case IFNET_SCHED_MODEL_DRIVER_MANAGED:
 183                 if (ifclassq_sched_fq_codel != 0) {
 184                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 185                             ifq->ifcq_sflags, ptype);
 186                 } else {
 187                         err = pktsched_setup(ifq, PKTSCHEDT_TCQ,
 188                             ifq->ifcq_sflags, ptype);
 189                 }
 190                 break;
 191
 192         case IFNET_SCHED_MODEL_NORMAL:
 193                 if (ifclassq_sched_fq_codel != 0) {
 194                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 195                             ifq->ifcq_sflags, ptype);
 196                 } else {
 197                         err = pktsched_setup(ifq, PKTSCHEDT_QFQ,
 198                             ifq->ifcq_sflags, ptype);
 199                 }
 200                 break;
 201         case IFNET_SCHED_MODEL_FQ_CODEL:
 202                 err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 203                     ifq->ifcq_sflags, ptype);
 204                 break;
 205         default:
 206                 VERIFY(0);
 207                 /* NOTREACHED */
 208         }
 209
 210         return err;
 211 }
 212
 213 void
 214 ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen)
 215 {
 216         IFCQ_LOCK(ifq);
 217         if (maxqlen == 0) {
 218                 maxqlen = if_sndq_maxlen;
 219         }
 220         IFCQ_SET_MAXLEN(ifq, maxqlen);
 221         IFCQ_UNLOCK(ifq);
 222 }
 223
 224 u_int32_t
 225 ifclassq_get_maxlen(struct ifclassq *ifq)
 226 {
 227         return IFCQ_MAXLEN(ifq);
 228 }
 229
 230 int
 231 ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets,
 232     u_int32_t *bytes)
 233 {
 234         int err = 0;
 235
 236         IFCQ_LOCK(ifq);
 237         if (sc == MBUF_SC_UNSPEC) {
 238                 VERIFY(packets != NULL);
 239                 *packets = IFCQ_LEN(ifq);
 240         } else {
 241                 VERIFY(MBUF_VALID_SC(sc));
 242                 VERIFY(packets != NULL && bytes != NULL);
 243                 IFCQ_LEN_SC(ifq, sc, packets, bytes, err);
 244         }
 245         IFCQ_UNLOCK(ifq);
 246
 247         return err;
 248 }
 249
 250 inline void
 251 ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp,
 252     void *p, classq_pkt_type_t ptype)
 253 {
 254         if (!IFNET_IS_CELLULAR(ifp)) {
 255                 return;
 256         }
 257
 258         switch (ptype) {
 259         case QP_MBUF: {
 260                 struct mbuf *m = p;
 261                 m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA;
 262                 m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq);
 263                 m->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent;
 264                 break;
 265         }
 266
 267
 268         default:
 269                 VERIFY(0);
 270                 /* NOTREACHED */
 271         }
 272 }
 273
 274 errno_t
 275 ifclassq_enqueue(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype,
 276     boolean_t *pdrop)
 277 {
 278         errno_t err;
 279
 280         switch (ptype) {
 281         case QP_MBUF:
 282                 IFCQ_LOCK_SPIN(ifq);
 283                 break;
 284
 285         default:
 286                 IFCQ_LOCK(ifq);
 287                 break;
 288         }
 289
 290         IFCQ_ENQUEUE(ifq, p, ptype, err, pdrop);
 291         IFCQ_UNLOCK(ifq);
 292         return err;
 293 }
 294
 295 errno_t
 296 ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit,
 297     u_int32_t byte_limit, void **head, void **tail,
 298     u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype)
 299 {
 300         return ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit,
 301                    byte_limit, head, tail, cnt, len, FALSE, ptype);
 302 }
 303
 304 errno_t
 305 ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 306     u_int32_t pkt_limit, u_int32_t byte_limit, void **head, void **tail,
 307     u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype)
 308 {
 309         return ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit,
 310                    head, tail, cnt, len, TRUE, ptype);
 311 }
 312
 313 static errno_t
 314 ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 315     u_int32_t pkt_limit, u_int32_t byte_limit, void **head,
 316     void **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt,
 317     classq_pkt_type_t *ptype)
 318 {
 319         struct ifnet *ifp = ifq->ifcq_ifp;
 320         u_int32_t i = 0, l = 0, lock_spin = 1;
 321         void **first, *last;
 322
 323         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 324
 325         *ptype = 0;
 326
 327
 328         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 329                 goto dequeue_loop;
 330         }
 331
 332         /*
 333          * If the scheduler support dequeueing multiple packets at the
 334          * same time, call that one instead.
 335          */
 336         if (drvmgt && ifq->ifcq_dequeue_sc_multi != NULL) {
 337                 int err;
 338
 339                 if (lock_spin) {
 340                         IFCQ_LOCK_SPIN(ifq);
 341                 } else {
 342                         IFCQ_LOCK(ifq);
 343                 }
 344                 err = ifq->ifcq_dequeue_sc_multi(ifq, sc, pkt_limit,
 345                     byte_limit, head, tail, cnt, len, ptype);
 346                 IFCQ_UNLOCK(ifq);
 347
 348                 if (err == 0 && (*head) == NULL) {
 349                         err = EAGAIN;
 350                 }
 351                 return err;
 352         } else if (ifq->ifcq_dequeue_multi != NULL) {
 353                 int err;
 354
 355                 if (lock_spin) {
 356                         IFCQ_LOCK_SPIN(ifq);
 357                 } else {
 358                         IFCQ_LOCK(ifq);
 359                 }
 360
 361                 err = ifq->ifcq_dequeue_multi(ifq, pkt_limit, byte_limit,
 362                     head, tail, cnt, len, ptype);
 363                 IFCQ_UNLOCK(ifq);
 364
 365                 if (err == 0 && (*head) == NULL) {
 366                         err = EAGAIN;
 367                 }
 368                 return err;
 369         }
 370
 371 dequeue_loop:
 372         *head = NULL;
 373         first = &(*head);
 374         last = NULL;
 375
 376         if (lock_spin) {
 377                 IFCQ_LOCK_SPIN(ifq);
 378         } else {
 379                 IFCQ_LOCK(ifq);
 380         }
 381
 382         while (i < pkt_limit && l < byte_limit) {
 383                 classq_pkt_type_t tmp_ptype;
 384                 if (drvmgt) {
 385                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 386                                 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype);
 387                         } else {
 388                                 IFCQ_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype);
 389                         }
 390                 } else {
 391                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 392                                 IFCQ_TBR_DEQUEUE(ifq, *head, &tmp_ptype);
 393                         } else {
 394                                 IFCQ_DEQUEUE(ifq, *head, &tmp_ptype);
 395                         }
 396                 }
 397
 398                 if (*head == NULL) {
 399                         break;
 400                 }
 401
 402                 switch (tmp_ptype) {
 403                 case QP_MBUF:
 404                         (*((mbuf_t *)head))->m_nextpkt = NULL;
 405                         last = *head;
 406                         l += (*((mbuf_t *)head))->m_pkthdr.len;
 407                         ifclassq_set_packet_metadata(ifq, ifp, (*head),
 408                             QP_MBUF);
 409                         head = (void **)&(*((mbuf_t *)head))->m_nextpkt;
 410                         break;
 411
 412
 413                 default:
 414                         VERIFY(0);
 415                         /* NOTREACHED */
 416                 }
 417
 418                 *ptype = tmp_ptype;
 419                 i++;
 420         }
 421
 422         IFCQ_UNLOCK(ifq);
 423
 424         if (tail != NULL) {
 425                 *tail = last;
 426         }
 427         if (cnt != NULL) {
 428                 *cnt = i;
 429         }
 430         if (len != NULL) {
 431                 *len = l;
 432         }
 433
 434         return (*first != NULL) ? 0 : EAGAIN;
 435 }
 436
 437 void
 438 ifclassq_update(struct ifclassq *ifq, cqev_t ev)
 439 {
 440         IFCQ_LOCK_ASSERT_HELD(ifq);
 441         VERIFY(IFCQ_IS_READY(ifq));
 442         IFCQ_UPDATE(ifq, ev);
 443 }
 444
 445 int
 446 ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline,
 447     ifclassq_enq_func enqueue, ifclassq_deq_func dequeue,
 448     ifclassq_deq_sc_func dequeue_sc, ifclassq_deq_multi_func dequeue_multi,
 449     ifclassq_deq_sc_multi_func dequeue_sc_multi, ifclassq_req_func request)
 450 {
 451         IFCQ_LOCK_ASSERT_HELD(ifq);
 452
 453         VERIFY(ifq->ifcq_disc == NULL);
 454         VERIFY(enqueue != NULL);
 455         VERIFY(request != NULL);
 456
 457         ifq->ifcq_type = type;
 458         ifq->ifcq_disc = discipline;
 459         ifq->ifcq_enqueue = enqueue;
 460         ifq->ifcq_dequeue = dequeue;
 461         ifq->ifcq_dequeue_sc = dequeue_sc;
 462         ifq->ifcq_dequeue_multi = dequeue_multi;
 463         ifq->ifcq_dequeue_sc_multi = dequeue_sc_multi;
 464         ifq->ifcq_request = request;
 465
 466         return 0;
 467 }
 468
 469 int
 470 ifclassq_detach(struct ifclassq *ifq)
 471 {
 472         IFCQ_LOCK_ASSERT_HELD(ifq);
 473
 474         VERIFY(ifq->ifcq_disc == NULL);
 475
 476         ifq->ifcq_type = PKTSCHEDT_NONE;
 477         ifq->ifcq_disc = NULL;
 478         ifq->ifcq_enqueue = NULL;
 479         ifq->ifcq_dequeue = NULL;
 480         ifq->ifcq_dequeue_sc = NULL;
 481         ifq->ifcq_request = NULL;
 482
 483         return 0;
 484 }
 485
 486 int
 487 ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf,
 488     u_int32_t *nbytes)
 489 {
 490         struct if_ifclassq_stats *ifqs;
 491         int err;
 492
 493         if (*nbytes < sizeof(*ifqs)) {
 494                 return EINVAL;
 495         }
 496
 497         ifqs = _MALLOC(sizeof(*ifqs), M_TEMP, M_WAITOK | M_ZERO);
 498         if (ifqs == NULL) {
 499                 return ENOMEM;
 500         }
 501
 502         IFCQ_LOCK(ifq);
 503         if (!IFCQ_IS_READY(ifq)) {
 504                 IFCQ_UNLOCK(ifq);
 505                 _FREE(ifqs, M_TEMP);
 506                 return ENXIO;
 507         }
 508
 509         ifqs->ifqs_len = IFCQ_LEN(ifq);
 510         ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq);
 511         *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt);
 512         *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt);
 513         ifqs->ifqs_scheduler = ifq->ifcq_type;
 514
 515         err = pktsched_getqstats(ifq, qid, ifqs);
 516         IFCQ_UNLOCK(ifq);
 517
 518         if (err == 0 && (err = copyout((caddr_t)ifqs,
 519             (user_addr_t)(uintptr_t)ubuf, sizeof(*ifqs))) == 0) {
 520                 *nbytes = sizeof(*ifqs);
 521         }
 522
 523         _FREE(ifqs, M_TEMP);
 524
 525         return err;
 526 }
 527
 528 const char *
 529 ifclassq_ev2str(cqev_t ev)
 530 {
 531         const char *c;
 532
 533         switch (ev) {
 534         case CLASSQ_EV_LINK_BANDWIDTH:
 535                 c = "LINK_BANDWIDTH";
 536                 break;
 537
 538         case CLASSQ_EV_LINK_LATENCY:
 539                 c = "LINK_LATENCY";
 540                 break;
 541
 542         case CLASSQ_EV_LINK_MTU:
 543                 c = "LINK_MTU";
 544                 break;
 545
 546         case CLASSQ_EV_LINK_UP:
 547                 c = "LINK_UP";
 548                 break;
 549
 550         case CLASSQ_EV_LINK_DOWN:
 551                 c = "LINK_DOWN";
 552                 break;
 553
 554         default:
 555                 c = "UNKNOWN";
 556                 break;
 557         }
 558
 559         return c;
 560 }
 561
 562 /*
 563  * internal representation of token bucket parameters
 564  *      rate:   byte_per_unittime << 32
 565  *              (((bits_per_sec) / 8) << 32) / machclk_freq
 566  *      depth:  byte << 32
 567  *
 568  */
 569 #define TBR_SHIFT       32
 570 #define TBR_SCALE(x)    ((int64_t)(x) << TBR_SHIFT)
 571 #define TBR_UNSCALE(x)  ((x) >> TBR_SHIFT)
 572
 573 void *
 574 ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_type_t *ptype)
 575 {
 576         return ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, ptype);
 577 }
 578
 579 void *
 580 ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 581     classq_pkt_type_t *ptype)
 582 {
 583         return ifclassq_tbr_dequeue_common(ifq, sc, TRUE, ptype);
 584 }
 585
 586 static void *
 587 ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 588     boolean_t drvmgt, classq_pkt_type_t *ptype)
 589 {
 590         struct tb_regulator *tbr;
 591         void *p;
 592         int64_t interval;
 593         u_int64_t now;
 594
 595         IFCQ_LOCK_ASSERT_HELD(ifq);
 596
 597         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 598         VERIFY(IFCQ_TBR_IS_ENABLED(ifq));
 599
 600         tbr = &ifq->ifcq_tbr;
 601         /* update token only when it is negative */
 602         if (tbr->tbr_token <= 0) {
 603                 now = read_machclk();
 604                 interval = now - tbr->tbr_last;
 605                 if (interval >= tbr->tbr_filluptime) {
 606                         tbr->tbr_token = tbr->tbr_depth;
 607                 } else {
 608                         tbr->tbr_token += interval * tbr->tbr_rate;
 609                         if (tbr->tbr_token > tbr->tbr_depth) {
 610                                 tbr->tbr_token = tbr->tbr_depth;
 611                         }
 612                 }
 613                 tbr->tbr_last = now;
 614         }
 615         /* if token is still negative, don't allow dequeue */
 616         if (tbr->tbr_token <= 0) {
 617                 return NULL;
 618         }
 619
 620         /*
 621          * ifclassq takes precedence over ALTQ queue;
 622          * ifcq_drain count is adjusted by the caller.
 623          */
 624         if (drvmgt) {
 625                 IFCQ_DEQUEUE_SC(ifq, sc, p, ptype);
 626         } else {
 627                 IFCQ_DEQUEUE(ifq, p, ptype);
 628         }
 629
 630         if (p != NULL) {
 631                 switch (*ptype) {
 632                 case QP_MBUF:
 633                         tbr->tbr_token -= TBR_SCALE(m_pktlen((mbuf_t)p));
 634                         break;
 635
 636
 637                 default:
 638                         VERIFY(0);
 639                         /* NOTREACHED */
 640                 }
 641         }
 642
 643         return p;
 644 }
 645
 646 /*
 647  * set a token bucket regulator.
 648  * if the specified rate is zero, the token bucket regulator is deleted.
 649  */
 650 int
 651 ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile,
 652     boolean_t update)
 653 {
 654         struct tb_regulator *tbr;
 655         struct ifnet *ifp = ifq->ifcq_ifp;
 656         u_int64_t rate, old_rate;
 657
 658         IFCQ_LOCK_ASSERT_HELD(ifq);
 659         VERIFY(IFCQ_IS_READY(ifq));
 660
 661         VERIFY(machclk_freq != 0);
 662
 663         tbr = &ifq->ifcq_tbr;
 664         old_rate = tbr->tbr_rate_raw;
 665
 666         rate = profile->rate;
 667         if (profile->percent > 0) {
 668                 u_int64_t eff_rate;
 669
 670                 if (profile->percent > 100) {
 671                         return EINVAL;
 672                 }
 673                 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0) {
 674                         return ENODEV;
 675                 }
 676                 rate = (eff_rate * profile->percent) / 100;
 677         }
 678
 679         if (rate == 0) {
 680                 if (!IFCQ_TBR_IS_ENABLED(ifq)) {
 681                         return ENOENT;
 682                 }
 683
 684                 if (pktsched_verbose) {
 685                         printf("%s: TBR disabled\n", if_name(ifp));
 686                 }
 687
 688                 /* disable this TBR */
 689                 ifq->ifcq_flags &= ~IFCQF_TBR;
 690                 bzero(tbr, sizeof(*tbr));
 691                 ifnet_set_start_cycle(ifp, NULL);
 692                 if (update) {
 693                         ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 694                 }
 695                 return 0;
 696         }
 697
 698         if (pktsched_verbose) {
 699                 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp),
 700                     (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" :
 701                     "enabled", rate, profile->depth);
 702         }
 703
 704         /* set the new TBR */
 705         bzero(tbr, sizeof(*tbr));
 706         tbr->tbr_rate_raw = rate;
 707         tbr->tbr_percent = profile->percent;
 708         ifq->ifcq_flags |= IFCQF_TBR;
 709
 710         /*
 711          * Note that the TBR fill up time (hence the ifnet restart time)
 712          * is directly related to the specified TBR depth.  The ideal
 713          * depth value should be computed such that the interval time
 714          * between each successive wakeup is adequately spaced apart,
 715          * in order to reduce scheduling overheads.  A target interval
 716          * of 10 ms seems to provide good performance balance.  This can be
 717          * overridden by specifying the depth profile.  Values smaller than
 718          * the ideal depth will reduce delay at the expense of CPU cycles.
 719          */
 720         tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq;
 721         if (tbr->tbr_rate > 0) {
 722                 u_int32_t mtu = ifp->if_mtu;
 723                 int64_t ival, idepth = 0;
 724                 int i;
 725
 726                 if (mtu < IF_MINMTU) {
 727                         mtu = IF_MINMTU;
 728                 }
 729
 730                 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */
 731
 732                 for (i = 1;; i++) {
 733                         idepth = TBR_SCALE(i * mtu);
 734                         if ((idepth / tbr->tbr_rate) > ival) {
 735                                 break;
 736                         }
 737                 }
 738                 VERIFY(idepth > 0);
 739
 740                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 741                 if (tbr->tbr_depth == 0) {
 742                         tbr->tbr_filluptime = idepth / tbr->tbr_rate;
 743                         /* a little fudge factor to get closer to rate */
 744                         tbr->tbr_depth = idepth + (idepth >> 3);
 745                 } else {
 746                         tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate;
 747                 }
 748         } else {
 749                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 750                 tbr->tbr_filluptime = 0xffffffffffffffffLL;
 751         }
 752         tbr->tbr_token = tbr->tbr_depth;
 753         tbr->tbr_last = read_machclk();
 754
 755         if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) {
 756                 struct timespec ts =
 757                 { 0, pktsched_abs_to_nsecs(tbr->tbr_filluptime) };
 758                 if (pktsched_verbose) {
 759                         printf("%s: TBR calculated tokens %lld "
 760                             "filluptime %llu ns\n", if_name(ifp),
 761                             TBR_UNSCALE(tbr->tbr_token),
 762                             pktsched_abs_to_nsecs(tbr->tbr_filluptime));
 763                 }
 764                 ifnet_set_start_cycle(ifp, &ts);
 765         } else {
 766                 if (pktsched_verbose) {
 767                         if (tbr->tbr_rate == 0) {
 768                                 printf("%s: TBR calculated tokens %lld "
 769                                     "infinite filluptime\n", if_name(ifp),
 770                                     TBR_UNSCALE(tbr->tbr_token));
 771                         } else if (!(ifp->if_flags & IFF_UP)) {
 772                                 printf("%s: TBR suspended (link is down)\n",
 773                                     if_name(ifp));
 774                         }
 775                 }
 776                 ifnet_set_start_cycle(ifp, NULL);
 777         }
 778         if (update && tbr->tbr_rate_raw != old_rate) {
 779                 ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 780         }
 781
 782         return 0;
 783 }
 784
 785 void
 786 ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay)
 787 {
 788         u_int64_t qdelay = 0;
 789         qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd);
 790
 791         if (ifclassq_target_qdelay != 0) {
 792                 qdelay = ifclassq_target_qdelay;
 793         }
 794
 795         /*
 796          * If we do not know the effective bandwidth, use the default
 797          * target queue delay.
 798          */
 799         if (qdelay == 0) {
 800                 qdelay = IFQ_TARGET_DELAY;
 801         }
 802
 803         /*
 804          * If a delay has been added to ifnet start callback for
 805          * coalescing, we have to add that to the pre-set target delay
 806          * because the packets can be in the queue longer.
 807          */
 808         if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
 809             ifp->if_start_delay_timeout > 0) {
 810                 qdelay += ifp->if_start_delay_timeout;
 811         }
 812
 813         *(if_target_qdelay) = qdelay;
 814 }
 815
 816 void
 817 ifclassq_calc_update_interval(u_int64_t *update_interval)
 818 {
 819         u_int64_t uint = 0;
 820
 821         /* If the system level override is set, use it */
 822         if (ifclassq_update_interval != 0) {
 823                 uint = ifclassq_update_interval;
 824         }
 825
 826         /* Otherwise use the default value */
 827         if (uint == 0) {
 828                 uint = IFQ_UPDATE_INTERVAL;
 829         }
 830
 831         *update_interval = uint;
 832 }
 833
 834 void
 835 ifclassq_reap_caches(boolean_t purge)
 836 {
 837         fq_codel_reap_caches(purge);
 838 }