bsd/net/classq/classq_subr.c

   1 /*
   2  * Copyright (c) 2011-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include <sys/param.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/errno.h>
  33 #include <sys/random.h>
  34 #include <sys/kernel_types.h>
  35 #include <sys/sysctl.h>
  36
  37 #include <kern/zalloc.h>
  38
  39 #include <net/if.h>
  40 #include <net/net_osdep.h>
  41 #include <net/classq/classq.h>
  42 #include <pexpert/pexpert.h>
  43 #include <net/classq/classq_sfb.h>
  44 #include <net/classq/classq_fq_codel.h>
  45 #include <net/pktsched/pktsched.h>
  46 #include <net/pktsched/pktsched_fq_codel.h>
  47 #include <net/flowadv.h>
  48
  49 #include <libkern/libkern.h>
  50
  51
  52 static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  53     u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *,
  54     u_int32_t *, boolean_t);
  55 static void ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  56     boolean_t, classq_pkt_t *);
  57
  58 static u_int64_t ifclassq_target_qdelay = 0;
  59 SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED,
  60     &ifclassq_target_qdelay, "target queue delay in nanoseconds");
  61
  62 static u_int64_t ifclassq_update_interval = 0;
  63 SYSCTL_QUAD(_net_classq, OID_AUTO, update_interval,
  64     CTLFLAG_RW | CTLFLAG_LOCKED, &ifclassq_update_interval,
  65     "update interval in nanoseconds");
  66
  67 #if DEBUG || DEVELOPMENT
  68 uint32_t ifclassq_flow_control_adv = 1; /* flow control advisory */
  69 SYSCTL_UINT(_net_classq, OID_AUTO, flow_control_adv,
  70     CTLFLAG_RW | CTLFLAG_LOCKED, &ifclassq_flow_control_adv, 1,
  71     "enable/disable flow control advisory");
  72
  73 uint16_t fq_codel_quantum = 0;
  74 #endif /* DEBUG || DEVELOPMENT */
  75
  76 void
  77 classq_init(void)
  78 {
  79         _CASSERT(MBUF_TC_BE == 0);
  80         _CASSERT(MBUF_SC_BE == 0);
  81         _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES);
  82 #if DEBUG || DEVELOPMENT
  83         PE_parse_boot_argn("fq_codel_quantum", &fq_codel_quantum,
  84             sizeof(fq_codel_quantum));
  85         PE_parse_boot_argn("ifclassq_target_qdelay", &ifclassq_target_qdelay,
  86             sizeof(ifclassq_target_qdelay));
  87         PE_parse_boot_argn("ifclassq_update_interval",
  88             &ifclassq_update_interval, sizeof(ifclassq_update_interval));
  89 #endif /* DEBUG || DEVELOPMENT */
  90         fq_codel_init();
  91 }
  92
  93 int
  94 ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse)
  95 {
  96 #pragma unused(reuse)
  97         struct ifclassq *ifq = &ifp->if_snd;
  98         int err = 0;
  99
 100         IFCQ_LOCK(ifq);
 101         VERIFY(IFCQ_IS_EMPTY(ifq));
 102         ifq->ifcq_ifp = ifp;
 103         IFCQ_LEN(ifq) = 0;
 104         IFCQ_BYTES(ifq) = 0;
 105         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
 106         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
 107
 108         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 109         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 110         VERIFY(ifq->ifcq_flags == 0);
 111         VERIFY(ifq->ifcq_sflags == 0);
 112         VERIFY(ifq->ifcq_disc == NULL);
 113
 114         if (ifp->if_eflags & IFEF_TXSTART) {
 115                 u_int32_t maxlen = 0;
 116
 117                 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) {
 118                         maxlen = if_sndq_maxlen;
 119                 }
 120                 IFCQ_SET_MAXLEN(ifq, maxlen);
 121
 122                 if (IFCQ_MAXLEN(ifq) != if_sndq_maxlen &&
 123                     IFCQ_TARGET_QDELAY(ifq) == 0) {
 124                         /*
 125                          * Choose static queues because the interface has
 126                          * maximum queue size set
 127                          */
 128                         sflags &= ~PKTSCHEDF_QALG_DELAYBASED;
 129                 }
 130                 ifq->ifcq_sflags = sflags;
 131                 err = ifclassq_pktsched_setup(ifq);
 132                 if (err == 0) {
 133                         ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED);
 134                 }
 135         }
 136         IFCQ_UNLOCK(ifq);
 137         return err;
 138 }
 139
 140 void
 141 ifclassq_teardown(struct ifnet *ifp)
 142 {
 143         struct ifclassq *ifq = &ifp->if_snd;
 144
 145         IFCQ_LOCK(ifq);
 146
 147         if (IFCQ_IS_READY(ifq)) {
 148                 if (IFCQ_TBR_IS_ENABLED(ifq)) {
 149                         struct tb_profile tb = { .rate = 0, .percent = 0, .depth = 0 };
 150                         (void) ifclassq_tbr_set(ifq, &tb, FALSE);
 151                 }
 152                 pktsched_teardown(ifq);
 153                 ifq->ifcq_flags = 0;
 154         }
 155         ifq->ifcq_sflags = 0;
 156
 157         VERIFY(IFCQ_IS_EMPTY(ifq));
 158         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 159         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 160         VERIFY(ifq->ifcq_flags == 0);
 161         VERIFY(ifq->ifcq_sflags == 0);
 162         VERIFY(ifq->ifcq_disc == NULL);
 163         IFCQ_LEN(ifq) = 0;
 164         IFCQ_BYTES(ifq) = 0;
 165         IFCQ_MAXLEN(ifq) = 0;
 166         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
 167         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
 168
 169         IFCQ_UNLOCK(ifq);
 170 }
 171
 172 int
 173 ifclassq_pktsched_setup(struct ifclassq *ifq)
 174 {
 175         struct ifnet *ifp = ifq->ifcq_ifp;
 176         classq_pkt_type_t ptype = QP_MBUF;
 177         int err = 0;
 178
 179         IFCQ_LOCK_ASSERT_HELD(ifq);
 180         VERIFY(ifp->if_eflags & IFEF_TXSTART);
 181
 182         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL, ifq->ifcq_sflags, ptype);
 183
 184         return err;
 185 }
 186
 187 void
 188 ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen)
 189 {
 190         IFCQ_LOCK(ifq);
 191         if (maxqlen == 0) {
 192                 maxqlen = if_sndq_maxlen;
 193         }
 194         IFCQ_SET_MAXLEN(ifq, maxqlen);
 195         IFCQ_UNLOCK(ifq);
 196 }
 197
 198 u_int32_t
 199 ifclassq_get_maxlen(struct ifclassq *ifq)
 200 {
 201         return IFCQ_MAXLEN(ifq);
 202 }
 203
 204 int
 205 ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets,
 206     u_int32_t *bytes)
 207 {
 208         int err = 0;
 209
 210         IFCQ_LOCK(ifq);
 211         if (sc == MBUF_SC_UNSPEC) {
 212                 VERIFY(packets != NULL);
 213                 *packets = IFCQ_LEN(ifq);
 214         } else {
 215                 cqrq_stat_sc_t req = { sc, 0, 0 };
 216
 217                 VERIFY(MBUF_VALID_SC(sc));
 218                 VERIFY(packets != NULL && bytes != NULL);
 219
 220                 err = fq_if_request_classq(ifq, CLASSQRQ_STAT_SC, &req);
 221                 if (packets != NULL) {
 222                         *packets = req.packets;
 223                 }
 224                 if (bytes != NULL) {
 225                         *bytes = req.bytes;
 226                 }
 227         }
 228         IFCQ_UNLOCK(ifq);
 229
 230
 231         return err;
 232 }
 233
 234 inline void
 235 ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp,
 236     classq_pkt_t *p)
 237 {
 238         if (!IFNET_IS_CELLULAR(ifp)) {
 239                 return;
 240         }
 241
 242         switch (p->cp_ptype) {
 243         case QP_MBUF: {
 244                 struct mbuf *m = p->cp_mbuf;
 245                 m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA;
 246                 m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq);
 247                 m->m_pkthdr.bufstatus_sndbuf = (uint32_t)ifp->if_sndbyte_unsent;
 248                 break;
 249         }
 250
 251
 252         default:
 253                 VERIFY(0);
 254                 /* NOTREACHED */
 255                 __builtin_unreachable();
 256         }
 257 }
 258
 259 errno_t
 260 ifclassq_enqueue(struct ifclassq *ifq, classq_pkt_t *head, classq_pkt_t *tail,
 261     u_int32_t cnt, u_int32_t bytes, boolean_t *pdrop)
 262 {
 263         return fq_if_enqueue_classq(ifq, head, tail, cnt, bytes, pdrop);
 264 }
 265
 266 errno_t
 267 ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit,
 268     u_int32_t byte_limit, classq_pkt_t *head, classq_pkt_t *tail,
 269     u_int32_t *cnt, u_int32_t *len)
 270 {
 271         return ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit,
 272                    byte_limit, head, tail, cnt, len, FALSE);
 273 }
 274
 275 errno_t
 276 ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 277     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 278     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len)
 279 {
 280         return ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit,
 281                    head, tail, cnt, len, TRUE);
 282 }
 283
 284 static errno_t
 285 ifclassq_dequeue_common_default(struct ifclassq *ifq, mbuf_svc_class_t sc,
 286     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 287     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt)
 288 {
 289         struct ifnet *ifp = ifq->ifcq_ifp;
 290         u_int32_t i = 0, l = 0;
 291         classq_pkt_t first = CLASSQ_PKT_INITIALIZER(first);
 292         classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
 293
 294         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 295
 296         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 297                 goto dequeue_loop;
 298         }
 299
 300         /*
 301          * If the scheduler support dequeueing multiple packets at the
 302          * same time, call that one instead.
 303          */
 304         if (drvmgt) {
 305                 int err;
 306
 307                 IFCQ_LOCK_SPIN(ifq);
 308                 err = fq_if_dequeue_sc_classq_multi(ifq, sc, pkt_limit,
 309                     byte_limit, head, tail, cnt, len);
 310                 IFCQ_UNLOCK(ifq);
 311
 312                 if (err == 0 && head->cp_mbuf == NULL) {
 313                         err = EAGAIN;
 314                 }
 315                 return err;
 316         } else {
 317                 int err;
 318
 319                 IFCQ_LOCK_SPIN(ifq);
 320                 err = fq_if_dequeue_classq_multi(ifq, pkt_limit, byte_limit,
 321                     head, tail, cnt, len);
 322                 IFCQ_UNLOCK(ifq);
 323
 324                 if (err == 0 && head->cp_mbuf == NULL) {
 325                         err = EAGAIN;
 326                 }
 327                 return err;
 328         }
 329
 330 dequeue_loop:
 331
 332         IFCQ_LOCK_SPIN(ifq);
 333
 334         while (i < pkt_limit && l < byte_limit) {
 335                 if (drvmgt) {
 336                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 337                                 IFCQ_TBR_DEQUEUE_SC(ifq, sc, head);
 338                         } else {
 339                                 fq_if_dequeue_sc_classq(ifq, sc, head);
 340                         }
 341                 } else {
 342                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 343                                 IFCQ_TBR_DEQUEUE(ifq, head);
 344                         } else {
 345                                 fq_if_dequeue_classq(ifq, head);
 346                         }
 347                 }
 348
 349                 if (head->cp_mbuf == NULL) {
 350                         break;
 351                 }
 352
 353                 if (first.cp_mbuf == NULL) {
 354                         first = *head;
 355                 }
 356
 357                 switch (head->cp_ptype) {
 358                 case QP_MBUF:
 359                         head->cp_mbuf->m_nextpkt = NULL;
 360                         l += head->cp_mbuf->m_pkthdr.len;
 361                         ifclassq_set_packet_metadata(ifq, ifp, head);
 362                         if (last.cp_mbuf != NULL) {
 363                                 last.cp_mbuf->m_nextpkt = head->cp_mbuf;
 364                         }
 365                         break;
 366
 367
 368                 default:
 369                         VERIFY(0);
 370                         /* NOTREACHED */
 371                         __builtin_unreachable();
 372                 }
 373
 374                 last = *head;
 375                 i++;
 376         }
 377
 378         IFCQ_UNLOCK(ifq);
 379
 380         if (tail != NULL) {
 381                 *tail = last;
 382         }
 383         if (cnt != NULL) {
 384                 *cnt = i;
 385         }
 386         if (len != NULL) {
 387                 *len = l;
 388         }
 389
 390         *head = first;
 391         return (first.cp_mbuf != NULL) ? 0 : EAGAIN;
 392 }
 393
 394 static errno_t
 395 ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 396     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 397     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt)
 398 {
 399         return ifclassq_dequeue_common_default(ifq, sc,
 400                    pkt_limit, byte_limit, head, tail, cnt, len, drvmgt);
 401 }
 402
 403 void
 404 ifclassq_update(struct ifclassq *ifq, cqev_t ev)
 405 {
 406         IFCQ_LOCK_ASSERT_HELD(ifq);
 407         VERIFY(IFCQ_IS_READY(ifq));
 408         fq_if_request_classq(ifq, CLASSQRQ_EVENT, (void *)ev);
 409 }
 410
 411 int
 412 ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline)
 413 {
 414         IFCQ_LOCK_ASSERT_HELD(ifq);
 415
 416         VERIFY(ifq->ifcq_disc == NULL);
 417
 418         ifq->ifcq_type = type;
 419         ifq->ifcq_disc = discipline;
 420
 421         return 0;
 422 }
 423
 424 void
 425 ifclassq_detach(struct ifclassq *ifq)
 426 {
 427         IFCQ_LOCK_ASSERT_HELD(ifq);
 428
 429         VERIFY(ifq->ifcq_disc == NULL);
 430
 431         ifq->ifcq_type = PKTSCHEDT_NONE;
 432 }
 433
 434 int
 435 ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf,
 436     u_int32_t *nbytes)
 437 {
 438         struct if_ifclassq_stats *ifqs;
 439         int err;
 440
 441         if (*nbytes < sizeof(*ifqs)) {
 442                 return EINVAL;
 443         }
 444
 445         ifqs = _MALLOC(sizeof(*ifqs), M_TEMP, M_WAITOK | M_ZERO);
 446         if (ifqs == NULL) {
 447                 return ENOMEM;
 448         }
 449
 450         IFCQ_LOCK(ifq);
 451         if (!IFCQ_IS_READY(ifq)) {
 452                 IFCQ_UNLOCK(ifq);
 453                 _FREE(ifqs, M_TEMP);
 454                 return ENXIO;
 455         }
 456
 457         ifqs->ifqs_len = IFCQ_LEN(ifq);
 458         ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq);
 459         *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt);
 460         *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt);
 461         ifqs->ifqs_scheduler = ifq->ifcq_type;
 462
 463         err = pktsched_getqstats(ifq, qid, ifqs);
 464         IFCQ_UNLOCK(ifq);
 465
 466         if (err == 0 && (err = copyout((caddr_t)ifqs,
 467             (user_addr_t)(uintptr_t)ubuf, sizeof(*ifqs))) == 0) {
 468                 *nbytes = sizeof(*ifqs);
 469         }
 470
 471         _FREE(ifqs, M_TEMP);
 472
 473         return err;
 474 }
 475
 476 const char *
 477 ifclassq_ev2str(cqev_t ev)
 478 {
 479         const char *c;
 480
 481         switch (ev) {
 482         case CLASSQ_EV_LINK_BANDWIDTH:
 483                 c = "LINK_BANDWIDTH";
 484                 break;
 485
 486         case CLASSQ_EV_LINK_LATENCY:
 487                 c = "LINK_LATENCY";
 488                 break;
 489
 490         case CLASSQ_EV_LINK_MTU:
 491                 c = "LINK_MTU";
 492                 break;
 493
 494         case CLASSQ_EV_LINK_UP:
 495                 c = "LINK_UP";
 496                 break;
 497
 498         case CLASSQ_EV_LINK_DOWN:
 499                 c = "LINK_DOWN";
 500                 break;
 501
 502         default:
 503                 c = "UNKNOWN";
 504                 break;
 505         }
 506
 507         return c;
 508 }
 509
 510 /*
 511  * internal representation of token bucket parameters
 512  *      rate:   byte_per_unittime << 32
 513  *              (((bits_per_sec) / 8) << 32) / machclk_freq
 514  *      depth:  byte << 32
 515  *
 516  */
 517 #define TBR_SHIFT       32
 518 #define TBR_SCALE(x)    ((int64_t)(x) << TBR_SHIFT)
 519 #define TBR_UNSCALE(x)  ((x) >> TBR_SHIFT)
 520
 521 void
 522 ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_t *pkt)
 523 {
 524         ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, pkt);
 525 }
 526
 527 void
 528 ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 529     classq_pkt_t *pkt)
 530 {
 531         ifclassq_tbr_dequeue_common(ifq, sc, TRUE, pkt);
 532 }
 533
 534 static void
 535 ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 536     boolean_t drvmgt, classq_pkt_t *pkt)
 537 {
 538         struct tb_regulator *tbr;
 539         int64_t interval;
 540         u_int64_t now;
 541
 542         IFCQ_LOCK_ASSERT_HELD(ifq);
 543
 544         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 545         VERIFY(IFCQ_TBR_IS_ENABLED(ifq));
 546
 547         *pkt = CLASSQ_PKT_INITIALIZER(*pkt);
 548         tbr = &ifq->ifcq_tbr;
 549         /* update token only when it is negative */
 550         if (tbr->tbr_token <= 0) {
 551                 now = read_machclk();
 552                 interval = now - tbr->tbr_last;
 553                 if (interval >= tbr->tbr_filluptime) {
 554                         tbr->tbr_token = tbr->tbr_depth;
 555                 } else {
 556                         tbr->tbr_token += interval * tbr->tbr_rate;
 557                         if (tbr->tbr_token > tbr->tbr_depth) {
 558                                 tbr->tbr_token = tbr->tbr_depth;
 559                         }
 560                 }
 561                 tbr->tbr_last = now;
 562         }
 563         /* if token is still negative, don't allow dequeue */
 564         if (tbr->tbr_token <= 0) {
 565                 return;
 566         }
 567
 568         /*
 569          * ifclassq takes precedence over ALTQ queue;
 570          * ifcq_drain count is adjusted by the caller.
 571          */
 572         if (drvmgt) {
 573                 fq_if_dequeue_sc_classq(ifq, sc, pkt);
 574         } else {
 575                 fq_if_dequeue_classq(ifq, pkt);
 576         }
 577
 578         if (pkt->cp_mbuf != NULL) {
 579                 switch (pkt->cp_ptype) {
 580                 case QP_MBUF:
 581                         tbr->tbr_token -= TBR_SCALE(m_pktlen(pkt->cp_mbuf));
 582                         break;
 583
 584
 585                 default:
 586                         VERIFY(0);
 587                         /* NOTREACHED */
 588                 }
 589         }
 590 }
 591
 592 /*
 593  * set a token bucket regulator.
 594  * if the specified rate is zero, the token bucket regulator is deleted.
 595  */
 596 int
 597 ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile,
 598     boolean_t update)
 599 {
 600         struct tb_regulator *tbr;
 601         struct ifnet *ifp = ifq->ifcq_ifp;
 602         u_int64_t rate, old_rate;
 603
 604         IFCQ_LOCK_ASSERT_HELD(ifq);
 605         VERIFY(IFCQ_IS_READY(ifq));
 606
 607         VERIFY(machclk_freq != 0);
 608
 609         tbr = &ifq->ifcq_tbr;
 610         old_rate = tbr->tbr_rate_raw;
 611
 612         rate = profile->rate;
 613         if (profile->percent > 0) {
 614                 u_int64_t eff_rate;
 615
 616                 if (profile->percent > 100) {
 617                         return EINVAL;
 618                 }
 619                 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0) {
 620                         return ENODEV;
 621                 }
 622                 rate = (eff_rate * profile->percent) / 100;
 623         }
 624
 625         if (rate == 0) {
 626                 if (!IFCQ_TBR_IS_ENABLED(ifq)) {
 627                         return 0;
 628                 }
 629
 630                 if (pktsched_verbose) {
 631                         printf("%s: TBR disabled\n", if_name(ifp));
 632                 }
 633
 634                 /* disable this TBR */
 635                 ifq->ifcq_flags &= ~IFCQF_TBR;
 636                 bzero(tbr, sizeof(*tbr));
 637                 ifnet_set_start_cycle(ifp, NULL);
 638                 if (update) {
 639                         ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 640                 }
 641                 return 0;
 642         }
 643
 644         if (pktsched_verbose) {
 645                 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp),
 646                     (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" :
 647                     "enabled", rate, profile->depth);
 648         }
 649
 650         /* set the new TBR */
 651         bzero(tbr, sizeof(*tbr));
 652         tbr->tbr_rate_raw = rate;
 653         tbr->tbr_percent = profile->percent;
 654         ifq->ifcq_flags |= IFCQF_TBR;
 655
 656         /*
 657          * Note that the TBR fill up time (hence the ifnet restart time)
 658          * is directly related to the specified TBR depth.  The ideal
 659          * depth value should be computed such that the interval time
 660          * between each successive wakeup is adequately spaced apart,
 661          * in order to reduce scheduling overheads.  A target interval
 662          * of 10 ms seems to provide good performance balance.  This can be
 663          * overridden by specifying the depth profile.  Values smaller than
 664          * the ideal depth will reduce delay at the expense of CPU cycles.
 665          */
 666         tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq;
 667         if (tbr->tbr_rate > 0) {
 668                 u_int32_t mtu = ifp->if_mtu;
 669                 int64_t ival, idepth = 0;
 670                 int i;
 671
 672                 if (mtu < IF_MINMTU) {
 673                         mtu = IF_MINMTU;
 674                 }
 675
 676                 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */
 677
 678                 for (i = 1;; i++) {
 679                         idepth = TBR_SCALE(i * mtu);
 680                         if ((idepth / tbr->tbr_rate) > ival) {
 681                                 break;
 682                         }
 683                 }
 684                 VERIFY(idepth > 0);
 685
 686                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 687                 if (tbr->tbr_depth == 0) {
 688                         tbr->tbr_filluptime = idepth / tbr->tbr_rate;
 689                         /* a little fudge factor to get closer to rate */
 690                         tbr->tbr_depth = idepth + (idepth >> 3);
 691                 } else {
 692                         tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate;
 693                 }
 694         } else {
 695                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 696                 tbr->tbr_filluptime = 0xffffffffffffffffLL;
 697         }
 698         tbr->tbr_token = tbr->tbr_depth;
 699         tbr->tbr_last = read_machclk();
 700
 701         if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) {
 702                 struct timespec ts =
 703                 { 0, (long)pktsched_abs_to_nsecs(tbr->tbr_filluptime) };
 704                 if (pktsched_verbose) {
 705                         printf("%s: TBR calculated tokens %lld "
 706                             "filluptime %llu ns\n", if_name(ifp),
 707                             TBR_UNSCALE(tbr->tbr_token),
 708                             pktsched_abs_to_nsecs(tbr->tbr_filluptime));
 709                 }
 710                 ifnet_set_start_cycle(ifp, &ts);
 711         } else {
 712                 if (pktsched_verbose) {
 713                         if (tbr->tbr_rate == 0) {
 714                                 printf("%s: TBR calculated tokens %lld "
 715                                     "infinite filluptime\n", if_name(ifp),
 716                                     TBR_UNSCALE(tbr->tbr_token));
 717                         } else if (!(ifp->if_flags & IFF_UP)) {
 718                                 printf("%s: TBR suspended (link is down)\n",
 719                                     if_name(ifp));
 720                         }
 721                 }
 722                 ifnet_set_start_cycle(ifp, NULL);
 723         }
 724         if (update && tbr->tbr_rate_raw != old_rate) {
 725                 ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 726         }
 727
 728         return 0;
 729 }
 730
 731 void
 732 ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay)
 733 {
 734         u_int64_t qdelay = 0;
 735         qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd);
 736
 737         if (ifclassq_target_qdelay != 0) {
 738                 qdelay = ifclassq_target_qdelay;
 739         }
 740
 741         /*
 742          * If we do not know the effective bandwidth, use the default
 743          * target queue delay.
 744          */
 745         if (qdelay == 0) {
 746                 qdelay = IFQ_TARGET_DELAY;
 747         }
 748
 749         /*
 750          * If a delay has been added to ifnet start callback for
 751          * coalescing, we have to add that to the pre-set target delay
 752          * because the packets can be in the queue longer.
 753          */
 754         if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
 755             ifp->if_start_delay_timeout > 0) {
 756                 qdelay += ifp->if_start_delay_timeout;
 757         }
 758
 759         *(if_target_qdelay) = qdelay;
 760 }
 761
 762 void
 763 ifclassq_calc_update_interval(u_int64_t *update_interval)
 764 {
 765         u_int64_t uint = 0;
 766
 767         /* If the system level override is set, use it */
 768         if (ifclassq_update_interval != 0) {
 769                 uint = ifclassq_update_interval;
 770         }
 771
 772         /* Otherwise use the default value */
 773         if (uint == 0) {
 774                 uint = IFQ_UPDATE_INTERVAL;
 775         }
 776
 777         *update_interval = uint;
 778 }
 779
 780 void
 781 ifclassq_reap_caches(boolean_t purge)
 782 {
 783         fq_codel_reap_caches(purge);
 784         flowadv_reap_caches(purge);
 785 }