bsd/net/classq/classq_subr.c

   1 /*
   2  * Copyright (c) 2011-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include <sys/param.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/errno.h>
  33 #include <sys/random.h>
  34 #include <sys/kernel_types.h>
  35 #include <sys/sysctl.h>
  36
  37 #include <kern/zalloc.h>
  38
  39 #include <net/if.h>
  40 #include <net/net_osdep.h>
  41 #include <net/classq/classq.h>
  42 #include <pexpert/pexpert.h>
  43 #include <net/classq/classq_sfb.h>
  44 #include <net/classq/classq_fq_codel.h>
  45 #include <net/pktsched/pktsched.h>
  46 #include <net/pktsched/pktsched_fq_codel.h>
  47
  48 #include <libkern/libkern.h>
  49
  50
  51 static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  52     u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *,
  53     u_int32_t *, boolean_t);
  54 static void ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  55     boolean_t, classq_pkt_t *);
  56
  57 static u_int64_t ifclassq_target_qdelay = 0;
  58 SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW | CTLFLAG_LOCKED,
  59     &ifclassq_target_qdelay, "target queue delay in nanoseconds");
  60
  61 static u_int64_t ifclassq_update_interval = 0;
  62 SYSCTL_QUAD(_net_classq, OID_AUTO, update_interval,
  63     CTLFLAG_RW | CTLFLAG_LOCKED, &ifclassq_update_interval,
  64     "update interval in nanoseconds");
  65
  66 static int32_t ifclassq_sched_fq_codel;
  67
  68 void
  69 classq_init(void)
  70 {
  71         _CASSERT(MBUF_TC_BE == 0);
  72         _CASSERT(MBUF_SC_BE == 0);
  73         _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES);
  74
  75         sfb_init();
  76         fq_codel_scheduler_init();
  77
  78         if (!PE_parse_boot_argn("fq_codel", &ifclassq_sched_fq_codel,
  79             sizeof(ifclassq_sched_fq_codel))) {
  80                 ifclassq_sched_fq_codel = 1;
  81         }
  82 }
  83
  84 int
  85 ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse)
  86 {
  87 #pragma unused(reuse)
  88         struct ifclassq *ifq = &ifp->if_snd;
  89         int err = 0;
  90
  91         IFCQ_LOCK(ifq);
  92         VERIFY(IFCQ_IS_EMPTY(ifq));
  93         ifq->ifcq_ifp = ifp;
  94         IFCQ_LEN(ifq) = 0;
  95         IFCQ_BYTES(ifq) = 0;
  96         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
  97         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
  98
  99         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 100         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 101         VERIFY(ifq->ifcq_flags == 0);
 102         VERIFY(ifq->ifcq_sflags == 0);
 103         VERIFY(ifq->ifcq_disc == NULL);
 104         VERIFY(ifq->ifcq_enqueue == NULL);
 105         VERIFY(ifq->ifcq_dequeue == NULL);
 106         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 107         VERIFY(ifq->ifcq_request == NULL);
 108
 109         if (ifp->if_eflags & IFEF_TXSTART) {
 110                 u_int32_t maxlen = 0;
 111
 112                 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0) {
 113                         maxlen = if_sndq_maxlen;
 114                 }
 115                 IFCQ_SET_MAXLEN(ifq, maxlen);
 116
 117                 if (IFCQ_MAXLEN(ifq) != if_sndq_maxlen &&
 118                     IFCQ_TARGET_QDELAY(ifq) == 0) {
 119                         /*
 120                          * Choose static queues because the interface has
 121                          * maximum queue size set
 122                          */
 123                         sflags &= ~PKTSCHEDF_QALG_DELAYBASED;
 124                 }
 125                 ifq->ifcq_sflags = sflags;
 126                 err = ifclassq_pktsched_setup(ifq);
 127                 if (err == 0) {
 128                         ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED);
 129                 }
 130         }
 131         IFCQ_UNLOCK(ifq);
 132         return err;
 133 }
 134
 135 void
 136 ifclassq_teardown(struct ifnet *ifp)
 137 {
 138         struct ifclassq *ifq = &ifp->if_snd;
 139
 140         IFCQ_LOCK(ifq);
 141
 142         if (IFCQ_IS_READY(ifq)) {
 143                 if (IFCQ_TBR_IS_ENABLED(ifq)) {
 144                         struct tb_profile tb = { .rate = 0, .percent = 0, .depth = 0 };
 145                         (void) ifclassq_tbr_set(ifq, &tb, FALSE);
 146                 }
 147                 (void) pktsched_teardown(ifq);
 148                 ifq->ifcq_flags = 0;
 149         }
 150         ifq->ifcq_sflags = 0;
 151
 152         VERIFY(IFCQ_IS_EMPTY(ifq));
 153         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 154         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 155         VERIFY(ifq->ifcq_flags == 0);
 156         VERIFY(ifq->ifcq_sflags == 0);
 157         VERIFY(ifq->ifcq_disc == NULL);
 158         VERIFY(ifq->ifcq_enqueue == NULL);
 159         VERIFY(ifq->ifcq_dequeue == NULL);
 160         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 161         VERIFY(ifq->ifcq_request == NULL);
 162         IFCQ_LEN(ifq) = 0;
 163         IFCQ_BYTES(ifq) = 0;
 164         IFCQ_MAXLEN(ifq) = 0;
 165         bzero(&ifq->ifcq_xmitcnt, sizeof(ifq->ifcq_xmitcnt));
 166         bzero(&ifq->ifcq_dropcnt, sizeof(ifq->ifcq_dropcnt));
 167
 168         IFCQ_UNLOCK(ifq);
 169 }
 170
 171 int
 172 ifclassq_pktsched_setup(struct ifclassq *ifq)
 173 {
 174         struct ifnet *ifp = ifq->ifcq_ifp;
 175         classq_pkt_type_t ptype = QP_MBUF;
 176         int err = 0;
 177
 178         IFCQ_LOCK_ASSERT_HELD(ifq);
 179         VERIFY(ifp->if_eflags & IFEF_TXSTART);
 180
 181         switch (ifp->if_output_sched_model) {
 182         case IFNET_SCHED_MODEL_DRIVER_MANAGED:
 183                 if (ifclassq_sched_fq_codel != 0) {
 184                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 185                             ifq->ifcq_sflags, ptype);
 186                 } else {
 187                         err = pktsched_setup(ifq, PKTSCHEDT_TCQ,
 188                             ifq->ifcq_sflags, ptype);
 189                 }
 190                 break;
 191
 192         case IFNET_SCHED_MODEL_NORMAL:
 193                 if (ifclassq_sched_fq_codel != 0) {
 194                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 195                             ifq->ifcq_sflags, ptype);
 196                 } else {
 197                         err = pktsched_setup(ifq, PKTSCHEDT_QFQ,
 198                             ifq->ifcq_sflags, ptype);
 199                 }
 200                 break;
 201         case IFNET_SCHED_MODEL_FQ_CODEL:
 202                 err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 203                     ifq->ifcq_sflags, ptype);
 204                 break;
 205         default:
 206                 VERIFY(0);
 207                 /* NOTREACHED */
 208         }
 209
 210         return err;
 211 }
 212
 213 void
 214 ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen)
 215 {
 216         IFCQ_LOCK(ifq);
 217         if (maxqlen == 0) {
 218                 maxqlen = if_sndq_maxlen;
 219         }
 220         IFCQ_SET_MAXLEN(ifq, maxqlen);
 221         IFCQ_UNLOCK(ifq);
 222 }
 223
 224 u_int32_t
 225 ifclassq_get_maxlen(struct ifclassq *ifq)
 226 {
 227         return IFCQ_MAXLEN(ifq);
 228 }
 229
 230 int
 231 ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets,
 232     u_int32_t *bytes)
 233 {
 234         int err = 0;
 235
 236         IFCQ_LOCK(ifq);
 237         if (sc == MBUF_SC_UNSPEC) {
 238                 VERIFY(packets != NULL);
 239                 *packets = IFCQ_LEN(ifq);
 240         } else {
 241                 VERIFY(MBUF_VALID_SC(sc));
 242                 VERIFY(packets != NULL && bytes != NULL);
 243                 IFCQ_LEN_SC(ifq, sc, packets, bytes, err);
 244         }
 245         IFCQ_UNLOCK(ifq);
 246
 247
 248         return err;
 249 }
 250
 251 inline void
 252 ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp,
 253     classq_pkt_t *p)
 254 {
 255         if (!IFNET_IS_CELLULAR(ifp)) {
 256                 return;
 257         }
 258
 259         switch (p->cp_ptype) {
 260         case QP_MBUF: {
 261                 struct mbuf *m = p->cp_mbuf;
 262                 m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA;
 263                 m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq);
 264                 m->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent;
 265                 break;
 266         }
 267
 268
 269         default:
 270                 VERIFY(0);
 271                 /* NOTREACHED */
 272                 __builtin_unreachable();
 273         }
 274 }
 275
 276 errno_t
 277 ifclassq_enqueue(struct ifclassq *ifq, classq_pkt_t *p, boolean_t *pdrop)
 278 {
 279         errno_t err;
 280
 281         switch (p->cp_ptype) {
 282         case QP_MBUF:
 283                 IFCQ_LOCK_SPIN(ifq);
 284                 break;
 285
 286         default:
 287                 IFCQ_LOCK(ifq);
 288                 break;
 289         }
 290
 291         IFCQ_ENQUEUE(ifq, p, err, pdrop);
 292         IFCQ_UNLOCK(ifq);
 293         return err;
 294 }
 295
 296 errno_t
 297 ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit,
 298     u_int32_t byte_limit, classq_pkt_t *head, classq_pkt_t *tail,
 299     u_int32_t *cnt, u_int32_t *len)
 300 {
 301         return ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit,
 302                    byte_limit, head, tail, cnt, len, FALSE);
 303 }
 304
 305 errno_t
 306 ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 307     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 308     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len)
 309 {
 310         return ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit,
 311                    head, tail, cnt, len, TRUE);
 312 }
 313
 314 static errno_t
 315 ifclassq_dequeue_common_default(struct ifclassq *ifq, mbuf_svc_class_t sc,
 316     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 317     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt)
 318 {
 319         struct ifnet *ifp = ifq->ifcq_ifp;
 320         u_int32_t i = 0, l = 0, lock_spin = 1;
 321         classq_pkt_t first = CLASSQ_PKT_INITIALIZER(first);
 322         classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
 323
 324         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 325
 326
 327         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 328                 goto dequeue_loop;
 329         }
 330
 331         /*
 332          * If the scheduler support dequeueing multiple packets at the
 333          * same time, call that one instead.
 334          */
 335         if (drvmgt && ifq->ifcq_dequeue_sc_multi != NULL) {
 336                 int err;
 337
 338                 if (lock_spin) {
 339                         IFCQ_LOCK_SPIN(ifq);
 340                 } else {
 341                         IFCQ_LOCK(ifq);
 342                 }
 343                 err = ifq->ifcq_dequeue_sc_multi(ifq, sc, pkt_limit,
 344                     byte_limit, head, tail, cnt, len);
 345                 IFCQ_UNLOCK(ifq);
 346
 347                 if (err == 0 && head->cp_mbuf == NULL) {
 348                         err = EAGAIN;
 349                 }
 350                 return err;
 351         } else if (ifq->ifcq_dequeue_multi != NULL) {
 352                 int err;
 353
 354                 if (lock_spin) {
 355                         IFCQ_LOCK_SPIN(ifq);
 356                 } else {
 357                         IFCQ_LOCK(ifq);
 358                 }
 359
 360                 err = ifq->ifcq_dequeue_multi(ifq, pkt_limit, byte_limit,
 361                     head, tail, cnt, len);
 362                 IFCQ_UNLOCK(ifq);
 363
 364                 if (err == 0 && head->cp_mbuf == NULL) {
 365                         err = EAGAIN;
 366                 }
 367                 return err;
 368         }
 369
 370 dequeue_loop:
 371
 372         if (lock_spin) {
 373                 IFCQ_LOCK_SPIN(ifq);
 374         } else {
 375                 IFCQ_LOCK(ifq);
 376         }
 377
 378         while (i < pkt_limit && l < byte_limit) {
 379                 if (drvmgt) {
 380                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 381                                 IFCQ_TBR_DEQUEUE_SC(ifq, sc, head);
 382                         } else {
 383                                 IFCQ_DEQUEUE_SC(ifq, sc, head);
 384                         }
 385                 } else {
 386                         if (IFCQ_TBR_IS_ENABLED(ifq)) {
 387                                 IFCQ_TBR_DEQUEUE(ifq, head);
 388                         } else {
 389                                 IFCQ_DEQUEUE(ifq, head);
 390                         }
 391                 }
 392
 393                 if (head->cp_mbuf == NULL) {
 394                         break;
 395                 }
 396
 397                 if (first.cp_mbuf == NULL) {
 398                         first = *head;
 399                 }
 400
 401                 switch (head->cp_ptype) {
 402                 case QP_MBUF:
 403                         head->cp_mbuf->m_nextpkt = NULL;
 404                         l += head->cp_mbuf->m_pkthdr.len;
 405                         ifclassq_set_packet_metadata(ifq, ifp, head);
 406                         if (last.cp_mbuf != NULL) {
 407                                 last.cp_mbuf->m_nextpkt = head->cp_mbuf;
 408                         }
 409                         break;
 410
 411
 412                 default:
 413                         VERIFY(0);
 414                         /* NOTREACHED */
 415                         __builtin_unreachable();
 416                 }
 417
 418                 last = *head;
 419                 i++;
 420         }
 421
 422         IFCQ_UNLOCK(ifq);
 423
 424         if (tail != NULL) {
 425                 *tail = last;
 426         }
 427         if (cnt != NULL) {
 428                 *cnt = i;
 429         }
 430         if (len != NULL) {
 431                 *len = l;
 432         }
 433
 434         *head = first;
 435         return (first.cp_mbuf != NULL) ? 0 : EAGAIN;
 436 }
 437
 438 static errno_t
 439 ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 440     u_int32_t pkt_limit, u_int32_t byte_limit, classq_pkt_t *head,
 441     classq_pkt_t *tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt)
 442 {
 443         return ifclassq_dequeue_common_default(ifq, sc,
 444                    pkt_limit, byte_limit, head, tail, cnt, len, drvmgt);
 445 }
 446
 447 void
 448 ifclassq_update(struct ifclassq *ifq, cqev_t ev)
 449 {
 450         IFCQ_LOCK_ASSERT_HELD(ifq);
 451         VERIFY(IFCQ_IS_READY(ifq));
 452         IFCQ_UPDATE(ifq, ev);
 453 }
 454
 455 int
 456 ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline,
 457     ifclassq_enq_func enqueue, ifclassq_deq_func dequeue,
 458     ifclassq_deq_sc_func dequeue_sc, ifclassq_deq_multi_func dequeue_multi,
 459     ifclassq_deq_sc_multi_func dequeue_sc_multi, ifclassq_req_func request)
 460 {
 461         IFCQ_LOCK_ASSERT_HELD(ifq);
 462
 463         VERIFY(ifq->ifcq_disc == NULL);
 464         VERIFY(enqueue != NULL);
 465         VERIFY(request != NULL);
 466
 467         ifq->ifcq_type = type;
 468         ifq->ifcq_disc = discipline;
 469         ifq->ifcq_enqueue = enqueue;
 470         ifq->ifcq_dequeue = dequeue;
 471         ifq->ifcq_dequeue_sc = dequeue_sc;
 472         ifq->ifcq_dequeue_multi = dequeue_multi;
 473         ifq->ifcq_dequeue_sc_multi = dequeue_sc_multi;
 474         ifq->ifcq_request = request;
 475
 476         return 0;
 477 }
 478
 479 int
 480 ifclassq_detach(struct ifclassq *ifq)
 481 {
 482         IFCQ_LOCK_ASSERT_HELD(ifq);
 483
 484         VERIFY(ifq->ifcq_disc == NULL);
 485
 486         ifq->ifcq_type = PKTSCHEDT_NONE;
 487         ifq->ifcq_disc = NULL;
 488         ifq->ifcq_enqueue = NULL;
 489         ifq->ifcq_dequeue = NULL;
 490         ifq->ifcq_dequeue_sc = NULL;
 491         ifq->ifcq_request = NULL;
 492
 493         return 0;
 494 }
 495
 496 int
 497 ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf,
 498     u_int32_t *nbytes)
 499 {
 500         struct if_ifclassq_stats *ifqs;
 501         int err;
 502
 503         if (*nbytes < sizeof(*ifqs)) {
 504                 return EINVAL;
 505         }
 506
 507         ifqs = _MALLOC(sizeof(*ifqs), M_TEMP, M_WAITOK | M_ZERO);
 508         if (ifqs == NULL) {
 509                 return ENOMEM;
 510         }
 511
 512         IFCQ_LOCK(ifq);
 513         if (!IFCQ_IS_READY(ifq)) {
 514                 IFCQ_UNLOCK(ifq);
 515                 _FREE(ifqs, M_TEMP);
 516                 return ENXIO;
 517         }
 518
 519         ifqs->ifqs_len = IFCQ_LEN(ifq);
 520         ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq);
 521         *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt);
 522         *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt);
 523         ifqs->ifqs_scheduler = ifq->ifcq_type;
 524
 525         err = pktsched_getqstats(ifq, qid, ifqs);
 526         IFCQ_UNLOCK(ifq);
 527
 528         if (err == 0 && (err = copyout((caddr_t)ifqs,
 529             (user_addr_t)(uintptr_t)ubuf, sizeof(*ifqs))) == 0) {
 530                 *nbytes = sizeof(*ifqs);
 531         }
 532
 533         _FREE(ifqs, M_TEMP);
 534
 535         return err;
 536 }
 537
 538 const char *
 539 ifclassq_ev2str(cqev_t ev)
 540 {
 541         const char *c;
 542
 543         switch (ev) {
 544         case CLASSQ_EV_LINK_BANDWIDTH:
 545                 c = "LINK_BANDWIDTH";
 546                 break;
 547
 548         case CLASSQ_EV_LINK_LATENCY:
 549                 c = "LINK_LATENCY";
 550                 break;
 551
 552         case CLASSQ_EV_LINK_MTU:
 553                 c = "LINK_MTU";
 554                 break;
 555
 556         case CLASSQ_EV_LINK_UP:
 557                 c = "LINK_UP";
 558                 break;
 559
 560         case CLASSQ_EV_LINK_DOWN:
 561                 c = "LINK_DOWN";
 562                 break;
 563
 564         default:
 565                 c = "UNKNOWN";
 566                 break;
 567         }
 568
 569         return c;
 570 }
 571
 572 /*
 573  * internal representation of token bucket parameters
 574  *      rate:   byte_per_unittime << 32
 575  *              (((bits_per_sec) / 8) << 32) / machclk_freq
 576  *      depth:  byte << 32
 577  *
 578  */
 579 #define TBR_SHIFT       32
 580 #define TBR_SCALE(x)    ((int64_t)(x) << TBR_SHIFT)
 581 #define TBR_UNSCALE(x)  ((x) >> TBR_SHIFT)
 582
 583 void
 584 ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_t *pkt)
 585 {
 586         ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, pkt);
 587 }
 588
 589 void
 590 ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 591     classq_pkt_t *pkt)
 592 {
 593         ifclassq_tbr_dequeue_common(ifq, sc, TRUE, pkt);
 594 }
 595
 596 static void
 597 ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 598     boolean_t drvmgt, classq_pkt_t *pkt)
 599 {
 600         struct tb_regulator *tbr;
 601         int64_t interval;
 602         u_int64_t now;
 603
 604         IFCQ_LOCK_ASSERT_HELD(ifq);
 605
 606         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 607         VERIFY(IFCQ_TBR_IS_ENABLED(ifq));
 608
 609         *pkt = CLASSQ_PKT_INITIALIZER(*pkt);
 610         tbr = &ifq->ifcq_tbr;
 611         /* update token only when it is negative */
 612         if (tbr->tbr_token <= 0) {
 613                 now = read_machclk();
 614                 interval = now - tbr->tbr_last;
 615                 if (interval >= tbr->tbr_filluptime) {
 616                         tbr->tbr_token = tbr->tbr_depth;
 617                 } else {
 618                         tbr->tbr_token += interval * tbr->tbr_rate;
 619                         if (tbr->tbr_token > tbr->tbr_depth) {
 620                                 tbr->tbr_token = tbr->tbr_depth;
 621                         }
 622                 }
 623                 tbr->tbr_last = now;
 624         }
 625         /* if token is still negative, don't allow dequeue */
 626         if (tbr->tbr_token <= 0) {
 627                 return;
 628         }
 629
 630         /*
 631          * ifclassq takes precedence over ALTQ queue;
 632          * ifcq_drain count is adjusted by the caller.
 633          */
 634         if (drvmgt) {
 635                 IFCQ_DEQUEUE_SC(ifq, sc, pkt);
 636         } else {
 637                 IFCQ_DEQUEUE(ifq, pkt);
 638         }
 639
 640         if (pkt->cp_mbuf != NULL) {
 641                 switch (pkt->cp_ptype) {
 642                 case QP_MBUF:
 643                         tbr->tbr_token -= TBR_SCALE(m_pktlen(pkt->cp_mbuf));
 644                         break;
 645
 646
 647                 default:
 648                         VERIFY(0);
 649                         /* NOTREACHED */
 650                 }
 651         }
 652 }
 653
 654 /*
 655  * set a token bucket regulator.
 656  * if the specified rate is zero, the token bucket regulator is deleted.
 657  */
 658 int
 659 ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile,
 660     boolean_t update)
 661 {
 662         struct tb_regulator *tbr;
 663         struct ifnet *ifp = ifq->ifcq_ifp;
 664         u_int64_t rate, old_rate;
 665
 666         IFCQ_LOCK_ASSERT_HELD(ifq);
 667         VERIFY(IFCQ_IS_READY(ifq));
 668
 669         VERIFY(machclk_freq != 0);
 670
 671         tbr = &ifq->ifcq_tbr;
 672         old_rate = tbr->tbr_rate_raw;
 673
 674         rate = profile->rate;
 675         if (profile->percent > 0) {
 676                 u_int64_t eff_rate;
 677
 678                 if (profile->percent > 100) {
 679                         return EINVAL;
 680                 }
 681                 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0) {
 682                         return ENODEV;
 683                 }
 684                 rate = (eff_rate * profile->percent) / 100;
 685         }
 686
 687         if (rate == 0) {
 688                 if (!IFCQ_TBR_IS_ENABLED(ifq)) {
 689                         return 0;
 690                 }
 691
 692                 if (pktsched_verbose) {
 693                         printf("%s: TBR disabled\n", if_name(ifp));
 694                 }
 695
 696                 /* disable this TBR */
 697                 ifq->ifcq_flags &= ~IFCQF_TBR;
 698                 bzero(tbr, sizeof(*tbr));
 699                 ifnet_set_start_cycle(ifp, NULL);
 700                 if (update) {
 701                         ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 702                 }
 703                 return 0;
 704         }
 705
 706         if (pktsched_verbose) {
 707                 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp),
 708                     (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" :
 709                     "enabled", rate, profile->depth);
 710         }
 711
 712         /* set the new TBR */
 713         bzero(tbr, sizeof(*tbr));
 714         tbr->tbr_rate_raw = rate;
 715         tbr->tbr_percent = profile->percent;
 716         ifq->ifcq_flags |= IFCQF_TBR;
 717
 718         /*
 719          * Note that the TBR fill up time (hence the ifnet restart time)
 720          * is directly related to the specified TBR depth.  The ideal
 721          * depth value should be computed such that the interval time
 722          * between each successive wakeup is adequately spaced apart,
 723          * in order to reduce scheduling overheads.  A target interval
 724          * of 10 ms seems to provide good performance balance.  This can be
 725          * overridden by specifying the depth profile.  Values smaller than
 726          * the ideal depth will reduce delay at the expense of CPU cycles.
 727          */
 728         tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq;
 729         if (tbr->tbr_rate > 0) {
 730                 u_int32_t mtu = ifp->if_mtu;
 731                 int64_t ival, idepth = 0;
 732                 int i;
 733
 734                 if (mtu < IF_MINMTU) {
 735                         mtu = IF_MINMTU;
 736                 }
 737
 738                 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */
 739
 740                 for (i = 1;; i++) {
 741                         idepth = TBR_SCALE(i * mtu);
 742                         if ((idepth / tbr->tbr_rate) > ival) {
 743                                 break;
 744                         }
 745                 }
 746                 VERIFY(idepth > 0);
 747
 748                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 749                 if (tbr->tbr_depth == 0) {
 750                         tbr->tbr_filluptime = idepth / tbr->tbr_rate;
 751                         /* a little fudge factor to get closer to rate */
 752                         tbr->tbr_depth = idepth + (idepth >> 3);
 753                 } else {
 754                         tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate;
 755                 }
 756         } else {
 757                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 758                 tbr->tbr_filluptime = 0xffffffffffffffffLL;
 759         }
 760         tbr->tbr_token = tbr->tbr_depth;
 761         tbr->tbr_last = read_machclk();
 762
 763         if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) {
 764                 struct timespec ts =
 765                 { 0, pktsched_abs_to_nsecs(tbr->tbr_filluptime) };
 766                 if (pktsched_verbose) {
 767                         printf("%s: TBR calculated tokens %lld "
 768                             "filluptime %llu ns\n", if_name(ifp),
 769                             TBR_UNSCALE(tbr->tbr_token),
 770                             pktsched_abs_to_nsecs(tbr->tbr_filluptime));
 771                 }
 772                 ifnet_set_start_cycle(ifp, &ts);
 773         } else {
 774                 if (pktsched_verbose) {
 775                         if (tbr->tbr_rate == 0) {
 776                                 printf("%s: TBR calculated tokens %lld "
 777                                     "infinite filluptime\n", if_name(ifp),
 778                                     TBR_UNSCALE(tbr->tbr_token));
 779                         } else if (!(ifp->if_flags & IFF_UP)) {
 780                                 printf("%s: TBR suspended (link is down)\n",
 781                                     if_name(ifp));
 782                         }
 783                 }
 784                 ifnet_set_start_cycle(ifp, NULL);
 785         }
 786         if (update && tbr->tbr_rate_raw != old_rate) {
 787                 ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 788         }
 789
 790         return 0;
 791 }
 792
 793 void
 794 ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay)
 795 {
 796         u_int64_t qdelay = 0;
 797         qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd);
 798
 799         if (ifclassq_target_qdelay != 0) {
 800                 qdelay = ifclassq_target_qdelay;
 801         }
 802
 803         /*
 804          * If we do not know the effective bandwidth, use the default
 805          * target queue delay.
 806          */
 807         if (qdelay == 0) {
 808                 qdelay = IFQ_TARGET_DELAY;
 809         }
 810
 811         /*
 812          * If a delay has been added to ifnet start callback for
 813          * coalescing, we have to add that to the pre-set target delay
 814          * because the packets can be in the queue longer.
 815          */
 816         if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
 817             ifp->if_start_delay_timeout > 0) {
 818                 qdelay += ifp->if_start_delay_timeout;
 819         }
 820
 821         *(if_target_qdelay) = qdelay;
 822 }
 823
 824 void
 825 ifclassq_calc_update_interval(u_int64_t *update_interval)
 826 {
 827         u_int64_t uint = 0;
 828
 829         /* If the system level override is set, use it */
 830         if (ifclassq_update_interval != 0) {
 831                 uint = ifclassq_update_interval;
 832         }
 833
 834         /* Otherwise use the default value */
 835         if (uint == 0) {
 836                 uint = IFQ_UPDATE_INTERVAL;
 837         }
 838
 839         *update_interval = uint;
 840 }
 841
 842 void
 843 ifclassq_reap_caches(boolean_t purge)
 844 {
 845         fq_codel_reap_caches(purge);
 846 }