bsd/net/classq/classq_subr.c

   1 /*
   2  * Copyright (c) 2011-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include <sys/param.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/errno.h>
  33 #include <sys/random.h>
  34 #include <sys/kernel_types.h>
  35 #include <sys/sysctl.h>
  36
  37 #include <kern/zalloc.h>
  38
  39 #include <net/if.h>
  40 #include <net/net_osdep.h>
  41 #include <net/classq/classq.h>
  42 #include <pexpert/pexpert.h>
  43 #include <net/classq/classq_sfb.h>
  44 #include <net/pktsched/pktsched.h>
  45 #include <net/pktsched/pktsched_fq_codel.h>
  46
  47 #include <libkern/libkern.h>
  48
  49
  50 static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  51     u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *,
  52     boolean_t, classq_pkt_type_t *);
  53 static void *ifclassq_tbr_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
  54     boolean_t, classq_pkt_type_t *);
  55
  56 static u_int64_t ifclassq_target_qdelay = 0;
  57 SYSCTL_QUAD(_net_classq, OID_AUTO, target_qdelay, CTLFLAG_RW|CTLFLAG_LOCKED,
  58     &ifclassq_target_qdelay, "target queue delay in nanoseconds");
  59
  60 static u_int64_t ifclassq_update_interval = 0;
  61 SYSCTL_QUAD(_net_classq, OID_AUTO, update_interval,
  62     CTLFLAG_RW|CTLFLAG_LOCKED, &ifclassq_update_interval,
  63     "update interval in nanoseconds");
  64
  65 static int32_t ifclassq_sched_fq_codel;
  66
  67 void
  68 classq_init(void)
  69 {
  70         _CASSERT(MBUF_TC_BE == 0);
  71         _CASSERT(MBUF_SC_BE == 0);
  72         _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES);
  73
  74         sfb_init();
  75         fq_codel_scheduler_init();
  76
  77         if (!PE_parse_boot_argn("fq_codel", &ifclassq_sched_fq_codel,
  78             sizeof (ifclassq_sched_fq_codel)))
  79                 ifclassq_sched_fq_codel = 1;
  80 }
  81
  82 int
  83 ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse)
  84 {
  85 #pragma unused(reuse)
  86         struct ifclassq *ifq = &ifp->if_snd;
  87         int err = 0;
  88
  89         IFCQ_LOCK(ifq);
  90         VERIFY(IFCQ_IS_EMPTY(ifq));
  91         ifq->ifcq_ifp = ifp;
  92         IFCQ_LEN(ifq) = 0;
  93         IFCQ_BYTES(ifq) = 0;
  94         bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt));
  95         bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt));
  96
  97         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
  98         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
  99         VERIFY(ifq->ifcq_flags == 0);
 100         VERIFY(ifq->ifcq_sflags == 0);
 101         VERIFY(ifq->ifcq_disc == NULL);
 102         VERIFY(ifq->ifcq_enqueue == NULL);
 103         VERIFY(ifq->ifcq_dequeue == NULL);
 104         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 105         VERIFY(ifq->ifcq_request == NULL);
 106
 107         if (ifp->if_eflags & IFEF_TXSTART) {
 108                 u_int32_t maxlen = 0;
 109
 110                 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0)
 111                         maxlen = if_sndq_maxlen;
 112                 IFCQ_SET_MAXLEN(ifq, maxlen);
 113
 114                 if (IFCQ_MAXLEN(ifq) != if_sndq_maxlen &&
 115                     IFCQ_TARGET_QDELAY(ifq) == 0) {
 116                         /*
 117                          * Choose static queues because the interface has
 118                          * maximum queue size set
 119                          */
 120                         sflags &= ~PKTSCHEDF_QALG_DELAYBASED;
 121                 }
 122                 ifq->ifcq_sflags = sflags;
 123                 err = ifclassq_pktsched_setup(ifq);
 124                 if (err == 0)
 125                         ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED);
 126         }
 127         IFCQ_UNLOCK(ifq);
 128         return (err);
 129 }
 130
 131 void
 132 ifclassq_teardown(struct ifnet *ifp)
 133 {
 134         struct ifclassq *ifq = &ifp->if_snd;
 135
 136         IFCQ_LOCK(ifq);
 137
 138         if (IFCQ_IS_READY(ifq)) {
 139                 if (IFCQ_TBR_IS_ENABLED(ifq)) {
 140                         struct tb_profile tb = { 0, 0, 0 };
 141                         (void) ifclassq_tbr_set(ifq, &tb, FALSE);
 142                 }
 143                 (void) pktsched_teardown(ifq);
 144                 ifq->ifcq_flags = 0;
 145         }
 146         ifq->ifcq_sflags = 0;
 147
 148         VERIFY(IFCQ_IS_EMPTY(ifq));
 149         VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
 150         VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
 151         VERIFY(ifq->ifcq_flags == 0);
 152         VERIFY(ifq->ifcq_sflags == 0);
 153         VERIFY(ifq->ifcq_disc == NULL);
 154         VERIFY(ifq->ifcq_enqueue == NULL);
 155         VERIFY(ifq->ifcq_dequeue == NULL);
 156         VERIFY(ifq->ifcq_dequeue_sc == NULL);
 157         VERIFY(ifq->ifcq_request == NULL);
 158         IFCQ_LEN(ifq) = 0;
 159         IFCQ_BYTES(ifq) = 0;
 160         IFCQ_MAXLEN(ifq) = 0;
 161         bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt));
 162         bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt));
 163
 164         IFCQ_UNLOCK(ifq);
 165 }
 166
 167 int
 168 ifclassq_pktsched_setup(struct ifclassq *ifq)
 169 {
 170         struct ifnet *ifp = ifq->ifcq_ifp;
 171         classq_pkt_type_t ptype = QP_MBUF;
 172         int err = 0;
 173
 174         IFCQ_LOCK_ASSERT_HELD(ifq);
 175         VERIFY(ifp->if_eflags & IFEF_TXSTART);
 176
 177         switch (ifp->if_output_sched_model) {
 178         case IFNET_SCHED_MODEL_DRIVER_MANAGED:
 179                 if (ifclassq_sched_fq_codel != 0) {
 180                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 181                             ifq->ifcq_sflags, ptype);
 182                 } else {
 183                         err = pktsched_setup(ifq, PKTSCHEDT_TCQ,
 184                             ifq->ifcq_sflags, ptype);
 185                 }
 186                 break;
 187
 188         case IFNET_SCHED_MODEL_NORMAL:
 189                 if (ifclassq_sched_fq_codel != 0) {
 190                         err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 191                             ifq->ifcq_sflags, ptype);
 192                 } else {
 193                         err = pktsched_setup(ifq, PKTSCHEDT_QFQ,
 194                             ifq->ifcq_sflags, ptype);
 195                 }
 196                 break;
 197         case IFNET_SCHED_MODEL_FQ_CODEL:
 198                 err = pktsched_setup(ifq, PKTSCHEDT_FQ_CODEL,
 199                     ifq->ifcq_sflags, ptype);
 200                 break;
 201         default:
 202                 VERIFY(0);
 203                 /* NOTREACHED */
 204         }
 205
 206         return (err);
 207 }
 208
 209 void
 210 ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen)
 211 {
 212         IFCQ_LOCK(ifq);
 213         if (maxqlen == 0)
 214                 maxqlen = if_sndq_maxlen;
 215         IFCQ_SET_MAXLEN(ifq, maxqlen);
 216         IFCQ_UNLOCK(ifq);
 217 }
 218
 219 u_int32_t
 220 ifclassq_get_maxlen(struct ifclassq *ifq)
 221 {
 222         return (IFCQ_MAXLEN(ifq));
 223 }
 224
 225 int
 226 ifclassq_get_len(struct ifclassq *ifq, mbuf_svc_class_t sc, u_int32_t *packets,
 227     u_int32_t *bytes)
 228 {
 229         int err = 0;
 230
 231         IFCQ_LOCK(ifq);
 232         if (sc == MBUF_SC_UNSPEC) {
 233                 VERIFY(packets != NULL);
 234                 *packets = IFCQ_LEN(ifq);
 235         } else {
 236                 VERIFY(MBUF_VALID_SC(sc));
 237                 VERIFY(packets != NULL && bytes != NULL);
 238                 IFCQ_LEN_SC(ifq, sc, packets, bytes, err);
 239         }
 240         IFCQ_UNLOCK(ifq);
 241
 242         return (err);
 243 }
 244
 245 inline void
 246 ifclassq_set_packet_metadata(struct ifclassq *ifq, struct ifnet *ifp,
 247     void *p, classq_pkt_type_t ptype)
 248 {
 249         if (!IFNET_IS_CELLULAR(ifp))
 250                 return;
 251
 252         switch (ptype) {
 253         case QP_MBUF: {
 254                 struct mbuf *m = p;
 255                 m->m_pkthdr.pkt_flags |= PKTF_VALID_UNSENT_DATA;
 256                 m->m_pkthdr.bufstatus_if = IFCQ_BYTES(ifq);
 257                 m->m_pkthdr.bufstatus_sndbuf = ifp->if_sndbyte_unsent;
 258                 break;
 259         }
 260
 261
 262         default:
 263                 VERIFY(0);
 264                 /* NOTREACHED */
 265         }
 266 }
 267
 268 errno_t
 269 ifclassq_enqueue(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype,
 270     boolean_t *pdrop)
 271 {
 272         errno_t err;
 273
 274         switch (ptype) {
 275         case QP_MBUF:
 276                 IFCQ_LOCK_SPIN(ifq);
 277                 break;
 278
 279         default:
 280                 IFCQ_LOCK(ifq);
 281                 break;
 282         }
 283
 284         IFCQ_ENQUEUE(ifq, p, ptype, err, pdrop);
 285         IFCQ_UNLOCK(ifq);
 286         return (err);
 287 }
 288
 289 errno_t
 290 ifclassq_dequeue(struct ifclassq *ifq, u_int32_t pkt_limit,
 291     u_int32_t byte_limit, void **head, void **tail,
 292     u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype)
 293 {
 294         return (ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, pkt_limit,
 295             byte_limit, head, tail, cnt, len, FALSE, ptype));
 296 }
 297
 298 errno_t
 299 ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 300     u_int32_t pkt_limit, u_int32_t byte_limit, void **head, void **tail,
 301     u_int32_t *cnt, u_int32_t *len, classq_pkt_type_t *ptype)
 302 {
 303         return (ifclassq_dequeue_common(ifq, sc, pkt_limit, byte_limit,
 304             head, tail, cnt, len, TRUE, ptype));
 305 }
 306
 307 static errno_t
 308 ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 309     u_int32_t pkt_limit, u_int32_t byte_limit, void **head,
 310     void **tail, u_int32_t *cnt, u_int32_t *len, boolean_t drvmgt,
 311     classq_pkt_type_t *ptype)
 312 {
 313         struct ifnet *ifp = ifq->ifcq_ifp;
 314         u_int32_t i = 0, l = 0, lock_spin = 1 ;
 315         void **first, *last;
 316
 317         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 318
 319         *ptype = 0;
 320
 321
 322         if (IFCQ_TBR_IS_ENABLED(ifq))
 323                 goto dequeue_loop;
 324
 325         /*
 326          * If the scheduler support dequeueing multiple packets at the
 327          * same time, call that one instead.
 328          */
 329         if (drvmgt && ifq->ifcq_dequeue_sc_multi != NULL) {
 330                 int err;
 331
 332                 if (lock_spin)
 333                         IFCQ_LOCK_SPIN(ifq);
 334                 else
 335                         IFCQ_LOCK(ifq);
 336                 err = ifq->ifcq_dequeue_sc_multi(ifq, sc, pkt_limit,
 337                     byte_limit, head, tail, cnt, len, ptype);
 338                 IFCQ_UNLOCK(ifq);
 339
 340                 if (err == 0 && (*head) == NULL)
 341                         err = EAGAIN;
 342                 return (err);
 343         } else if (ifq->ifcq_dequeue_multi != NULL) {
 344                 int err;
 345
 346                 if (lock_spin)
 347                         IFCQ_LOCK_SPIN(ifq);
 348                 else
 349                         IFCQ_LOCK(ifq);
 350
 351                 err = ifq->ifcq_dequeue_multi(ifq, pkt_limit, byte_limit,
 352                     head, tail, cnt, len, ptype);
 353                 IFCQ_UNLOCK(ifq);
 354
 355                 if (err == 0 && (*head) == NULL)
 356                         err = EAGAIN;
 357                 return (err);
 358         }
 359
 360 dequeue_loop:
 361         *head = NULL;
 362         first = &(*head);
 363         last = NULL;
 364
 365         if (lock_spin)
 366                 IFCQ_LOCK_SPIN(ifq);
 367         else
 368                 IFCQ_LOCK(ifq);
 369
 370         while (i < pkt_limit && l < byte_limit) {
 371                 classq_pkt_type_t tmp_ptype;
 372                 if (drvmgt) {
 373                         if (IFCQ_TBR_IS_ENABLED(ifq))
 374                                 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype);
 375                         else
 376                                 IFCQ_DEQUEUE_SC(ifq, sc, *head, &tmp_ptype);
 377                 } else {
 378                         if (IFCQ_TBR_IS_ENABLED(ifq))
 379                                 IFCQ_TBR_DEQUEUE(ifq, *head, &tmp_ptype);
 380                         else
 381                                 IFCQ_DEQUEUE(ifq, *head, &tmp_ptype);
 382                 }
 383
 384                 if (*head == NULL)
 385                         break;
 386
 387                 switch (tmp_ptype) {
 388                 case QP_MBUF:
 389                         (*((mbuf_t *)head))->m_nextpkt = NULL;
 390                         last = *head;
 391                         l += (*((mbuf_t *)head))->m_pkthdr.len;
 392                         ifclassq_set_packet_metadata(ifq, ifp, (*head),
 393                             QP_MBUF);
 394                         head = (void **)&(*((mbuf_t *)head))->m_nextpkt;
 395                         break;
 396
 397
 398                 default:
 399                         VERIFY(0);
 400                         /* NOTREACHED */
 401                 }
 402
 403                 *ptype = tmp_ptype;
 404                 i++;
 405         }
 406
 407         IFCQ_UNLOCK(ifq);
 408
 409         if (tail != NULL)
 410                 *tail = last;
 411         if (cnt != NULL)
 412                 *cnt = i;
 413         if (len != NULL)
 414                 *len = l;
 415
 416         return ((*first != NULL) ? 0 : EAGAIN);
 417 }
 418
 419 void
 420 ifclassq_update(struct ifclassq *ifq, cqev_t ev)
 421 {
 422         IFCQ_LOCK_ASSERT_HELD(ifq);
 423         VERIFY(IFCQ_IS_READY(ifq));
 424         IFCQ_UPDATE(ifq, ev);
 425 }
 426
 427 int
 428 ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline,
 429     ifclassq_enq_func enqueue, ifclassq_deq_func dequeue,
 430     ifclassq_deq_sc_func dequeue_sc, ifclassq_deq_multi_func dequeue_multi,
 431     ifclassq_deq_sc_multi_func dequeue_sc_multi, ifclassq_req_func request)
 432 {
 433         IFCQ_LOCK_ASSERT_HELD(ifq);
 434
 435         VERIFY(ifq->ifcq_disc == NULL);
 436         VERIFY(enqueue != NULL);
 437         VERIFY(request != NULL);
 438
 439         ifq->ifcq_type = type;
 440         ifq->ifcq_disc = discipline;
 441         ifq->ifcq_enqueue = enqueue;
 442         ifq->ifcq_dequeue = dequeue;
 443         ifq->ifcq_dequeue_sc = dequeue_sc;
 444         ifq->ifcq_dequeue_multi = dequeue_multi;
 445         ifq->ifcq_dequeue_sc_multi = dequeue_sc_multi;
 446         ifq->ifcq_request = request;
 447
 448         return (0);
 449 }
 450
 451 int
 452 ifclassq_detach(struct ifclassq *ifq)
 453 {
 454         IFCQ_LOCK_ASSERT_HELD(ifq);
 455
 456         VERIFY(ifq->ifcq_disc == NULL);
 457
 458         ifq->ifcq_type = PKTSCHEDT_NONE;
 459         ifq->ifcq_disc = NULL;
 460         ifq->ifcq_enqueue = NULL;
 461         ifq->ifcq_dequeue = NULL;
 462         ifq->ifcq_dequeue_sc = NULL;
 463         ifq->ifcq_request = NULL;
 464
 465         return (0);
 466 }
 467
 468 int
 469 ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf,
 470     u_int32_t *nbytes)
 471 {
 472         struct if_ifclassq_stats *ifqs;
 473         int err;
 474
 475         if (*nbytes < sizeof (*ifqs))
 476                 return (EINVAL);
 477
 478         ifqs = _MALLOC(sizeof (*ifqs), M_TEMP, M_WAITOK | M_ZERO);
 479         if (ifqs == NULL)
 480                 return (ENOMEM);
 481
 482         IFCQ_LOCK(ifq);
 483         if (!IFCQ_IS_READY(ifq)) {
 484                 IFCQ_UNLOCK(ifq);
 485                 _FREE(ifqs, M_TEMP);
 486                 return (ENXIO);
 487         }
 488
 489         ifqs->ifqs_len = IFCQ_LEN(ifq);
 490         ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq);
 491         *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt);
 492         *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt);
 493         ifqs->ifqs_scheduler = ifq->ifcq_type;
 494
 495         err = pktsched_getqstats(ifq, qid, ifqs);
 496         IFCQ_UNLOCK(ifq);
 497
 498         if (err == 0 && (err = copyout((caddr_t)ifqs,
 499             (user_addr_t)(uintptr_t)ubuf, sizeof (*ifqs))) == 0)
 500                 *nbytes = sizeof (*ifqs);
 501
 502         _FREE(ifqs, M_TEMP);
 503
 504         return (err);
 505 }
 506
 507 const char *
 508 ifclassq_ev2str(cqev_t ev)
 509 {
 510         const char *c;
 511
 512         switch (ev) {
 513         case CLASSQ_EV_LINK_BANDWIDTH:
 514                 c = "LINK_BANDWIDTH";
 515                 break;
 516
 517         case CLASSQ_EV_LINK_LATENCY:
 518                 c = "LINK_LATENCY";
 519                 break;
 520
 521         case CLASSQ_EV_LINK_MTU:
 522                 c = "LINK_MTU";
 523                 break;
 524
 525         case CLASSQ_EV_LINK_UP:
 526                 c = "LINK_UP";
 527                 break;
 528
 529         case CLASSQ_EV_LINK_DOWN:
 530                 c = "LINK_DOWN";
 531                 break;
 532
 533         default:
 534                 c = "UNKNOWN";
 535                 break;
 536         }
 537
 538         return (c);
 539 }
 540
 541 /*
 542  * internal representation of token bucket parameters
 543  *      rate:   byte_per_unittime << 32
 544  *              (((bits_per_sec) / 8) << 32) / machclk_freq
 545  *      depth:  byte << 32
 546  *
 547  */
 548 #define TBR_SHIFT       32
 549 #define TBR_SCALE(x)    ((int64_t)(x) << TBR_SHIFT)
 550 #define TBR_UNSCALE(x)  ((x) >> TBR_SHIFT)
 551
 552 void *
 553 ifclassq_tbr_dequeue(struct ifclassq *ifq, classq_pkt_type_t *ptype)
 554 {
 555         return (ifclassq_tbr_dequeue_common(ifq, MBUF_SC_UNSPEC, FALSE, ptype));
 556 }
 557
 558 void *
 559 ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
 560     classq_pkt_type_t *ptype)
 561 {
 562         return (ifclassq_tbr_dequeue_common(ifq, sc, TRUE, ptype));
 563 }
 564
 565 static void *
 566 ifclassq_tbr_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
 567     boolean_t drvmgt, classq_pkt_type_t *ptype)
 568 {
 569         struct tb_regulator *tbr;
 570         void *p;
 571         int64_t interval;
 572         u_int64_t now;
 573
 574         IFCQ_LOCK_ASSERT_HELD(ifq);
 575
 576         VERIFY(!drvmgt || MBUF_VALID_SC(sc));
 577         VERIFY(IFCQ_TBR_IS_ENABLED(ifq));
 578
 579         tbr = &ifq->ifcq_tbr;
 580         /* update token only when it is negative */
 581         if (tbr->tbr_token <= 0) {
 582                 now = read_machclk();
 583                 interval = now - tbr->tbr_last;
 584                 if (interval >= tbr->tbr_filluptime) {
 585                         tbr->tbr_token = tbr->tbr_depth;
 586                 } else {
 587                         tbr->tbr_token += interval * tbr->tbr_rate;
 588                         if (tbr->tbr_token > tbr->tbr_depth)
 589                                 tbr->tbr_token = tbr->tbr_depth;
 590                 }
 591                 tbr->tbr_last = now;
 592         }
 593         /* if token is still negative, don't allow dequeue */
 594         if (tbr->tbr_token <= 0)
 595                 return (NULL);
 596
 597         /*
 598          * ifclassq takes precedence over ALTQ queue;
 599          * ifcq_drain count is adjusted by the caller.
 600          */
 601                 if (drvmgt)
 602                         IFCQ_DEQUEUE_SC(ifq, sc, p, ptype);
 603                 else
 604                         IFCQ_DEQUEUE(ifq, p, ptype);
 605
 606         if (p != NULL) {
 607                 switch (*ptype) {
 608                 case QP_MBUF:
 609                         tbr->tbr_token -= TBR_SCALE(m_pktlen((mbuf_t)p));
 610                         break;
 611
 612
 613                 default:
 614                         VERIFY(0);
 615                         /* NOTREACHED */
 616                 }
 617         }
 618
 619         return (p);
 620 }
 621
 622 /*
 623  * set a token bucket regulator.
 624  * if the specified rate is zero, the token bucket regulator is deleted.
 625  */
 626 int
 627 ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile,
 628     boolean_t update)
 629 {
 630         struct tb_regulator *tbr;
 631         struct ifnet *ifp = ifq->ifcq_ifp;
 632         u_int64_t rate, old_rate;
 633
 634         IFCQ_LOCK_ASSERT_HELD(ifq);
 635         VERIFY(IFCQ_IS_READY(ifq));
 636
 637         VERIFY(machclk_freq != 0);
 638
 639         tbr = &ifq->ifcq_tbr;
 640         old_rate = tbr->tbr_rate_raw;
 641
 642         rate = profile->rate;
 643         if (profile->percent > 0) {
 644                 u_int64_t eff_rate;
 645
 646                 if (profile->percent > 100)
 647                         return (EINVAL);
 648                 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0)
 649                         return (ENODEV);
 650                 rate = (eff_rate * profile->percent) / 100;
 651         }
 652
 653         if (rate == 0) {
 654                 if (!IFCQ_TBR_IS_ENABLED(ifq))
 655                         return (ENOENT);
 656
 657                 if (pktsched_verbose)
 658                         printf("%s: TBR disabled\n", if_name(ifp));
 659
 660                 /* disable this TBR */
 661                 ifq->ifcq_flags &= ~IFCQF_TBR;
 662                 bzero(tbr, sizeof (*tbr));
 663                 ifnet_set_start_cycle(ifp, NULL);
 664                 if (update)
 665                         ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 666                 return (0);
 667         }
 668
 669         if (pktsched_verbose) {
 670                 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp),
 671                     (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" :
 672                     "enabled", rate, profile->depth);
 673         }
 674
 675         /* set the new TBR */
 676         bzero(tbr, sizeof (*tbr));
 677         tbr->tbr_rate_raw = rate;
 678         tbr->tbr_percent = profile->percent;
 679         ifq->ifcq_flags |= IFCQF_TBR;
 680
 681         /*
 682          * Note that the TBR fill up time (hence the ifnet restart time)
 683          * is directly related to the specified TBR depth.  The ideal
 684          * depth value should be computed such that the interval time
 685          * between each successive wakeup is adequately spaced apart,
 686          * in order to reduce scheduling overheads.  A target interval
 687          * of 10 ms seems to provide good performance balance.  This can be
 688          * overridden by specifying the depth profile.  Values smaller than
 689          * the ideal depth will reduce delay at the expense of CPU cycles.
 690          */
 691         tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq;
 692         if (tbr->tbr_rate > 0) {
 693                 u_int32_t mtu = ifp->if_mtu;
 694                 int64_t ival, idepth = 0;
 695                 int i;
 696
 697                 if (mtu < IF_MINMTU)
 698                         mtu = IF_MINMTU;
 699
 700                 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */
 701
 702                 for (i = 1; ; i++) {
 703                         idepth = TBR_SCALE(i * mtu);
 704                         if ((idepth / tbr->tbr_rate) > ival)
 705                                 break;
 706                 }
 707                 VERIFY(idepth > 0);
 708
 709                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 710                 if (tbr->tbr_depth == 0) {
 711                         tbr->tbr_filluptime = idepth / tbr->tbr_rate;
 712                         /* a little fudge factor to get closer to rate */
 713                         tbr->tbr_depth = idepth + (idepth >> 3);
 714                 } else {
 715                         tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate;
 716                 }
 717         } else {
 718                 tbr->tbr_depth = TBR_SCALE(profile->depth);
 719                 tbr->tbr_filluptime = 0xffffffffffffffffLL;
 720         }
 721         tbr->tbr_token = tbr->tbr_depth;
 722         tbr->tbr_last = read_machclk();
 723
 724         if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) {
 725                 struct timespec ts =
 726                     { 0, pktsched_abs_to_nsecs(tbr->tbr_filluptime) };
 727                 if (pktsched_verbose) {
 728                         printf("%s: TBR calculated tokens %lld "
 729                             "filluptime %llu ns\n", if_name(ifp),
 730                             TBR_UNSCALE(tbr->tbr_token),
 731                             pktsched_abs_to_nsecs(tbr->tbr_filluptime));
 732                 }
 733                 ifnet_set_start_cycle(ifp, &ts);
 734         } else {
 735                 if (pktsched_verbose) {
 736                         if (tbr->tbr_rate == 0) {
 737                                 printf("%s: TBR calculated tokens %lld "
 738                                     "infinite filluptime\n", if_name(ifp),
 739                                     TBR_UNSCALE(tbr->tbr_token));
 740                         } else if (!(ifp->if_flags & IFF_UP)) {
 741                                 printf("%s: TBR suspended (link is down)\n",
 742                                     if_name(ifp));
 743                         }
 744                 }
 745                 ifnet_set_start_cycle(ifp, NULL);
 746         }
 747         if (update && tbr->tbr_rate_raw != old_rate)
 748                 ifclassq_update(ifq, CLASSQ_EV_LINK_BANDWIDTH);
 749
 750         return (0);
 751 }
 752
 753 void
 754 ifclassq_calc_target_qdelay(struct ifnet *ifp, u_int64_t *if_target_qdelay)
 755 {
 756         u_int64_t qdelay = 0;
 757         qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd);
 758
 759         if (ifclassq_target_qdelay != 0)
 760                 qdelay = ifclassq_target_qdelay;
 761
 762         /*
 763          * If we do not know the effective bandwidth, use the default
 764          * target queue delay.
 765          */
 766         if (qdelay == 0)
 767                 qdelay = IFQ_TARGET_DELAY;
 768
 769         /*
 770          * If a delay has been added to ifnet start callback for
 771          * coalescing, we have to add that to the pre-set target delay
 772          * because the packets can be in the queue longer.
 773          */
 774         if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
 775             ifp->if_start_delay_timeout > 0)
 776                 qdelay += ifp->if_start_delay_timeout;
 777
 778         *(if_target_qdelay) = qdelay;
 779 }
 780
 781 void
 782 ifclassq_calc_update_interval(u_int64_t *update_interval)
 783 {
 784         u_int64_t uint = 0;
 785
 786         /* If the system level override is set, use it */
 787         if (ifclassq_update_interval != 0)
 788                 uint = ifclassq_update_interval;
 789
 790         /* Otherwise use the default value */
 791         if (uint == 0)
 792                 uint = IFQ_UPDATE_INTERVAL;
 793
 794         *update_interval = uint;
 795 }