]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/pktsched/pktsched_tcq.c
projects / apple / xnu.git / blob
? search:


ecadb8bbd6672202ccb5a3ce42572abaa6b1d291
[apple/xnu.git] / bsd / net / pktsched / pktsched_tcq.c
1 /*
2 * Copyright (c) 2011-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * traffic class queue
31 */
32
33 #include <sys/cdefs.h>
34 #include <sys/param.h>
35 #include <sys/malloc.h>
36 #include <sys/mbuf.h>
37 #include <sys/systm.h>
38 #include <sys/errno.h>
39 #include <sys/kernel.h>
40 #include <sys/syslog.h>
41
42 #include <kern/zalloc.h>
43
44 #include <net/if.h>
45 #include <net/net_osdep.h>
46
47 #include <net/pktsched/pktsched_tcq.h>
48 #include <netinet/in.h>
49
50 /*
51 * function prototypes
52 */
53 static int tcq_enqueue_ifclassq(struct ifclassq *, struct mbuf *);
54 static struct mbuf *tcq_dequeue_tc_ifclassq(struct ifclassq *,
55 mbuf_svc_class_t, cqdq_op_t);
56 static int tcq_request_ifclassq(struct ifclassq *, cqrq_t, void *);
57 static int tcq_clear_interface(struct tcq_if *);
58 static struct tcq_class *tcq_class_create(struct tcq_if *, int, u_int32_t,
59 int, u_int32_t);
60 static int tcq_class_destroy(struct tcq_if *, struct tcq_class *);
61 static int tcq_destroy_locked(struct tcq_if *);
62 static inline int tcq_addq(struct tcq_class *, struct mbuf *,
63 struct pf_mtag *);
64 static inline struct mbuf *tcq_getq(struct tcq_class *);
65 static inline struct mbuf *tcq_pollq(struct tcq_class *);
66 static void tcq_purgeq(struct tcq_if *, struct tcq_class *, u_int32_t,
67 u_int32_t *, u_int32_t *);
68 static void tcq_purge_sc(struct tcq_if *, cqrq_purge_sc_t *);
69 static void tcq_updateq(struct tcq_if *, struct tcq_class *, cqev_t);
70 static int tcq_throttle(struct tcq_if *, cqrq_throttle_t *);
71 static int tcq_resumeq(struct tcq_if *, struct tcq_class *);
72 static int tcq_suspendq(struct tcq_if *, struct tcq_class *);
73 static int tcq_stat_sc(struct tcq_if *, cqrq_stat_sc_t *);
74 static struct mbuf *tcq_dequeue_cl(struct tcq_if *, struct tcq_class *,
75 mbuf_svc_class_t, cqdq_op_t);
76 static inline struct tcq_class *tcq_clh_to_clp(struct tcq_if *, u_int32_t);
77 static const char *tcq_style(struct tcq_if *);
78
79 #define TCQ_ZONE_MAX 32 /* maximum elements in zone */
80 #define TCQ_ZONE_NAME "pktsched_tcq" /* zone name */
81
82 static unsigned int tcq_size; /* size of zone element */
83 static struct zone *tcq_zone; /* zone for tcq */
84
85 #define TCQ_CL_ZONE_MAX 32 /* maximum elements in zone */
86 #define TCQ_CL_ZONE_NAME "pktsched_tcq_cl" /* zone name */
87
88 static unsigned int tcq_cl_size; /* size of zone element */
89 static struct zone *tcq_cl_zone; /* zone for tcq_class */
90
91 void
92 tcq_init(void)
93 {
94 tcq_size = sizeof (struct tcq_if);
95 tcq_zone = zinit(tcq_size, TCQ_ZONE_MAX * tcq_size,
96 0, TCQ_ZONE_NAME);
97 if (tcq_zone == NULL) {
98 panic("%s: failed allocating %s", __func__, TCQ_ZONE_NAME);
99 /* NOTREACHED */
100 }
101 zone_change(tcq_zone, Z_EXPAND, TRUE);
102 zone_change(tcq_zone, Z_CALLERACCT, TRUE);
103
104 tcq_cl_size = sizeof (struct tcq_class);
105 tcq_cl_zone = zinit(tcq_cl_size, TCQ_CL_ZONE_MAX * tcq_cl_size,
106 0, TCQ_CL_ZONE_NAME);
107 if (tcq_cl_zone == NULL) {
108 panic("%s: failed allocating %s", __func__, TCQ_CL_ZONE_NAME);
109 /* NOTREACHED */
110 }
111 zone_change(tcq_cl_zone, Z_EXPAND, TRUE);
112 zone_change(tcq_cl_zone, Z_CALLERACCT, TRUE);
113 }
114
115 struct tcq_if *
116 tcq_alloc(struct ifnet *ifp, int how, boolean_t altq)
117 {
118 struct tcq_if *tif;
119
120 tif = (how == M_WAITOK) ? zalloc(tcq_zone) : zalloc_noblock(tcq_zone);
121 if (tif == NULL)
122 return (NULL);
123
124 bzero(tif, tcq_size);
125 tif->tif_maxpri = -1;
126 tif->tif_ifq = &ifp->if_snd;
127 if (altq)
128 tif->tif_flags |= TCQIFF_ALTQ;
129
130 if (pktsched_verbose) {
131 log(LOG_DEBUG, "%s: %s scheduler allocated\n",
132 if_name(ifp), tcq_style(tif));
133 }
134
135 return (tif);
136 }
137
138 int
139 tcq_destroy(struct tcq_if *tif)
140 {
141 struct ifclassq *ifq = tif->tif_ifq;
142 int err;
143
144 IFCQ_LOCK(ifq);
145 err = tcq_destroy_locked(tif);
146 IFCQ_UNLOCK(ifq);
147
148 return (err);
149 }
150
151 static int
152 tcq_destroy_locked(struct tcq_if *tif)
153 {
154 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
155
156 (void) tcq_clear_interface(tif);
157
158 if (pktsched_verbose) {
159 log(LOG_DEBUG, "%s: %s scheduler destroyed\n",
160 if_name(TCQIF_IFP(tif)), tcq_style(tif));
161 }
162
163 zfree(tcq_zone, tif);
164
165 return (0);
166 }
167
168 /*
169 * bring the interface back to the initial state by discarding
170 * all the filters and classes.
171 */
172 static int
173 tcq_clear_interface(struct tcq_if *tif)
174 {
175 struct tcq_class *cl;
176 int pri;
177
178 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
179
180 /* clear out the classes */
181 for (pri = 0; pri <= tif->tif_maxpri; pri++)
182 if ((cl = tif->tif_classes[pri]) != NULL)
183 tcq_class_destroy(tif, cl);
184
185 return (0);
186 }
187
188 /* discard all the queued packets on the interface */
189 void
190 tcq_purge(struct tcq_if *tif)
191 {
192 struct tcq_class *cl;
193 int pri;
194
195 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
196
197 for (pri = 0; pri <= tif->tif_maxpri; pri++) {
198 if ((cl = tif->tif_classes[pri]) != NULL && !qempty(&cl->cl_q))
199 tcq_purgeq(tif, cl, 0, NULL, NULL);
200 }
201 #if !PF_ALTQ
202 /*
203 * This assertion is safe to be made only when PF_ALTQ is not
204 * configured; otherwise, IFCQ_LEN represents the sum of the
205 * packets managed by ifcq_disc and altq_disc instances, which
206 * is possible when transitioning between the two.
207 */
208 VERIFY(IFCQ_LEN(tif->tif_ifq) == 0);
209 #endif /* !PF_ALTQ */
210 }
211
212 static void
213 tcq_purge_sc(struct tcq_if *tif, cqrq_purge_sc_t *pr)
214 {
215 struct ifclassq *ifq = tif->tif_ifq;
216 u_int32_t i;
217
218 IFCQ_LOCK_ASSERT_HELD(ifq);
219
220 VERIFY(pr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(pr->sc));
221 VERIFY(pr->flow != 0);
222
223 if (pr->sc != MBUF_SC_UNSPEC) {
224 i = MBUF_SCIDX(pr->sc);
225 VERIFY(i < IFCQ_SC_MAX);
226
227 tcq_purgeq(tif, ifq->ifcq_disc_slots[i].cl,
228 pr->flow, &pr->packets, &pr->bytes);
229 } else {
230 u_int32_t cnt, len;
231
232 pr->packets = 0;
233 pr->bytes = 0;
234
235 for (i = 0; i < IFCQ_SC_MAX; i++) {
236 tcq_purgeq(tif, ifq->ifcq_disc_slots[i].cl,
237 pr->flow, &cnt, &len);
238 pr->packets += cnt;
239 pr->bytes += len;
240 }
241 }
242 }
243
244 void
245 tcq_event(struct tcq_if *tif, cqev_t ev)
246 {
247 struct tcq_class *cl;
248 int pri;
249
250 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
251
252 for (pri = 0; pri <= tif->tif_maxpri; pri++)
253 if ((cl = tif->tif_classes[pri]) != NULL)
254 tcq_updateq(tif, cl, ev);
255 }
256
257 int
258 tcq_add_queue(struct tcq_if *tif, int priority, u_int32_t qlimit,
259 int flags, u_int32_t qid, struct tcq_class **clp)
260 {
261 struct tcq_class *cl;
262
263 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
264
265 /* check parameters */
266 if (priority >= TCQ_MAXPRI)
267 return (EINVAL);
268 if (tif->tif_classes[priority] != NULL)
269 return (EBUSY);
270 if (tcq_clh_to_clp(tif, qid) != NULL)
271 return (EBUSY);
272
273 cl = tcq_class_create(tif, priority, qlimit, flags, qid);
274 if (cl == NULL)
275 return (ENOMEM);
276
277 if (clp != NULL)
278 *clp = cl;
279
280 return (0);
281 }
282
283 static struct tcq_class *
284 tcq_class_create(struct tcq_if *tif, int pri, u_int32_t qlimit,
285 int flags, u_int32_t qid)
286 {
287 struct ifnet *ifp;
288 struct ifclassq *ifq;
289 struct tcq_class *cl;
290
291 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
292
293 /* Sanitize flags unless internally configured */
294 if (tif->tif_flags & TCQIFF_ALTQ)
295 flags &= TQCF_USERFLAGS;
296
297 #if !CLASSQ_RED
298 if (flags & TQCF_RED) {
299 log(LOG_ERR, "%s: %s RED not available!\n",
300 if_name(TCQIF_IFP(tif)), tcq_style(tif));
301 return (NULL);
302 }
303 #endif /* !CLASSQ_RED */
304
305 #if !CLASSQ_RIO
306 if (flags & TQCF_RIO) {
307 log(LOG_ERR, "%s: %s RIO not available!\n",
308 if_name(TCQIF_IFP(tif)), tcq_style(tif));
309 return (NULL);
310 }
311 #endif /* CLASSQ_RIO */
312
313 #if !CLASSQ_BLUE
314 if (flags & TQCF_BLUE) {
315 log(LOG_ERR, "%s: %s BLUE not available!\n",
316 if_name(TCQIF_IFP(tif)), tcq_style(tif));
317 return (NULL);
318 }
319 #endif /* CLASSQ_BLUE */
320
321 /* These are mutually exclusive */
322 if ((flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) &&
323 (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RED &&
324 (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_RIO &&
325 (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_BLUE &&
326 (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) != TQCF_SFB) {
327 log(LOG_ERR, "%s: %s more than one RED|RIO|BLUE|SFB\n",
328 if_name(TCQIF_IFP(tif)), tcq_style(tif));
329 return (NULL);
330 }
331
332 ifq = tif->tif_ifq;
333 ifp = TCQIF_IFP(tif);
334
335 if ((cl = tif->tif_classes[pri]) != NULL) {
336 /* modify the class instead of creating a new one */
337 if (!qempty(&cl->cl_q))
338 tcq_purgeq(tif, cl, 0, NULL, NULL);
339 #if CLASSQ_RIO
340 if (q_is_rio(&cl->cl_q))
341 rio_destroy(cl->cl_rio);
342 #endif /* CLASSQ_RIO */
343 #if CLASSQ_RED
344 if (q_is_red(&cl->cl_q))
345 red_destroy(cl->cl_red);
346 #endif /* CLASSQ_RED */
347 #if CLASSQ_BLUE
348 if (q_is_blue(&cl->cl_q))
349 blue_destroy(cl->cl_blue);
350 #endif /* CLASSQ_BLUE */
351 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
352 sfb_destroy(cl->cl_sfb);
353 cl->cl_qalg.ptr = NULL;
354 qtype(&cl->cl_q) = Q_DROPTAIL;
355 qstate(&cl->cl_q) = QS_RUNNING;
356 } else {
357 cl = zalloc(tcq_cl_zone);
358 if (cl == NULL)
359 return (NULL);
360
361 bzero(cl, tcq_cl_size);
362 }
363
364 tif->tif_classes[pri] = cl;
365 if (flags & TQCF_DEFAULTCLASS)
366 tif->tif_default = cl;
367 if (qlimit == 0 || qlimit > IFCQ_MAXLEN(ifq)) {
368 qlimit = IFCQ_MAXLEN(ifq);
369 if (qlimit == 0)
370 qlimit = DEFAULT_QLIMIT; /* use default */
371 }
372 _qinit(&cl->cl_q, Q_DROPTAIL, qlimit);
373 cl->cl_flags = flags;
374 cl->cl_pri = pri;
375 if (pri > tif->tif_maxpri)
376 tif->tif_maxpri = pri;
377 cl->cl_tif = tif;
378 cl->cl_handle = qid;
379
380 if (flags & (TQCF_RED|TQCF_RIO|TQCF_BLUE|TQCF_SFB)) {
381 #if CLASSQ_RED || CLASSQ_RIO
382 u_int64_t ifbandwidth = ifnet_output_linkrate(ifp);
383 int pkttime;
384 #endif /* CLASSQ_RED || CLASSQ_RIO */
385
386 cl->cl_qflags = 0;
387 if (flags & TQCF_ECN) {
388 if (flags & TQCF_BLUE)
389 cl->cl_qflags |= BLUEF_ECN;
390 else if (flags & TQCF_SFB)
391 cl->cl_qflags |= SFBF_ECN;
392 else if (flags & TQCF_RED)
393 cl->cl_qflags |= REDF_ECN;
394 else if (flags & TQCF_RIO)
395 cl->cl_qflags |= RIOF_ECN;
396 }
397 if (flags & TQCF_FLOWCTL) {
398 if (flags & TQCF_SFB)
399 cl->cl_qflags |= SFBF_FLOWCTL;
400 }
401 if (flags & TQCF_DELAYBASED) {
402 if (flags & TQCF_SFB)
403 cl->cl_qflags |= SFBF_DELAYBASED;
404 }
405 if (flags & TQCF_CLEARDSCP) {
406 if (flags & TQCF_RIO)
407 cl->cl_qflags |= RIOF_CLEARDSCP;
408 }
409 #if CLASSQ_RED || CLASSQ_RIO
410 /*
411 * XXX: RED & RIO should be watching link speed and MTU
412 * events and recompute pkttime accordingly.
413 */
414 if (ifbandwidth < 8)
415 pkttime = 1000 * 1000 * 1000; /* 1 sec */
416 else
417 pkttime = (int64_t)ifp->if_mtu * 1000 * 1000 * 1000 /
418 (ifbandwidth / 8);
419
420 /* Test for exclusivity {RED,RIO,BLUE,SFB} was done above */
421 #if CLASSQ_RED
422 if (flags & TQCF_RED) {
423 cl->cl_red = red_alloc(ifp, 0, 0,
424 qlimit(&cl->cl_q) * 10/100,
425 qlimit(&cl->cl_q) * 30/100,
426 cl->cl_qflags, pkttime);
427 if (cl->cl_red != NULL)
428 qtype(&cl->cl_q) = Q_RED;
429 }
430 #endif /* CLASSQ_RED */
431 #if CLASSQ_RIO
432 if (flags & TQCF_RIO) {
433 cl->cl_rio =
434 rio_alloc(ifp, 0, NULL, cl->cl_qflags, pkttime);
435 if (cl->cl_rio != NULL)
436 qtype(&cl->cl_q) = Q_RIO;
437 }
438 #endif /* CLASSQ_RIO */
439 #endif /* CLASSQ_RED || CLASSQ_RIO */
440 #if CLASSQ_BLUE
441 if (flags & TQCF_BLUE) {
442 cl->cl_blue = blue_alloc(ifp, 0, 0, cl->cl_qflags);
443 if (cl->cl_blue != NULL)
444 qtype(&cl->cl_q) = Q_BLUE;
445 }
446 #endif /* CLASSQ_BLUE */
447 if (flags & TQCF_SFB) {
448 if (!(cl->cl_flags & TQCF_LAZY))
449 cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
450 qlimit(&cl->cl_q), cl->cl_qflags);
451 if (cl->cl_sfb != NULL || (cl->cl_flags & TQCF_LAZY))
452 qtype(&cl->cl_q) = Q_SFB;
453 }
454 }
455
456 if (pktsched_verbose) {
457 log(LOG_DEBUG, "%s: %s created qid=%d pri=%d qlimit=%d "
458 "flags=%b\n", if_name(ifp), tcq_style(tif),
459 cl->cl_handle, cl->cl_pri, qlimit, flags, TQCF_BITS);
460 }
461
462 return (cl);
463 }
464
465 int
466 tcq_remove_queue(struct tcq_if *tif, u_int32_t qid)
467 {
468 struct tcq_class *cl;
469
470 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
471
472 if ((cl = tcq_clh_to_clp(tif, qid)) == NULL)
473 return (EINVAL);
474
475 return (tcq_class_destroy(tif, cl));
476 }
477
478 static int
479 tcq_class_destroy(struct tcq_if *tif, struct tcq_class *cl)
480 {
481 struct ifclassq *ifq = tif->tif_ifq;
482 int pri;
483
484 IFCQ_LOCK_ASSERT_HELD(ifq);
485
486 if (!qempty(&cl->cl_q))
487 tcq_purgeq(tif, cl, 0, NULL, NULL);
488
489 tif->tif_classes[cl->cl_pri] = NULL;
490 if (tif->tif_maxpri == cl->cl_pri) {
491 for (pri = cl->cl_pri; pri >= 0; pri--)
492 if (tif->tif_classes[pri] != NULL) {
493 tif->tif_maxpri = pri;
494 break;
495 }
496 if (pri < 0)
497 tif->tif_maxpri = -1;
498 }
499
500 if (tif->tif_default == cl)
501 tif->tif_default = NULL;
502
503 if (cl->cl_qalg.ptr != NULL) {
504 #if CLASSQ_RIO
505 if (q_is_rio(&cl->cl_q))
506 rio_destroy(cl->cl_rio);
507 #endif /* CLASSQ_RIO */
508 #if CLASSQ_RED
509 if (q_is_red(&cl->cl_q))
510 red_destroy(cl->cl_red);
511 #endif /* CLASSQ_RED */
512 #if CLASSQ_BLUE
513 if (q_is_blue(&cl->cl_q))
514 blue_destroy(cl->cl_blue);
515 #endif /* CLASSQ_BLUE */
516 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
517 sfb_destroy(cl->cl_sfb);
518 cl->cl_qalg.ptr = NULL;
519 qtype(&cl->cl_q) = Q_DROPTAIL;
520 qstate(&cl->cl_q) = QS_RUNNING;
521 }
522
523 if (pktsched_verbose) {
524 log(LOG_DEBUG, "%s: %s destroyed qid=%d pri=%d\n",
525 if_name(TCQIF_IFP(tif)), tcq_style(tif),
526 cl->cl_handle, cl->cl_pri);
527 }
528
529 zfree(tcq_cl_zone, cl);
530 return (0);
531 }
532
533 int
534 tcq_enqueue(struct tcq_if *tif, struct tcq_class *cl, struct mbuf *m,
535 struct pf_mtag *t)
536 {
537 struct ifclassq *ifq = tif->tif_ifq;
538 int len, ret;
539
540 IFCQ_LOCK_ASSERT_HELD(ifq);
541 VERIFY(cl == NULL || cl->cl_tif == tif);
542
543 if (cl == NULL) {
544 #if PF_ALTQ
545 cl = tcq_clh_to_clp(tif, t->pftag_qid);
546 #else /* !PF_ALTQ */
547 cl = tcq_clh_to_clp(tif, 0);
548 #endif /* !PF_ALTQ */
549 if (cl == NULL) {
550 cl = tif->tif_default;
551 if (cl == NULL) {
552 IFCQ_CONVERT_LOCK(ifq);
553 m_freem(m);
554 return (ENOBUFS);
555 }
556 }
557 }
558
559 len = m_pktlen(m);
560
561 ret = tcq_addq(cl, m, t);
562 if (ret != 0) {
563 if (ret == CLASSQEQ_SUCCESS_FC) {
564 /* packet enqueued, return advisory feedback */
565 ret = EQFULL;
566 } else {
567 VERIFY(ret == CLASSQEQ_DROPPED ||
568 ret == CLASSQEQ_DROPPED_FC ||
569 ret == CLASSQEQ_DROPPED_SP);
570 /* packet has been freed in tcq_addq */
571 PKTCNTR_ADD(&cl->cl_dropcnt, 1, len);
572 IFCQ_DROP_ADD(ifq, 1, len);
573 switch (ret) {
574 case CLASSQEQ_DROPPED:
575 return (ENOBUFS);
576 case CLASSQEQ_DROPPED_FC:
577 return (EQFULL);
578 case CLASSQEQ_DROPPED_SP:
579 return (EQSUSPENDED);
580 }
581 /* NOT REACHED */
582 }
583 }
584 IFCQ_INC_LEN(ifq);
585
586 /* successfully queued. */
587 return (ret);
588 }
589
590 /*
591 * note: CLASSQDQ_POLL returns the next packet without removing the packet
592 * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
593 * CLASSQDQ_REMOVE must return the same packet if called immediately
594 * after CLASSQDQ_POLL.
595 */
596 struct mbuf *
597 tcq_dequeue_tc(struct tcq_if *tif, mbuf_svc_class_t sc, cqdq_op_t op)
598 {
599 return (tcq_dequeue_cl(tif, NULL, sc, op));
600 }
601
602 static struct mbuf *
603 tcq_dequeue_cl(struct tcq_if *tif, struct tcq_class *cl,
604 mbuf_svc_class_t sc, cqdq_op_t op)
605 {
606 struct ifclassq *ifq = tif->tif_ifq;
607 struct mbuf *m;
608
609 IFCQ_LOCK_ASSERT_HELD(ifq);
610
611 if (cl == NULL) {
612 cl = tcq_clh_to_clp(tif, MBUF_SCIDX(sc));
613 if (cl == NULL)
614 return (NULL);
615 }
616
617 if (qempty(&cl->cl_q))
618 return (NULL);
619
620 VERIFY(!IFCQ_IS_EMPTY(ifq));
621
622 if (op == CLASSQDQ_POLL)
623 return (tcq_pollq(cl));
624
625 m = tcq_getq(cl);
626 if (m != NULL) {
627 IFCQ_DEC_LEN(ifq);
628 if (qempty(&cl->cl_q))
629 cl->cl_period++;
630 PKTCNTR_ADD(&cl->cl_xmitcnt, 1, m_pktlen(m));
631 IFCQ_XMIT_ADD(ifq, 1, m_pktlen(m));
632 }
633 return (m);
634 }
635
636 static inline int
637 tcq_addq(struct tcq_class *cl, struct mbuf *m, struct pf_mtag *t)
638 {
639 struct tcq_if *tif = cl->cl_tif;
640 struct ifclassq *ifq = tif->tif_ifq;
641
642 IFCQ_LOCK_ASSERT_HELD(ifq);
643
644 #if CLASSQ_RIO
645 if (q_is_rio(&cl->cl_q))
646 return (rio_addq(cl->cl_rio, &cl->cl_q, m, t));
647 else
648 #endif /* CLASSQ_RIO */
649 #if CLASSQ_RED
650 if (q_is_red(&cl->cl_q))
651 return (red_addq(cl->cl_red, &cl->cl_q, m, t));
652 else
653 #endif /* CLASSQ_RED */
654 #if CLASSQ_BLUE
655 if (q_is_blue(&cl->cl_q))
656 return (blue_addq(cl->cl_blue, &cl->cl_q, m, t));
657 else
658 #endif /* CLASSQ_BLUE */
659 if (q_is_sfb(&cl->cl_q)) {
660 if (cl->cl_sfb == NULL) {
661 struct ifnet *ifp = TCQIF_IFP(tif);
662
663 VERIFY(cl->cl_flags & TQCF_LAZY);
664 cl->cl_flags &= ~TQCF_LAZY;
665 IFCQ_CONVERT_LOCK(ifq);
666
667 cl->cl_sfb = sfb_alloc(ifp, cl->cl_handle,
668 qlimit(&cl->cl_q), cl->cl_qflags);
669 if (cl->cl_sfb == NULL) {
670 /* fall back to droptail */
671 qtype(&cl->cl_q) = Q_DROPTAIL;
672 cl->cl_flags &= ~TQCF_SFB;
673 cl->cl_qflags &= ~(SFBF_ECN | SFBF_FLOWCTL);
674
675 log(LOG_ERR, "%s: %s SFB lazy allocation "
676 "failed for qid=%d pri=%d, falling back "
677 "to DROPTAIL\n", if_name(ifp),
678 tcq_style(tif), cl->cl_handle,
679 cl->cl_pri);
680 } else if (tif->tif_throttle != IFNET_THROTTLE_OFF) {
681 /* if there's pending throttling, set it */
682 cqrq_throttle_t tr = { 1, tif->tif_throttle };
683 int err = tcq_throttle(tif, &tr);
684
685 if (err == EALREADY)
686 err = 0;
687 if (err != 0) {
688 tr.level = IFNET_THROTTLE_OFF;
689 (void) tcq_throttle(tif, &tr);
690 }
691 }
692 }
693 if (cl->cl_sfb != NULL)
694 return (sfb_addq(cl->cl_sfb, &cl->cl_q, m, t));
695 } else if (qlen(&cl->cl_q) >= qlimit(&cl->cl_q)) {
696 IFCQ_CONVERT_LOCK(ifq);
697 m_freem(m);
698 return (CLASSQEQ_DROPPED);
699 }
700
701 #if PF_ECN
702 if (cl->cl_flags & TQCF_CLEARDSCP)
703 write_dsfield(m, t, 0);
704 #endif /* PF_ECN */
705
706 _addq(&cl->cl_q, m);
707
708 return (0);
709 }
710
711 static inline struct mbuf *
712 tcq_getq(struct tcq_class *cl)
713 {
714 IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq);
715
716 #if CLASSQ_RIO
717 if (q_is_rio(&cl->cl_q))
718 return (rio_getq(cl->cl_rio, &cl->cl_q));
719 else
720 #endif /* CLASSQ_RIO */
721 #if CLASSQ_RED
722 if (q_is_red(&cl->cl_q))
723 return (red_getq(cl->cl_red, &cl->cl_q));
724 else
725 #endif /* CLASSQ_RED */
726 #if CLASSQ_BLUE
727 if (q_is_blue(&cl->cl_q))
728 return (blue_getq(cl->cl_blue, &cl->cl_q));
729 else
730 #endif /* CLASSQ_BLUE */
731 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
732 return (sfb_getq(cl->cl_sfb, &cl->cl_q));
733
734 return (_getq(&cl->cl_q));
735 }
736
737 static inline struct mbuf *
738 tcq_pollq(struct tcq_class *cl)
739 {
740 IFCQ_LOCK_ASSERT_HELD(cl->cl_tif->tif_ifq);
741
742 return (qhead(&cl->cl_q));
743 }
744
745 static void
746 tcq_purgeq(struct tcq_if *tif, struct tcq_class *cl, u_int32_t flow,
747 u_int32_t *packets, u_int32_t *bytes)
748 {
749 struct ifclassq *ifq = tif->tif_ifq;
750 u_int32_t cnt = 0, len = 0, qlen;
751
752 IFCQ_LOCK_ASSERT_HELD(ifq);
753
754 if ((qlen = qlen(&cl->cl_q)) == 0)
755 goto done;
756
757 /* become regular mutex before freeing mbufs */
758 IFCQ_CONVERT_LOCK(ifq);
759
760 #if CLASSQ_RIO
761 if (q_is_rio(&cl->cl_q))
762 rio_purgeq(cl->cl_rio, &cl->cl_q, flow, &cnt, &len);
763 else
764 #endif /* CLASSQ_RIO */
765 #if CLASSQ_RED
766 if (q_is_red(&cl->cl_q))
767 red_purgeq(cl->cl_red, &cl->cl_q, flow, &cnt, &len);
768 else
769 #endif /* CLASSQ_RED */
770 #if CLASSQ_BLUE
771 if (q_is_blue(&cl->cl_q))
772 blue_purgeq(cl->cl_blue, &cl->cl_q, flow, &cnt, &len);
773 else
774 #endif /* CLASSQ_BLUE */
775 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
776 sfb_purgeq(cl->cl_sfb, &cl->cl_q, flow, &cnt, &len);
777 else
778 _flushq_flow(&cl->cl_q, flow, &cnt, &len);
779
780 if (cnt > 0) {
781 VERIFY(qlen(&cl->cl_q) == (qlen - cnt));
782
783 PKTCNTR_ADD(&cl->cl_dropcnt, cnt, len);
784 IFCQ_DROP_ADD(ifq, cnt, len);
785
786 VERIFY(((signed)IFCQ_LEN(ifq) - cnt) >= 0);
787 IFCQ_LEN(ifq) -= cnt;
788
789 if (pktsched_verbose) {
790 log(LOG_DEBUG, "%s: %s purge qid=%d pri=%d "
791 "qlen=[%d,%d] cnt=%d len=%d flow=0x%x\n",
792 if_name(TCQIF_IFP(tif)), tcq_style(tif),
793 cl->cl_handle, cl->cl_pri, qlen, qlen(&cl->cl_q),
794 cnt, len, flow);
795 }
796 }
797 done:
798 if (packets != NULL)
799 *packets = cnt;
800 if (bytes != NULL)
801 *bytes = len;
802 }
803
804 static void
805 tcq_updateq(struct tcq_if *tif, struct tcq_class *cl, cqev_t ev)
806 {
807 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
808
809 if (pktsched_verbose) {
810 log(LOG_DEBUG, "%s: %s update qid=%d pri=%d event=%s\n",
811 if_name(TCQIF_IFP(tif)), tcq_style(tif),
812 cl->cl_handle, cl->cl_pri, ifclassq_ev2str(ev));
813 }
814
815 #if CLASSQ_RIO
816 if (q_is_rio(&cl->cl_q))
817 return (rio_updateq(cl->cl_rio, ev));
818 #endif /* CLASSQ_RIO */
819 #if CLASSQ_RED
820 if (q_is_red(&cl->cl_q))
821 return (red_updateq(cl->cl_red, ev));
822 #endif /* CLASSQ_RED */
823 #if CLASSQ_BLUE
824 if (q_is_blue(&cl->cl_q))
825 return (blue_updateq(cl->cl_blue, ev));
826 #endif /* CLASSQ_BLUE */
827 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
828 return (sfb_updateq(cl->cl_sfb, ev));
829 }
830
831 int
832 tcq_get_class_stats(struct tcq_if *tif, u_int32_t qid,
833 struct tcq_classstats *sp)
834 {
835 struct tcq_class *cl;
836
837 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
838
839 if ((cl = tcq_clh_to_clp(tif, qid)) == NULL)
840 return (EINVAL);
841
842 sp->class_handle = cl->cl_handle;
843 sp->priority = cl->cl_pri;
844 sp->qlength = qlen(&cl->cl_q);
845 sp->qlimit = qlimit(&cl->cl_q);
846 sp->period = cl->cl_period;
847 sp->xmitcnt = cl->cl_xmitcnt;
848 sp->dropcnt = cl->cl_dropcnt;
849
850 sp->qtype = qtype(&cl->cl_q);
851 sp->qstate = qstate(&cl->cl_q);
852 #if CLASSQ_RED
853 if (q_is_red(&cl->cl_q))
854 red_getstats(cl->cl_red, &sp->red[0]);
855 #endif /* CLASSQ_RED */
856 #if CLASSQ_RIO
857 if (q_is_rio(&cl->cl_q))
858 rio_getstats(cl->cl_rio, &sp->red[0]);
859 #endif /* CLASSQ_RIO */
860 #if CLASSQ_BLUE
861 if (q_is_blue(&cl->cl_q))
862 blue_getstats(cl->cl_blue, &sp->blue);
863 #endif /* CLASSQ_BLUE */
864 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
865 sfb_getstats(cl->cl_sfb, &sp->sfb);
866
867 return (0);
868 }
869
870 static int
871 tcq_stat_sc(struct tcq_if *tif, cqrq_stat_sc_t *sr)
872 {
873 struct ifclassq *ifq = tif->tif_ifq;
874 struct tcq_class *cl;
875 u_int32_t i;
876
877 IFCQ_LOCK_ASSERT_HELD(ifq);
878
879 VERIFY(sr->sc == MBUF_SC_UNSPEC || MBUF_VALID_SC(sr->sc));
880
881 i = MBUF_SCIDX(sr->sc);
882 VERIFY(i < IFCQ_SC_MAX);
883
884 cl = ifq->ifcq_disc_slots[i].cl;
885 sr->packets = qlen(&cl->cl_q);
886 sr->bytes = qsize(&cl->cl_q);
887
888 return (0);
889 }
890
891 /* convert a class handle to the corresponding class pointer */
892 static inline struct tcq_class *
893 tcq_clh_to_clp(struct tcq_if *tif, u_int32_t chandle)
894 {
895 struct tcq_class *cl;
896 int idx;
897
898 IFCQ_LOCK_ASSERT_HELD(tif->tif_ifq);
899
900 for (idx = tif->tif_maxpri; idx >= 0; idx--)
901 if ((cl = tif->tif_classes[idx]) != NULL &&
902 cl->cl_handle == chandle)
903 return (cl);
904
905 return (NULL);
906 }
907
908 static const char *
909 tcq_style(struct tcq_if *tif)
910 {
911 return ((tif->tif_flags & TCQIFF_ALTQ) ? "ALTQ_TCQ" : "TCQ");
912 }
913
914 /*
915 * tcq_enqueue_ifclassq is an enqueue function to be registered to
916 * (*ifcq_enqueue) in struct ifclassq.
917 */
918 static int
919 tcq_enqueue_ifclassq(struct ifclassq *ifq, struct mbuf *m)
920 {
921 u_int32_t i;
922
923 IFCQ_LOCK_ASSERT_HELD(ifq);
924
925 if (!(m->m_flags & M_PKTHDR)) {
926 /* should not happen */
927 log(LOG_ERR, "%s: packet does not have pkthdr\n",
928 if_name(ifq->ifcq_ifp));
929 IFCQ_CONVERT_LOCK(ifq);
930 m_freem(m);
931 return (ENOBUFS);
932 }
933
934 i = MBUF_SCIDX(mbuf_get_service_class(m));
935 VERIFY((u_int32_t)i < IFCQ_SC_MAX);
936
937 return (tcq_enqueue(ifq->ifcq_disc,
938 ifq->ifcq_disc_slots[i].cl, m, m_pftag(m)));
939 }
940
941 /*
942 * tcq_dequeue_tc_ifclassq is a dequeue function to be registered to
943 * (*ifcq_dequeue) in struct ifclass.
944 *
945 * note: CLASSQDQ_POLL returns the next packet without removing the packet
946 * from the queue. CLASSQDQ_REMOVE is a normal dequeue operation.
947 * CLASSQDQ_REMOVE must return the same packet if called immediately
948 * after CLASSQDQ_POLL.
949 */
950 static struct mbuf *
951 tcq_dequeue_tc_ifclassq(struct ifclassq *ifq, mbuf_svc_class_t sc,
952 cqdq_op_t op)
953 {
954 u_int32_t i = MBUF_SCIDX(sc);
955
956 VERIFY((u_int32_t)i < IFCQ_SC_MAX);
957
958 return (tcq_dequeue_cl(ifq->ifcq_disc,
959 ifq->ifcq_disc_slots[i].cl, sc, op));
960 }
961
962 static int
963 tcq_request_ifclassq(struct ifclassq *ifq, cqrq_t req, void *arg)
964 {
965 struct tcq_if *tif = (struct tcq_if *)ifq->ifcq_disc;
966 int err = 0;
967
968 IFCQ_LOCK_ASSERT_HELD(ifq);
969
970 switch (req) {
971 case CLASSQRQ_PURGE:
972 tcq_purge(tif);
973 break;
974
975 case CLASSQRQ_PURGE_SC:
976 tcq_purge_sc(tif, (cqrq_purge_sc_t *)arg);
977 break;
978
979 case CLASSQRQ_EVENT:
980 tcq_event(tif, (cqev_t)arg);
981 break;
982
983 case CLASSQRQ_THROTTLE:
984 err = tcq_throttle(tif, (cqrq_throttle_t *)arg);
985 break;
986
987 case CLASSQRQ_STAT_SC:
988 err = tcq_stat_sc(tif, (cqrq_stat_sc_t *)arg);
989 break;
990 }
991 return (err);
992 }
993
994 int
995 tcq_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags)
996 {
997 struct ifnet *ifp = ifq->ifcq_ifp;
998 struct tcq_class *cl0, *cl1, *cl2, *cl3;
999 struct tcq_if *tif;
1000 u_int32_t maxlen = 0, qflags = 0;
1001 int err = 0;
1002
1003 IFCQ_LOCK_ASSERT_HELD(ifq);
1004 VERIFY(ifq->ifcq_disc == NULL);
1005 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
1006
1007 if (flags & PKTSCHEDF_QALG_RED)
1008 qflags |= TQCF_RED;
1009 if (flags & PKTSCHEDF_QALG_RIO)
1010 qflags |= TQCF_RIO;
1011 if (flags & PKTSCHEDF_QALG_BLUE)
1012 qflags |= TQCF_BLUE;
1013 if (flags & PKTSCHEDF_QALG_SFB)
1014 qflags |= TQCF_SFB;
1015 if (flags & PKTSCHEDF_QALG_ECN)
1016 qflags |= TQCF_ECN;
1017 if (flags & PKTSCHEDF_QALG_FLOWCTL)
1018 qflags |= TQCF_FLOWCTL;
1019 if (flags & PKTSCHEDF_QALG_DELAYBASED)
1020 qflags |= TQCF_DELAYBASED;
1021
1022 tif = tcq_alloc(ifp, M_WAITOK, FALSE);
1023 if (tif == NULL)
1024 return (ENOMEM);
1025
1026 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0)
1027 maxlen = if_sndq_maxlen;
1028
1029 if ((err = tcq_add_queue(tif, 0, maxlen,
1030 qflags | PRCF_LAZY, SCIDX_BK, &cl0)) != 0)
1031 goto cleanup;
1032
1033 if ((err = tcq_add_queue(tif, 1, maxlen,
1034 qflags | TQCF_DEFAULTCLASS, SCIDX_BE, &cl1)) != 0)
1035 goto cleanup;
1036
1037 if ((err = tcq_add_queue(tif, 2, maxlen,
1038 qflags | PRCF_LAZY, SCIDX_VI, &cl2)) != 0)
1039 goto cleanup;
1040
1041 if ((err = tcq_add_queue(tif, 3, maxlen,
1042 qflags, SCIDX_VO, &cl3)) != 0)
1043 goto cleanup;
1044
1045 err = ifclassq_attach(ifq, PKTSCHEDT_TCQ, tif,
1046 tcq_enqueue_ifclassq, NULL, tcq_dequeue_tc_ifclassq,
1047 tcq_request_ifclassq);
1048
1049 /* cache these for faster lookup */
1050 if (err == 0) {
1051 /* Map {BK_SYS,BK} to TC_BK */
1052 ifq->ifcq_disc_slots[SCIDX_BK_SYS].qid = SCIDX_BK;
1053 ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl = cl0;
1054
1055 ifq->ifcq_disc_slots[SCIDX_BK].qid = SCIDX_BK;
1056 ifq->ifcq_disc_slots[SCIDX_BK].cl = cl0;
1057
1058 /* Map {BE,RD,OAM} to TC_BE */
1059 ifq->ifcq_disc_slots[SCIDX_BE].qid = SCIDX_BE;
1060 ifq->ifcq_disc_slots[SCIDX_BE].cl = cl1;
1061
1062 ifq->ifcq_disc_slots[SCIDX_RD].qid = SCIDX_BE;
1063 ifq->ifcq_disc_slots[SCIDX_RD].cl = cl1;
1064
1065 ifq->ifcq_disc_slots[SCIDX_OAM].qid = SCIDX_BE;
1066 ifq->ifcq_disc_slots[SCIDX_OAM].cl = cl1;
1067
1068 /* Map {AV,RV,VI} to TC_VI */
1069 ifq->ifcq_disc_slots[SCIDX_AV].qid = SCIDX_VI;
1070 ifq->ifcq_disc_slots[SCIDX_AV].cl = cl2;
1071
1072 ifq->ifcq_disc_slots[SCIDX_RV].qid = SCIDX_VI;
1073 ifq->ifcq_disc_slots[SCIDX_RV].cl = cl2;
1074
1075 ifq->ifcq_disc_slots[SCIDX_VI].qid = SCIDX_VI;
1076 ifq->ifcq_disc_slots[SCIDX_VI].cl = cl2;
1077
1078 /* Map {VO,CTL} to TC_VO */
1079 ifq->ifcq_disc_slots[SCIDX_VO].qid = SCIDX_VO;
1080 ifq->ifcq_disc_slots[SCIDX_VO].cl = cl3;
1081
1082 ifq->ifcq_disc_slots[SCIDX_CTL].qid = SCIDX_VO;
1083 ifq->ifcq_disc_slots[SCIDX_CTL].cl = cl3;
1084 }
1085
1086 cleanup:
1087 if (err != 0)
1088 (void) tcq_destroy_locked(tif);
1089
1090 return (err);
1091 }
1092
1093 int
1094 tcq_teardown_ifclassq(struct ifclassq *ifq)
1095 {
1096 struct tcq_if *tif = ifq->ifcq_disc;
1097 int i;
1098
1099 IFCQ_LOCK_ASSERT_HELD(ifq);
1100 VERIFY(tif != NULL && ifq->ifcq_type == PKTSCHEDT_TCQ);
1101
1102 (void) tcq_destroy_locked(tif);
1103
1104 ifq->ifcq_disc = NULL;
1105 for (i = 0; i < IFCQ_SC_MAX; i++) {
1106 ifq->ifcq_disc_slots[i].qid = 0;
1107 ifq->ifcq_disc_slots[i].cl = NULL;
1108 }
1109
1110 return (ifclassq_detach(ifq));
1111 }
1112
1113 int
1114 tcq_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t slot,
1115 struct if_ifclassq_stats *ifqs)
1116 {
1117 struct tcq_if *tif = ifq->ifcq_disc;
1118
1119 IFCQ_LOCK_ASSERT_HELD(ifq);
1120 VERIFY(ifq->ifcq_type == PKTSCHEDT_TCQ);
1121
1122 if (slot >= IFCQ_SC_MAX)
1123 return (EINVAL);
1124
1125 return (tcq_get_class_stats(tif, ifq->ifcq_disc_slots[slot].qid,
1126 &ifqs->ifqs_tcq_stats));
1127 }
1128
1129 static int
1130 tcq_throttle(struct tcq_if *tif, cqrq_throttle_t *tr)
1131 {
1132 struct ifclassq *ifq = tif->tif_ifq;
1133 struct tcq_class *cl;
1134 int err = 0;
1135
1136 IFCQ_LOCK_ASSERT_HELD(ifq);
1137 VERIFY(!(tif->tif_flags & TCQIFF_ALTQ));
1138
1139 if (!tr->set) {
1140 tr->level = tif->tif_throttle;
1141 return (0);
1142 }
1143
1144 if (tr->level == tif->tif_throttle)
1145 return (EALREADY);
1146
1147 /* Current throttling levels only involve BK_SYS class */
1148 cl = ifq->ifcq_disc_slots[SCIDX_BK_SYS].cl;
1149
1150 switch (tr->level) {
1151 case IFNET_THROTTLE_OFF:
1152 err = tcq_resumeq(tif, cl);
1153 break;
1154
1155 case IFNET_THROTTLE_OPPORTUNISTIC:
1156 err = tcq_suspendq(tif, cl);
1157 break;
1158
1159 default:
1160 VERIFY(0);
1161 /* NOTREACHED */
1162 }
1163
1164 if (err == 0 || err == ENXIO) {
1165 if (pktsched_verbose) {
1166 log(LOG_DEBUG, "%s: %s throttling %slevel set %d->%d\n",
1167 if_name(TCQIF_IFP(tif)), tcq_style(tif),
1168 (err == 0) ? "" : "lazy ", tif->tif_throttle,
1169 tr->level);
1170 }
1171 tif->tif_throttle = tr->level;
1172 if (err != 0)
1173 err = 0;
1174 else
1175 tcq_purgeq(tif, cl, 0, NULL, NULL);
1176 } else {
1177 log(LOG_ERR, "%s: %s unable to set throttling level "
1178 "%d->%d [error=%d]\n", if_name(TCQIF_IFP(tif)),
1179 tcq_style(tif), tif->tif_throttle, tr->level, err);
1180 }
1181
1182 return (err);
1183 }
1184
1185 static int
1186 tcq_resumeq(struct tcq_if *tif, struct tcq_class *cl)
1187 {
1188 struct ifclassq *ifq = tif->tif_ifq;
1189 int err = 0;
1190
1191 IFCQ_LOCK_ASSERT_HELD(ifq);
1192
1193 #if CLASSQ_RIO
1194 if (q_is_rio(&cl->cl_q))
1195 err = rio_suspendq(cl->cl_rio, &cl->cl_q, FALSE);
1196 else
1197 #endif /* CLASSQ_RIO */
1198 #if CLASSQ_RED
1199 if (q_is_red(&cl->cl_q))
1200 err = red_suspendq(cl->cl_red, &cl->cl_q, FALSE);
1201 else
1202 #endif /* CLASSQ_RED */
1203 #if CLASSQ_BLUE
1204 if (q_is_blue(&cl->cl_q))
1205 err = blue_suspendq(cl->cl_blue, &cl->cl_q, FALSE);
1206 else
1207 #endif /* CLASSQ_BLUE */
1208 if (q_is_sfb(&cl->cl_q) && cl->cl_sfb != NULL)
1209 err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, FALSE);
1210
1211 if (err == 0)
1212 qstate(&cl->cl_q) = QS_RUNNING;
1213
1214 return (err);
1215 }
1216
1217 static int
1218 tcq_suspendq(struct tcq_if *tif, struct tcq_class *cl)
1219 {
1220 struct ifclassq *ifq = tif->tif_ifq;
1221 int err = 0;
1222
1223 IFCQ_LOCK_ASSERT_HELD(ifq);
1224
1225 #if CLASSQ_RIO
1226 if (q_is_rio(&cl->cl_q))
1227 err = rio_suspendq(cl->cl_rio, &cl->cl_q, TRUE);
1228 else
1229 #endif /* CLASSQ_RIO */
1230 #if CLASSQ_RED
1231 if (q_is_red(&cl->cl_q))
1232 err = red_suspendq(cl->cl_red, &cl->cl_q, TRUE);
1233 else
1234 #endif /* CLASSQ_RED */
1235 #if CLASSQ_BLUE
1236 if (q_is_blue(&cl->cl_q))
1237 err = blue_suspendq(cl->cl_blue, &cl->cl_q, TRUE);
1238 else
1239 #endif /* CLASSQ_BLUE */
1240 if (q_is_sfb(&cl->cl_q)) {
1241 if (cl->cl_sfb != NULL) {
1242 err = sfb_suspendq(cl->cl_sfb, &cl->cl_q, TRUE);
1243 } else {
1244 VERIFY(cl->cl_flags & TQCF_LAZY);
1245 err = ENXIO; /* delayed throttling */
1246 }
1247 }
1248
1249 if (err == 0 || err == ENXIO)
1250 qstate(&cl->cl_q) = QS_SUSPENDED;
1251
1252 return (err);
1253 }
mirror of XNU