]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/classq/classq_subr.c
projects / apple / xnu.git / blob
? search:


738c86e2371e2d96ed258c9699b1561ec61f385e
[apple/xnu.git] / bsd / net / classq / classq_subr.c
1 /*
2 * Copyright (c) 2011 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/mbuf.h>
32 #include <sys/errno.h>
33 #include <sys/random.h>
34 #include <sys/kernel_types.h>
35 #include <sys/sysctl.h>
36
37 #include <kern/zalloc.h>
38
39 #include <net/if.h>
40 #include <net/net_osdep.h>
41 #include <net/classq/classq.h>
42 #if CLASSQ_RED
43 #include <net/classq/classq_red.h>
44 #endif /* CLASSQ_RED */
45 #if CLASSQ_RIO
46 #include <net/classq/classq_rio.h>
47 #endif /* CLASSQ_RIO */
48 #if CLASSQ_BLUE
49 #include <net/classq/classq_blue.h>
50 #endif /* CLASSQ_BLUE */
51 #include <net/classq/classq_sfb.h>
52 #include <net/pktsched/pktsched.h>
53
54 #include <libkern/libkern.h>
55
56 #if PF_ALTQ
57 #include <net/altq/altq.h>
58 #endif /* PF_ALTQ */
59
60 static errno_t ifclassq_dequeue_common(struct ifclassq *, mbuf_svc_class_t,
61 u_int32_t, struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *,
62 boolean_t);
63 static struct mbuf *ifclassq_poll_common(struct ifclassq *,
64 mbuf_svc_class_t, boolean_t);
65 static struct mbuf *ifclassq_tbr_dequeue_common(struct ifclassq *, int,
66 mbuf_svc_class_t, boolean_t);
67
68 void
69 classq_init(void)
70 {
71 _CASSERT(MBUF_TC_BE == 0);
72 _CASSERT(MBUF_SC_BE == 0);
73 _CASSERT(IFCQ_SC_MAX == MBUF_SC_MAX_CLASSES);
74
75 #if CLASSQ_RED
76 red_init();
77 #endif /* CLASSQ_RED */
78 #if CLASSQ_RIO
79 rio_init();
80 #endif /* CLASSQ_RIO */
81 #if CLASSQ_BLUE
82 blue_init();
83 #endif /* CLASSQ_BLUE */
84 sfb_init();
85 }
86
87 int
88 ifclassq_setup(struct ifnet *ifp, u_int32_t sflags, boolean_t reuse)
89 {
90 #pragma unused(reuse)
91 struct ifclassq *ifq = &ifp->if_snd;
92 int err = 0;
93
94 IFCQ_LOCK(ifq);
95 VERIFY(IFCQ_IS_EMPTY(ifq));
96 ifq->ifcq_ifp = ifp;
97 IFCQ_LEN(ifq) = 0;
98 bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt));
99 bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt));
100
101 VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
102 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
103 VERIFY(ifq->ifcq_flags == 0);
104 VERIFY(ifq->ifcq_sflags == 0);
105 VERIFY(ifq->ifcq_disc == NULL);
106 VERIFY(ifq->ifcq_enqueue == NULL);
107 VERIFY(ifq->ifcq_dequeue == NULL);
108 VERIFY(ifq->ifcq_dequeue_sc == NULL);
109 VERIFY(ifq->ifcq_request == NULL);
110
111 if (ifp->if_eflags & IFEF_TXSTART) {
112 u_int32_t maxlen = 0;
113
114 if ((maxlen = IFCQ_MAXLEN(ifq)) == 0)
115 maxlen = if_sndq_maxlen;
116 IFCQ_SET_MAXLEN(ifq, maxlen);
117
118 ifq->ifcq_sflags = sflags;
119 err = ifclassq_pktsched_setup(ifq);
120 if (err == 0)
121 ifq->ifcq_flags = (IFCQF_READY | IFCQF_ENABLED);
122 }
123
124 #if PF_ALTQ
125 ifq->ifcq_drain = 0;
126 IFCQ_ALTQ(ifq)->altq_ifcq = ifq;
127 VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE);
128 VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0);
129 VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL);
130 VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL);
131 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL);
132 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL);
133 VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL);
134
135 if ((ifp->if_eflags & IFEF_TXSTART) &&
136 ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED)
137 ALTQ_SET_READY(IFCQ_ALTQ(ifq));
138 else
139 ALTQ_CLEAR_READY(IFCQ_ALTQ(ifq));
140 #endif /* PF_ALTQ */
141 IFCQ_UNLOCK(ifq);
142
143 return (err);
144 }
145
146 void
147 ifclassq_teardown(struct ifnet *ifp)
148 {
149 struct ifclassq *ifq = &ifp->if_snd;
150
151 IFCQ_LOCK(ifq);
152 #if PF_ALTQ
153 if (ALTQ_IS_READY(IFCQ_ALTQ(ifq))) {
154 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq)))
155 altq_disable(IFCQ_ALTQ(ifq));
156 if (ALTQ_IS_ATTACHED(IFCQ_ALTQ(ifq)))
157 altq_detach(IFCQ_ALTQ(ifq));
158 IFCQ_ALTQ(ifq)->altq_flags = 0;
159 }
160 ifq->ifcq_drain = 0;
161 IFCQ_ALTQ(ifq)->altq_ifcq = NULL;
162 VERIFY(IFCQ_ALTQ(ifq)->altq_type == ALTQT_NONE);
163 VERIFY(IFCQ_ALTQ(ifq)->altq_flags == 0);
164 VERIFY(IFCQ_ALTQ(ifq)->altq_disc == NULL);
165 VERIFY(IFCQ_ALTQ(ifq)->altq_enqueue == NULL);
166 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue == NULL);
167 VERIFY(IFCQ_ALTQ(ifq)->altq_dequeue_sc == NULL);
168 VERIFY(IFCQ_ALTQ(ifq)->altq_request == NULL);
169 #endif /* PF_ALTQ */
170
171 if (IFCQ_IS_READY(ifq)) {
172 if (IFCQ_TBR_IS_ENABLED(ifq)) {
173 struct tb_profile tb = { 0, 0, 0 };
174 (void) ifclassq_tbr_set(ifq, &tb, FALSE);
175 }
176 (void) pktsched_teardown(ifq);
177 ifq->ifcq_flags = 0;
178 }
179 ifq->ifcq_sflags = 0;
180
181 VERIFY(IFCQ_IS_EMPTY(ifq));
182 VERIFY(!IFCQ_TBR_IS_ENABLED(ifq));
183 VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
184 VERIFY(ifq->ifcq_flags == 0);
185 VERIFY(ifq->ifcq_sflags == 0);
186 VERIFY(ifq->ifcq_disc == NULL);
187 VERIFY(ifq->ifcq_enqueue == NULL);
188 VERIFY(ifq->ifcq_dequeue == NULL);
189 VERIFY(ifq->ifcq_dequeue_sc == NULL);
190 VERIFY(ifq->ifcq_request == NULL);
191 IFCQ_LEN(ifq) = 0;
192 IFCQ_MAXLEN(ifq) = 0;
193 bzero(&ifq->ifcq_xmitcnt, sizeof (ifq->ifcq_xmitcnt));
194 bzero(&ifq->ifcq_dropcnt, sizeof (ifq->ifcq_dropcnt));
195
196 IFCQ_UNLOCK(ifq);
197 }
198
199 int
200 ifclassq_pktsched_setup(struct ifclassq *ifq)
201 {
202 struct ifnet *ifp = ifq->ifcq_ifp;
203 int err = 0;
204
205 IFCQ_LOCK_ASSERT_HELD(ifq);
206 VERIFY(ifp->if_eflags & IFEF_TXSTART);
207
208 switch (ifp->if_output_sched_model) {
209 case IFNET_SCHED_MODEL_DRIVER_MANAGED:
210 err = pktsched_setup(ifq, PKTSCHEDT_TCQ, ifq->ifcq_sflags);
211 break;
212
213 case IFNET_SCHED_MODEL_NORMAL:
214 err = pktsched_setup(ifq, PKTSCHEDT_QFQ, ifq->ifcq_sflags);
215 break;
216
217 default:
218 VERIFY(0);
219 /* NOTREACHED */
220 }
221
222 return (err);
223 }
224
225 void
226 ifclassq_set_maxlen(struct ifclassq *ifq, u_int32_t maxqlen)
227 {
228 IFCQ_LOCK(ifq);
229 if (maxqlen == 0)
230 maxqlen = if_sndq_maxlen;
231 IFCQ_SET_MAXLEN(ifq, maxqlen);
232 IFCQ_UNLOCK(ifq);
233 }
234
235 u_int32_t
236 ifclassq_get_maxlen(struct ifclassq *ifq)
237 {
238 return (IFCQ_MAXLEN(ifq));
239 }
240
241 u_int32_t
242 ifclassq_get_len(struct ifclassq *ifq)
243 {
244 return (IFCQ_LEN(ifq));
245 }
246
247 errno_t
248 ifclassq_enqueue(struct ifclassq *ifq, struct mbuf *m)
249 {
250 errno_t err;
251
252 IFCQ_LOCK_SPIN(ifq);
253
254 #if PF_ALTQ
255 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq))) {
256 ALTQ_ENQUEUE(IFCQ_ALTQ(ifq), m, err);
257 } else {
258 u_int32_t qlen = IFCQ_LEN(ifq);
259 IFCQ_ENQUEUE(ifq, m, err);
260 if (IFCQ_LEN(ifq) > qlen)
261 ifq->ifcq_drain += (IFCQ_LEN(ifq) - qlen);
262 }
263 #else /* !PF_ALTQ */
264 IFCQ_ENQUEUE(ifq, m, err);
265 #endif /* PF_ALTQ */
266
267 IFCQ_UNLOCK(ifq);
268
269 return (err);
270 }
271
272 errno_t
273 ifclassq_dequeue(struct ifclassq *ifq, u_int32_t limit, struct mbuf **head,
274 struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
275 {
276 return (ifclassq_dequeue_common(ifq, MBUF_SC_UNSPEC, limit, head, tail,
277 cnt, len, FALSE));
278 }
279
280 errno_t
281 ifclassq_dequeue_sc(struct ifclassq *ifq, mbuf_svc_class_t sc,
282 u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
283 u_int32_t *len)
284 {
285 return (ifclassq_dequeue_common(ifq, sc, limit, head, tail,
286 cnt, len, TRUE));
287 }
288
289 static errno_t
290 ifclassq_dequeue_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
291 u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
292 u_int32_t *len, boolean_t drvmgt)
293 {
294 struct ifnet *ifp = ifq->ifcq_ifp;
295 u_int32_t i = 0, l = 0;
296 struct mbuf **first, *last;
297 #if PF_ALTQ
298 struct ifaltq *altq = IFCQ_ALTQ(ifq);
299 boolean_t draining;
300 #endif /* PF_ALTQ */
301
302 VERIFY(!drvmgt || MBUF_VALID_SC(sc));
303
304 *head = NULL;
305 first = &(*head);
306 last = NULL;
307
308 ifq = &ifp->if_snd;
309 IFCQ_LOCK_SPIN(ifq);
310
311 while (i < limit) {
312 u_int64_t pktlen;
313 #if PF_ALTQ
314 u_int32_t qlen;
315
316 qlen = IFCQ_LEN(ifq);
317 draining = IFCQ_IS_DRAINING(ifq);
318
319 if (drvmgt) {
320 if (IFCQ_TBR_IS_ENABLED(ifq))
321 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head);
322 else if (draining)
323 IFCQ_DEQUEUE_SC(ifq, sc, *head);
324 else if (ALTQ_IS_ENABLED(altq))
325 ALTQ_DEQUEUE_SC(altq, sc, *head);
326 else
327 *head = NULL;
328 } else {
329 if (IFCQ_TBR_IS_ENABLED(ifq))
330 IFCQ_TBR_DEQUEUE(ifq, *head);
331 else if (draining)
332 IFCQ_DEQUEUE(ifq, *head);
333 else if (ALTQ_IS_ENABLED(altq))
334 ALTQ_DEQUEUE(altq, *head);
335 else
336 *head = NULL;
337 }
338
339 if (draining && *head != NULL) {
340 VERIFY(ifq->ifcq_drain >= (qlen - IFCQ_LEN(ifq)));
341 ifq->ifcq_drain -= (qlen - IFCQ_LEN(ifq));
342 }
343 #else /* ! PF_ALTQ */
344 if (drvmgt) {
345 if (IFCQ_TBR_IS_ENABLED(ifq))
346 IFCQ_TBR_DEQUEUE_SC(ifq, sc, *head);
347 else
348 IFCQ_DEQUEUE_SC(ifq, sc, *head);
349 } else {
350 if (IFCQ_TBR_IS_ENABLED(ifq))
351 IFCQ_TBR_DEQUEUE(ifq, *head);
352 else
353 IFCQ_DEQUEUE(ifq, *head);
354 }
355 #endif /* !PF_ALTQ */
356
357 if (*head == NULL)
358 break;
359
360 (*head)->m_nextpkt = NULL;
361 last = *head;
362
363 l += (*head)->m_pkthdr.len;
364 pktlen = (*head)->m_pkthdr.len;
365
366 (*head)->m_pkthdr.pf_mtag.pftag_pktseq =
367 atomic_add_64_ov(&(ifp->if_bw.cur_seq), pktlen);
368
369 head = &(*head)->m_nextpkt;
370 i++;
371 }
372
373 IFCQ_UNLOCK(ifq);
374
375 if (tail != NULL)
376 *tail = last;
377 if (cnt != NULL)
378 *cnt = i;
379 if (len != NULL)
380 *len = l;
381
382 return ((*first != NULL) ? 0 : EAGAIN);
383 }
384
385 struct mbuf *
386 ifclassq_poll(struct ifclassq *ifq)
387 {
388 return (ifclassq_poll_common(ifq, MBUF_SC_UNSPEC, FALSE));
389 }
390
391 struct mbuf *
392 ifclassq_poll_sc(struct ifclassq *ifq, mbuf_svc_class_t sc)
393 {
394 return (ifclassq_poll_common(ifq, sc, TRUE));
395 }
396
397 static struct mbuf *
398 ifclassq_poll_common(struct ifclassq *ifq, mbuf_svc_class_t sc,
399 boolean_t drvmgt)
400 {
401 #if PF_ALTQ
402 struct ifaltq *altq = IFCQ_ALTQ(ifq);
403 #endif /* PF_ALTQ */
404 struct mbuf *m;
405
406 VERIFY(!drvmgt || MBUF_VALID_SC(sc));
407
408 #if PF_ALTQ
409 if (drvmgt) {
410 if (IFCQ_TBR_IS_ENABLED(ifq))
411 IFCQ_TBR_POLL_SC(ifq, sc, m);
412 else if (IFCQ_IS_DRAINING(ifq))
413 IFCQ_POLL_SC(ifq, sc, m);
414 else if (ALTQ_IS_ENABLED(altq))
415 ALTQ_POLL_SC(altq, sc, m);
416 else
417 m = NULL;
418 } else {
419 if (IFCQ_TBR_IS_ENABLED(ifq))
420 IFCQ_TBR_POLL(ifq, m);
421 else if (IFCQ_IS_DRAINING(ifq))
422 IFCQ_POLL(ifq, m);
423 else if (ALTQ_IS_ENABLED(altq))
424 ALTQ_POLL(altq, m);
425 else
426 m = NULL;
427 }
428 #else /* ! PF_ALTQ */
429 if (drvmgt) {
430 if (IFCQ_TBR_IS_ENABLED(ifq))
431 IFCQ_TBR_POLL_SC(ifq, sc, m);
432 else
433 IFCQ_POLL_SC(ifq, sc, m);
434 } else {
435 if (IFCQ_TBR_IS_ENABLED(ifq))
436 IFCQ_TBR_POLL(ifq, m);
437 else
438 IFCQ_POLL(ifq, m);
439 }
440 #endif /* !PF_ALTQ */
441
442 return (m);
443 }
444
445 void
446 ifclassq_update(struct ifclassq *ifq, cqev_t ev)
447 {
448 IFCQ_LOCK_ASSERT_HELD(ifq);
449 VERIFY(IFCQ_IS_READY(ifq));
450
451 #if PF_ALTQ
452 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq)))
453 ALTQ_UPDATE(IFCQ_ALTQ(ifq), ev);
454 #endif /* PF_ALTQ */
455 IFCQ_UPDATE(ifq, ev);
456 }
457
458 int
459 ifclassq_attach(struct ifclassq *ifq, u_int32_t type, void *discipline,
460 ifclassq_enq_func enqueue, ifclassq_deq_func dequeue,
461 ifclassq_deq_sc_func dequeue_sc, ifclassq_req_func request)
462 {
463 IFCQ_LOCK_ASSERT_HELD(ifq);
464
465 VERIFY(ifq->ifcq_disc == NULL);
466 VERIFY(enqueue != NULL);
467 VERIFY(!(dequeue != NULL && dequeue_sc != NULL));
468 VERIFY(request != NULL);
469
470 ifq->ifcq_type = type;
471 ifq->ifcq_disc = discipline;
472 ifq->ifcq_enqueue = enqueue;
473 ifq->ifcq_dequeue = dequeue;
474 ifq->ifcq_dequeue_sc = dequeue_sc;
475 ifq->ifcq_request = request;
476
477 return (0);
478 }
479
480 int
481 ifclassq_detach(struct ifclassq *ifq)
482 {
483 IFCQ_LOCK_ASSERT_HELD(ifq);
484
485 VERIFY(ifq->ifcq_disc == NULL);
486
487 ifq->ifcq_type = PKTSCHEDT_NONE;
488 ifq->ifcq_disc = NULL;
489 ifq->ifcq_enqueue = NULL;
490 ifq->ifcq_dequeue = NULL;
491 ifq->ifcq_dequeue_sc = NULL;
492 ifq->ifcq_request = NULL;
493
494 return (0);
495 }
496
497 int
498 ifclassq_getqstats(struct ifclassq *ifq, u_int32_t qid, void *ubuf,
499 u_int32_t *nbytes)
500 {
501 struct if_ifclassq_stats *ifqs;
502 int err;
503
504 if (*nbytes < sizeof (*ifqs))
505 return (EINVAL);
506
507 ifqs = _MALLOC(sizeof (*ifqs), M_TEMP, M_WAITOK | M_ZERO);
508 if (ifqs == NULL)
509 return (ENOMEM);
510
511 IFCQ_LOCK(ifq);
512 if (!IFCQ_IS_READY(ifq)) {
513 IFCQ_UNLOCK(ifq);
514 _FREE(ifqs, M_TEMP);
515 return (ENXIO);
516 }
517
518 ifqs->ifqs_len = IFCQ_LEN(ifq);
519 ifqs->ifqs_maxlen = IFCQ_MAXLEN(ifq);
520 *(&ifqs->ifqs_xmitcnt) = *(&ifq->ifcq_xmitcnt);
521 *(&ifqs->ifqs_dropcnt) = *(&ifq->ifcq_dropcnt);
522 ifqs->ifqs_scheduler = ifq->ifcq_type;
523
524 err = pktsched_getqstats(ifq, qid, ifqs);
525 IFCQ_UNLOCK(ifq);
526
527 if (err == 0 && (err = copyout((caddr_t)ifqs,
528 (user_addr_t)(uintptr_t)ubuf, sizeof (*ifqs))) == 0)
529 *nbytes = sizeof (*ifqs);
530
531 _FREE(ifqs, M_TEMP);
532
533 return (err);
534 }
535
536 const char *
537 ifclassq_ev2str(cqev_t ev)
538 {
539 const char *c;
540
541 switch (ev) {
542 case CLASSQ_EV_LINK_SPEED:
543 c = "LINK_SPEED";
544 break;
545
546 case CLASSQ_EV_LINK_MTU:
547 c = "LINK_MTU";
548 break;
549
550 case CLASSQ_EV_LINK_UP:
551 c = "LINK_UP";
552 break;
553
554 case CLASSQ_EV_LINK_DOWN:
555 c = "LINK_DOWN";
556 break;
557
558 default:
559 c = "UNKNOWN";
560 break;
561 }
562
563 return (c);
564 }
565
566 /*
567 * internal representation of token bucket parameters
568 * rate: byte_per_unittime << 32
569 * (((bits_per_sec) / 8) << 32) / machclk_freq
570 * depth: byte << 32
571 *
572 */
573 #define TBR_SHIFT 32
574 #define TBR_SCALE(x) ((int64_t)(x) << TBR_SHIFT)
575 #define TBR_UNSCALE(x) ((x) >> TBR_SHIFT)
576
577 struct mbuf *
578 ifclassq_tbr_dequeue(struct ifclassq *ifq, int op)
579 {
580 return (ifclassq_tbr_dequeue_common(ifq, op, MBUF_SC_UNSPEC, FALSE));
581 }
582
583 struct mbuf *
584 ifclassq_tbr_dequeue_sc(struct ifclassq *ifq, int op, mbuf_svc_class_t sc)
585 {
586 return (ifclassq_tbr_dequeue_common(ifq, op, sc, TRUE));
587 }
588
589 static struct mbuf *
590 ifclassq_tbr_dequeue_common(struct ifclassq *ifq, int op,
591 mbuf_svc_class_t sc, boolean_t drvmgt)
592 {
593 struct tb_regulator *tbr;
594 struct mbuf *m;
595 int64_t interval;
596 u_int64_t now;
597
598 IFCQ_LOCK_ASSERT_HELD(ifq);
599
600 VERIFY(!drvmgt || MBUF_VALID_SC(sc));
601 VERIFY(IFCQ_TBR_IS_ENABLED(ifq));
602
603 tbr = &ifq->ifcq_tbr;
604 if (op == CLASSQDQ_REMOVE && tbr->tbr_lastop == CLASSQDQ_POLL) {
605 /* if this is a remove after poll, bypass tbr check */
606 } else {
607 /* update token only when it is negative */
608 if (tbr->tbr_token <= 0) {
609 now = read_machclk();
610 interval = now - tbr->tbr_last;
611 if (interval >= tbr->tbr_filluptime) {
612 tbr->tbr_token = tbr->tbr_depth;
613 } else {
614 tbr->tbr_token += interval * tbr->tbr_rate;
615 if (tbr->tbr_token > tbr->tbr_depth)
616 tbr->tbr_token = tbr->tbr_depth;
617 }
618 tbr->tbr_last = now;
619 }
620 /* if token is still negative, don't allow dequeue */
621 if (tbr->tbr_token <= 0)
622 return (NULL);
623 }
624
625 /*
626 * ifclassq takes precedence over ALTQ queue;
627 * ifcq_drain count is adjusted by the caller.
628 */
629 #if PF_ALTQ
630 if (IFCQ_IS_DRAINING(ifq)) {
631 #endif /* PF_ALTQ */
632 if (op == CLASSQDQ_POLL) {
633 if (drvmgt)
634 IFCQ_POLL_SC(ifq, sc, m);
635 else
636 IFCQ_POLL(ifq, m);
637 } else {
638 if (drvmgt)
639 IFCQ_DEQUEUE_SC(ifq, sc, m);
640 else
641 IFCQ_DEQUEUE(ifq, m);
642 }
643 #if PF_ALTQ
644 } else {
645 struct ifaltq *altq = IFCQ_ALTQ(ifq);
646 if (ALTQ_IS_ENABLED(altq)) {
647 if (drvmgt)
648 m = (*altq->altq_dequeue_sc)(altq, sc, op);
649 else
650 m = (*altq->altq_dequeue)(altq, op);
651 } else {
652 m = NULL;
653 }
654 }
655 #endif /* PF_ALTQ */
656
657 if (m != NULL && op == CLASSQDQ_REMOVE)
658 tbr->tbr_token -= TBR_SCALE(m_pktlen(m));
659 tbr->tbr_lastop = op;
660
661 return (m);
662 }
663
664 /*
665 * set a token bucket regulator.
666 * if the specified rate is zero, the token bucket regulator is deleted.
667 */
668 int
669 ifclassq_tbr_set(struct ifclassq *ifq, struct tb_profile *profile,
670 boolean_t update)
671 {
672 struct tb_regulator *tbr;
673 struct ifnet *ifp = ifq->ifcq_ifp;
674 u_int64_t rate, old_rate;
675
676 IFCQ_LOCK_ASSERT_HELD(ifq);
677 VERIFY(IFCQ_IS_READY(ifq));
678
679 VERIFY(machclk_freq != 0);
680
681 tbr = &ifq->ifcq_tbr;
682 old_rate = tbr->tbr_rate_raw;
683
684 rate = profile->rate;
685 if (profile->percent > 0) {
686 u_int64_t eff_rate;
687
688 if (profile->percent > 100)
689 return (EINVAL);
690 if ((eff_rate = ifp->if_output_bw.eff_bw) == 0)
691 return (ENODEV);
692 rate = (eff_rate * profile->percent) / 100;
693 }
694
695 if (rate == 0) {
696 if (!IFCQ_TBR_IS_ENABLED(ifq))
697 return (ENOENT);
698
699 if (pktsched_verbose)
700 printf("%s: TBR disabled\n", if_name(ifp));
701
702 /* disable this TBR */
703 ifq->ifcq_flags &= ~IFCQF_TBR;
704 bzero(tbr, sizeof (*tbr));
705 ifnet_set_start_cycle(ifp, NULL);
706 if (update)
707 ifclassq_update(ifq, CLASSQ_EV_LINK_SPEED);
708 return (0);
709 }
710
711 if (pktsched_verbose) {
712 printf("%s: TBR %s (rate %llu bps depth %u)\n", if_name(ifp),
713 (ifq->ifcq_flags & IFCQF_TBR) ? "reconfigured" :
714 "enabled", rate, profile->depth);
715 }
716
717 /* set the new TBR */
718 bzero(tbr, sizeof (*tbr));
719 tbr->tbr_rate_raw = rate;
720 tbr->tbr_percent = profile->percent;
721 ifq->ifcq_flags |= IFCQF_TBR;
722
723 /*
724 * Note that the TBR fill up time (hence the ifnet restart time)
725 * is directly related to the specified TBR depth. The ideal
726 * depth value should be computed such that the interval time
727 * between each successive wakeup is adequately spaced apart,
728 * in order to reduce scheduling overheads. A target interval
729 * of 10 ms seems to provide good performance balance. This can be
730 * overridden by specifying the depth profile. Values smaller than
731 * the ideal depth will reduce delay at the expense of CPU cycles.
732 */
733 tbr->tbr_rate = TBR_SCALE(rate / 8) / machclk_freq;
734 if (tbr->tbr_rate > 0) {
735 u_int32_t mtu = ifp->if_mtu;
736 int64_t ival, idepth = 0;
737 int i;
738
739 if (mtu < IF_MINMTU)
740 mtu = IF_MINMTU;
741
742 ival = pktsched_nsecs_to_abstime(10 * NSEC_PER_MSEC); /* 10ms */
743
744 for (i = 1; ; i++) {
745 idepth = TBR_SCALE(i * mtu);
746 if ((idepth / tbr->tbr_rate) > ival)
747 break;
748 }
749 VERIFY(idepth > 0);
750
751 tbr->tbr_depth = TBR_SCALE(profile->depth);
752 if (tbr->tbr_depth == 0) {
753 tbr->tbr_filluptime = idepth / tbr->tbr_rate;
754 /* a little fudge factor to get closer to rate */
755 tbr->tbr_depth = idepth + (idepth >> 3);
756 } else {
757 tbr->tbr_filluptime = tbr->tbr_depth / tbr->tbr_rate;
758 }
759 } else {
760 tbr->tbr_depth = TBR_SCALE(profile->depth);
761 tbr->tbr_filluptime = 0xffffffffffffffffLL;
762 }
763 tbr->tbr_token = tbr->tbr_depth;
764 tbr->tbr_last = read_machclk();
765 tbr->tbr_lastop = CLASSQDQ_REMOVE;
766
767 if (tbr->tbr_rate > 0 && (ifp->if_flags & IFF_UP)) {
768 struct timespec ts =
769 { 0, pktsched_abs_to_nsecs(tbr->tbr_filluptime) };
770 if (pktsched_verbose) {
771 printf("%s: TBR calculated tokens %lld "
772 "filluptime %llu ns\n", if_name(ifp),
773 TBR_UNSCALE(tbr->tbr_token),
774 pktsched_abs_to_nsecs(tbr->tbr_filluptime));
775 }
776 ifnet_set_start_cycle(ifp, &ts);
777 } else {
778 if (pktsched_verbose) {
779 if (tbr->tbr_rate == 0) {
780 printf("%s: TBR calculated tokens %lld "
781 "infinite filluptime\n", if_name(ifp),
782 TBR_UNSCALE(tbr->tbr_token));
783 } else if (!(ifp->if_flags & IFF_UP)) {
784 printf("%s: TBR suspended (link is down)\n",
785 if_name(ifp));
786 }
787 }
788 ifnet_set_start_cycle(ifp, NULL);
789 }
790 if (update && tbr->tbr_rate_raw != old_rate)
791 ifclassq_update(ifq, CLASSQ_EV_LINK_SPEED);
792
793 return (0);
794 }
mirror of XNU