/*
- * Copyright (c) 2016-2017 Apple Inc. All rights reserved.
+ * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <net/classq/classq_fq_codel.h>
#include <net/pktsched/pktsched_fq_codel.h>
-static size_t fq_if_size;
-static struct zone *fq_if_zone;
+static ZONE_DECLARE(fq_if_zone, "pktsched_fq_if", sizeof(fq_if_t), ZC_ZFREE_CLEARMEM);
static fq_if_t *fq_if_alloc(struct ifnet *, classq_pkt_type_t);
static void fq_if_destroy(fq_if_t *fqs);
-static void fq_if_classq_init(fq_if_t *fqs, u_int32_t priority,
- u_int32_t quantum, u_int32_t drr_max, u_int32_t svc_class);
-static int fq_if_enqueue_classq(struct ifclassq *ifq, void *p,
- classq_pkt_type_t ptype, boolean_t *pdrop);
-static void *fq_if_dequeue_classq(struct ifclassq *, classq_pkt_type_t *);
-static int fq_if_dequeue_classq_multi(struct ifclassq *, u_int32_t,
- u_int32_t, void **, void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *);
-static void *fq_if_dequeue_sc_classq(struct ifclassq *, mbuf_svc_class_t,
- classq_pkt_type_t *);
-static int fq_if_dequeue_sc_classq_multi(struct ifclassq *,
- mbuf_svc_class_t, u_int32_t, u_int32_t, void **,
- void **, u_int32_t *, u_int32_t *, classq_pkt_type_t *);
-static void fq_if_dequeue(fq_if_t *, fq_if_classq_t *, u_int32_t,
- u_int32_t, void **, void **, u_int32_t *, u_int32_t *,
- boolean_t drvmgmt, classq_pkt_type_t *);
-static int fq_if_request_classq(struct ifclassq *ifq, cqrq_t op, void *arg);
+static void fq_if_classq_init(fq_if_t *fqs, uint32_t priority,
+ uint16_t quantum, uint32_t drr_max, uint32_t svc_class);
+static void fq_if_dequeue(fq_if_t *, fq_if_classq_t *, uint32_t,
+ int64_t, classq_pkt_t *, classq_pkt_t *, uint32_t *,
+ uint32_t *, boolean_t drvmgmt);
void fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat);
static void fq_if_purge(fq_if_t *);
static void fq_if_purge_classq(fq_if_t *, fq_if_classq_t *);
static void fq_if_empty_old_flow(fq_if_t *fqs, fq_if_classq_t *fq_cl,
fq_t *fq, bool remove_hash);
-#define FQ_IF_ZONE_MAX 32 /* Maximum elements in zone */
-#define FQ_IF_ZONE_NAME "pktsched_fq_if" /* zone for fq_if class */
-
#define FQ_IF_FLOW_HASH_ID(_flowid_) \
(((_flowid_) >> FQ_IF_HASH_TAG_SHIFT) & FQ_IF_HASH_TAG_MASK)
(STAILQ_EMPTY(&(_fcl_)->fcl_new_flows) && \
STAILQ_EMPTY(&(_fcl_)->fcl_old_flows))
-typedef void (* fq_if_append_pkt_t)(void *, void *);
+typedef void (* fq_if_append_pkt_t)(classq_pkt_t *, classq_pkt_t *);
typedef boolean_t (* fq_getq_flow_t)(fq_if_t *, fq_if_classq_t *, fq_t *,
- u_int32_t, u_int32_t, void **, void **, u_int32_t *, u_int32_t *,
- boolean_t *, u_int32_t);
+ int64_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *,
+ u_int32_t *, boolean_t *, u_int32_t);
static void
-fq_if_append_mbuf(void *pkt, void *next_pkt)
+fq_if_append_mbuf(classq_pkt_t *pkt, classq_pkt_t *next_pkt)
{
- ((mbuf_t)pkt)->m_nextpkt = (mbuf_t)next_pkt;
+ pkt->cp_mbuf->m_nextpkt = next_pkt->cp_mbuf;
}
static boolean_t
fq_getq_flow_mbuf(fq_if_t *fqs, fq_if_classq_t *fq_cl, fq_t *fq,
- u_int32_t byte_limit, u_int32_t pkt_limit, void **top, void **last,
- u_int32_t *byte_cnt, u_int32_t *pkt_cnt, boolean_t *qempty,
- u_int32_t pflags)
+ int64_t byte_limit, u_int32_t pkt_limit, classq_pkt_t *top,
+ classq_pkt_t *last, u_int32_t *byte_cnt, u_int32_t *pkt_cnt,
+ boolean_t *qempty, u_int32_t pflags)
{
- struct mbuf *m;
u_int32_t plen;
pktsched_pkt_t pkt;
boolean_t limit_reached = FALSE;
while (fq->fq_deficit > 0 && limit_reached == FALSE &&
!MBUFQ_EMPTY(&fq->fq_mbufq)) {
_PKTSCHED_PKT_INIT(&pkt);
- m = fq_getq_flow(fqs, fq, &pkt);
+ fq_getq_flow(fqs, fq, &pkt);
ASSERT(pkt.pktsched_ptype == QP_MBUF);
plen = pktsched_get_pkt_len(&pkt);
fq->fq_deficit -= plen;
- m->m_pkthdr.pkt_flags |= pflags;
+ pkt.pktsched_pkt_mbuf->m_pkthdr.pkt_flags |= pflags;
- if (*top == NULL) {
- *top = m;
+ if (top->cp_mbuf == NULL) {
+ *top = pkt.pktsched_pkt;
} else {
- ASSERT(*last != NULL);
- ASSERT((*(struct mbuf **)last)->m_nextpkt == NULL);
- (*(struct mbuf **)last)->m_nextpkt = m;
+ ASSERT(last->cp_mbuf != NULL);
+ ASSERT(last->cp_mbuf->m_nextpkt == NULL);
+ last->cp_mbuf->m_nextpkt = pkt.pktsched_pkt_mbuf;
}
- *last = m;
- (*(mbuf_t *)last)->m_nextpkt = NULL;
+ *last = pkt.pktsched_pkt;
+ last->cp_mbuf->m_nextpkt = NULL;
fq_cl->fcl_stat.fcl_dequeue++;
fq_cl->fcl_stat.fcl_dequeue_bytes += plen;
*pkt_cnt += 1;
*byte_cnt += plen;
- ifclassq_set_packet_metadata(ifq, ifp, m, QP_MBUF);
+ ifclassq_set_packet_metadata(ifq, ifp, &pkt.pktsched_pkt);
/* Check if the limit is reached */
if (*pkt_cnt >= pkt_limit || *byte_cnt >= byte_limit) {
return limit_reached;
}
-void
-fq_codel_scheduler_init(void)
-{
- /* Initialize the zone for flow queue structures */
- fq_codel_init();
-
- fq_if_size = sizeof(fq_if_t);
- fq_if_zone = zinit(fq_if_size, (FQ_IF_ZONE_MAX * fq_if_size), 0,
- FQ_IF_ZONE_NAME);
- if (fq_if_zone == NULL) {
- panic("%s: failed allocating from %s", __func__,
- (FQ_IF_ZONE_NAME));
- }
- zone_change(fq_if_zone, Z_EXPAND, TRUE);
- zone_change(fq_if_zone, Z_CALLERACCT, TRUE);
-}
-
fq_if_t *
fq_if_alloc(struct ifnet *ifp, classq_pkt_type_t ptype)
{
fq_if_t *fqs;
- fqs = zalloc(fq_if_zone);
- if (fqs == NULL) {
- return NULL;
- }
- bzero(fqs, fq_if_size);
+ fqs = zalloc_flags(fq_if_zone, Z_WAITOK | Z_ZERO);
fqs->fqs_ifq = &ifp->if_snd;
fqs->fqs_ptype = ptype;
zfree(fq_if_zone, fqs);
}
-static inline u_int32_t
+static inline uint8_t
fq_if_service_to_priority(fq_if_t *fqs, mbuf_svc_class_t svc)
{
- u_int32_t pri;
+ uint8_t pri;
if (fqs->fqs_flags & FQS_DRIVER_MANAGED) {
switch (svc) {
return pri;
}
-void
-fq_if_classq_init(fq_if_t *fqs, u_int32_t pri, u_int32_t quantum,
- u_int32_t drr_max, u_int32_t svc_class)
+static void
+fq_if_classq_init(fq_if_t *fqs, uint32_t pri, uint16_t quantum,
+ uint32_t drr_max, uint32_t svc_class)
{
fq_if_classq_t *fq_cl;
-
+ VERIFY(pri < FQ_IF_MAX_CLASSES);
fq_cl = &fqs->fqs_classq[pri];
- VERIFY(pri >= 0 && pri < FQ_IF_MAX_CLASSES &&
- fq_cl->fcl_quantum == 0);
+ VERIFY(fq_cl->fcl_quantum == 0);
fq_cl->fcl_quantum = quantum;
fq_cl->fcl_pri = pri;
fq_cl->fcl_drr_max = drr_max;
}
int
-fq_if_enqueue_classq(struct ifclassq *ifq, void *p, classq_pkt_type_t ptype,
- boolean_t *pdrop)
+fq_if_enqueue_classq(struct ifclassq *ifq, classq_pkt_t *head,
+ classq_pkt_t *tail, uint32_t cnt, uint32_t bytes, boolean_t *pdrop)
{
- u_int32_t pri;
+ uint8_t pri;
fq_if_t *fqs;
fq_if_classq_t *fq_cl;
- int ret, len;
+ int ret;
mbuf_svc_class_t svc;
pktsched_pkt_t pkt;
- IFCQ_LOCK_ASSERT_HELD(ifq);
- if ((ptype == QP_MBUF) && !(((mbuf_t)p)->m_flags & M_PKTHDR)) {
- IFCQ_CONVERT_LOCK(ifq);
- m_freem((mbuf_t)p);
- *pdrop = TRUE;
- return ENOBUFS;
- }
- pktsched_pkt_encap(&pkt, ptype, p);
+ pktsched_pkt_encap_chain(&pkt, head, tail, cnt, bytes);
fqs = (fq_if_t *)ifq->ifcq_disc;
svc = pktsched_get_pkt_svc(&pkt);
pri = fq_if_service_to_priority(fqs, svc);
- VERIFY(pri >= 0 && pri < FQ_IF_MAX_CLASSES);
+ VERIFY(pri < FQ_IF_MAX_CLASSES);
fq_cl = &fqs->fqs_classq[pri];
- if (svc == MBUF_SC_BK_SYS && fqs->fqs_throttle == 1) {
+ if (__improbable(svc == MBUF_SC_BK_SYS && fqs->fqs_throttle == 1)) {
/* BK_SYS is currently throttled */
- fq_cl->fcl_stat.fcl_throttle_drops++;
- IFCQ_CONVERT_LOCK(ifq);
+ atomic_add_32(&fq_cl->fcl_stat.fcl_throttle_drops, 1);
pktsched_free_pkt(&pkt);
*pdrop = TRUE;
- return EQSUSPENDED;
+ ret = EQSUSPENDED;
+ goto done;
}
- len = pktsched_get_pkt_len(&pkt);
+ IFCQ_LOCK_SPIN(ifq);
ret = fq_addq(fqs, &pkt, fq_cl);
if (!(fqs->fqs_flags & FQS_DRIVER_MANAGED) &&
!FQ_IF_CLASSQ_IDLE(fq_cl)) {
}
}
- if (ret != 0) {
+ if (__improbable(ret != 0)) {
if (ret == CLASSQEQ_SUCCESS_FC) {
/* packet enqueued, return advisory feedback */
ret = EQFULL;
*pdrop = FALSE;
+ } else if (ret == CLASSQEQ_COMPRESSED) {
+ ret = 0;
+ *pdrop = FALSE;
} else {
+ IFCQ_UNLOCK(ifq);
*pdrop = TRUE;
- VERIFY(ret == CLASSQEQ_DROP ||
- ret == CLASSQEQ_DROP_FC ||
- ret == CLASSQEQ_DROP_SP);
pktsched_free_pkt(&pkt);
switch (ret) {
case CLASSQEQ_DROP:
- return ENOBUFS;
+ ret = ENOBUFS;
+ goto done;
case CLASSQEQ_DROP_FC:
- return EQFULL;
+ ret = EQFULL;
+ goto done;
case CLASSQEQ_DROP_SP:
- return EQSUSPENDED;
+ ret = EQSUSPENDED;
+ goto done;
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ __builtin_unreachable();
}
+ /* NOTREACHED */
+ __builtin_unreachable();
}
} else {
*pdrop = FALSE;
}
- IFCQ_INC_LEN(ifq);
- IFCQ_INC_BYTES(ifq, len);
+ IFCQ_ADD_LEN(ifq, cnt);
+ IFCQ_INC_BYTES(ifq, bytes);
+ IFCQ_UNLOCK(ifq);
+done:
return ret;
}
-static void *
-fq_if_dequeue_classq(struct ifclassq *ifq, classq_pkt_type_t *ptype)
+void
+fq_if_dequeue_classq(struct ifclassq *ifq, classq_pkt_t *pkt)
{
- void *top;
-
(void) fq_if_dequeue_classq_multi(ifq, 1,
- CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, &top, NULL, NULL, NULL, ptype);
- return top;
+ CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, pkt, NULL, NULL, NULL);
}
-static void *
+void
fq_if_dequeue_sc_classq(struct ifclassq *ifq, mbuf_svc_class_t svc,
- classq_pkt_type_t *ptype)
+ classq_pkt_t *pkt)
{
- void *top;
fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc;
+ uint32_t total_pktcnt = 0, total_bytecnt = 0;
fq_if_classq_t *fq_cl;
- u_int32_t pri;
+ uint8_t pri;
pri = fq_if_service_to_priority(fqs, svc);
fq_cl = &fqs->fqs_classq[pri];
fq_if_dequeue(fqs, fq_cl, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT,
- &top, NULL, NULL, NULL, TRUE, ptype);
- return top;
+ pkt, NULL, &total_pktcnt, &total_bytecnt, TRUE);
+
+ IFCQ_XMIT_ADD(ifq, total_pktcnt, total_bytecnt);
}
int
fq_if_dequeue_classq_multi(struct ifclassq *ifq, u_int32_t maxpktcnt,
- u_int32_t maxbytecnt, void **first_packet,
- void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt,
- classq_pkt_type_t *ptype)
+ u_int32_t maxbytecnt, classq_pkt_t *first_packet,
+ classq_pkt_t *last_packet, u_int32_t *retpktcnt,
+ u_int32_t *retbytecnt)
{
- void *top = NULL, *tail = NULL, *first, *last;
- u_int32_t pktcnt = 0, bytecnt = 0, total_pktcnt, total_bytecnt;
- fq_if_t *fqs;
+ u_int32_t pktcnt = 0, bytecnt = 0, total_pktcnt = 0, total_bytecnt = 0;
+ classq_pkt_t first = CLASSQ_PKT_INITIALIZER(fisrt);
+ classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
+ classq_pkt_t tmp = CLASSQ_PKT_INITIALIZER(tmp);
+ fq_if_append_pkt_t append_pkt;
fq_if_classq_t *fq_cl;
+ fq_if_t *fqs;
int pri;
- fq_if_append_pkt_t append_pkt;
IFCQ_LOCK_ASSERT_HELD(ifq);
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
- first = last = NULL;
- total_pktcnt = total_bytecnt = 0;
- *ptype = fqs->fqs_ptype;
-
for (;;) {
- classq_pkt_type_t tmp_ptype;
+ classq_pkt_t top = CLASSQ_PKT_INITIALIZER(top);
+ classq_pkt_t tail = CLASSQ_PKT_INITIALIZER(tail);
+
if (fqs->fqs_bitmaps[FQ_IF_ER] == 0 &&
fqs->fqs_bitmaps[FQ_IF_EB] == 0) {
fqs->fqs_bitmaps[FQ_IF_EB] = fqs->fqs_bitmaps[FQ_IF_IB];
}
fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt),
(maxbytecnt - total_bytecnt), &top, &tail, &pktcnt,
- &bytecnt, FALSE, &tmp_ptype);
- if (top != NULL) {
- ASSERT(tmp_ptype == *ptype);
+ &bytecnt, FALSE);
+ if (top.cp_mbuf != NULL) {
ASSERT(pktcnt > 0 && bytecnt > 0);
- if (first == NULL) {
+ if (first.cp_mbuf == NULL) {
first = top;
- last = tail;
total_pktcnt = pktcnt;
total_bytecnt = bytecnt;
} else {
- append_pkt(last, top);
- last = tail;
+ ASSERT(last.cp_mbuf != NULL);
+ append_pkt(&last, &top);
total_pktcnt += pktcnt;
total_bytecnt += bytecnt;
}
- append_pkt(last, NULL);
+ last = tail;
+ append_pkt(&last, &tmp);
fq_cl->fcl_budget -= bytecnt;
pktcnt = 0;
bytecnt = 0;
break;
}
}
- if (first != NULL) {
- if (first_packet != NULL) {
- *first_packet = first;
- }
- if (last_packet != NULL) {
- *last_packet = last;
- }
- if (retpktcnt != NULL) {
- *retpktcnt = total_pktcnt;
- }
- if (retbytecnt != NULL) {
- *retbytecnt = total_bytecnt;
- }
- IFCQ_XMIT_ADD(ifq, total_pktcnt, total_bytecnt);
- } else {
- if (first_packet != NULL) {
- *first_packet = NULL;
- }
- if (last_packet != NULL) {
- *last_packet = NULL;
- }
- if (retpktcnt != NULL) {
- *retpktcnt = 0;
- }
- if (retbytecnt != NULL) {
- *retbytecnt = 0;
- }
+
+ if (__probable(first_packet != NULL)) {
+ *first_packet = first;
+ }
+ if (last_packet != NULL) {
+ *last_packet = last;
+ }
+ if (retpktcnt != NULL) {
+ *retpktcnt = total_pktcnt;
+ }
+ if (retbytecnt != NULL) {
+ *retbytecnt = total_bytecnt;
}
+
+ IFCQ_XMIT_ADD(ifq, total_pktcnt, total_bytecnt);
return 0;
}
int
fq_if_dequeue_sc_classq_multi(struct ifclassq *ifq, mbuf_svc_class_t svc,
- u_int32_t maxpktcnt, u_int32_t maxbytecnt, void **first_packet,
- void **last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt,
- classq_pkt_type_t *ptype)
+ u_int32_t maxpktcnt, u_int32_t maxbytecnt, classq_pkt_t *first_packet,
+ classq_pkt_t *last_packet, u_int32_t *retpktcnt, u_int32_t *retbytecnt)
{
-#pragma unused(maxpktcnt, maxbytecnt, first_packet, last_packet, retpktcnt, retbytecnt)
fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc;
- u_int32_t pri;
+ uint8_t pri;
u_int32_t total_pktcnt = 0, total_bytecnt = 0;
fq_if_classq_t *fq_cl;
- void *first = NULL, *last = NULL;
+ classq_pkt_t first = CLASSQ_PKT_INITIALIZER(fisrt);
+ classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
fq_if_append_pkt_t append_pkt;
switch (fqs->fqs_ptype) {
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
pri = fq_if_service_to_priority(fqs, svc);
fq_cl = &fqs->fqs_classq[pri];
-
/*
* Now we have the queue for a particular service class. We need
* to dequeue as many packets as needed, first from the new flows
*/
while (total_pktcnt < maxpktcnt && total_bytecnt < maxbytecnt &&
fq_cl->fcl_stat.fcl_pkt_cnt > 0) {
- void *top, *tail;
+ classq_pkt_t top = CLASSQ_PKT_INITIALIZER(top);
+ classq_pkt_t tail = CLASSQ_PKT_INITIALIZER(tail);
u_int32_t pktcnt = 0, bytecnt = 0;
+
fq_if_dequeue(fqs, fq_cl, (maxpktcnt - total_pktcnt),
(maxbytecnt - total_bytecnt), &top, &tail, &pktcnt,
- &bytecnt, TRUE, ptype);
- if (first == NULL) {
- first = top;
- total_pktcnt = pktcnt;
- total_bytecnt = bytecnt;
- } else {
- append_pkt(last, top);
- total_pktcnt += pktcnt;
- total_bytecnt += bytecnt;
+ &bytecnt, TRUE);
+ if (top.cp_mbuf != NULL) {
+ if (first.cp_mbuf == NULL) {
+ first = top;
+ total_pktcnt = pktcnt;
+ total_bytecnt = bytecnt;
+ } else {
+ ASSERT(last.cp_mbuf != NULL);
+ append_pkt(&last, &top);
+ total_pktcnt += pktcnt;
+ total_bytecnt += bytecnt;
+ }
+ last = tail;
}
- last = tail;
}
- if (first != NULL) {
- if (first_packet != NULL) {
- *first_packet = first;
- }
- if (last_packet != NULL) {
- *last_packet = last;
- }
- if (retpktcnt != NULL) {
- *retpktcnt = total_pktcnt;
- }
- if (retbytecnt != NULL) {
- *retbytecnt = total_bytecnt;
- }
- } else {
- if (first_packet != NULL) {
- *first_packet = NULL;
- }
- if (last_packet != NULL) {
- *last_packet = NULL;
- }
- if (retpktcnt != NULL) {
- *retpktcnt = 0;
- }
- if (retbytecnt != NULL) {
- *retbytecnt = 0;
- }
+
+ if (__probable(first_packet != NULL)) {
+ *first_packet = first;
+ }
+ if (last_packet != NULL) {
+ *last_packet = last;
}
+ if (retpktcnt != NULL) {
+ *retpktcnt = total_pktcnt;
+ }
+ if (retbytecnt != NULL) {
+ *retbytecnt = total_bytecnt;
+ }
+
+ IFCQ_XMIT_ADD(ifq, total_pktcnt, total_bytecnt);
+
return 0;
}
fq_cl = &fqs->fqs_classq[fq->fq_sc_index];
pkts = bytes = 0;
_PKTSCHED_PKT_INIT(&pkt);
- while (fq_getq_flow(fqs, fq, &pkt) != NULL) {
+ for (;;) {
+ fq_getq_flow(fqs, fq, &pkt);
+ if (pkt.pktsched_pkt_mbuf == NULL) {
+ VERIFY(pkt.pktsched_ptype == QP_INVALID);
+ break;
+ }
pkts++;
bytes += pktsched_get_pkt_len(&pkt);
pktsched_free_pkt(&pkt);
fq_if_throttle(fq_if_t *fqs, cqrq_throttle_t *tr)
{
struct ifclassq *ifq = fqs->fqs_ifq;
- int index;
+ uint8_t index;
#if !MACH_ASSERT
#pragma unused(ifq)
#endif
void
fq_if_stat_sc(fq_if_t *fqs, cqrq_stat_sc_t *stat)
{
- u_int32_t pri;
+ uint8_t pri;
fq_if_classq_t *fq_cl;
if (stat == NULL) {
pri = fq_if_service_to_priority(fqs, stat->sc);
fq_cl = &fqs->fqs_classq[pri];
- stat->packets = fq_cl->fcl_stat.fcl_pkt_cnt;
- stat->bytes = fq_cl->fcl_stat.fcl_byte_cnt;
+ stat->packets = (uint32_t)fq_cl->fcl_stat.fcl_pkt_cnt;
+ stat->bytes = (uint32_t)fq_cl->fcl_stat.fcl_byte_cnt;
}
int
8, MBUF_SC_CTL);
}
- err = ifclassq_attach(ifq, PKTSCHEDT_FQ_CODEL, fqs,
- fq_if_enqueue_classq, fq_if_dequeue_classq,
- fq_if_dequeue_sc_classq, fq_if_dequeue_classq_multi,
- fq_if_dequeue_sc_classq_multi, fq_if_request_classq);
+ err = ifclassq_attach(ifq, PKTSCHEDT_FQ_CODEL, fqs);
if (err != 0) {
printf("%s: error from ifclassq_attach, "
fq_t *fq = fqs->fqs_large_flow;
fq_if_classq_t *fq_cl;
pktsched_pkt_t pkt;
- uint32_t *pkt_flags;
+ volatile uint32_t *pkt_flags;
uint64_t *pkt_timestamp;
if (fq == NULL) {
fq_cl = &fqs->fqs_classq[fq->fq_sc_index];
_PKTSCHED_PKT_INIT(&pkt);
- (void)fq_getq_flow_internal(fqs, fq, &pkt);
+ fq_getq_flow_internal(fqs, fq, &pkt);
+ ASSERT(pkt.pktsched_ptype != QP_INVALID);
pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL,
NULL, NULL);
IFCQ_CONVERT_LOCK(fqs->fqs_ifq);
*pkt_timestamp = 0;
- if (pkt.pktsched_ptype == QP_MBUF) {
+ switch (pkt.pktsched_ptype) {
+ case QP_MBUF:
*pkt_flags &= ~PKTF_PRIV_GUARDED;
+ break;
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ __builtin_unreachable();
}
if (fq_empty(fq)) {
}
void
-fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, u_int32_t pktlimit,
- u_int32_t bytelimit, void **top, void **tail,
- u_int32_t *retpktcnt, u_int32_t *retbytecnt, boolean_t drvmgmt,
- classq_pkt_type_t *ptype)
+fq_if_dequeue(fq_if_t *fqs, fq_if_classq_t *fq_cl, uint32_t pktlimit,
+ int64_t bytelimit, classq_pkt_t *top, classq_pkt_t *tail,
+ uint32_t *retpktcnt, uint32_t *retbytecnt, boolean_t drvmgmt)
{
fq_t *fq = NULL, *tfq = NULL;
flowq_stailq_t temp_stailq;
u_int32_t pktcnt, bytecnt;
boolean_t qempty, limit_reached = FALSE;
- void *last = NULL;
+ classq_pkt_t last = CLASSQ_PKT_INITIALIZER(last);
fq_getq_flow_t fq_getq_flow_fn;
switch (fqs->fqs_ptype) {
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
/*
* maximum byte limit should not be greater than the budget for
* this class
*/
- if ((int32_t)bytelimit > fq_cl->fcl_budget && !drvmgmt) {
+ if (bytelimit > fq_cl->fcl_budget && !drvmgmt) {
bytelimit = fq_cl->fcl_budget;
}
VERIFY(pktlimit > 0 && bytelimit > 0 && top != NULL);
-
- *top = NULL;
- *ptype = fqs->fqs_ptype;
pktcnt = bytecnt = 0;
STAILQ_INIT(&temp_stailq);
fq_cl->fcl_old_flows = temp_stailq;
}
- if (last != NULL) {
- VERIFY(*top != NULL);
+ if (last.cp_mbuf != NULL) {
+ VERIFY(top->cp_mbuf != NULL);
if (tail != NULL) {
*tail = last;
}
}
}
-int
+void
fq_if_teardown_ifclassq(struct ifclassq *ifq)
{
fq_if_t *fqs = (fq_if_t *)ifq->ifcq_disc;
fq_if_destroy(fqs);
ifq->ifcq_disc = NULL;
- return ifclassq_detach(ifq);
+ ifclassq_detach(ifq);
}
static void
struct fq_codel_flowstats *flowstat)
{
bzero(flowstat, sizeof(*flowstat));
- flowstat->fqst_min_qdelay = fq->fq_min_qdelay;
+ flowstat->fqst_min_qdelay = (uint32_t)fq->fq_min_qdelay;
flowstat->fqst_bytes = fq->fq_bytes;
flowstat->fqst_flowhash = fq->fq_flowhash;
if (fq->fq_flags & FQF_NEW_FLOW) {
fcls->fcls_throttle_off = fq_cl->fcl_stat.fcl_throttle_off;
fcls->fcls_throttle_drops = fq_cl->fcl_stat.fcl_throttle_drops;
fcls->fcls_dup_rexmts = fq_cl->fcl_stat.fcl_dup_rexmts;
+ fcls->fcls_pkts_compressible = fq_cl->fcl_stat.fcl_pkts_compressible;
+ fcls->fcls_pkts_compressed = fq_cl->fcl_stat.fcl_pkts_compressed;
/* Gather per flow stats */
flowstat_cnt = min((fcls->fcls_newflows_cnt +
projects / apple / xnu.git / blobdiff