/*
- * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
+ * Copyright (c) 2007-2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <libkern/libkern.h>
-u_int32_t classq_verbose = 0; /* more noise if greater than 1 */
+u_int32_t classq_verbose = 0; /* more noise if greater than 1 */
-SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "classq");
+SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "classq");
-SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED,
- &classq_verbose, 0, "Class queue verbosity level");
+SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
+ &classq_verbose, 0, "Class queue verbosity level");
void
_qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype)
/* add a packet at the tail of the queue */
void
-_addq(class_queue_t *q, void *pkt)
+_addq(class_queue_t *q, classq_pkt_t *pkt)
{
uint32_t size = 0;
+ ASSERT(pkt->cp_ptype == qptype(q));
+
switch (qptype(q)) {
case QP_MBUF: {
- struct mbuf *m = pkt;
+ struct mbuf *m = pkt->cp_mbuf;
MBUFQ_ENQUEUE(&qmbufq(q), m);
size = m_length(m);
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
qlen(q)++;
/* add one or more packets at the tail of the queue */
void
-_addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail,
+_addq_multi(class_queue_t *q, classq_pkt_t *pkt_head, classq_pkt_t *pkt_tail,
u_int32_t cnt, u_int32_t size)
{
+ ASSERT(pkt_head->cp_ptype == qptype(q));
+ ASSERT(pkt_tail->cp_ptype == qptype(q));
switch (qptype(q)) {
case QP_MBUF: {
- struct mbuf *m_head = pkt_head;
- struct mbuf *m_tail = pkt_tail;
+ struct mbuf *m_head = pkt_head->cp_mbuf;
+ struct mbuf *m_tail = pkt_tail->cp_mbuf;
MBUFQ_ENQUEUE_MULTI(&qmbufq(q), m_head, m_tail);
break;
}
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
qlen(q) += cnt;
}
/* get a packet at the head of the queue */
-void *
-_getq(class_queue_t *q)
+void
+_getq(class_queue_t *q, classq_pkt_t *pkt)
{
- void *pkt = NULL;
uint32_t pkt_len;
switch (qptype(q)) {
case QP_MBUF: {
- struct mbuf *m;
- MBUFQ_DEQUEUE(&qmbufq(q), m);
- if (m != NULL) {
- pkt_len = m_length(m);
- pkt = m;
+ MBUFQ_DEQUEUE(&qmbufq(q), pkt->cp_mbuf);
+ if (__probable(pkt->cp_mbuf != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+ pkt_len = m_length(pkt->cp_mbuf);
}
break;
}
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
- if (pkt == NULL) {
+ if (pkt->cp_mbuf == NULL) {
VERIFY(qlen(q) == 0);
- if (qsize(q) > 0)
+ if (qsize(q) > 0) {
qsize(q) = 0;
- return (NULL);
+ }
+ return;
}
VERIFY(qlen(q) > 0);
qlen(q)--;
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - pkt_len) > 0)
+ if (((int)qsize(q) - pkt_len) > 0) {
qsize(q) -= pkt_len;
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
-
- return (pkt);
+ }
}
-static void *
-_getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid)
+static void
+_getq_flow_or_scidx(class_queue_t *q, classq_pkt_t *pkt, u_int32_t val,
+ boolean_t isflowid)
{
- void *pkt = NULL;
uint32_t pkt_len;
switch (qptype(q)) {
break;
}
}
- if (m != NULL) {
- pkt = m;
+ if (__probable(m != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(pkt, m);
pkt_len = m_length(m);
}
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
- if (pkt != NULL) {
+ if (pkt->cp_mbuf != NULL) {
VERIFY(qlen(q) > 0);
qlen(q)--;
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - pkt_len) > 0)
+ if (((int)qsize(q) - pkt_len) > 0) {
qsize(q) -= pkt_len;
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
+ }
}
-
- return (pkt);
}
/* get a packet of a specific flow beginning from the head of the queue */
-void *
-_getq_flow(class_queue_t *q, u_int32_t flow)
+void
+_getq_flow(class_queue_t *q, classq_pkt_t *pkt, u_int32_t flow)
{
- return (_getq_flow_or_scidx(q, flow, TRUE));
+ return _getq_flow_or_scidx(q, pkt, flow, TRUE);
}
/* Get a packet whose MBUF_SCIDX() < scidx from head of queue */
-void *
-_getq_scidx_lt(class_queue_t *q, u_int32_t scidx)
+void
+_getq_scidx_lt(class_queue_t *q, classq_pkt_t *pkt, u_int32_t scidx)
{
- return (_getq_flow_or_scidx(q, scidx, FALSE));
+ return _getq_flow_or_scidx(q, pkt, scidx, FALSE);
}
/* get all packets (chained) starting from the head of the queue */
-void *
-_getq_all(class_queue_t *q, void **last, u_int32_t *qlenp,
- u_int64_t *qsizep)
+void
+_getq_all(class_queue_t *q, classq_pkt_t *first, classq_pkt_t *last,
+ u_int32_t *qlenp, u_int64_t *qsizep)
{
- void *pkt = NULL;
-
switch (qptype(q)) {
case QP_MBUF:
- pkt = MBUFQ_FIRST(&qmbufq(q));
- if (last != NULL)
- *last = MBUFQ_LAST(&qmbufq(q));
+ first->cp_mbuf = MBUFQ_FIRST(&qmbufq(q));
+ if (__probable(first->cp_mbuf != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(first, first->cp_mbuf);
+ }
+ if (last != NULL) {
+ last->cp_mbuf = MBUFQ_LAST(&qmbufq(q));
+ if (__probable(last->cp_mbuf != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(last, last->cp_mbuf);
+ }
+ }
MBUFQ_INIT(&qmbufq(q));
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
- if (qlenp != NULL)
+ if (qlenp != NULL) {
*qlenp = qlen(q);
- if (qsizep != NULL)
+ }
+ if (qsizep != NULL) {
*qsizep = qsize(q);
+ }
qlen(q) = 0;
qsize(q) = 0;
-
- return (pkt);
}
static inline struct mbuf *
--qlen(q);
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - m_length(m)) > 0)
+ if (((int)qsize(q) - m_length(m)) > 0) {
qsize(q) -= m_length(m);
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
+ }
if (qempty(q)) {
VERIFY(m == MBUFQ_FIRST(head));
head->mq_last = &MBUFQ_NEXT(n);
}
}
- return (m);
+ return m;
}
/* drop a packet at the tail of the queue */
-void *
-_getq_tail(class_queue_t *q)
+void
+_getq_tail(class_queue_t *q, classq_pkt_t *pkt)
{
- void *t = NULL;
-
switch (qptype(q)) {
case QP_MBUF:
- t = _getq_tail_mbuf(q);
+ pkt->cp_mbuf = _getq_tail_mbuf(q);
+ if (__probable(pkt->cp_mbuf != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+ }
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
-
- return (t);
}
static inline struct mbuf *
n = qlen(q);
if (n == 0) {
VERIFY(MBUFQ_EMPTY(head));
- if (qsize(q) > 0)
+ if (qsize(q) > 0) {
qsize(q) = 0;
- return (NULL);
+ }
+ return NULL;
}
m = MBUFQ_FIRST(head);
- read_frandom(&rnd, sizeof (rnd));
+ read_frandom(&rnd, sizeof(rnd));
n = (rnd % n) + 1;
if (n == 1) {
- if ((MBUFQ_FIRST(head) = MBUFQ_NEXT(m)) == NULL)
+ if ((MBUFQ_FIRST(head) = MBUFQ_NEXT(m)) == NULL) {
(head)->mq_last = &MBUFQ_FIRST(head);
+ }
} else {
struct mbuf *p = NULL;
VERIFY(n > 1);
while (n--) {
- if (MBUFQ_NEXT(m) == NULL)
+ if (MBUFQ_NEXT(m) == NULL) {
break;
+ }
p = m;
m = MBUFQ_NEXT(m);
}
VERIFY(p != NULL && MBUFQ_NEXT(p) == m);
- if ((MBUFQ_NEXT(p) = MBUFQ_NEXT(m)) == NULL)
+ if ((MBUFQ_NEXT(p) = MBUFQ_NEXT(m)) == NULL) {
(head)->mq_last = &MBUFQ_NEXT(p);
+ }
}
VERIFY(qlen(q) > 0);
--qlen(q);
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - m_length(m)) > 0)
+ if (((int)qsize(q) - m_length(m)) > 0) {
qsize(q) -= m_length(m);
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
+ }
MBUFQ_NEXT(m) = NULL;
- return (m);
+ return m;
}
/* randomly select a packet in the queue */
-void *
-_getq_random(class_queue_t *q)
+void
+_getq_random(class_queue_t *q, classq_pkt_t *pkt)
{
- void *r = NULL;
-
switch (qptype(q)) {
case QP_MBUF:
- r = _getq_random_mbuf(q);
+ pkt->cp_mbuf = _getq_random_mbuf(q);
+ if (__probable(pkt->cp_mbuf != NULL)) {
+ CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+ }
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
-
- return (r);
}
static inline void
struct mbuf *m0, **mtail;
m0 = MBUFQ_FIRST(head);
- if (m0 == NULL)
+ if (m0 == NULL) {
return;
+ }
if (m0 != m) {
- while (MBUFQ_NEXT(m0) != m) {
- if (m0 == NULL)
- return;
+ while (m0 != NULL && MBUFQ_NEXT(m0) != m) {
m0 = MBUFQ_NEXT(m0);
}
+ if (m0 == NULL) {
+ return;
+ }
+
mtail = &MBUFQ_NEXT(m0);
} else {
mtail = &MBUFQ_FIRST(head);
}
*mtail = MBUFQ_NEXT(m);
- if (*mtail == NULL)
+ if (*mtail == NULL) {
head->mq_last = mtail;
+ }
VERIFY(qlen(q) > 0);
--qlen(q);
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - m_length(m)) > 0)
+ if (((int)qsize(q) - m_length(m)) > 0) {
qsize(q) -= m_length(m);
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
+ }
MBUFQ_NEXT(m) = NULL;
}
/* remove a packet from the queue */
void
-_removeq(class_queue_t *q, void *pkt)
+_removeq(class_queue_t *q, classq_pkt_t *pkt)
{
switch (qptype(q)) {
case QP_MBUF:
- _removeq_mbuf(q, pkt);
+ ASSERT(pkt->cp_ptype == QP_MBUF);
+ _removeq_mbuf(q, pkt->cp_mbuf);
break;
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
}
qlen(q) -= c;
/* qsize is an approximation, so adjust if necessary */
- if (((int)qsize(q) - l) > 0)
+ if (((int)qsize(q) - l) > 0) {
qsize(q) -= l;
- else if (qsize(q) != 0)
+ } else if (qsize(q) != 0) {
qsize(q) = 0;
+ }
}
- if (!MBUFQ_EMPTY(&freeq))
+ if (!MBUFQ_EMPTY(&freeq)) {
m_freem_list(MBUFQ_FIRST(&freeq));
+ }
- if (cnt != NULL)
+ if (cnt != NULL) {
*cnt = c;
- if (len != NULL)
+ }
+ if (len != NULL) {
*len = l;
+ }
}
+
void
_flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len)
{
default:
VERIFY(0);
/* NOTREACHED */
+ __builtin_unreachable();
}
}
projects / apple / xnu.git / blobdiff