xnu-6153.41.3.tar.gz

[apple/xnu.git] / bsd / net / classq / classq.c

diff --git a/bsd/net/classq/classq.c b/bsd/net/classq/classq.c
index 35d86188fd8907c40335ecfee1d60687b4910218..dad2552c3212792de985f072d7464efa17414da8 100644 (file)
--- a/bsd/net/classq/classq.c
+++ b/bsd/net/classq/classq.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
+ * Copyright (c) 2007-2019 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -74,12 +74,12 @@
 #include <libkern/libkern.h>
 
 
 #include <libkern/libkern.h>
 
 

-u_int32_t classq_verbose = 0;  /* more noise if greater than 1 */
+u_int32_t classq_verbose = 0;   /* more noise if greater than 1 */

 
 

-SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "classq");
+SYSCTL_NODE(_net, OID_AUTO, classq, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "classq");

 
 

-SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW|CTLFLAG_LOCKED,
-       &classq_verbose, 0, "Class queue verbosity level");
+SYSCTL_UINT(_net_classq, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
+    &classq_verbose, 0, "Class queue verbosity level");

 
 void
 _qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype)
 
 void
 _qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype)


@@ -105,13 +105,15 @@ _qinit(class_queue_t *q, int type, int lim, classq_pkt_type_t ptype)
 
 /* add a packet at the tail of the queue */
 void
 
 /* add a packet at the tail of the queue */
 void

-_addq(class_queue_t *q, void *pkt)
+_addq(class_queue_t *q, classq_pkt_t *pkt)

 {
        uint32_t size = 0;
 
 {
        uint32_t size = 0;
 

+       ASSERT(pkt->cp_ptype == qptype(q));
+

        switch (qptype(q)) {
        case QP_MBUF: {
        switch (qptype(q)) {
        case QP_MBUF: {

-               struct mbuf *m = pkt;
+               struct mbuf *m = pkt->cp_mbuf;

                MBUFQ_ENQUEUE(&qmbufq(q), m);
                size = m_length(m);
                break;
                MBUFQ_ENQUEUE(&qmbufq(q), m);
                size = m_length(m);
                break;


@@ -121,6 +123,7 @@ _addq(class_queue_t *q, void *pkt)
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        qlen(q)++;
        }
 
        qlen(q)++;


@@ -130,13 +133,15 @@ _addq(class_queue_t *q, void *pkt)
 
 /* add one or more packets at the tail of the queue */
 void
 
 /* add one or more packets at the tail of the queue */
 void

-_addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail,
+_addq_multi(class_queue_t *q, classq_pkt_t *pkt_head, classq_pkt_t *pkt_tail,

     u_int32_t cnt, u_int32_t size)
 {
     u_int32_t cnt, u_int32_t size)
 {

+       ASSERT(pkt_head->cp_ptype == qptype(q));
+       ASSERT(pkt_tail->cp_ptype == qptype(q));

        switch (qptype(q)) {
        case QP_MBUF: {
        switch (qptype(q)) {
        case QP_MBUF: {

-               struct mbuf *m_head = pkt_head;
-               struct mbuf *m_tail = pkt_tail;
+               struct mbuf *m_head = pkt_head->cp_mbuf;
+               struct mbuf *m_tail = pkt_tail->cp_mbuf;

                MBUFQ_ENQUEUE_MULTI(&qmbufq(q), m_head, m_tail);
                break;
        }
                MBUFQ_ENQUEUE_MULTI(&qmbufq(q), m_head, m_tail);
                break;
        }


@@ -145,6 +150,7 @@ _addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail,
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        qlen(q) += cnt;
        }
 
        qlen(q) += cnt;


@@ -152,19 +158,17 @@ _addq_multi(class_queue_t *q, void *pkt_head, void *pkt_tail,
 }
 
 /* get a packet at the head of the queue */
 }
 
 /* get a packet at the head of the queue */

-void *
-_getq(class_queue_t *q)
+void
+_getq(class_queue_t *q, classq_pkt_t *pkt)

 {
 {

-       void *pkt = NULL;

        uint32_t pkt_len;
 
        switch (qptype(q)) {
        case QP_MBUF: {
        uint32_t pkt_len;
 
        switch (qptype(q)) {
        case QP_MBUF: {

-               struct mbuf *m;
-               MBUFQ_DEQUEUE(&qmbufq(q), m);
-               if (m != NULL) {
-                       pkt_len = m_length(m);
-                       pkt = m;
+               MBUFQ_DEQUEUE(&qmbufq(q), pkt->cp_mbuf);
+               if (__probable(pkt->cp_mbuf != NULL)) {
+                       CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+                       pkt_len = m_length(pkt->cp_mbuf);

                }
                break;
        }
                }
                break;
        }


@@ -173,30 +177,31 @@ _getq(class_queue_t *q)
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        }
 

-       if (pkt == NULL) {
+       if (pkt->cp_mbuf == NULL) {

                VERIFY(qlen(q) == 0);
                VERIFY(qlen(q) == 0);

-               if (qsize(q) > 0)
+               if (qsize(q) > 0) {

                        qsize(q) = 0;
                        qsize(q) = 0;

-               return (NULL);
+               }
+               return;

        }
        VERIFY(qlen(q) > 0);
        qlen(q)--;
 
        /* qsize is an approximation, so adjust if necessary */
        }
        VERIFY(qlen(q) > 0);
        qlen(q)--;
 
        /* qsize is an approximation, so adjust if necessary */

-       if (((int)qsize(q) - pkt_len) > 0)
+       if (((int)qsize(q) - pkt_len) > 0) {

                qsize(q) -= pkt_len;
                qsize(q) -= pkt_len;

-       else if (qsize(q) != 0)
+       } else if (qsize(q) != 0) {

                qsize(q) = 0;
                qsize(q) = 0;

-
-       return (pkt);
+       }

 }
 
 }
 

-static void *
-_getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid)
+static void
+_getq_flow_or_scidx(class_queue_t *q, classq_pkt_t *pkt, u_int32_t val,
+    boolean_t isflowid)

 {
 {

-       void *pkt = NULL;

        uint32_t pkt_len;
 
        switch (qptype(q)) {
        uint32_t pkt_len;
 
        switch (qptype(q)) {


@@ -215,8 +220,8 @@ _getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid)
                                break;
                        }
                }
                                break;
                        }
                }

-               if (m != NULL) {
-                       pkt = m;
+               if (__probable(m != NULL)) {
+                       CLASSQ_PKT_INIT_MBUF(pkt, m);

                        pkt_len = m_length(m);
                }
                break;
                        pkt_len = m_length(m);
                }
                break;


@@ -226,48 +231,53 @@ _getq_flow_or_scidx(class_queue_t *q, u_int32_t val, boolean_t isflowid)
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        }
 

-       if (pkt != NULL) {
+       if (pkt->cp_mbuf != NULL) {

                VERIFY(qlen(q) > 0);
                qlen(q)--;
 
                /* qsize is an approximation, so adjust if necessary */
                VERIFY(qlen(q) > 0);
                qlen(q)--;
 
                /* qsize is an approximation, so adjust if necessary */

-               if (((int)qsize(q) - pkt_len) > 0)
+               if (((int)qsize(q) - pkt_len) > 0) {

                        qsize(q) -= pkt_len;
                        qsize(q) -= pkt_len;

-               else if (qsize(q) != 0)
+               } else if (qsize(q) != 0) {

                        qsize(q) = 0;
                        qsize(q) = 0;

+               }

        }
        }

-
-       return (pkt);

 }
 
 /* get a packet of a specific flow beginning from the head of the queue */
 }
 
 /* get a packet of a specific flow beginning from the head of the queue */

-void *
-_getq_flow(class_queue_t *q, u_int32_t flow)
+void
+_getq_flow(class_queue_t *q, classq_pkt_t *pkt, u_int32_t flow)

 {
 {

-       return (_getq_flow_or_scidx(q, flow, TRUE));
+       return _getq_flow_or_scidx(q, pkt, flow, TRUE);

 }
 
 /* Get a packet whose MBUF_SCIDX() < scidx from head of queue */
 }
 
 /* Get a packet whose MBUF_SCIDX() < scidx from head of queue */

-void *
-_getq_scidx_lt(class_queue_t *q, u_int32_t scidx)
+void
+_getq_scidx_lt(class_queue_t *q, classq_pkt_t *pkt, u_int32_t scidx)

 {
 {

-       return (_getq_flow_or_scidx(q, scidx, FALSE));
+       return _getq_flow_or_scidx(q, pkt, scidx, FALSE);

 }
 
 /* get all packets (chained) starting from the head of the queue */
 }
 
 /* get all packets (chained) starting from the head of the queue */

-void *
-_getq_all(class_queue_t *q, void **last, u_int32_t *qlenp,
-    u_int64_t *qsizep)
+void
+_getq_all(class_queue_t *q, classq_pkt_t *first, classq_pkt_t *last,
+    u_int32_t *qlenp, u_int64_t *qsizep)

 {
 {

-       void *pkt = NULL;
-

        switch (qptype(q)) {
        case QP_MBUF:
        switch (qptype(q)) {
        case QP_MBUF:

-               pkt = MBUFQ_FIRST(&qmbufq(q));
-               if (last != NULL)
-                       *last = MBUFQ_LAST(&qmbufq(q));
+               first->cp_mbuf = MBUFQ_FIRST(&qmbufq(q));
+               if (__probable(first->cp_mbuf != NULL)) {
+                       CLASSQ_PKT_INIT_MBUF(first, first->cp_mbuf);
+               }
+               if (last != NULL) {
+                       last->cp_mbuf = MBUFQ_LAST(&qmbufq(q));
+                       if (__probable(last->cp_mbuf != NULL)) {
+                               CLASSQ_PKT_INIT_MBUF(last, last->cp_mbuf);
+                       }
+               }

                MBUFQ_INIT(&qmbufq(q));
                break;
 
                MBUFQ_INIT(&qmbufq(q));
                break;
 


@@ -275,17 +285,18 @@ _getq_all(class_queue_t *q, void **last, u_int32_t *qlenp,
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 
        }
 

-       if (qlenp != NULL)
+       if (qlenp != NULL) {

                *qlenp = qlen(q);
                *qlenp = qlen(q);

-       if (qsizep != NULL)
+       }
+       if (qsizep != NULL) {

                *qsizep = qsize(q);
                *qsizep = qsize(q);

+       }

 
        qlen(q) = 0;
        qsize(q) = 0;
 
        qlen(q) = 0;
        qsize(q) = 0;

-
-       return (pkt);

 }
 
 static inline struct mbuf *
 }
 
 static inline struct mbuf *


@@ -311,10 +322,11 @@ _getq_tail_mbuf(class_queue_t *q)
                --qlen(q);
 
                /* qsize is an approximation, so adjust if necessary */
                --qlen(q);
 
                /* qsize is an approximation, so adjust if necessary */

-               if (((int)qsize(q) - m_length(m)) > 0)
+               if (((int)qsize(q) - m_length(m)) > 0) {

                        qsize(q) -= m_length(m);
                        qsize(q) -= m_length(m);

-               else if (qsize(q) != 0)
+               } else if (qsize(q) != 0) {

                        qsize(q) = 0;
                        qsize(q) = 0;

+               }

 
                if (qempty(q)) {
                        VERIFY(m == MBUFQ_FIRST(head));
 
                if (qempty(q)) {
                        VERIFY(m == MBUFQ_FIRST(head));


@@ -324,26 +336,26 @@ _getq_tail_mbuf(class_queue_t *q)
                        head->mq_last = &MBUFQ_NEXT(n);
                }
        }
                        head->mq_last = &MBUFQ_NEXT(n);
                }
        }

-       return (m);
+       return m;

 }
 
 /* drop a packet at the tail of the queue */
 }
 
 /* drop a packet at the tail of the queue */

-void *
-_getq_tail(class_queue_t *q)
+void
+_getq_tail(class_queue_t *q, classq_pkt_t *pkt)

 {
 {

-       void *t = NULL;
-

        switch (qptype(q)) {
        case QP_MBUF:
        switch (qptype(q)) {
        case QP_MBUF:

-               t = _getq_tail_mbuf(q);
+               pkt->cp_mbuf = _getq_tail_mbuf(q);
+               if (__probable(pkt->cp_mbuf != NULL)) {
+                       CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+               }

                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */
                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
        }

-
-       return (t);

 }
 
 static inline struct mbuf *
 }
 
 static inline struct mbuf *


@@ -360,65 +372,70 @@ _getq_random_mbuf(class_queue_t *q)
        n = qlen(q);
        if (n == 0) {
                VERIFY(MBUFQ_EMPTY(head));
        n = qlen(q);
        if (n == 0) {
                VERIFY(MBUFQ_EMPTY(head));

-               if (qsize(q) > 0)
+               if (qsize(q) > 0) {

                        qsize(q) = 0;
                        qsize(q) = 0;

-               return (NULL);
+               }
+               return NULL;

        }
 
        m = MBUFQ_FIRST(head);
        }
 
        m = MBUFQ_FIRST(head);

-       read_frandom(&rnd, sizeof (rnd));
+       read_frandom(&rnd, sizeof(rnd));

        n = (rnd % n) + 1;
 
        if (n == 1) {
        n = (rnd % n) + 1;
 
        if (n == 1) {

-               if ((MBUFQ_FIRST(head) = MBUFQ_NEXT(m)) == NULL)
+               if ((MBUFQ_FIRST(head) = MBUFQ_NEXT(m)) == NULL) {

                        (head)->mq_last = &MBUFQ_FIRST(head);
                        (head)->mq_last = &MBUFQ_FIRST(head);

+               }

        } else {
                struct mbuf *p = NULL;
 
                VERIFY(n > 1);
                while (n--) {
        } else {
                struct mbuf *p = NULL;
 
                VERIFY(n > 1);
                while (n--) {

-                       if (MBUFQ_NEXT(m) == NULL)
+                       if (MBUFQ_NEXT(m) == NULL) {

                                break;
                                break;

+                       }

                        p = m;
                        m = MBUFQ_NEXT(m);
                }
                VERIFY(p != NULL && MBUFQ_NEXT(p) == m);
 
                        p = m;
                        m = MBUFQ_NEXT(m);
                }
                VERIFY(p != NULL && MBUFQ_NEXT(p) == m);
 

-               if ((MBUFQ_NEXT(p) = MBUFQ_NEXT(m)) == NULL)
+               if ((MBUFQ_NEXT(p) = MBUFQ_NEXT(m)) == NULL) {

                        (head)->mq_last = &MBUFQ_NEXT(p);
                        (head)->mq_last = &MBUFQ_NEXT(p);

+               }

        }
 
        VERIFY(qlen(q) > 0);
        --qlen(q);
 
        /* qsize is an approximation, so adjust if necessary */
        }
 
        VERIFY(qlen(q) > 0);
        --qlen(q);
 
        /* qsize is an approximation, so adjust if necessary */

-       if (((int)qsize(q) - m_length(m)) > 0)
+       if (((int)qsize(q) - m_length(m)) > 0) {

                qsize(q) -= m_length(m);
                qsize(q) -= m_length(m);

-       else if (qsize(q) != 0)
+       } else if (qsize(q) != 0) {

                qsize(q) = 0;
                qsize(q) = 0;

+       }

 
        MBUFQ_NEXT(m) = NULL;
 
 
        MBUFQ_NEXT(m) = NULL;
 

-       return (m);
+       return m;

 }
 
 /* randomly select a packet in the queue */
 }
 
 /* randomly select a packet in the queue */

-void *
-_getq_random(class_queue_t *q)
+void
+_getq_random(class_queue_t *q, classq_pkt_t *pkt)

 {
 {

-       void *r = NULL;
-

        switch (qptype(q)) {
        case QP_MBUF:
        switch (qptype(q)) {
        case QP_MBUF:

-               r = _getq_random_mbuf(q);
+               pkt->cp_mbuf = _getq_random_mbuf(q);
+               if (__probable(pkt->cp_mbuf != NULL)) {
+                       CLASSQ_PKT_INIT_MBUF(pkt, pkt->cp_mbuf);
+               }

                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */
                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
        }

-
-       return (r);

 }
 
 static inline void
 }
 
 static inline void


@@ -428,48 +445,55 @@ _removeq_mbuf(class_queue_t *q, struct mbuf *m)
        struct mbuf *m0, **mtail;
 
        m0 = MBUFQ_FIRST(head);
        struct mbuf *m0, **mtail;
 
        m0 = MBUFQ_FIRST(head);

-       if (m0 == NULL)
+       if (m0 == NULL) {

                return;
                return;

+       }

 
        if (m0 != m) {
 
        if (m0 != m) {

-               while (MBUFQ_NEXT(m0) != m) {
-                       if (m0 == NULL)
-                               return;
+               while (m0 != NULL && MBUFQ_NEXT(m0) != m) {

                        m0 = MBUFQ_NEXT(m0);
                }
                        m0 = MBUFQ_NEXT(m0);
                }

+               if (m0 == NULL) {
+                       return;
+               }
+

                mtail = &MBUFQ_NEXT(m0);
        } else {
                mtail = &MBUFQ_FIRST(head);
        }
 
        *mtail = MBUFQ_NEXT(m);
                mtail = &MBUFQ_NEXT(m0);
        } else {
                mtail = &MBUFQ_FIRST(head);
        }
 
        *mtail = MBUFQ_NEXT(m);

-       if (*mtail == NULL)
+       if (*mtail == NULL) {

                head->mq_last = mtail;
                head->mq_last = mtail;

+       }

 
        VERIFY(qlen(q) > 0);
        --qlen(q);
 
        /* qsize is an approximation, so adjust if necessary */
 
        VERIFY(qlen(q) > 0);
        --qlen(q);
 
        /* qsize is an approximation, so adjust if necessary */

-       if (((int)qsize(q) - m_length(m)) > 0)
+       if (((int)qsize(q) - m_length(m)) > 0) {

                qsize(q) -= m_length(m);
                qsize(q) -= m_length(m);

-       else if (qsize(q) != 0)
+       } else if (qsize(q) != 0) {

                qsize(q) = 0;
                qsize(q) = 0;

+       }

 
        MBUFQ_NEXT(m) = NULL;
 }
 
 /* remove a packet from the queue */
 void
 
        MBUFQ_NEXT(m) = NULL;
 }
 
 /* remove a packet from the queue */
 void

-_removeq(class_queue_t *q, void *pkt)
+_removeq(class_queue_t *q, classq_pkt_t *pkt)

 {
        switch (qptype(q)) {
        case QP_MBUF:
 {
        switch (qptype(q)) {
        case QP_MBUF:

-               _removeq_mbuf(q, pkt);
+               ASSERT(pkt->cp_ptype == QP_MBUF);
+               _removeq_mbuf(q, pkt->cp_mbuf);

                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */
                break;
 
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 }
 
        }
 }
 


@@ -510,21 +534,26 @@ _flushq_flow_mbuf(class_queue_t *q, u_int32_t flow, u_int32_t *cnt,
                qlen(q) -= c;
 
                /* qsize is an approximation, so adjust if necessary */
                qlen(q) -= c;
 
                /* qsize is an approximation, so adjust if necessary */

-               if (((int)qsize(q) - l) > 0)
+               if (((int)qsize(q) - l) > 0) {

                        qsize(q) -= l;
                        qsize(q) -= l;

-               else if (qsize(q) != 0)
+               } else if (qsize(q) != 0) {

                        qsize(q) = 0;
                        qsize(q) = 0;

+               }

        }
 
        }
 

-       if (!MBUFQ_EMPTY(&freeq))
+       if (!MBUFQ_EMPTY(&freeq)) {

                m_freem_list(MBUFQ_FIRST(&freeq));
                m_freem_list(MBUFQ_FIRST(&freeq));

+       }

 
 

-       if (cnt != NULL)
+       if (cnt != NULL) {

                *cnt = c;
                *cnt = c;

-       if (len != NULL)
+       }
+       if (len != NULL) {

                *len = l;
                *len = l;

+       }

 }
 
 }
 

+

 void
 _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len)
 {
 void
 _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len)
 {


@@ -536,5 +565,6 @@ _flushq_flow(class_queue_t *q, u_int32_t flow, u_int32_t *cnt, u_int32_t *len)
        default:
                VERIFY(0);
                /* NOTREACHED */
        default:
                VERIFY(0);
                /* NOTREACHED */

+               __builtin_unreachable();

        }
 }
        }
 }