| | 1 | /* |
| | 2 | * Copyright (c) 2011-2018 Apple Inc. All rights reserved. |
| | 3 | * |
| | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
| | 5 | * |
| | 6 | * This file contains Original Code and/or Modifications of Original Code |
| | 7 | * as defined in and that are subject to the Apple Public Source License |
| | 8 | * Version 2.0 (the 'License'). You may not use this file except in |
| | 9 | * compliance with the License. The rights granted to you under the License |
| | 10 | * may not be used to create, or enable the creation or redistribution of, |
| | 11 | * unlawful or unlicensed copies of an Apple operating system, or to |
| | 12 | * circumvent, violate, or enable the circumvention or violation of, any |
| | 13 | * terms of an Apple operating system software license agreement. |
| | 14 | * |
| | 15 | * Please obtain a copy of the License at |
| | 16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
| | 17 | * |
| | 18 | * The Original Code and all software distributed under the License are |
| | 19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
| | 20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
| | 21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
| | 22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
| | 23 | * Please see the License for the specific language governing rights and |
| | 24 | * limitations under the License. |
| | 25 | * |
| | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
| | 27 | */ |
| | 28 | |
| | 29 | #ifndef _NET_CLASSQ_IF_CLASSQ_H_ |
| | 30 | #define _NET_CLASSQ_IF_CLASSQ_H_ |
| | 31 | |
| | 32 | #ifdef PRIVATE |
| | 33 | #define IFCQ_SC_MAX 10 /* max number of queues */ |
| | 34 | |
| | 35 | #ifdef BSD_KERNEL_PRIVATE |
| | 36 | #include <net/classq/classq.h> |
| | 37 | |
| | 38 | /* maximum number of packets stored across all queues */ |
| | 39 | #define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048 |
| | 40 | |
| | 41 | /* classq request types */ |
| | 42 | typedef enum cqrq { |
| | 43 | CLASSQRQ_PURGE = 1, /* purge all packets */ |
| | 44 | CLASSQRQ_PURGE_SC = 2, /* purge service class (and flow) */ |
| | 45 | CLASSQRQ_EVENT = 3, /* interface events */ |
| | 46 | CLASSQRQ_THROTTLE = 4, /* throttle packets */ |
| | 47 | CLASSQRQ_STAT_SC = 5, /* get service class queue stats */ |
| | 48 | } cqrq_t; |
| | 49 | |
| | 50 | /* classq purge_sc request argument */ |
| | 51 | typedef struct cqrq_purge_sc { |
| | 52 | mbuf_svc_class_t sc; /* (in) service class */ |
| | 53 | u_int32_t flow; /* (in) 0 means all flows */ |
| | 54 | u_int32_t packets; /* (out) purged packets */ |
| | 55 | u_int32_t bytes; /* (out) purged bytes */ |
| | 56 | } cqrq_purge_sc_t; |
| | 57 | |
| | 58 | /* classq throttle request argument */ |
| | 59 | typedef struct cqrq_throttle { |
| | 60 | u_int32_t set; /* set or get */ |
| | 61 | u_int32_t level; /* (in/out) throttling level */ |
| | 62 | } cqrq_throttle_t; |
| | 63 | |
| | 64 | /* classq service class stats request argument */ |
| | 65 | typedef struct cqrq_stat_sc { |
| | 66 | mbuf_svc_class_t sc; /* (in) service class */ |
| | 67 | u_int32_t packets; /* (out) packets enqueued */ |
| | 68 | u_int32_t bytes; /* (out) bytes enqueued */ |
| | 69 | } cqrq_stat_sc_t; |
| | 70 | |
| | 71 | /* |
| | 72 | * A token-bucket regulator limits the rate that a network driver can |
| | 73 | * dequeue packets from the output queue. Modern cards are able to buffer |
| | 74 | * a large amount of packets and dequeue too many packets at a time. This |
| | 75 | * bursty dequeue behavior makes it impossible to schedule packets by |
| | 76 | * queueing disciplines. A token-bucket is used to control the burst size |
| | 77 | * in a device independent manner. |
| | 78 | */ |
| | 79 | struct tb_regulator { |
| | 80 | u_int64_t tbr_rate_raw; /* (unscaled) token bucket rate */ |
| | 81 | u_int32_t tbr_percent; /* token bucket rate in percentage */ |
| | 82 | int64_t tbr_rate; /* (scaled) token bucket rate */ |
| | 83 | int64_t tbr_depth; /* (scaled) token bucket depth */ |
| | 84 | |
| | 85 | int64_t tbr_token; /* (scaled) current token */ |
| | 86 | int64_t tbr_filluptime; /* (scaled) time to fill up bucket */ |
| | 87 | u_int64_t tbr_last; /* last time token was updated */ |
| | 88 | |
| | 89 | /* needed for poll-and-dequeue */ |
| | 90 | }; |
| | 91 | |
| | 92 | /* simple token bucket meter profile */ |
| | 93 | struct tb_profile { |
| | 94 | u_int64_t rate; /* rate in bit-per-sec */ |
| | 95 | u_int32_t percent; /* rate in percentage */ |
| | 96 | u_int32_t depth; /* depth in bytes */ |
| | 97 | }; |
| | 98 | |
| | 99 | struct ifclassq; |
| | 100 | enum cqdq_op; |
| | 101 | enum cqrq; |
| | 102 | |
| | 103 | #if DEBUG || DEVELOPMENT |
| | 104 | extern uint32_t ifclassq_flow_control_adv; |
| | 105 | #endif /* DEBUG || DEVELOPMENT */ |
| | 106 | |
| | 107 | typedef int (*ifclassq_enq_func)(struct ifclassq *, classq_pkt_t *, |
| | 108 | boolean_t *); |
| | 109 | typedef void (*ifclassq_deq_func)(struct ifclassq *, classq_pkt_t *); |
| | 110 | typedef void (*ifclassq_deq_sc_func)(struct ifclassq *, mbuf_svc_class_t, |
| | 111 | classq_pkt_t *); |
| | 112 | typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t, |
| | 113 | u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *, u_int32_t *); |
| | 114 | typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *, |
| | 115 | mbuf_svc_class_t, u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, |
| | 116 | u_int32_t *, u_int32_t *); |
| | 117 | typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *); |
| | 118 | |
| | 119 | /* |
| | 120 | * Structure defining a queue for a network interface. |
| | 121 | */ |
| | 122 | struct ifclassq { |
| | 123 | decl_lck_mtx_data(, ifcq_lock); |
| | 124 | |
| | 125 | struct ifnet *ifcq_ifp; /* back pointer to interface */ |
| | 126 | u_int32_t ifcq_len; /* packet count */ |
| | 127 | u_int32_t ifcq_maxlen; |
| | 128 | struct pktcntr ifcq_xmitcnt; |
| | 129 | struct pktcntr ifcq_dropcnt; |
| | 130 | |
| | 131 | u_int32_t ifcq_type; /* scheduler type */ |
| | 132 | u_int32_t ifcq_flags; /* flags */ |
| | 133 | u_int32_t ifcq_sflags; /* scheduler flags */ |
| | 134 | u_int32_t ifcq_target_qdelay; /* target queue delay */ |
| | 135 | u_int32_t ifcq_bytes; /* bytes count */ |
| | 136 | u_int32_t ifcq_pkt_drop_limit; |
| | 137 | void *ifcq_disc; /* for scheduler-specific use */ |
| | 138 | /* |
| | 139 | * ifcq_disc_slots[] represents the leaf classes configured for the |
| | 140 | * corresponding discpline/scheduler, ordered by their corresponding |
| | 141 | * service class index. Each slot holds the queue ID used to identify |
| | 142 | * the class instance, as well as the class instance pointer itself. |
| | 143 | * The latter is used during enqueue and dequeue in order to avoid the |
| | 144 | * costs associated with looking up the class pointer based on the |
| | 145 | * queue ID. The queue ID is used when querying the statistics from |
| | 146 | * user space. |
| | 147 | * |
| | 148 | * Avoiding the use of queue ID during enqueue and dequeue is made |
| | 149 | * possible by virtue of knowing the particular mbuf service class |
| | 150 | * associated with the packets. The service class index of the |
| | 151 | * packet is used as the index to ifcq_disc_slots[]. |
| | 152 | * |
| | 153 | * ifcq_disc_slots[] therefore also acts as a lookup table which |
| | 154 | * provides for the mapping between MBUF_SC values and the actual |
| | 155 | * scheduler classes. |
| | 156 | */ |
| | 157 | struct ifclassq_disc_slot { |
| | 158 | u_int32_t qid; |
| | 159 | void *cl; |
| | 160 | } ifcq_disc_slots[IFCQ_SC_MAX]; /* for discipline use */ |
| | 161 | |
| | 162 | /* token bucket regulator */ |
| | 163 | struct tb_regulator ifcq_tbr; /* TBR */ |
| | 164 | }; |
| | 165 | |
| | 166 | /* ifcq_flags */ |
| | 167 | #define IFCQF_READY 0x01 /* ifclassq supports discipline */ |
| | 168 | #define IFCQF_ENABLED 0x02 /* ifclassq is in use */ |
| | 169 | #define IFCQF_TBR 0x04 /* Token Bucket Regulator is in use */ |
| | 170 | |
| | 171 | #define IFCQ_IS_READY(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_READY) |
| | 172 | #define IFCQ_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_ENABLED) |
| | 173 | #define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR) |
| | 174 | |
| | 175 | /* classq enqueue return value */ |
| | 176 | /* packet has to be dropped */ |
| | 177 | #define CLASSQEQ_DROP (-1) |
| | 178 | /* packet successfully enqueued */ |
| | 179 | #define CLASSQEQ_SUCCESS 0 |
| | 180 | /* packet enqueued; give flow control feedback */ |
| | 181 | #define CLASSQEQ_SUCCESS_FC 1 |
| | 182 | /* packet needs to be dropped due to flowcontrol; give flow control feedback */ |
| | 183 | #define CLASSQEQ_DROP_FC 2 |
| | 184 | /* packet needs to be dropped due to suspension; give flow control feedback */ |
| | 185 | #define CLASSQEQ_DROP_SP 3 |
| | 186 | /* packet has been compressed with another one */ |
| | 187 | #define CLASSQEQ_COMPRESSED 4 |
| | 188 | |
| | 189 | /* interface event argument for CLASSQRQ_EVENT */ |
| | 190 | typedef enum cqev { |
| | 191 | CLASSQ_EV_INIT = 0, |
| | 192 | CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */ |
| | 193 | CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */ |
| | 194 | CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */ |
| | 195 | CLASSQ_EV_LINK_UP = 4, /* link is now up */ |
| | 196 | CLASSQ_EV_LINK_DOWN = 5, /* link is now down */ |
| | 197 | } cqev_t; |
| | 198 | #endif /* BSD_KERNEL_PRIVATE */ |
| | 199 | |
| | 200 | #include <net/classq/classq.h> |
| | 201 | #include <net/pktsched/pktsched_fq_codel.h> |
| | 202 | |
| | 203 | #ifdef __cplusplus |
| | 204 | extern "C" { |
| | 205 | #endif |
| | 206 | struct if_ifclassq_stats { |
| | 207 | u_int32_t ifqs_len; |
| | 208 | u_int32_t ifqs_maxlen; |
| | 209 | struct pktcntr ifqs_xmitcnt; |
| | 210 | struct pktcntr ifqs_dropcnt; |
| | 211 | u_int32_t ifqs_scheduler; |
| | 212 | struct fq_codel_classstats ifqs_fq_codel_stats; |
| | 213 | } __attribute__((aligned(8))); |
| | 214 | |
| | 215 | #ifdef __cplusplus |
| | 216 | } |
| | 217 | #endif |
| | 218 | |
| | 219 | #ifdef BSD_KERNEL_PRIVATE |
| | 220 | /* |
| | 221 | * For ifclassq lock |
| | 222 | */ |
| | 223 | #define IFCQ_LOCK_ASSERT_HELD(_ifcq) \ |
| | 224 | LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) |
| | 225 | |
| | 226 | #define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \ |
| | 227 | LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) |
| | 228 | |
| | 229 | #define IFCQ_LOCK(_ifcq) \ |
| | 230 | lck_mtx_lock(&(_ifcq)->ifcq_lock) |
| | 231 | |
| | 232 | #define IFCQ_LOCK_SPIN(_ifcq) \ |
| | 233 | lck_mtx_lock_spin(&(_ifcq)->ifcq_lock) |
| | 234 | |
| | 235 | #define IFCQ_CONVERT_LOCK(_ifcq) do { \ |
| | 236 | IFCQ_LOCK_ASSERT_HELD(_ifcq); \ |
| | 237 | lck_mtx_convert_spin(&(_ifcq)->ifcq_lock); \ |
| | 238 | } while (0) |
| | 239 | |
| | 240 | #define IFCQ_UNLOCK(_ifcq) \ |
| | 241 | lck_mtx_unlock(&(_ifcq)->ifcq_lock) |
| | 242 | |
| | 243 | /* |
| | 244 | * For ifclassq operations |
| | 245 | */ |
| | 246 | #define IFCQ_TBR_DEQUEUE(_ifcq, _p) do { \ |
| | 247 | ifclassq_tbr_dequeue(_ifcq, _p); \ |
| | 248 | } while (0) |
| | 249 | |
| | 250 | #define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p) do { \ |
| | 251 | ifclassq_tbr_dequeue_sc(_ifcq, _sc, _p); \ |
| | 252 | } while (0) |
| | 253 | |
| | 254 | #define IFCQ_LEN(_ifcq) ((_ifcq)->ifcq_len) |
| | 255 | #define IFCQ_QFULL(_ifcq) (IFCQ_LEN(_ifcq) >= (_ifcq)->ifcq_maxlen) |
| | 256 | #define IFCQ_IS_EMPTY(_ifcq) (IFCQ_LEN(_ifcq) == 0) |
| | 257 | #define IFCQ_INC_LEN(_ifcq) (IFCQ_LEN(_ifcq)++) |
| | 258 | #define IFCQ_DEC_LEN(_ifcq) (IFCQ_LEN(_ifcq)--) |
| | 259 | #define IFCQ_ADD_LEN(_ifcq, _len) (IFCQ_LEN(_ifcq) += (_len)) |
| | 260 | #define IFCQ_SUB_LEN(_ifcq, _len) (IFCQ_LEN(_ifcq) -= (_len)) |
| | 261 | #define IFCQ_MAXLEN(_ifcq) ((_ifcq)->ifcq_maxlen) |
| | 262 | #define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len)) |
| | 263 | #define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay) |
| | 264 | #define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes) |
| | 265 | #define IFCQ_INC_BYTES(_ifcq, _len) \ |
| | 266 | ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len)) |
| | 267 | #define IFCQ_DEC_BYTES(_ifcq, _len) \ |
| | 268 | ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len)) |
| | 269 | |
| | 270 | #define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \ |
| | 271 | PKTCNTR_ADD(&(_ifcq)->ifcq_xmitcnt, _pkt, _len); \ |
| | 272 | } while (0) |
| | 273 | |
| | 274 | #define IFCQ_DROP_ADD(_ifcq, _pkt, _len) do { \ |
| | 275 | PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \ |
| | 276 | } while (0) |
| | 277 | |
| | 278 | #define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit) |
| | 279 | |
| | 280 | extern int ifclassq_setup(struct ifnet *, u_int32_t, boolean_t); |
| | 281 | extern void ifclassq_teardown(struct ifnet *); |
| | 282 | extern int ifclassq_pktsched_setup(struct ifclassq *); |
| | 283 | extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t); |
| | 284 | extern u_int32_t ifclassq_get_maxlen(struct ifclassq *); |
| | 285 | extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t, |
| | 286 | u_int32_t *, u_int32_t *); |
| | 287 | extern errno_t ifclassq_enqueue(struct ifclassq *, classq_pkt_t *, |
| | 288 | classq_pkt_t *, u_int32_t, u_int32_t, boolean_t *); |
| | 289 | extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t, |
| | 290 | classq_pkt_t *, classq_pkt_t *, u_int32_t *, u_int32_t *); |
| | 291 | extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, |
| | 292 | u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *, |
| | 293 | u_int32_t *); |
| | 294 | extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *); |
| | 295 | extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t, |
| | 296 | classq_pkt_type_t *); |
| | 297 | extern void ifclassq_update(struct ifclassq *, cqev_t); |
| | 298 | extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *); |
| | 299 | extern void ifclassq_detach(struct ifclassq *); |
| | 300 | extern int ifclassq_getqstats(struct ifclassq *, u_int32_t, |
| | 301 | void *, u_int32_t *); |
| | 302 | extern const char *ifclassq_ev2str(cqev_t); |
| | 303 | extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t); |
| | 304 | extern void ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_t *); |
| | 305 | extern void ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, |
| | 306 | classq_pkt_t *); |
| | 307 | extern void ifclassq_calc_target_qdelay(struct ifnet *ifp, |
| | 308 | u_int64_t *if_target_qdelay); |
| | 309 | extern void ifclassq_calc_update_interval(u_int64_t *update_interval); |
| | 310 | extern void ifclassq_set_packet_metadata(struct ifclassq *ifq, |
| | 311 | struct ifnet *ifp, classq_pkt_t *p); |
| | 312 | extern void ifclassq_reap_caches(boolean_t); |
| | 313 | |
| | 314 | #endif /* BSD_KERNEL_PRIVATE */ |
| | 315 | #endif /* PRIVATE */ |
| | 316 | #endif /* _NET_CLASSQ_IF_CLASSQ_H_ */ |
projects / apple / xnu.git / blame_incremental