X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4bd07ac2140668789aa3ee8ec4dde4a3e0a3bba5..f427ee49d309d8fc33ebf3042c3a775f2f530ded:/bsd/net/classq/classq_sfb.c diff --git a/bsd/net/classq/classq_sfb.c b/bsd/net/classq/classq_sfb.c index 5831f968f..ffdc20922 100644 --- a/bsd/net/classq/classq_sfb.c +++ b/bsd/net/classq/classq_sfb.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Apple Inc. All rights reserved. + * Copyright (c) 2011-2020 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -50,9 +50,7 @@ #include #include #include -#if INET6 #include -#endif #include #include @@ -95,19 +93,19 @@ * across platforms for 1-word key (32-bit flowhash value). See flowhash.h * for other alternatives. We only need 16-bit hash output. */ -#define SFB_HASH net_flowhash_mh3_x86_32 -#define SFB_HASHMASK HASHMASK(16) +#define SFB_HASH net_flowhash_mh3_x86_32 +#define SFB_HASHMASK HASHMASK(16) -#define SFB_BINMASK(_x) \ +#define SFB_BINMASK(_x) \ ((_x) & HASHMASK(SFB_BINS_SHIFT)) -#define SFB_BINST(_sp, _l, _n, _c) \ +#define SFB_BINST(_sp, _l, _n, _c) \ (&(*(_sp)->sfb_bins)[_c].stats[_l][_n]) -#define SFB_BINFT(_sp, _l, _n, _c) \ +#define SFB_BINFT(_sp, _l, _n, _c) \ (&(*(_sp)->sfb_bins)[_c].freezetime[_l][_n]) -#define SFB_FC_LIST(_sp, _n) \ +#define SFB_FC_LIST(_sp, _n) \ (&(*(_sp)->sfb_fc_lists)[_n]) /* @@ -116,9 +114,9 @@ * uplink speed is not known, a default value is chosen and is randomized * to be within the following range. */ -#define HOLDTIME_BASE (100ULL * 1000 * 1000) /* 100ms */ -#define HOLDTIME_MIN (10ULL * 1000 * 1000) /* 10ms */ -#define HOLDTIME_MAX (100ULL * 1000 * 1000) /* 100ms */ +#define HOLDTIME_BASE (100ULL * 1000 * 1000) /* 100ms */ +#define HOLDTIME_MIN (10ULL * 1000 * 1000) /* 10ms */ +#define HOLDTIME_MAX (100ULL * 1000 * 1000) /* 100ms */ /* * The pboxtime parameter determines the bandwidth allocated for rogue @@ -126,34 +124,34 @@ * is not known, a default value is chosen and is randomized to be within * the following range. */ -#define PBOXTIME_BASE (300ULL * 1000 * 1000) /* 300ms */ -#define PBOXTIME_MIN (30ULL * 1000 * 1000) /* 30ms */ -#define PBOXTIME_MAX (300ULL * 1000 * 1000) /* 300ms */ +#define PBOXTIME_BASE (300ULL * 1000 * 1000) /* 300ms */ +#define PBOXTIME_MIN (30ULL * 1000 * 1000) /* 30ms */ +#define PBOXTIME_MAX (300ULL * 1000 * 1000) /* 300ms */ /* * Target queueing delay is the amount of extra delay that can be added * to accommodate variations in the link bandwidth. The queue should be * large enough to induce this much delay and nothing more than that. */ -#define TARGET_QDELAY_BASE (10ULL * 1000 * 1000) /* 10ms */ -#define TARGET_QDELAY_MIN (10ULL * 1000) /* 10us */ -#define TARGET_QDELAY_MAX (20ULL * 1000 * 1000 * 1000) /* 20s */ +#define TARGET_QDELAY_BASE (10ULL * 1000 * 1000) /* 10ms */ +#define TARGET_QDELAY_MIN (10ULL * 1000) /* 10us */ +#define TARGET_QDELAY_MAX (20ULL * 1000 * 1000 * 1000) /* 20s */ /* * Update interval for checking the extra delay added by the queue. This - * should be 90-95 percentile of RTT experienced by any TCP connection + * should be 90-95 percentile of RTT experienced by any TCP connection * so that it will take care of the burst traffic. */ -#define UPDATE_INTERVAL_BASE (100ULL * 1000 * 1000) /* 100ms */ -#define UPDATE_INTERVAL_MIN (100ULL * 1000 * 1000) /* 100ms */ -#define UPDATE_INTERVAL_MAX (10ULL * 1000 * 1000 * 1000) /* 10s */ +#define UPDATE_INTERVAL_BASE (100ULL * 1000 * 1000) /* 100ms */ +#define UPDATE_INTERVAL_MIN (100ULL * 1000 * 1000) /* 100ms */ +#define UPDATE_INTERVAL_MAX (10ULL * 1000 * 1000 * 1000) /* 10s */ -#define SFB_RANDOM(sp, tmin, tmax) ((sfb_random(sp) % (tmax)) + (tmin)) +#define SFB_RANDOM(sp, tmin, tmax) ((sfb_random(sp) % (tmax)) + (tmin)) -#define SFB_PKT_PBOX 0x1 /* in penalty box */ +#define SFB_PKT_PBOX 0x1 /* in penalty box */ /* The following mantissa values are in SFB_FP_SHIFT Q format */ -#define SFB_MAX_PMARK (1 << SFB_FP_SHIFT) /* Q14 representation of 1.00 */ +#define SFB_MAX_PMARK (1 << SFB_FP_SHIFT) /* Q14 representation of 1.00 */ /* * These are d1 (increment) and d2 (decrement) parameters, used to determine @@ -165,207 +163,148 @@ * against packet loss, it can quickly reach to a substantial increase in * traffic load. */ -#define SFB_INCREMENT 82 /* Q14 representation of 0.005 */ -#define SFB_DECREMENT 16 /* Q14 representation of 0.001 */ +#define SFB_INCREMENT 82 /* Q14 representation of 0.005 */ +#define SFB_DECREMENT 16 /* Q14 representation of 0.001 */ -#define SFB_PMARK_TH 16056 /* Q14 representation of 0.98 */ -#define SFB_PMARK_WARM 3276 /* Q14 representation of 0.2 */ +#define SFB_PMARK_TH 16056 /* Q14 representation of 0.98 */ +#define SFB_PMARK_WARM 3276 /* Q14 representation of 0.2 */ -#define SFB_PMARK_INC(_bin) do { \ - (_bin)->pmark += sfb_increment; \ - if ((_bin)->pmark > SFB_MAX_PMARK) \ - (_bin)->pmark = SFB_MAX_PMARK; \ +#define SFB_PMARK_INC(_bin) do { \ + (_bin)->pmark += sfb_increment; \ + if ((_bin)->pmark > SFB_MAX_PMARK) \ + (_bin)->pmark = SFB_MAX_PMARK; \ } while (0) -#define SFB_PMARK_DEC(_bin) do { \ - if ((_bin)->pmark > 0) { \ - (_bin)->pmark -= sfb_decrement; \ - if ((_bin)->pmark < 0) \ - (_bin)->pmark = 0; \ - } \ +#define SFB_PMARK_DEC(_bin) do { \ + if ((_bin)->pmark > 0) { \ + (_bin)->pmark -= sfb_decrement; \ + if ((_bin)->pmark < 0) \ + (_bin)->pmark = 0; \ + } \ } while (0) /* Minimum nuber of bytes in queue to get flow controlled */ -#define SFB_MIN_FC_THRESHOLD_BYTES 7500 +#define SFB_MIN_FC_THRESHOLD_BYTES 7500 -#define SFB_SET_DELAY_HIGH(_sp_, _q_) do { \ - (_sp_)->sfb_flags |= SFBF_DELAYHIGH; \ - (_sp_)->sfb_fc_threshold = max(SFB_MIN_FC_THRESHOLD_BYTES, \ - (qsize((_q_)) >> 3)); \ +#define SFB_SET_DELAY_HIGH(_sp_, _q_) do { \ + (_sp_)->sfb_flags |= SFBF_DELAYHIGH; \ + (_sp_)->sfb_fc_threshold = ulmax(SFB_MIN_FC_THRESHOLD_BYTES, \ + (uint32_t)(qsize((_q_)) >> 3)); \ } while (0) -#define SFB_QUEUE_DELAYBASED(_sp_) ((_sp_)->sfb_flags & SFBF_DELAYBASED) +#define SFB_QUEUE_DELAYBASED(_sp_) ((_sp_)->sfb_flags & SFBF_DELAYBASED) #define SFB_IS_DELAYHIGH(_sp_) ((_sp_)->sfb_flags & SFBF_DELAYHIGH) -#define SFB_QUEUE_DELAYBASED_MAXSIZE 2048 /* max pkts */ +#define SFB_QUEUE_DELAYBASED_MAXSIZE 2048 /* max pkts */ -#define HINTERVAL_MIN (10) /* 10 seconds */ -#define HINTERVAL_MAX (20) /* 20 seconds */ -#define SFB_HINTERVAL(sp) ((sfb_random(sp) % HINTERVAL_MAX) + HINTERVAL_MIN) +#define HINTERVAL_MIN (10) /* 10 seconds */ +#define HINTERVAL_MAX (20) /* 20 seconds */ +#define SFB_HINTERVAL(sp) ((sfb_random(sp) % HINTERVAL_MAX) + HINTERVAL_MIN) -#define DEQUEUE_DECAY 7 /* ilog2 of EWMA decay rate, (128) */ -#define DEQUEUE_SPIKE(_new, _old) \ +#define DEQUEUE_DECAY 7 /* ilog2 of EWMA decay rate, (128) */ +#define DEQUEUE_SPIKE(_new, _old) \ ((u_int64_t)ABS((int64_t)(_new) - (int64_t)(_old)) > ((_old) << 11)) -#define ABS(v) (((v) > 0) ? (v) : -(v)) - -#define SFB_ZONE_MAX 32 /* maximum elements in zone */ -#define SFB_ZONE_NAME "classq_sfb" /* zone name */ - -#define SFB_BINS_ZONE_MAX 32 /* maximum elements in zone */ -#define SFB_BINS_ZONE_NAME "classq_sfb_bins" /* zone name */ - -#define SFB_FCL_ZONE_MAX 32 /* maximum elements in zone */ -#define SFB_FCL_ZONE_NAME "classq_sfb_fcl" /* zone name */ - /* Place the flow control entries in current bin on level 0 */ -#define SFB_FC_LEVEL 0 - -/* Store SFB hash and flags in the module private scratch space */ -#define pkt_sfb_hash8 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val8 -#define pkt_sfb_hash16 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val16 -#define pkt_sfb_hash32 pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 -#define pkt_sfb_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 +#define SFB_FC_LEVEL 0 -static unsigned int sfb_size; /* size of zone element */ -static struct zone *sfb_zone; /* zone for sfb */ +static ZONE_DECLARE(sfb_zone, "classq_sfb", + sizeof(struct sfb), ZC_ZFREE_CLEARMEM); -static unsigned int sfb_bins_size; /* size of zone element */ -static struct zone *sfb_bins_zone; /* zone for sfb_bins */ +static ZONE_DECLARE(sfb_bins_zone, "classq_sfb_bins", + sizeof(struct sfb_bins), ZC_ZFREE_CLEARMEM); -static unsigned int sfb_fcl_size; /* size of zone element */ -static struct zone *sfb_fcl_zone; /* zone for sfb_fc_lists */ +static ZONE_DECLARE(sfb_fcl_zone, "classq_sfb_fcl", + sizeof(struct sfb_fcl), ZC_ZFREE_CLEARMEM); /* internal function prototypes */ static u_int32_t sfb_random(struct sfb *); -static struct mbuf *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, - boolean_t); +static void *sfb_getq_flow(struct sfb *, class_queue_t *, u_int32_t, boolean_t, + pktsched_pkt_t *); static void sfb_resetq(struct sfb *, cqev_t); static void sfb_calc_holdtime(struct sfb *, u_int64_t); static void sfb_calc_pboxtime(struct sfb *, u_int64_t); static void sfb_calc_hinterval(struct sfb *, u_int64_t *); -static void sfb_calc_target_qdelay(struct sfb *, u_int64_t); static void sfb_calc_update_interval(struct sfb *, u_int64_t); static void sfb_swap_bins(struct sfb *, u_int32_t); -static inline int sfb_pcheck(struct sfb *, struct pkthdr *); -static int sfb_penalize(struct sfb *, struct pkthdr *, struct timespec *); +static inline int sfb_pcheck(struct sfb *, uint32_t); +static int sfb_penalize(struct sfb *, uint32_t, uint32_t *, struct timespec *); static void sfb_adjust_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *, boolean_t); static void sfb_decrement_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); static void sfb_increment_bin(struct sfb *, struct sfbbinstats *, struct timespec *, struct timespec *); -static inline void sfb_dq_update_bins(struct sfb *, struct pkthdr *, - struct timespec *, u_int32_t qsize); -static inline void sfb_eq_update_bins(struct sfb *, struct pkthdr *); -static int sfb_drop_early(struct sfb *, struct pkthdr *, u_int16_t *, +static inline void sfb_dq_update_bins(struct sfb *, uint32_t, uint32_t, + struct timespec *, u_int64_t qsize); +static inline void sfb_eq_update_bins(struct sfb *, uint32_t, uint32_t); +static int sfb_drop_early(struct sfb *, uint32_t, u_int16_t *, struct timespec *); -static boolean_t sfb_bin_addfcentry(struct sfb *, struct pkthdr *); +static boolean_t sfb_bin_addfcentry(struct sfb *, pktsched_pkt_t *, + uint32_t, uint8_t, uint32_t); static void sfb_fclist_append(struct sfb *, struct sfb_fcl *); static void sfb_fclists_clean(struct sfb *sp); static int sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin); static void sfb_detect_dequeue_stall(struct sfb *sp, class_queue_t *, struct timespec *); -SYSCTL_NODE(_net_classq, OID_AUTO, sfb, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "SFB"); +SYSCTL_NODE(_net_classq, OID_AUTO, sfb, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "SFB"); -static u_int64_t sfb_holdtime = 0; /* 0 indicates "automatic" */ -SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, holdtime, CTLFLAG_RW|CTLFLAG_LOCKED, +static u_int64_t sfb_holdtime = 0; /* 0 indicates "automatic" */ +SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, holdtime, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_holdtime, "SFB freeze time in nanoseconds"); -static u_int64_t sfb_pboxtime = 0; /* 0 indicates "automatic" */ -SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, pboxtime, CTLFLAG_RW|CTLFLAG_LOCKED, +static u_int64_t sfb_pboxtime = 0; /* 0 indicates "automatic" */ +SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, pboxtime, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_pboxtime, "SFB penalty box time in nanoseconds"); static u_int64_t sfb_hinterval; -SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, hinterval, CTLFLAG_RW|CTLFLAG_LOCKED, +SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, hinterval, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_hinterval, "SFB hash interval in nanoseconds"); -static u_int64_t sfb_target_qdelay = 0; -SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, target_qdelay, CTLFLAG_RW|CTLFLAG_LOCKED, - &sfb_target_qdelay, "SFB target queue delay in nanoseconds"); - -static u_int64_t sfb_update_interval; -SYSCTL_QUAD(_net_classq_sfb, OID_AUTO, update_interval, - CTLFLAG_RW|CTLFLAG_LOCKED, &sfb_update_interval, "SFB update interval"); - static u_int32_t sfb_increment = SFB_INCREMENT; -SYSCTL_UINT(_net_classq_sfb, OID_AUTO, increment, CTLFLAG_RW|CTLFLAG_LOCKED, +SYSCTL_UINT(_net_classq_sfb, OID_AUTO, increment, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_increment, SFB_INCREMENT, "SFB increment [d1]"); static u_int32_t sfb_decrement = SFB_DECREMENT; -SYSCTL_UINT(_net_classq_sfb, OID_AUTO, decrement, CTLFLAG_RW|CTLFLAG_LOCKED, +SYSCTL_UINT(_net_classq_sfb, OID_AUTO, decrement, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_decrement, SFB_DECREMENT, "SFB decrement [d2]"); -static u_int32_t sfb_allocation = 0; /* 0 means "automatic" */ -SYSCTL_UINT(_net_classq_sfb, OID_AUTO, allocation, CTLFLAG_RW|CTLFLAG_LOCKED, +static u_int32_t sfb_allocation = 0; /* 0 means "automatic" */ +SYSCTL_UINT(_net_classq_sfb, OID_AUTO, allocation, CTLFLAG_RW | CTLFLAG_LOCKED, &sfb_allocation, 0, "SFB bin allocation"); static u_int32_t sfb_ratelimit = 0; -SYSCTL_UINT(_net_classq_sfb, OID_AUTO, ratelimit, CTLFLAG_RW|CTLFLAG_LOCKED, - &sfb_ratelimit, 0, "SFB rate limit"); +SYSCTL_UINT(_net_classq_sfb, OID_AUTO, ratelimit, CTLFLAG_RW | CTLFLAG_LOCKED, + &sfb_ratelimit, 0, "SFB rate limit"); -#define KBPS (1ULL * 1000) /* 1 Kbits per second */ -#define MBPS (1ULL * 1000 * 1000) /* 1 Mbits per second */ -#define GBPS (MBPS * 1000) /* 1 Gbits per second */ +#define KBPS (1ULL * 1000) /* 1 Kbits per second */ +#define MBPS (1ULL * 1000 * 1000) /* 1 Mbits per second */ +#define GBPS (MBPS * 1000) /* 1 Gbits per second */ struct sfb_time_tbl { - u_int64_t speed; /* uplink speed */ - u_int64_t holdtime; /* hold time */ - u_int64_t pboxtime; /* penalty box time */ + u_int64_t speed; /* uplink speed */ + u_int64_t holdtime; /* hold time */ + u_int64_t pboxtime; /* penalty box time */ }; static struct sfb_time_tbl sfb_ttbl[] = { - { 1 * MBPS, HOLDTIME_BASE * 1000, PBOXTIME_BASE * 1000 }, - { 10 * MBPS, HOLDTIME_BASE * 100, PBOXTIME_BASE * 100 }, - { 100 * MBPS, HOLDTIME_BASE * 10, PBOXTIME_BASE * 10 }, - { 1 * GBPS, HOLDTIME_BASE, PBOXTIME_BASE }, - { 10 * GBPS, HOLDTIME_BASE / 10, PBOXTIME_BASE / 10 }, - { 100 * GBPS, HOLDTIME_BASE / 100, PBOXTIME_BASE / 100 }, - { 0, 0, 0 } + { .speed = 1 * MBPS, .holdtime = HOLDTIME_BASE * 1000, .pboxtime = PBOXTIME_BASE * 1000}, + { .speed = 10 * MBPS, .holdtime = HOLDTIME_BASE * 100, .pboxtime = PBOXTIME_BASE * 100 }, + { .speed = 100 * MBPS, .holdtime = HOLDTIME_BASE * 10, .pboxtime = PBOXTIME_BASE * 10 }, + { .speed = 1 * GBPS, .holdtime = HOLDTIME_BASE, .pboxtime = PBOXTIME_BASE }, + { .speed = 10 * GBPS, .holdtime = HOLDTIME_BASE / 10, .pboxtime = PBOXTIME_BASE / 10 }, + { .speed = 100 * GBPS, .holdtime = HOLDTIME_BASE / 100, .pboxtime = PBOXTIME_BASE / 100 }, + { .speed = 0, .holdtime = 0, .pboxtime = 0 } }; -void -sfb_init(void) -{ - _CASSERT(SFBF_ECN4 == CLASSQF_ECN4); - _CASSERT(SFBF_ECN6 == CLASSQF_ECN6); - - sfb_size = sizeof (struct sfb); - sfb_zone = zinit(sfb_size, SFB_ZONE_MAX * sfb_size, - 0, SFB_ZONE_NAME); - if (sfb_zone == NULL) { - panic("%s: failed allocating %s", __func__, SFB_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(sfb_zone, Z_EXPAND, TRUE); - zone_change(sfb_zone, Z_CALLERACCT, TRUE); - - sfb_bins_size = sizeof (*((struct sfb *)0)->sfb_bins); - sfb_bins_zone = zinit(sfb_bins_size, SFB_BINS_ZONE_MAX * sfb_bins_size, - 0, SFB_BINS_ZONE_NAME); - if (sfb_bins_zone == NULL) { - panic("%s: failed allocating %s", __func__, SFB_BINS_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(sfb_bins_zone, Z_EXPAND, TRUE); - zone_change(sfb_bins_zone, Z_CALLERACCT, TRUE); - - sfb_fcl_size = sizeof (*((struct sfb *)0)->sfb_fc_lists); - sfb_fcl_zone = zinit(sfb_fcl_size, SFB_FCL_ZONE_MAX * sfb_fcl_size, - 0, SFB_FCL_ZONE_NAME); - if (sfb_fcl_zone == NULL) { - panic("%s: failed allocating %s", __func__, SFB_FCL_ZONE_NAME); - /* NOTREACHED */ - } - zone_change(sfb_fcl_zone, Z_EXPAND, TRUE); - zone_change(sfb_fcl_zone, Z_CALLERACCT, TRUE); -} +static_assert(SFBF_ECN4 == CLASSQF_ECN4); +static_assert(SFBF_ECN6 == CLASSQF_ECN6); static u_int32_t sfb_random(struct sfb *sp) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - return (RandomULong()); + return RandomULong(); } static void @@ -378,12 +317,13 @@ sfb_calc_holdtime(struct sfb *sp, u_int64_t outbw) } else if (outbw == 0) { holdtime = SFB_RANDOM(sp, HOLDTIME_MIN, HOLDTIME_MAX); } else { - unsigned int n, i; + uint64_t n, i; n = sfb_ttbl[0].holdtime; for (i = 0; sfb_ttbl[i].speed != 0; i++) { - if (outbw < sfb_ttbl[i].speed) + if (outbw < sfb_ttbl[i].speed) { break; + } n = sfb_ttbl[i].holdtime; } holdtime = n; @@ -401,12 +341,13 @@ sfb_calc_pboxtime(struct sfb *sp, u_int64_t outbw) } else if (outbw == 0) { pboxtime = SFB_RANDOM(sp, PBOXTIME_MIN, PBOXTIME_MAX); } else { - unsigned int n, i; + uint64_t n, i; n = sfb_ttbl[0].pboxtime; for (i = 0; sfb_ttbl[i].speed != 0; i++) { - if (outbw < sfb_ttbl[i].speed) + if (outbw < sfb_ttbl[i].speed) { break; + } n = sfb_ttbl[i].pboxtime; } pboxtime = n; @@ -418,7 +359,7 @@ sfb_calc_pboxtime(struct sfb *sp, u_int64_t outbw) static void sfb_calc_hinterval(struct sfb *sp, u_int64_t *t) { - u_int64_t hinterval; + u_int64_t hinterval = 0; struct timespec now; if (t != NULL) { @@ -428,10 +369,11 @@ sfb_calc_hinterval(struct sfb *sp, u_int64_t *t) hinterval = *t; } - if (sfb_hinterval != 0) + if (sfb_hinterval != 0) { hinterval = sfb_hinterval; - else if (t == NULL || hinterval == 0) + } else if (t == NULL || hinterval == 0) { hinterval = ((u_int64_t)SFB_HINTERVAL(sp) * NSEC_PER_SEC); + } net_nsectimer(&hinterval, &sp->sfb_hinterval); @@ -439,53 +381,12 @@ sfb_calc_hinterval(struct sfb *sp, u_int64_t *t) net_timeradd(&now, &sp->sfb_hinterval, &sp->sfb_nextreset); } -static void -sfb_calc_target_qdelay(struct sfb *sp, u_int64_t out_bw) -{ -#pragma unused(out_bw) - u_int64_t target_qdelay = 0; - struct ifnet *ifp = sp->sfb_ifp; - - target_qdelay = IFCQ_TARGET_QDELAY(&ifp->if_snd); - - if (sfb_target_qdelay != 0) - target_qdelay = sfb_target_qdelay; - - /* - * If we do not know the effective bandwidth, use the default - * target queue delay. - */ - if (target_qdelay == 0) - target_qdelay = IFQ_TARGET_DELAY; - - /* - * If a delay has been added to ifnet start callback for - * coalescing, we have to add that to the pre-set target delay - * because the packets can be in the queue longer. - */ - if ((ifp->if_eflags & IFEF_ENQUEUE_MULTI) && - ifp->if_start_delay_timeout > 0) - target_qdelay += ifp->if_start_delay_timeout; - - sp->sfb_target_qdelay = target_qdelay; -} - static void sfb_calc_update_interval(struct sfb *sp, u_int64_t out_bw) { #pragma unused(out_bw) u_int64_t update_interval = 0; - - /* If the system-level override is set, use it */ - if (sfb_update_interval != 0) - update_interval = sfb_update_interval; - /* - * If we do not know the effective bandwidth, use the default - * update interval. - */ - if (update_interval == 0) - update_interval = IFQ_UPDATE_INTERVAL; - + ifclassq_calc_update_interval(&update_interval); net_nsectimer(&update_interval, &sp->sfb_update_interval); } @@ -500,30 +401,13 @@ sfb_alloc(struct ifnet *ifp, u_int32_t qid, u_int32_t qlim, u_int32_t flags) VERIFY(ifp != NULL && qlim > 0); - sp = zalloc(sfb_zone); - if (sp == NULL) { - log(LOG_ERR, "%s: SFB unable to allocate\n", if_name(ifp)); - return (NULL); - } - bzero(sp, sfb_size); - - if ((sp->sfb_bins = zalloc(sfb_bins_zone)) == NULL) { - log(LOG_ERR, "%s: SFB unable to allocate bins\n", if_name(ifp)); - sfb_destroy(sp); - return (NULL); - } - bzero(sp->sfb_bins, sfb_bins_size); - - if ((sp->sfb_fc_lists = zalloc(sfb_fcl_zone)) == NULL) { - log(LOG_ERR, "%s: SFB unable to allocate flow control lists\n", - if_name(ifp)); - sfb_destroy(sp); - return(NULL); - } - bzero(sp->sfb_fc_lists, sfb_fcl_size); + sp = zalloc_flags(sfb_zone, Z_WAITOK | Z_ZERO); + sp->sfb_bins = zalloc_flags(sfb_bins_zone, Z_WAITOK | Z_ZERO); + sp->sfb_fc_lists = zalloc_flags(sfb_fcl_zone, Z_WAITOK | Z_ZERO); - for (i = 0; i < SFB_BINS; ++i) + for (i = 0; i < SFB_BINS; ++i) { STAILQ_INIT(&SFB_FC_LIST(sp, i)->fclist); + } sp->sfb_ifp = ifp; sp->sfb_qlim = qlim; @@ -537,16 +421,15 @@ sfb_alloc(struct ifnet *ifp, u_int32_t qid, u_int32_t qlim, u_int32_t flags) } #endif /* !PF_ECN */ - sfb_resetq(sp, -1); + sfb_resetq(sp, CLASSQ_EV_INIT); - return (sp); + return sp; } static void sfb_fclist_append(struct sfb *sp, struct sfb_fcl *fcl) { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - VERIFY(STAILQ_EMPTY(&fcl->fclist) || fcl->cnt > 0); sp->sfb_stats.flow_feedback += fcl->cnt; fcl->cnt = 0; @@ -563,8 +446,9 @@ sfb_fclists_clean(struct sfb *sp) /* Move all the flow control entries to the flowadv list */ for (i = 0; i < SFB_BINS; ++i) { struct sfb_fcl *fcl = SFB_FC_LIST(sp, i); - if (!STAILQ_EMPTY(&fcl->fclist)) + if (!STAILQ_EMPTY(&fcl->fclist)) { sfb_fclist_append(sp, fcl); + } } } @@ -594,8 +478,9 @@ sfb_resetq(struct sfb *sp, cqev_t ev) if (ev != CLASSQ_EV_LINK_DOWN) { (*sp->sfb_bins)[0].fudge = sfb_random(sp); (*sp->sfb_bins)[1].fudge = sfb_random(sp); - sp->sfb_allocation = ((sfb_allocation == 0) ? - (sp->sfb_qlim / 3) : sfb_allocation); + sp->sfb_allocation = sfb_allocation == 0 ? + (uint16_t)(sp->sfb_qlim / 3) : + (uint16_t)sfb_allocation; sp->sfb_drop_thresh = sp->sfb_allocation + (sp->sfb_allocation >> 1); } @@ -609,18 +494,20 @@ sfb_resetq(struct sfb *sp, cqev_t ev) sfb_calc_holdtime(sp, eff_rate); sfb_calc_pboxtime(sp, eff_rate); sfb_calc_hinterval(sp, NULL); - sfb_calc_target_qdelay(sp, eff_rate); + ifclassq_calc_target_qdelay(ifp, &sp->sfb_target_qdelay); sfb_calc_update_interval(sp, eff_rate); if (ev == CLASSQ_EV_LINK_DOWN || - ev == CLASSQ_EV_LINK_UP) + ev == CLASSQ_EV_LINK_UP) { sfb_fclists_clean(sp); + } - bzero(sp->sfb_bins, sizeof (*sp->sfb_bins)); - bzero(&sp->sfb_stats, sizeof (sp->sfb_stats)); + bzero(sp->sfb_bins, sizeof(*sp->sfb_bins)); + bzero(&sp->sfb_stats, sizeof(sp->sfb_stats)); - if (ev == CLASSQ_EV_LINK_DOWN || !classq_verbose) + if (ev == CLASSQ_EV_LINK_DOWN || !classq_verbose) { return; + } log(LOG_DEBUG, "%s: SFB qid=%d, holdtime=%llu nsec, " "pboxtime=%llu nsec, allocation=%d, drop_thresh=%d, " @@ -630,7 +517,7 @@ sfb_resetq(struct sfb *sp, cqev_t ev) if_name(ifp), sp->sfb_qid, (u_int64_t)sp->sfb_holdtime.tv_nsec, (u_int64_t)sp->sfb_pboxtime.tv_nsec, (u_int32_t)sp->sfb_allocation, (u_int32_t)sp->sfb_drop_thresh, - (int)sp->sfb_hinterval.tv_sec, (int)sizeof (*sp->sfb_bins), + (int)sp->sfb_hinterval.tv_sec, (int)sizeof(*sp->sfb_bins), eff_rate, (u_int64_t)sp->sfb_target_qdelay, (u_int64_t)sp->sfb_update_interval.tv_sec, (u_int64_t)sp->sfb_update_interval.tv_nsec, sp->sfb_flags); @@ -645,7 +532,7 @@ sfb_getstats(struct sfb *sp, struct sfb_stats *sps) sps->current = sp->sfb_current; sps->target_qdelay = sp->sfb_target_qdelay; sps->min_estdelay = sp->sfb_min_qdelay; - sps->delay_fcthreshold = sp->sfb_fc_threshold; + sps->delay_fcthreshold = (uint32_t)sp->sfb_fc_threshold; sps->flags = sp->sfb_flags; net_timernsec(&sp->sfb_holdtime, &sp->sfb_stats.hold_time); @@ -654,13 +541,13 @@ sfb_getstats(struct sfb *sp, struct sfb_stats *sps) net_timernsec(&sp->sfb_update_interval, &sps->update_interval); *(&(sps->sfbstats)) = *(&(sp->sfb_stats)); - _CASSERT(sizeof ((*sp->sfb_bins)[0].stats) == - sizeof (sps->binstats[0].stats)); + _CASSERT(sizeof((*sp->sfb_bins)[0].stats) == + sizeof(sps->binstats[0].stats)); bcopy(&(*sp->sfb_bins)[0].stats, &sps->binstats[0].stats, - sizeof (sps->binstats[0].stats)); + sizeof(sps->binstats[0].stats)); bcopy(&(*sp->sfb_bins)[1].stats, &sps->binstats[1].stats, - sizeof (sps->binstats[1].stats)); + sizeof(sps->binstats[1].stats)); } static void @@ -668,8 +555,9 @@ sfb_swap_bins(struct sfb *sp, u_int32_t len) { int i, j, s; - if (sp->sfb_flags & SFBF_SUSPENDED) + if (sp->sfb_flags & SFBF_SUSPENDED) { return; + } s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -678,7 +566,7 @@ sfb_swap_bins(struct sfb *sp, u_int32_t len) sp->sfb_clearpkts = len; sp->sfb_stats.num_rehash++; - s = (sp->sfb_current ^= 1); /* flip the bit (swap current) */ + s = (sp->sfb_current ^= 1); /* flip the bit (swap current) */ if (classq_verbose) { log(LOG_DEBUG, "%s: SFB qid=%d, set %d is now current, " @@ -687,44 +575,49 @@ sfb_swap_bins(struct sfb *sp, u_int32_t len) /* clear freezetime for all current bins */ bzero(&(*sp->sfb_bins)[s].freezetime, - sizeof ((*sp->sfb_bins)[s].freezetime)); + sizeof((*sp->sfb_bins)[s].freezetime)); /* clear/adjust bin statistics and flow control lists */ for (i = 0; i < SFB_BINS; i++) { struct sfb_fcl *fcl = SFB_FC_LIST(sp, i); - if (!STAILQ_EMPTY(&fcl->fclist)) + if (!STAILQ_EMPTY(&fcl->fclist)) { sfb_fclist_append(sp, fcl); + } for (j = 0; j < SFB_LEVELS; j++) { struct sfbbinstats *cbin, *wbin; - cbin = SFB_BINST(sp, j, i, s); /* current */ - wbin = SFB_BINST(sp, j, i, s ^ 1); /* warm-up */ + cbin = SFB_BINST(sp, j, i, s); /* current */ + wbin = SFB_BINST(sp, j, i, s ^ 1); /* warm-up */ cbin->pkts = 0; cbin->bytes = 0; - if (cbin->pmark > SFB_MAX_PMARK) + if (cbin->pmark > SFB_MAX_PMARK) { cbin->pmark = SFB_MAX_PMARK; - if (cbin->pmark < 0) + } + if (cbin->pmark < 0) { cbin->pmark = 0; + } /* * Keep pmark from before to identify * non-responsives immediately. */ - if (wbin->pmark > SFB_PMARK_WARM) + if (wbin->pmark > SFB_PMARK_WARM) { wbin->pmark = SFB_PMARK_WARM; + } } } } static inline int -sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) +sfb_pcheck(struct sfb *sp, uint32_t pkt_sfb_hash) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; int s; s = sp->sfb_current; @@ -739,33 +632,39 @@ sfb_pcheck(struct sfb *sp, struct pkthdr *pkt) * Level 0: bin index at [0] for set 0; [2] for set 1 * Level 1: bin index at [1] for set 0; [3] for set 1 */ - if (SFB_BINST(sp, 0, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), + if (SFB_BINST(sp, 0, SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s)->pmark < SFB_PMARK_TH || - SFB_BINST(sp, 1, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), - s)->pmark < SFB_PMARK_TH) - return (0); + SFB_BINST(sp, 1, SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), + s)->pmark < SFB_PMARK_TH) { + return 0; + } #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { - if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); - else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + if (s == 0) { /* set 0, bin index [0,1] */ + n = SFB_BINMASK(pkt_sfb_hash8[i]); + } else { /* set 1, bin index [2,3] */ + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); + } - if (SFB_BINST(sp, i, n, s)->pmark < SFB_PMARK_TH) - return (0); + if (SFB_BINST(sp, i, n, s)->pmark < SFB_PMARK_TH) { + return 0; + } } #endif /* SFB_LEVELS != 2 */ - return (1); + return 1; } static int -sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) +sfb_penalize(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t *pkt_sfb_flags, + struct timespec *now) { - struct timespec delta = { 0, 0 }; + struct timespec delta = { .tv_sec = 0, .tv_nsec = 0 }; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; /* If minimum pmark of current bins is < SFB_PMARK_TH, we're done */ - if (!sfb_ratelimit || !sfb_pcheck(sp, pkt)) - return (0); + if (!sfb_ratelimit || !sfb_pcheck(sp, pkt_sfb_hash)) { + return 0; + } net_timersub(now, &sp->sfb_pboxfreeze, &delta); if (net_timercmp(&delta, &sp->sfb_pboxtime, <)) { @@ -783,22 +682,25 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1)]); bin = SFB_BINST(sp, 0, n, w); - if (bin->pkts >= sp->sfb_allocation) + if (bin->pkts >= sp->sfb_allocation) { sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, w), now); + } /* Level 0: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(w << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(w << 1) + 1]); bin = SFB_BINST(sp, 1, n, w); - if (bin->pkts >= sp->sfb_allocation) + if (bin->pkts >= sp->sfb_allocation) { sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, w), now); + } #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { - if (w == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); - else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + if (w == 0) { /* set 0, bin index [0,1] */ + n = SFB_BINMASK(pkt_sfb_hash8[i]); + } else { /* set 1, bin index [2,3] */ + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); + } bin = SFB_BINST(sp, i, n, w); if (bin->pkts >= sp->sfb_allocation) { @@ -807,14 +709,14 @@ sfb_penalize(struct sfb *sp, struct pkthdr *pkt, struct timespec *now) } } #endif /* SFB_LEVELS != 2 */ - return (1); + return 1; } /* non-conformant or else misclassified flow; queue it anyway */ - pkt->pkt_sfb_flags |= SFB_PKT_PBOX; + *pkt_sfb_flags |= SFB_PKT_PBOX; *(&sp->sfb_pboxfreeze) = *now; - return (0); + return 0; } static void @@ -836,29 +738,30 @@ sfb_adjust_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft, /* increment/decrement marking probability */ *ft = *now; - if (inc) + if (inc) { SFB_PMARK_INC(bin); - else + } else { SFB_PMARK_DEC(bin); + } } static void sfb_decrement_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft, struct timespec *now) { - return (sfb_adjust_bin(sp, bin, ft, now, FALSE)); + return sfb_adjust_bin(sp, bin, ft, now, FALSE); } static void sfb_increment_bin(struct sfb *sp, struct sfbbinstats *bin, struct timespec *ft, struct timespec *now) { - return (sfb_adjust_bin(sp, bin, ft, now, TRUE)); + return sfb_adjust_bin(sp, bin, ft, now, TRUE); } static inline void -sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, - struct timespec *now, u_int32_t qsize) +sfb_dq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len, + struct timespec *now, u_int64_t qsize) { #if SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 int i; @@ -866,6 +769,7 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, struct sfbbinstats *bin; int s, n; struct sfb_fcl *fcl = NULL; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -875,78 +779,88 @@ sfb_dq_update_bins(struct sfb *sp, struct pkthdr *pkt, */ #if SFB_LEVELS == 2 && SFB_FC_LEVEL == 0 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int32_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; + bin->bytes -= pkt_len; - if (bin->pkts == 0) + if (bin->pkts == 0) { sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now); + } /* Deliver flow control feedback to the sockets */ if (SFB_QUEUE_DELAYBASED(sp)) { if (!(SFB_IS_DELAYHIGH(sp)) || bin->bytes <= sp->sfb_fc_threshold || - bin->pkts == 0 || qsize == 0) + bin->pkts == 0 || qsize == 0) { fcl = SFB_FC_LIST(sp, n); + } } else if (bin->pkts <= (sp->sfb_allocation >> 2)) { - fcl = SFB_FC_LIST(sp, n); + fcl = SFB_FC_LIST(sp, n); } - if (fcl != NULL && !STAILQ_EMPTY(&fcl->fclist)) + if (fcl != NULL && !STAILQ_EMPTY(&fcl->fclist)) { sfb_fclist_append(sp, fcl); + } fcl = NULL; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= (u_int64_t)pkt_len); bin->pkts--; - bin->bytes -= pkt->len; - if (bin->pkts == 0) + bin->bytes -= pkt_len; + if (bin->pkts == 0) { sfb_decrement_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now); + } #else /* SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 */ for (i = 0; i < SFB_LEVELS; i++) { - if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); - else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + if (s == 0) { /* set 0, bin index [0,1] */ + n = SFB_BINMASK(pkt_sfb_hash8[i]); + } else { /* set 1, bin index [2,3] */ + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); + } bin = SFB_BINST(sp, i, n, s); - VERIFY(bin->pkts > 0 && bin->bytes >= pkt->len); + VERIFY(bin->pkts > 0 && bin->bytes >= pkt_len); bin->pkts--; - bin->bytes -= pkt->len; - if (bin->pkts == 0) + bin->bytes -= pkt_len; + if (bin->pkts == 0) { sfb_decrement_bin(sp, bin, SFB_BINFT(sp, i, n, s), now); - if (i != SFB_FC_LEVEL) + } + if (i != SFB_FC_LEVEL) { continue; + } if (SFB_QUEUE_DELAYBASED(sp)) { if (!(SFB_IS_DELAYHIGH(sp)) || - bin->bytes <= sp->sfb_fc_threshold) + bin->bytes <= sp->sfb_fc_threshold) { fcl = SFB_FC_LIST(sp, n); + } } else if (bin->pkts <= (sp->sfb_allocation >> 2)) { fcl = SFB_FC_LIST(sp, n); } - if (fcl != NULL && !STAILQ_EMPTY(&fcl->fclist)) + if (fcl != NULL && !STAILQ_EMPTY(&fcl->fclist)) { sfb_fclist_append(sp, fcl); + } fcl = NULL; } #endif /* SFB_LEVELS != 2 || SFB_FC_LEVEL != 0 */ } static inline void -sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) +sfb_eq_update_bins(struct sfb *sp, uint32_t pkt_sfb_hash, uint32_t pkt_len) { #if SFB_LEVELS != 2 int i, n; #endif /* SFB_LEVELS != 2 */ int s; struct sfbbinstats *bin; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -956,73 +870,70 @@ sfb_eq_update_bins(struct sfb *sp, struct pkthdr *pkt) #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ bin = SFB_BINST(sp, 0, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1)]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; /* Level 1: bin index at [1] for set 0; [3] for set 1 */ bin = SFB_BINST(sp, 1, - SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]), s); + SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]), s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { - if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); - else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + if (s == 0) { /* set 0, bin index [0,1] */ + n = SFB_BINMASK(pkt_sfb_hash8[i]); + } else { /* set 1, bin index [2,3] */ + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); + } bin = SFB_BINST(sp, i, n, s); bin->pkts++; - bin->bytes += pkt->len; + bin->bytes += pkt_len; } #endif /* SFB_LEVELS != 2 */ } static boolean_t -sfb_bin_addfcentry(struct sfb *sp, struct pkthdr *pkt) +sfb_bin_addfcentry(struct sfb *sp, pktsched_pkt_t *pkt, uint32_t pkt_sfb_hash, + uint8_t flowsrc, uint32_t flowid) { struct flowadv_fcentry *fce; - u_int32_t flowsrc, flowid; struct sfb_fcl *fcl; int s; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - flowsrc = pkt->pkt_flowsrc; - flowid = pkt->pkt_flowid; - if (flowid == 0) { sp->sfb_stats.null_flowid++; - return (FALSE); + return FALSE; } /* * Use value at index 0 for set 0 and * value at index 2 for set 1 */ - fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)])); + fcl = SFB_FC_LIST(sp, SFB_BINMASK(pkt_sfb_hash8[(s << 1)])); STAILQ_FOREACH(fce, &fcl->fclist, fce_link) { - if (fce->fce_flowsrc == flowsrc && + if ((uint8_t)fce->fce_flowsrc_type == flowsrc && fce->fce_flowid == flowid) { /* Already on flow control list; just return */ - return (TRUE); + return TRUE; } } IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - fce = flowadv_alloc_entry(M_WAITOK); + fce = pktsched_alloc_fcentry(pkt, sp->sfb_ifp, M_WAITOK); if (fce != NULL) { - fce->fce_flowsrc = flowsrc; - fce->fce_flowid = flowid; STAILQ_INSERT_TAIL(&fcl->fclist, fce, fce_link); fcl->cnt++; sp->sfb_stats.flow_controlled++; } - return (fce != NULL); + return fce != NULL; } /* @@ -1039,21 +950,23 @@ sfb_bin_mark_or_drop(struct sfb *sp, struct sfbbinstats *bin) * bytes than the flowcontrol threshold. */ if (SFB_IS_DELAYHIGH(sp) && - bin->bytes >= (sp->sfb_fc_threshold << 1)) + bin->bytes >= (sp->sfb_fc_threshold << 1)) { ret = 1; + } } else { if (bin->pkts >= sp->sfb_allocation && - bin->pkts >= sp->sfb_drop_thresh) - ret = 1; /* drop or mark */ + bin->pkts >= sp->sfb_drop_thresh) { + ret = 1; /* drop or mark */ + } } - return (ret); + return ret; } /* * early-drop probability is kept in pmark of each bin of the flow */ static int -sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, +sfb_drop_early(struct sfb *sp, uint32_t pkt_sfb_hash, u_int16_t *pmin, struct timespec *now) { #if SFB_LEVELS != 2 @@ -1061,6 +974,7 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, #endif /* SFB_LEVELS != 2 */ struct sfbbinstats *bin; int s, n, ret = 0; + uint8_t *pkt_sfb_hash8 = (uint8_t *)&pkt_sfb_hash; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); @@ -1072,49 +986,57 @@ sfb_drop_early(struct sfb *sp, struct pkthdr *pkt, u_int16_t *pmin, */ #if SFB_LEVELS == 2 /* Level 0: bin index at [0] for set 0; [2] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1)]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1)]); bin = SFB_BINST(sp, 0, n, s); - if (*pmin > (u_int16_t)bin->pmark) + if (*pmin > (u_int16_t)bin->pmark) { *pmin = (u_int16_t)bin->pmark; + } /* Update SFB probability */ - if (bin->pkts >= sp->sfb_allocation) + if (bin->pkts >= sp->sfb_allocation) { sfb_increment_bin(sp, bin, SFB_BINFT(sp, 0, n, s), now); + } ret = sfb_bin_mark_or_drop(sp, bin); /* Level 1: bin index at [1] for set 0; [3] for set 1 */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[(s << 1) + 1]); + n = SFB_BINMASK(pkt_sfb_hash8[(s << 1) + 1]); bin = SFB_BINST(sp, 1, n, s); - if (*pmin > (u_int16_t)bin->pmark) + if (*pmin > (u_int16_t)bin->pmark) { *pmin = (u_int16_t)bin->pmark; + } - if (bin->pkts >= sp->sfb_allocation) + if (bin->pkts >= sp->sfb_allocation) { sfb_increment_bin(sp, bin, SFB_BINFT(sp, 1, n, s), now); + } #else /* SFB_LEVELS != 2 */ for (i = 0; i < SFB_LEVELS; i++) { - if (s == 0) /* set 0, bin index [0,1] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i]); - else /* set 1, bin index [2,3] */ - n = SFB_BINMASK(pkt->pkt_sfb_hash8[i + 2]); + if (s == 0) { /* set 0, bin index [0,1] */ + n = SFB_BINMASK(pkt_sfb_hash8[i]); + } else { /* set 1, bin index [2,3] */ + n = SFB_BINMASK(pkt_sfb_hash8[i + 2]); + } bin = SFB_BINST(sp, i, n, s); - if (*pmin > (u_int16_t)bin->pmark) + if (*pmin > (u_int16_t)bin->pmark) { *pmin = (u_int16_t)bin->pmark; + } - if (bin->pkts >= sp->sfb_allocation) + if (bin->pkts >= sp->sfb_allocation) { sfb_increment_bin(sp, bin, SFB_BINFT(sp, i, n, s), now); - if (i == SFB_FC_LEVEL) + } + if (i == SFB_FC_LEVEL) { ret = sfb_bin_mark_or_drop(sp, bin); + } } #endif /* SFB_LEVELS != 2 */ - if (sp->sfb_flags & SFBF_SUSPENDED) - ret = 1; /* drop or mark */ - - return (ret); + if (sp->sfb_flags & SFBF_SUSPENDED) { + ret = 1; /* drop or mark */ + } + return ret; } void @@ -1125,8 +1047,9 @@ sfb_detect_dequeue_stall(struct sfb *sp, class_queue_t *q, if (!SFB_QUEUE_DELAYBASED(sp) || SFB_IS_DELAYHIGH(sp) || qsize(q) <= SFB_MIN_FC_THRESHOLD_BYTES || - !net_timerisset(&sp->sfb_getqtime)) + !net_timerisset(&sp->sfb_getqtime)) { return; + } net_timeradd(&sp->sfb_getqtime, &sp->sfb_update_interval, &max_getqtime); @@ -1140,36 +1063,56 @@ sfb_detect_dequeue_stall(struct sfb *sp, class_queue_t *q, } } -#define DTYPE_NODROP 0 /* no drop */ -#define DTYPE_FORCED 1 /* a "forced" drop */ -#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ +#define DTYPE_NODROP 0 /* no drop */ +#define DTYPE_FORCED 1 /* a "forced" drop */ +#define DTYPE_EARLY 2 /* an "unforced" (early) drop */ int -sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) +sfb_addq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt, + struct pf_mtag *t) { #if !PF_ECN #pragma unused(t) #endif /* !PF_ECN */ - struct pkthdr *pkt = &m->m_pkthdr; struct timespec now; int droptype, s; - u_int16_t pmin; + uint16_t pmin; int fc_adv = 0; int ret = CLASSQEQ_SUCCESS; - u_int32_t maxqsize = 0; + uint32_t maxqsize = 0; + uint64_t *pkt_timestamp; + uint32_t *pkt_sfb_hash; + uint16_t *pkt_sfb_hash16; + uint32_t *pkt_sfb_flags; + uint32_t pkt_flowid; + volatile uint32_t *pkt_flags; + uint8_t pkt_proto, pkt_flowsrc; s = sp->sfb_current; VERIFY((s + (s ^ 1)) == 1); - /* See comments in */ - VERIFY(!(pkt->pkt_flags & PKTF_PRIV_GUARDED)); - pkt->pkt_flags |= PKTF_PRIV_GUARDED; + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid, + &pkt_flowsrc, &pkt_proto, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); + pkt_sfb_hash16 = (uint16_t *)pkt_sfb_hash; + + switch (pkt->pktsched_ptype) { + case QP_MBUF: + /* See comments in */ + VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED)); + *pkt_flags |= PKTF_PRIV_GUARDED; + break; + default: + VERIFY(0); + /* NOTREACHED */ + __builtin_unreachable(); + } - if (pkt->pkt_enqueue_ts > 0) { - net_nsectimer(&pkt->pkt_enqueue_ts, &now); + if (*pkt_timestamp > 0) { + net_nsectimer(pkt_timestamp, &now); } else { nanouptime(&now); - net_timernsec(&now, &pkt->pkt_enqueue_ts); + net_timernsec(&now, pkt_timestamp); } /* time to swap the bins? */ @@ -1189,15 +1132,16 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) * If getq time is not set because this is the first packet * or after idle time, set it now so that we can detect a stall. */ - if (qsize(q) == 0 && !net_timerisset(&sp->sfb_getqtime)) + if (qsize(q) == 0 && !net_timerisset(&sp->sfb_getqtime)) { *(&sp->sfb_getqtime) = *(&now); + } - pkt->pkt_sfb_flags = 0; - pkt->pkt_sfb_hash16[s] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + *pkt_sfb_flags = 0; + pkt_sfb_hash16[s] = + (SFB_HASH(&pkt_flowid, sizeof(pkt_flowid), (*sp->sfb_bins)[s].fudge) & SFB_HASHMASK); - pkt->pkt_sfb_hash16[s ^ 1] = - (SFB_HASH(&pkt->pkt_flowid, sizeof (pkt->pkt_flowid), + pkt_sfb_hash16[s ^ 1] = + (SFB_HASH(&pkt_flowid, sizeof(pkt_flowid), (*sp->sfb_bins)[s ^ 1].fudge) & SFB_HASHMASK); /* check if the queue has been stalled */ @@ -1205,21 +1149,23 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) /* see if we drop early */ droptype = DTYPE_NODROP; - if (sfb_drop_early(sp, pkt, &pmin, &now)) { + if (sfb_drop_early(sp, *pkt_sfb_hash, &pmin, &now)) { /* flow control, mark or drop by sfb */ if ((sp->sfb_flags & SFBF_FLOWCTL) && - (pkt->pkt_flags & PKTF_FLOW_ADV)) { + (*pkt_flags & PKTF_FLOW_ADV)) { fc_adv = 1; /* drop all during suspension or for non-TCP */ if ((sp->sfb_flags & SFBF_SUSPENDED) || - pkt->pkt_proto != IPPROTO_TCP) { + pkt_proto != IPPROTO_TCP) { droptype = DTYPE_EARLY; sp->sfb_stats.drop_early++; } } #if PF_ECN + /* XXX: only supported for mbuf */ else if ((sp->sfb_flags & SFBF_ECN) && - (pkt->pkt_proto == IPPROTO_TCP) && /* only for TCP */ + (pkt->pktsched_ptype == QP_MBUF) && + (pkt_proto == IPPROTO_TCP) && /* only for TCP */ ((sfb_random(sp) & SFB_MAX_PMARK) <= pmin) && mark_ecn(m, t, sp->sfb_flags) && !(sp->sfb_flags & SFBF_SUSPENDED)) { @@ -1235,24 +1181,26 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } /* non-responsive flow penalty? */ - if (droptype == DTYPE_NODROP && sfb_penalize(sp, pkt, &now)) { + if (droptype == DTYPE_NODROP && sfb_penalize(sp, *pkt_sfb_hash, + pkt_sfb_flags, &now)) { droptype = DTYPE_FORCED; sp->sfb_stats.drop_pbox++; } - if (SFB_QUEUE_DELAYBASED(sp)) + if (SFB_QUEUE_DELAYBASED(sp)) { maxqsize = SFB_QUEUE_DELAYBASED_MAXSIZE; - else + } else { maxqsize = qlimit(q); + } /* * When the queue length hits the queue limit, make it a forced * drop */ if (droptype == DTYPE_NODROP && qlen(q) >= maxqsize) { - if (pkt->pkt_proto == IPPROTO_TCP && + if (pkt_proto == IPPROTO_TCP && qlen(q) < (maxqsize + (maxqsize >> 1)) && - ((pkt->pkt_flags & PKTF_TCP_REXMT) || + ((*pkt_flags & PKTF_TCP_REXMT) || (sp->sfb_flags & SFBF_LAST_PKT_DROPPED))) { /* * At some level, dropping packets will make the @@ -1271,60 +1219,86 @@ sfb_addq(struct sfb *sp, class_queue_t *q, struct mbuf *m, struct pf_mtag *t) } if (fc_adv == 1 && droptype != DTYPE_FORCED && - sfb_bin_addfcentry(sp, pkt)) { + sfb_bin_addfcentry(sp, pkt, *pkt_sfb_hash, pkt_flowsrc, + pkt_flowid)) { /* deliver flow control advisory error */ if (droptype == DTYPE_NODROP) { ret = CLASSQEQ_SUCCESS_FC; VERIFY(!(sp->sfb_flags & SFBF_SUSPENDED)); } else if (sp->sfb_flags & SFBF_SUSPENDED) { - /* dropped due to suspension */ - ret = CLASSQEQ_DROPPED_SP; + /* drop due to suspension */ + ret = CLASSQEQ_DROP_SP; } else { - /* dropped due to flow-control */ - ret = CLASSQEQ_DROPPED_FC; + /* drop due to flow-control */ + ret = CLASSQEQ_DROP_FC; } } /* if successful enqueue this packet, else drop it */ if (droptype == DTYPE_NODROP) { - _addq(q, m); + VERIFY(pkt->pktsched_ptype == qptype(q)); + _addq(q, &pkt->pktsched_pkt); } else { IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - m_freem(m); - return ((ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROPPED); + return (ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP; } - if (!(pkt->pkt_sfb_flags & SFB_PKT_PBOX)) - sfb_eq_update_bins(sp, pkt); - else + if (!(*pkt_sfb_flags & SFB_PKT_PBOX)) { + sfb_eq_update_bins(sp, *pkt_sfb_hash, + pktsched_get_pkt_len(pkt)); + } else { sp->sfb_stats.pbox_packets++; + } /* successfully queued */ - return (ret); + return ret; } -static struct mbuf * -sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) +static void * +sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge, + pktsched_pkt_t *pkt) { struct timespec now; - struct mbuf *m; - struct pkthdr *pkt; - - if (!purge && (sp->sfb_flags & SFBF_SUSPENDED)) - return (NULL); + uint64_t *pkt_timestamp; + volatile uint32_t *pkt_flags; + uint32_t *pkt_sfb_flags; + uint32_t *pkt_sfb_hash; + classq_pkt_t p = CLASSQ_PKT_INITIALIZER(p); + + if (!purge && (sp->sfb_flags & SFBF_SUSPENDED)) { + return NULL; + } nanouptime(&now); /* flow of 0 means head of queue */ - if ((m = ((flow == 0) ? _getq(q) : _getq_flow(q, flow))) == NULL) { - if (!purge) + if (flow == 0) { + _getq(q, &p); + } else { + _getq_flow(q, &p, flow); + } + + if (p.cp_ptype == QP_INVALID) { + if (!purge) { net_timerclear(&sp->sfb_getqtime); - return (NULL); + } + return NULL; } - VERIFY(m->m_flags & M_PKTHDR); + pktsched_pkt_encap(pkt, &p); + pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, + NULL, NULL, NULL); + pkt_sfb_hash = pktsched_get_pkt_sfb_vars(pkt, &pkt_sfb_flags); - pkt = &m->m_pkthdr; - VERIFY(pkt->pkt_flags & PKTF_PRIV_GUARDED); + /* See comments in */ + switch (p.cp_ptype) { + case QP_MBUF: + VERIFY(*pkt_flags & PKTF_PRIV_GUARDED); + break; + default: + VERIFY(0); + /* NOTREACHED */ + __builtin_unreachable(); + } if (!purge) { /* calculate EWMA of dequeues */ @@ -1342,8 +1316,9 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) * average, weigh the average more against * the old value. */ - if (DEQUEUE_SPIKE(new, avg)) + if (DEQUEUE_SPIKE(new, avg)) { decay += 5; + } avg = (((avg << decay) - avg) + new) >> decay; } else { avg = new; @@ -1356,27 +1331,29 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) if (!purge && SFB_QUEUE_DELAYBASED(sp)) { u_int64_t dequeue_ns, queue_delay = 0; net_timernsec(&now, &dequeue_ns); - if (dequeue_ns > pkt->pkt_enqueue_ts) - queue_delay = dequeue_ns - pkt->pkt_enqueue_ts; + if (dequeue_ns > *pkt_timestamp) { + queue_delay = dequeue_ns - *pkt_timestamp; + } if (sp->sfb_min_qdelay == 0 || - (queue_delay > 0 && queue_delay < sp->sfb_min_qdelay)) + (queue_delay > 0 && queue_delay < sp->sfb_min_qdelay)) { sp->sfb_min_qdelay = queue_delay; + } if (net_timercmp(&now, &sp->sfb_update_time, >=)) { if (sp->sfb_min_qdelay > sp->sfb_target_qdelay) { - if (!SFB_IS_DELAYHIGH(sp)) + if (!SFB_IS_DELAYHIGH(sp)) { SFB_SET_DELAY_HIGH(sp, q); + } } else { sp->sfb_flags &= ~(SFBF_DELAYHIGH); sp->sfb_fc_threshold = 0; - } net_timeradd(&now, &sp->sfb_update_interval, &sp->sfb_update_time); sp->sfb_min_qdelay = 0; } } - pkt->pkt_enqueue_ts = 0; + *pkt_timestamp = 0; /* * Clearpkts are the ones which were in the queue when the hash @@ -1387,18 +1364,28 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) * this reason. A rule of thumb is to set it to K*D, where D is * the time taken to drain queue. */ - if (pkt->pkt_sfb_flags & SFB_PKT_PBOX) { - pkt->pkt_sfb_flags &= ~SFB_PKT_PBOX; - if (sp->sfb_clearpkts > 0) + if (*pkt_sfb_flags & SFB_PKT_PBOX) { + *pkt_sfb_flags &= ~SFB_PKT_PBOX; + if (sp->sfb_clearpkts > 0) { sp->sfb_clearpkts--; + } } else if (sp->sfb_clearpkts > 0) { sp->sfb_clearpkts--; } else { - sfb_dq_update_bins(sp, pkt, &now, qsize(q)); + sfb_dq_update_bins(sp, *pkt_sfb_hash, pktsched_get_pkt_len(pkt), + &now, qsize(q)); } - /* See comments in */ - pkt->pkt_flags &= ~PKTF_PRIV_GUARDED; + switch (p.cp_ptype) { + case QP_MBUF: + /* See comments in */ + *pkt_flags &= ~PKTF_PRIV_GUARDED; + break; + default: + VERIFY(0); + /* NOTREACHED */ + __builtin_unreachable(); + } /* * If the queue becomes empty before the update interval, reset @@ -1411,14 +1398,13 @@ sfb_getq_flow(struct sfb *sp, class_queue_t *q, u_int32_t flow, boolean_t purge) net_timerclear(&sp->sfb_update_time); net_timerclear(&sp->sfb_getqtime); } - - return (m); + return pkt->pktsched_pkt_mbuf; } -struct mbuf * -sfb_getq(struct sfb *sp, class_queue_t *q) +void +sfb_getq(struct sfb *sp, class_queue_t *q, pktsched_pkt_t *pkt) { - return (sfb_getq_flow(sp, q, 0, FALSE)); + sfb_getq_flow(sp, q, 0, FALSE, pkt); } void @@ -1426,20 +1412,21 @@ sfb_purgeq(struct sfb *sp, class_queue_t *q, u_int32_t flow, u_int32_t *packets, u_int32_t *bytes) { u_int32_t cnt = 0, len = 0; - struct mbuf *m; + pktsched_pkt_t pkt; IFCQ_CONVERT_LOCK(&sp->sfb_ifp->if_snd); - - while ((m = sfb_getq_flow(sp, q, flow, TRUE)) != NULL) { + while (sfb_getq_flow(sp, q, flow, TRUE, &pkt) != NULL) { cnt++; - len += m_pktlen(m); - m_freem(m); + len += pktsched_get_pkt_len(&pkt); + pktsched_free_pkt(&pkt); } - if (packets != NULL) + if (packets != NULL) { *packets = cnt; - if (bytes != NULL) + } + if (bytes != NULL) { *bytes = len; + } } void @@ -1454,8 +1441,9 @@ sfb_updateq(struct sfb *sp, cqev_t ev) u_int64_t eff_rate = ifnet_output_linkrate(ifp); /* update parameters only if rate has changed */ - if (eff_rate == sp->sfb_eff_rate) + if (eff_rate == sp->sfb_eff_rate) { break; + } if (classq_verbose) { log(LOG_DEBUG, "%s: SFB qid=%d, adapting to new " @@ -1464,7 +1452,7 @@ sfb_updateq(struct sfb *sp, cqev_t ev) } sfb_calc_holdtime(sp, eff_rate); sfb_calc_pboxtime(sp, eff_rate); - sfb_calc_target_qdelay(sp, eff_rate); + ifclassq_calc_target_qdelay(ifp, &sp->sfb_target_qdelay); sfb_calc_update_interval(sp, eff_rate); break; } @@ -1495,14 +1483,15 @@ sfb_suspendq(struct sfb *sp, class_queue_t *q, boolean_t on) VERIFY(ifp != NULL); if ((on && (sp->sfb_flags & SFBF_SUSPENDED)) || - (!on && !(sp->sfb_flags & SFBF_SUSPENDED))) - return (0); + (!on && !(sp->sfb_flags & SFBF_SUSPENDED))) { + return 0; + } if (!(sp->sfb_flags & SFBF_FLOWCTL)) { log(LOG_ERR, "%s: SFB qid=%d, unable to %s queue since " "flow-control is not enabled", if_name(ifp), sp->sfb_qid, (on ? "suspend" : "resume")); - return (ENOTSUP); + return ENOTSUP; } if (classq_verbose) { @@ -1517,5 +1506,5 @@ sfb_suspendq(struct sfb *sp, class_queue_t *q, boolean_t on) sfb_swap_bins(sp, qlen(q)); } - return (0); + return 0; }