[apple/xnu.git] / bsd / net / pktsched / pktsched_fq_codel.h

/*
 * Copyright (c) 2016-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _NET_PKTSCHED_FQ_CODEL_H_
#define _NET_PKTSCHED_FQ_CODEL_H_

#ifdef PRIVATE
#include <sys/types.h>
#include <sys/param.h>

#ifdef BSD_KERNEL_PRIVATE
#include <net/flowadv.h>
#include <net/pktsched/pktsched.h>
#endif /* BSD_KERNEL_PRIVATE */

#ifdef __cplusplus
extern "C" {
#endif

#ifdef BSD_KERNEL_PRIVATE
struct fcl_stat {
	u_int32_t fcl_flow_control;
	u_int32_t fcl_flow_feedback;
	u_int32_t fcl_dequeue_stall;
	u_int32_t fcl_flow_control_fail;
	u_int64_t fcl_drop_overflow;
	u_int64_t fcl_drop_early;
	u_int32_t fcl_drop_memfailure;
	u_int32_t fcl_flows_cnt;
	u_int32_t fcl_newflows_cnt;
	u_int32_t fcl_oldflows_cnt;
	u_int64_t fcl_pkt_cnt;
	u_int64_t fcl_dequeue;
	u_int64_t fcl_dequeue_bytes;
	u_int64_t fcl_byte_cnt;
	u_int32_t fcl_throttle_on;
	u_int32_t fcl_throttle_off;
	u_int32_t fcl_throttle_drops;
	u_int32_t fcl_dup_rexmts;
	u_int32_t fcl_pkts_compressible;
	u_int32_t fcl_pkts_compressed;
};

/*
 * Use the top most 8 bits of flow id as the tag for set associative
 * hashing
 */

#define FQ_IF_HASH_TAG_SIZE     8
#define FQ_IF_HASH_TAG_SHIFT    24
#define FQ_IF_HASH_TAG_MASK     0xFF
#define FQ_IF_HASH_TABLE_SIZE   (1 << FQ_IF_HASH_TAG_SIZE)

/* Set the quantum to be one MTU */
#define FQ_IF_DEFAULT_QUANTUM   1500

/* Max number of service classes currently supported */
#define FQ_IF_MAX_CLASSES       10
_Static_assert(FQ_IF_MAX_CLASSES < 127,
    "maximum number of classes needs to fit in a single byte");

#define FQ_IF_LARGE_FLOW_BYTE_LIMIT     15000

struct flowq;
typedef u_int32_t pktsched_bitmap_t;
struct if_ifclassq_stats;

enum fq_if_state {
	FQ_IF_ER = 0,           /* eligible, ready */
	FQ_IF_IR = 1,           /* ineligible, ready */
	FQ_IF_EB = 2,           /* eligible blocked */
	FQ_IF_IB = 3,           /* ineligible, blocked */
	FQ_IF_MAX_STATE
};

/*
 * This priority index is used for QFQ state bitmaps, lower index gets
 * higher priority
 */
#define FQ_IF_BK_SYS_INDEX      9
#define FQ_IF_BK_INDEX  8
#define FQ_IF_BE_INDEX  7
#define FQ_IF_RD_INDEX  6
#define FQ_IF_OAM_INDEX 5
#define FQ_IF_AV_INDEX  4
#define FQ_IF_RV_INDEX  3
#define FQ_IF_VI_INDEX  2
#define FQ_IF_SIG_INDEX 2
#define FQ_IF_VO_INDEX  1
#define FQ_IF_CTL_INDEX 0

typedef SLIST_HEAD(, flowq) flowq_list_t;
typedef STAILQ_HEAD(, flowq) flowq_stailq_t;
typedef struct fq_if_classq {
	uint32_t fcl_pri;      /* class priority, lower the better */
	uint32_t fcl_service_class;    /* service class */
	uint16_t fcl_quantum;          /* quantum in bytes */
	uint32_t fcl_drr_max;          /* max flows per class for DRR */
	int64_t fcl_budget;             /* budget for this classq */
	flowq_stailq_t fcl_new_flows;   /* List of new flows */
	flowq_stailq_t fcl_old_flows;   /* List of old flows */
	struct fcl_stat fcl_stat;
} fq_if_classq_t;

typedef struct fq_codel_sched_data {
	struct ifclassq *fqs_ifq;       /* back pointer to ifclassq */
	u_int64_t       fqs_target_qdelay;      /* Target queue delay (ns) */
	u_int64_t       fqs_update_interval;    /* update interval (ns) */
	flowq_list_t    fqs_flows[FQ_IF_HASH_TABLE_SIZE]; /* flows table */
	pktsched_bitmap_t       fqs_bitmaps[FQ_IF_MAX_STATE];
	u_int32_t       fqs_pkt_droplimit;      /* drop limit */
	u_int8_t        fqs_throttle;   /* throttle on or off */
	u_int8_t        fqs_flags;      /* flags */
#define FQS_DRIVER_MANAGED      0x1
	fq_if_classq_t  fqs_classq[FQ_IF_MAX_CLASSES]; /* class queues */
	struct flowadv_fclist   fqs_fclist; /* flow control state */
	struct flowq    *fqs_large_flow; /* flow has highest number of bytes */
	classq_pkt_type_t       fqs_ptype;
} fq_if_t;

#endif /* BSD_KERNEL_PRIVATE */

struct fq_codel_flowstats {
	u_int32_t       fqst_min_qdelay;
#define FQ_FLOWSTATS_OLD_FLOW   0x1
#define FQ_FLOWSTATS_NEW_FLOW   0x2
#define FQ_FLOWSTATS_LARGE_FLOW 0x4
#define FQ_FLOWSTATS_DELAY_HIGH 0x8
#define FQ_FLOWSTATS_FLOWCTL_ON 0x10
	u_int32_t       fqst_flags;
	u_int32_t       fqst_bytes;
	u_int32_t       fqst_flowhash;
};

#define FQ_IF_MAX_FLOWSTATS     20

struct fq_codel_classstats {
	u_int32_t       fcls_pri;
	u_int32_t       fcls_service_class;
	u_int32_t       fcls_quantum;
	u_int32_t       fcls_drr_max;
	int64_t         fcls_budget;
	u_int64_t       fcls_target_qdelay;
	u_int64_t       fcls_update_interval;
	u_int32_t       fcls_flow_control;
	u_int32_t       fcls_flow_feedback;
	u_int32_t       fcls_dequeue_stall;
	u_int32_t       fcls_flow_control_fail;
	u_int64_t       fcls_drop_overflow;
	u_int64_t       fcls_drop_early;
	u_int32_t       fcls_drop_memfailure;
	u_int32_t       fcls_flows_cnt;
	u_int32_t       fcls_newflows_cnt;
	u_int32_t       fcls_oldflows_cnt;
	u_int64_t       fcls_pkt_cnt;
	u_int64_t       fcls_dequeue;
	u_int64_t       fcls_dequeue_bytes;
	u_int64_t       fcls_byte_cnt;
	u_int32_t       fcls_throttle_on;
	u_int32_t       fcls_throttle_off;
	u_int32_t       fcls_throttle_drops;
	u_int32_t       fcls_dup_rexmts;
	u_int32_t       fcls_flowstats_cnt;
	struct fq_codel_flowstats fcls_flowstats[FQ_IF_MAX_FLOWSTATS];
	u_int32_t       fcls_pkts_compressible;
	u_int32_t       fcls_pkts_compressed;
};

#ifdef BSD_KERNEL_PRIVATE

extern void fq_codel_scheduler_init(void);
extern int fq_if_enqueue_classq(struct ifclassq *ifq, classq_pkt_t *h,
    classq_pkt_t *t, uint32_t cnt, uint32_t bytes, boolean_t *pdrop);
extern void fq_if_dequeue_classq(struct ifclassq *ifq, classq_pkt_t *pkt);
extern void fq_if_dequeue_sc_classq(struct ifclassq *ifq, mbuf_svc_class_t svc,
    classq_pkt_t *pkt);
extern int fq_if_dequeue_classq_multi(struct ifclassq *ifq, u_int32_t maxpktcnt,
    u_int32_t maxbytecnt, classq_pkt_t *first_packet, classq_pkt_t *last_packet,
    u_int32_t *retpktcnt, u_int32_t *retbytecnt);
extern int fq_if_dequeue_sc_classq_multi(struct ifclassq *ifq,
    mbuf_svc_class_t svc, u_int32_t maxpktcnt, u_int32_t maxbytecnt,
    classq_pkt_t *first_packet, classq_pkt_t *last_packet, u_int32_t *retpktcnt,
    u_int32_t *retbytecnt);
extern int fq_if_request_classq(struct ifclassq *ifq, cqrq_t rq, void *arg);
extern struct flowq *fq_if_hash_pkt(fq_if_t *, u_int32_t, mbuf_svc_class_t,
    u_int64_t, boolean_t, classq_pkt_type_t);
extern boolean_t fq_if_at_drop_limit(fq_if_t *);
extern void fq_if_drop_packet(fq_if_t *);
extern void fq_if_is_flow_heavy(fq_if_t *, struct flowq *);
extern boolean_t fq_if_add_fcentry(fq_if_t *, pktsched_pkt_t *, uint8_t,
    struct flowq *, fq_if_classq_t *);
extern void fq_if_flow_feedback(fq_if_t *, struct flowq *, fq_if_classq_t *);
extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags,
    classq_pkt_type_t ptype);
extern void fq_if_teardown_ifclassq(struct ifclassq *ifq);
extern int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid,
    struct if_ifclassq_stats *ifqs);
extern void fq_if_destroy_flow(fq_if_t *, fq_if_classq_t *,
    struct flowq *, bool);


#endif /* BSD_KERNEL_PRIVATE */

#ifdef __cplusplus
}
#endif

#endif /* PRIVATE */
#endif /* _NET_PKTSCHED_PKTSCHED_FQ_CODEL_H_ */
Commit	Line	Data
39037602	1	/*
f427ee49	2	* Copyright (c) 2016-2020 Apple Inc. All rights reserved.
39037602 A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef _NET_PKTSCHED_FQ_CODEL_H_
0a7de745	30	#define _NET_PKTSCHED_FQ_CODEL_H_
39037602 A	31
	32	#ifdef PRIVATE
	33	#include <sys/types.h>
	34	#include <sys/param.h>
	35
	36	#ifdef BSD_KERNEL_PRIVATE
	37	#include <net/flowadv.h>
f427ee49	38	#include <net/pktsched/pktsched.h>
39037602 A	39	#endif /* BSD_KERNEL_PRIVATE */
	40
	41	#ifdef __cplusplus
	42	extern "C" {
	43	#endif
	44
	45	#ifdef BSD_KERNEL_PRIVATE
	46	struct fcl_stat {
	47	u_int32_t fcl_flow_control;
	48	u_int32_t fcl_flow_feedback;
	49	u_int32_t fcl_dequeue_stall;
	50	u_int32_t fcl_flow_control_fail;
	51	u_int64_t fcl_drop_overflow;
	52	u_int64_t fcl_drop_early;
	53	u_int32_t fcl_drop_memfailure;
	54	u_int32_t fcl_flows_cnt;
	55	u_int32_t fcl_newflows_cnt;
	56	u_int32_t fcl_oldflows_cnt;
	57	u_int64_t fcl_pkt_cnt;
	58	u_int64_t fcl_dequeue;
	59	u_int64_t fcl_dequeue_bytes;
	60	u_int64_t fcl_byte_cnt;
	61	u_int32_t fcl_throttle_on;
	62	u_int32_t fcl_throttle_off;
	63	u_int32_t fcl_throttle_drops;
	64	u_int32_t fcl_dup_rexmts;
f427ee49 A	65	u_int32_t fcl_pkts_compressible;
f427ee49 A	66	u_int32_t fcl_pkts_compressed;
39037602 A	67	};
	68
	69	/*
	70	* Use the top most 8 bits of flow id as the tag for set associative
	71	* hashing
	72	*/
	73
0a7de745 A	74	#define FQ_IF_HASH_TAG_SIZE 8
	75	#define FQ_IF_HASH_TAG_SHIFT 24
	76	#define FQ_IF_HASH_TAG_MASK 0xFF
	77	#define FQ_IF_HASH_TABLE_SIZE (1 << FQ_IF_HASH_TAG_SIZE)
39037602	78
39037602	79	/* Set the quantum to be one MTU */
0a7de745	80	#define FQ_IF_DEFAULT_QUANTUM 1500
39037602 A	81
39037602 A	82	/* Max number of service classes currently supported */
0a7de745	83	#define FQ_IF_MAX_CLASSES 10
f427ee49 A	84	_Static_assert(FQ_IF_MAX_CLASSES < 127,
f427ee49 A	85	"maximum number of classes needs to fit in a single byte");
39037602	86
0a7de745	87	#define FQ_IF_LARGE_FLOW_BYTE_LIMIT 15000
5ba3f43e	88
39037602 A	89	struct flowq;
	90	typedef u_int32_t pktsched_bitmap_t;
	91	struct if_ifclassq_stats;
	92
	93	enum fq_if_state {
0a7de745 A	94	FQ_IF_ER = 0, /* eligible, ready */
	95	FQ_IF_IR = 1, /* ineligible, ready */
	96	FQ_IF_EB = 2, /* eligible blocked */
	97	FQ_IF_IB = 3, /* ineligible, blocked */
39037602 A	98	FQ_IF_MAX_STATE
	99	};
	100
	101	/*
	102	* This priority index is used for QFQ state bitmaps, lower index gets
	103	* higher priority
	104	*/
0a7de745 A	105	#define FQ_IF_BK_SYS_INDEX 9
	106	#define FQ_IF_BK_INDEX 8
	107	#define FQ_IF_BE_INDEX 7
	108	#define FQ_IF_RD_INDEX 6
	109	#define FQ_IF_OAM_INDEX 5
	110	#define FQ_IF_AV_INDEX 4
	111	#define FQ_IF_RV_INDEX 3
	112	#define FQ_IF_VI_INDEX 2
	113	#define FQ_IF_SIG_INDEX 2
	114	#define FQ_IF_VO_INDEX 1
	115	#define FQ_IF_CTL_INDEX 0
39037602 A	116
	117	typedef SLIST_HEAD(, flowq) flowq_list_t;
	118	typedef STAILQ_HEAD(, flowq) flowq_stailq_t;
	119	typedef struct fq_if_classq {
f427ee49 A	120	uint32_t fcl_pri; /* class priority, lower the better */
	121	uint32_t fcl_service_class; /* service class */
	122	uint16_t fcl_quantum; /* quantum in bytes */
	123	uint32_t fcl_drr_max; /* max flows per class for DRR */
0a7de745 A	124	int64_t fcl_budget; /* budget for this classq */
	125	flowq_stailq_t fcl_new_flows; /* List of new flows */
	126	flowq_stailq_t fcl_old_flows; /* List of old flows */
39037602 A	127	struct fcl_stat fcl_stat;
	128	} fq_if_classq_t;
	129
	130	typedef struct fq_codel_sched_data {
0a7de745 A	131	struct ifclassq fqs_ifq; / back pointer to ifclassq */
	132	u_int64_t fqs_target_qdelay; /* Target queue delay (ns) */
	133	u_int64_t fqs_update_interval; /* update interval (ns) */
	134	flowq_list_t fqs_flows[FQ_IF_HASH_TABLE_SIZE]; /* flows table */
	135	pktsched_bitmap_t fqs_bitmaps[FQ_IF_MAX_STATE];
	136	u_int32_t fqs_pkt_droplimit; /* drop limit */
	137	u_int8_t fqs_throttle; /* throttle on or off */
	138	u_int8_t fqs_flags; /* flags */
	139	#define FQS_DRIVER_MANAGED 0x1
	140	fq_if_classq_t fqs_classq[FQ_IF_MAX_CLASSES]; /* class queues */
	141	struct flowadv_fclist fqs_fclist; /* flow control state */
	142	struct flowq fqs_large_flow; / flow has highest number of bytes */
	143	classq_pkt_type_t fqs_ptype;
39037602 A	144	} fq_if_t;
	145
	146	#endif /* BSD_KERNEL_PRIVATE */
	147
5ba3f43e	148	struct fq_codel_flowstats {
0a7de745 A	149	u_int32_t fqst_min_qdelay;
	150	#define FQ_FLOWSTATS_OLD_FLOW 0x1
	151	#define FQ_FLOWSTATS_NEW_FLOW 0x2
	152	#define FQ_FLOWSTATS_LARGE_FLOW 0x4
	153	#define FQ_FLOWSTATS_DELAY_HIGH 0x8
	154	#define FQ_FLOWSTATS_FLOWCTL_ON 0x10
	155	u_int32_t fqst_flags;
	156	u_int32_t fqst_bytes;
	157	u_int32_t fqst_flowhash;
5ba3f43e A	158	};
5ba3f43e A	159
0a7de745	160	#define FQ_IF_MAX_FLOWSTATS 20
5ba3f43e	161
39037602	162	struct fq_codel_classstats {
0a7de745 A	163	u_int32_t fcls_pri;
	164	u_int32_t fcls_service_class;
	165	u_int32_t fcls_quantum;
	166	u_int32_t fcls_drr_max;
	167	int64_t fcls_budget;
	168	u_int64_t fcls_target_qdelay;
	169	u_int64_t fcls_update_interval;
	170	u_int32_t fcls_flow_control;
	171	u_int32_t fcls_flow_feedback;
	172	u_int32_t fcls_dequeue_stall;
	173	u_int32_t fcls_flow_control_fail;
	174	u_int64_t fcls_drop_overflow;
	175	u_int64_t fcls_drop_early;
	176	u_int32_t fcls_drop_memfailure;
	177	u_int32_t fcls_flows_cnt;
	178	u_int32_t fcls_newflows_cnt;
	179	u_int32_t fcls_oldflows_cnt;
	180	u_int64_t fcls_pkt_cnt;
	181	u_int64_t fcls_dequeue;
	182	u_int64_t fcls_dequeue_bytes;
	183	u_int64_t fcls_byte_cnt;
	184	u_int32_t fcls_throttle_on;
	185	u_int32_t fcls_throttle_off;
	186	u_int32_t fcls_throttle_drops;
	187	u_int32_t fcls_dup_rexmts;
	188	u_int32_t fcls_flowstats_cnt;
5ba3f43e	189	struct fq_codel_flowstats fcls_flowstats[FQ_IF_MAX_FLOWSTATS];
f427ee49 A	190	u_int32_t fcls_pkts_compressible;
f427ee49 A	191	u_int32_t fcls_pkts_compressed;
39037602 A	192	};
	193
	194	#ifdef BSD_KERNEL_PRIVATE
	195
	196	extern void fq_codel_scheduler_init(void);
f427ee49 A	197	extern int fq_if_enqueue_classq(struct ifclassq ifq, classq_pkt_t h,
	198	classq_pkt_t t, uint32_t cnt, uint32_t bytes, boolean_t pdrop);
	199	extern void fq_if_dequeue_classq(struct ifclassq ifq, classq_pkt_t pkt);
	200	extern void fq_if_dequeue_sc_classq(struct ifclassq *ifq, mbuf_svc_class_t svc,
	201	classq_pkt_t *pkt);
	202	extern int fq_if_dequeue_classq_multi(struct ifclassq *ifq, u_int32_t maxpktcnt,
	203	u_int32_t maxbytecnt, classq_pkt_t first_packet, classq_pkt_t last_packet,
	204	u_int32_t retpktcnt, u_int32_t retbytecnt);
	205	extern int fq_if_dequeue_sc_classq_multi(struct ifclassq *ifq,
	206	mbuf_svc_class_t svc, u_int32_t maxpktcnt, u_int32_t maxbytecnt,
	207	classq_pkt_t first_packet, classq_pkt_t last_packet, u_int32_t *retpktcnt,
	208	u_int32_t *retbytecnt);
	209	extern int fq_if_request_classq(struct ifclassq ifq, cqrq_t rq, void arg);
39037602	210	extern struct flowq fq_if_hash_pkt(fq_if_t , u_int32_t, mbuf_svc_class_t,
5ba3f43e	211	u_int64_t, boolean_t, classq_pkt_type_t);
39037602 A	212	extern boolean_t fq_if_at_drop_limit(fq_if_t *);
	213	extern void fq_if_drop_packet(fq_if_t *);
	214	extern void fq_if_is_flow_heavy(fq_if_t , struct flowq );
c3c9b80d A	215	extern boolean_t fq_if_add_fcentry(fq_if_t , pktsched_pkt_t , uint8_t,
c3c9b80d A	216	struct flowq , fq_if_classq_t );
39037602	217	extern void fq_if_flow_feedback(fq_if_t , struct flowq , fq_if_classq_t *);
5ba3f43e A	218	extern int fq_if_setup_ifclassq(struct ifclassq *ifq, u_int32_t flags,
5ba3f43e A	219	classq_pkt_type_t ptype);
f427ee49	220	extern void fq_if_teardown_ifclassq(struct ifclassq *ifq);
39037602 A	221	extern int fq_if_getqstats_ifclassq(struct ifclassq *ifq, u_int32_t qid,
39037602 A	222	struct if_ifclassq_stats *ifqs);
5ba3f43e	223	extern void fq_if_destroy_flow(fq_if_t , fq_if_classq_t ,
c3c9b80d	224	struct flowq *, bool);
39037602 A	225
	226
	227	#endif /* BSD_KERNEL_PRIVATE */
	228
	229	#ifdef __cplusplus
	230	}
	231	#endif
	232
	233	#endif /* PRIVATE */
	234	#endif /* _NET_PKTSCHED_PKTSCHED_FQ_CODEL_H_ */