[apple/xnu.git] / bsd / net / pktsched / pktsched.c

/*
 * Copyright (c) 2011-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <sys/cdefs.h>

#include <sys/param.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/errno.h>
#include <sys/mcache.h>
#include <sys/sysctl.h>

#include <dev/random/randomdev.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_dl.h>
#include <net/if_types.h>
#include <net/net_osdep.h>
#include <net/pktsched/pktsched.h>
#include <net/pktsched/pktsched_fq_codel.h>
#include <net/pktsched/pktsched_netem.h>

#include <pexpert/pexpert.h>


u_int32_t machclk_freq = 0;
u_int64_t machclk_per_sec = 0;
u_int32_t pktsched_verbose = 0; /* more noise if greater than 1 */

static void init_machclk(void);

SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "pktsched");

SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
    &pktsched_verbose, 0, "Packet scheduler verbosity level");

void
pktsched_init(void)
{
	init_machclk();
	if (machclk_freq == 0) {
		panic("%s: no CPU clock available!\n", __func__);
		/* NOTREACHED */
	}

	netem_init();
}

static void
init_machclk(void)
{
	/*
	 * Initialize machclk_freq using the timerbase frequency
	 * value from device specific info.
	 */
	machclk_freq = (uint32_t)gPEClockFrequencyInfo.timebase_frequency_hz;

	clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
	    &machclk_per_sec);
}

u_int64_t
pktsched_abs_to_nsecs(u_int64_t abstime)
{
	u_int64_t nsecs;

	absolutetime_to_nanoseconds(abstime, &nsecs);
	return nsecs;
}

u_int64_t
pktsched_nsecs_to_abstime(u_int64_t nsecs)
{
	u_int64_t abstime;

	nanoseconds_to_absolutetime(nsecs, &abstime);
	return abstime;
}

int
pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
    classq_pkt_type_t ptype)
{
	int error = 0;
	u_int32_t rflags;

	IFCQ_LOCK_ASSERT_HELD(ifq);

	VERIFY(machclk_freq != 0);

	/* Nothing to do unless the scheduler type changes */
	if (ifq->ifcq_type == scheduler) {
		return 0;
	}

	/*
	 * Remember the flags that need to be restored upon success, as
	 * they may be cleared when we tear down existing scheduler.
	 */
	rflags = (ifq->ifcq_flags & IFCQF_ENABLED);

	if (ifq->ifcq_type != PKTSCHEDT_NONE) {
		pktsched_teardown(ifq);

		/* Teardown should have succeeded */
		VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
		VERIFY(ifq->ifcq_disc == NULL);
	}

	error = fq_if_setup_ifclassq(ifq, sflags, ptype);
	if (error == 0) {
		ifq->ifcq_flags |= rflags;
	}

	return error;
}

void
pktsched_teardown(struct ifclassq *ifq)
{
	IFCQ_LOCK_ASSERT_HELD(ifq);

	if_qflush(ifq->ifcq_ifp, 1);
	VERIFY(IFCQ_IS_EMPTY(ifq));

	ifq->ifcq_flags &= ~IFCQF_ENABLED;

	if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
		/* Could be PKTSCHEDT_NONE */
		fq_if_teardown_ifclassq(ifq);
	}

	return;
}

int
pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
    struct if_ifclassq_stats *ifqs)
{
	int error = 0;

	IFCQ_LOCK_ASSERT_HELD(ifq);

	if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
		/* Could be PKTSCHEDT_NONE */
		error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
	}

	return error;
}

void
pktsched_pkt_encap(pktsched_pkt_t *pkt, classq_pkt_t *cpkt)
{
	pkt->pktsched_pkt = *cpkt;
	pkt->pktsched_tail = *cpkt;
	pkt->pktsched_pcnt = 1;

	switch (cpkt->cp_ptype) {
	case QP_MBUF:
		pkt->pktsched_plen =
		    (uint32_t)m_pktlen(pkt->pktsched_pkt_mbuf);
		break;


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}
}

void
pktsched_pkt_encap_chain(pktsched_pkt_t *pkt, classq_pkt_t *cpkt,
    classq_pkt_t *tail, uint32_t cnt, uint32_t bytes)
{
	pkt->pktsched_pkt = *cpkt;
	pkt->pktsched_tail = *tail;
	pkt->pktsched_pcnt = cnt;
	pkt->pktsched_plen = bytes;

	switch (cpkt->cp_ptype) {
	case QP_MBUF:
		break;


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}
}

int
pktsched_clone_pkt(pktsched_pkt_t *pkt1, pktsched_pkt_t *pkt2)
{
	struct mbuf *m1, *m2;

	ASSERT(pkt1 != NULL);
	ASSERT(pkt1->pktsched_pkt_mbuf != NULL);
	ASSERT(pkt1->pktsched_pcnt == 1);

	/* allow in place clone, but make sure pkt2->pktsched_pkt won't leak */
	ASSERT((pkt1 == pkt2 && pkt1->pktsched_pkt_mbuf ==
	    pkt2->pktsched_pkt_mbuf) || (pkt1 != pkt2 &&
	    pkt2->pktsched_pkt_mbuf == NULL));

	switch (pkt1->pktsched_ptype) {
	case QP_MBUF:
		m1 = (struct mbuf *)pkt1->pktsched_pkt_mbuf;
		m2 = m_dup(m1, M_NOWAIT);
		if (__improbable(m2 == NULL)) {
			return ENOBUFS;
		}
		pkt2->pktsched_pkt_mbuf = m2;
		break;


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}

	pkt2->pktsched_plen = pkt1->pktsched_plen;
	pkt2->pktsched_ptype = pkt1->pktsched_ptype;
	pkt2->pktsched_tail = pkt2->pktsched_pkt;
	pkt2->pktsched_pcnt = 1;
	return 0;
}

void
pktsched_corrupt_packet(pktsched_pkt_t *pkt)
{
	struct mbuf *m = NULL;
	uint8_t *data = NULL;
	uint32_t data_len = 0;
	uint32_t rand32, rand_off, rand_bit;

	switch (pkt->pktsched_ptype) {
	case QP_MBUF:
		m = pkt->pktsched_pkt_mbuf;
		data = mtod(m, uint8_t *);
		data_len = m->m_pkthdr.len;
		break;

	default:
		/* NOTREACHED */
		VERIFY(0);
		__builtin_unreachable();
	}

	read_frandom(&rand32, sizeof(rand32));
	rand_bit = rand32 & 0x8;
	rand_off = (rand32 >> 3) % data_len;
	data[rand_off] ^= 1 << rand_bit;
}

void
pktsched_free_pkt(pktsched_pkt_t *pkt)
{
	uint32_t cnt = pkt->pktsched_pcnt;
	ASSERT(cnt != 0);

	switch (pkt->pktsched_ptype) {
	case QP_MBUF: {
		struct mbuf *m;

		m = pkt->pktsched_pkt_mbuf;
		if (cnt == 1) {
			VERIFY(m->m_nextpkt == NULL);
		} else {
			VERIFY(m->m_nextpkt != NULL);
		}
		m_freem_list(m);
		break;
	}

	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}
	pkt->pktsched_pkt = CLASSQ_PKT_INITIALIZER(pkt->pktsched_pkt);
	pkt->pktsched_tail = CLASSQ_PKT_INITIALIZER(pkt->pktsched_tail);
	pkt->pktsched_plen = 0;
	pkt->pktsched_pcnt = 0;
}

mbuf_svc_class_t
pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
{
	mbuf_svc_class_t svc = MBUF_SC_UNSPEC;

	switch (pkt->pktsched_ptype) {
	case QP_MBUF:
		svc = m_get_service_class(pkt->pktsched_pkt_mbuf);
		break;


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}

	return svc;
}

void
pktsched_get_pkt_vars(pktsched_pkt_t *pkt, volatile uint32_t **flags,
    uint64_t **timestamp, uint32_t *flowid, uint8_t *flowsrc, uint8_t *proto,
    uint32_t *comp_gencnt)
{
	switch (pkt->pktsched_ptype) {
	case QP_MBUF: {
		struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);

		if (flags != NULL) {
			*flags = &pkth->pkt_flags;
		}
		if (timestamp != NULL) {
			*timestamp = &pkth->pkt_timestamp;
		}
		if (flowid != NULL) {
			*flowid = pkth->pkt_flowid;
		}
		if (flowsrc != NULL) {
			*flowsrc = pkth->pkt_flowsrc;
		}
		if (proto != NULL) {
			*proto = pkth->pkt_proto;
		}
		if (comp_gencnt != NULL) {
			*comp_gencnt = pkth->comp_gencnt;
		}

		break;
	}


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}
}

struct flowadv_fcentry *
pktsched_alloc_fcentry(pktsched_pkt_t *pkt, struct ifnet *ifp, int how)
{
#pragma unused(ifp)
	struct flowadv_fcentry *fce = NULL;

	switch (pkt->pktsched_ptype) {
	case QP_MBUF: {
		struct mbuf *m = pkt->pktsched_pkt_mbuf;

		fce = flowadv_alloc_entry(how);
		if (fce == NULL) {
			break;
		}

		_CASSERT(sizeof(m->m_pkthdr.pkt_flowid) ==
		    sizeof(fce->fce_flowid));

		fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
		fce->fce_flowid = m->m_pkthdr.pkt_flowid;
		break;
	}


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}

	return fce;
}

uint32_t *
pktsched_get_pkt_sfb_vars(pktsched_pkt_t *pkt, uint32_t **sfb_flags)
{
	uint32_t *hashp = NULL;

	switch (pkt->pktsched_ptype) {
	case QP_MBUF: {
		struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);

		_CASSERT(sizeof(pkth->pkt_mpriv_hash) == sizeof(uint32_t));
		_CASSERT(sizeof(pkth->pkt_mpriv_flags) == sizeof(uint32_t));
		*sfb_flags = &pkth->pkt_mpriv_flags;
		hashp = &pkth->pkt_mpriv_hash;
		break;
	}


	default:
		VERIFY(0);
		/* NOTREACHED */
		__builtin_unreachable();
	}

	return hashp;
}
Commit	Line	Data
316670eb	1	/*
f427ee49	2	* Copyright (c) 2011-2020 Apple Inc. All rights reserved.
316670eb A	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#include <sys/cdefs.h>
	30
	31	#include <sys/param.h>
	32	#include <sys/malloc.h>
	33	#include <sys/mbuf.h>
	34	#include <sys/systm.h>
	35	#include <sys/kernel.h>
	36	#include <sys/errno.h>
	37	#include <sys/mcache.h>
	38	#include <sys/sysctl.h>
	39
cb323159	40	#include <dev/random/randomdev.h>
316670eb A	41	#include <net/if.h>
	42	#include <net/if_var.h>
	43	#include <net/if_dl.h>
	44	#include <net/if_types.h>
	45	#include <net/net_osdep.h>
	46	#include <net/pktsched/pktsched.h>
39037602	47	#include <net/pktsched/pktsched_fq_codel.h>
cb323159	48	#include <net/pktsched/pktsched_netem.h>
316670eb A	49
	50	#include <pexpert/pexpert.h>
	51
5ba3f43e	52
316670eb A	53	u_int32_t machclk_freq = 0;
316670eb A	54	u_int64_t machclk_per_sec = 0;
cb323159	55	u_int32_t pktsched_verbose = 0; /* more noise if greater than 1 */
316670eb A	56
	57	static void init_machclk(void);
	58
0a7de745	59	SYSCTL_NODE(_net, OID_AUTO, pktsched, CTLFLAG_RW \| CTLFLAG_LOCKED, 0, "pktsched");
316670eb	60
0a7de745 A	61	SYSCTL_UINT(_net_pktsched, OID_AUTO, verbose, CTLFLAG_RW \| CTLFLAG_LOCKED,
0a7de745 A	62	&pktsched_verbose, 0, "Packet scheduler verbosity level");
316670eb A	63
	64	void
	65	pktsched_init(void)
	66	{
	67	init_machclk();
	68	if (machclk_freq == 0) {
	69	panic("%s: no CPU clock available!\n", __func__);
	70	/* NOTREACHED */
	71	}
	72
cb323159	73	netem_init();
316670eb A	74	}
	75
	76	static void
	77	init_machclk(void)
	78	{
	79	/*
	80	* Initialize machclk_freq using the timerbase frequency
	81	* value from device specific info.
	82	*/
f427ee49	83	machclk_freq = (uint32_t)gPEClockFrequencyInfo.timebase_frequency_hz;
316670eb A	84
	85	clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC,
	86	&machclk_per_sec);
	87	}
	88
	89	u_int64_t
	90	pktsched_abs_to_nsecs(u_int64_t abstime)
	91	{
	92	u_int64_t nsecs;
	93
	94	absolutetime_to_nanoseconds(abstime, &nsecs);
0a7de745	95	return nsecs;
316670eb A	96	}
	97
	98	u_int64_t
	99	pktsched_nsecs_to_abstime(u_int64_t nsecs)
	100	{
	101	u_int64_t abstime;
	102
	103	nanoseconds_to_absolutetime(nsecs, &abstime);
0a7de745	104	return abstime;
316670eb A	105	}
	106
	107	int
5ba3f43e A	108	pktsched_setup(struct ifclassq *ifq, u_int32_t scheduler, u_int32_t sflags,
5ba3f43e A	109	classq_pkt_type_t ptype)
316670eb A	110	{
316670eb A	111	int error = 0;
316670eb A	112	u_int32_t rflags;
	113
	114	IFCQ_LOCK_ASSERT_HELD(ifq);
	115
	116	VERIFY(machclk_freq != 0);
	117
	118	/* Nothing to do unless the scheduler type changes */
0a7de745 A	119	if (ifq->ifcq_type == scheduler) {
	120	return 0;
	121	}
316670eb	122
316670eb A	123	/*
	124	* Remember the flags that need to be restored upon success, as
	125	* they may be cleared when we tear down existing scheduler.
	126	*/
	127	rflags = (ifq->ifcq_flags & IFCQF_ENABLED);
	128
	129	if (ifq->ifcq_type != PKTSCHEDT_NONE) {
f427ee49	130	pktsched_teardown(ifq);
316670eb A	131
	132	/* Teardown should have succeeded */
	133	VERIFY(ifq->ifcq_type == PKTSCHEDT_NONE);
	134	VERIFY(ifq->ifcq_disc == NULL);
316670eb A	135	}
316670eb A	136
f427ee49	137	error = fq_if_setup_ifclassq(ifq, sflags, ptype);
0a7de745	138	if (error == 0) {
316670eb	139	ifq->ifcq_flags \|= rflags;
0a7de745	140	}
316670eb	141
0a7de745	142	return error;
316670eb A	143	}
316670eb A	144
f427ee49	145	void
316670eb A	146	pktsched_teardown(struct ifclassq *ifq)
316670eb A	147	{
316670eb A	148	IFCQ_LOCK_ASSERT_HELD(ifq);
	149
	150	if_qflush(ifq->ifcq_ifp, 1);
	151	VERIFY(IFCQ_IS_EMPTY(ifq));
	152
	153	ifq->ifcq_flags &= ~IFCQF_ENABLED;
	154
f427ee49 A	155	if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
	156	/* Could be PKTSCHEDT_NONE */
	157	fq_if_teardown_ifclassq(ifq);
316670eb	158	}
f427ee49 A	159
f427ee49 A	160	return;
316670eb A	161	}
	162
	163	int
	164	pktsched_getqstats(struct ifclassq *ifq, u_int32_t qid,
	165	struct if_ifclassq_stats *ifqs)
	166	{
f427ee49	167	int error = 0;
316670eb A	168
	169	IFCQ_LOCK_ASSERT_HELD(ifq);
	170
f427ee49 A	171	if (ifq->ifcq_type == PKTSCHEDT_FQ_CODEL) {
f427ee49 A	172	/* Could be PKTSCHEDT_NONE */
39037602	173	error = fq_if_getqstats_ifclassq(ifq, qid, ifqs);
316670eb A	174	}
316670eb A	175
0a7de745	176	return error;
316670eb	177	}
5ba3f43e A	178
5ba3f43e A	179	void
cb323159	180	pktsched_pkt_encap(pktsched_pkt_t pkt, classq_pkt_t cpkt)
5ba3f43e	181	{
cb323159	182	pkt->pktsched_pkt = *cpkt;
f427ee49 A	183	pkt->pktsched_tail = *cpkt;
f427ee49 A	184	pkt->pktsched_pcnt = 1;
5ba3f43e	185
cb323159	186	switch (cpkt->cp_ptype) {
5ba3f43e A	187	case QP_MBUF:
5ba3f43e A	188	pkt->pktsched_plen =
cb323159	189	(uint32_t)m_pktlen(pkt->pktsched_pkt_mbuf);
5ba3f43e A	190	break;
	191
	192
	193	default:
	194	VERIFY(0);
	195	/* NOTREACHED */
cb323159	196	__builtin_unreachable();
5ba3f43e A	197	}
	198	}
	199
f427ee49 A	200	void
	201	pktsched_pkt_encap_chain(pktsched_pkt_t pkt, classq_pkt_t cpkt,
	202	classq_pkt_t *tail, uint32_t cnt, uint32_t bytes)
	203	{
	204	pkt->pktsched_pkt = *cpkt;
	205	pkt->pktsched_tail = *tail;
	206	pkt->pktsched_pcnt = cnt;
	207	pkt->pktsched_plen = bytes;
	208
	209	switch (cpkt->cp_ptype) {
	210	case QP_MBUF:
	211	break;
	212
	213
	214	default:
	215	VERIFY(0);
	216	/* NOTREACHED */
	217	__builtin_unreachable();
	218	}
	219	}
	220
cb323159 A	221	int
	222	pktsched_clone_pkt(pktsched_pkt_t pkt1, pktsched_pkt_t pkt2)
	223	{
	224	struct mbuf m1, m2;
	225
	226	ASSERT(pkt1 != NULL);
	227	ASSERT(pkt1->pktsched_pkt_mbuf != NULL);
f427ee49 A	228	ASSERT(pkt1->pktsched_pcnt == 1);
f427ee49 A	229
cb323159 A	230	/* allow in place clone, but make sure pkt2->pktsched_pkt won't leak */
	231	ASSERT((pkt1 == pkt2 && pkt1->pktsched_pkt_mbuf ==
	232	pkt2->pktsched_pkt_mbuf) \|\| (pkt1 != pkt2 &&
	233	pkt2->pktsched_pkt_mbuf == NULL));
	234
	235	switch (pkt1->pktsched_ptype) {
	236	case QP_MBUF:
	237	m1 = (struct mbuf *)pkt1->pktsched_pkt_mbuf;
	238	m2 = m_dup(m1, M_NOWAIT);
	239	if (__improbable(m2 == NULL)) {
	240	return ENOBUFS;
	241	}
	242	pkt2->pktsched_pkt_mbuf = m2;
	243	break;
	244
	245
	246	default:
	247	VERIFY(0);
	248	/* NOTREACHED */
	249	__builtin_unreachable();
	250	}
	251
	252	pkt2->pktsched_plen = pkt1->pktsched_plen;
	253	pkt2->pktsched_ptype = pkt1->pktsched_ptype;
f427ee49 A	254	pkt2->pktsched_tail = pkt2->pktsched_pkt;
f427ee49 A	255	pkt2->pktsched_pcnt = 1;
cb323159 A	256	return 0;
	257	}
	258
	259	void
	260	pktsched_corrupt_packet(pktsched_pkt_t *pkt)
	261	{
	262	struct mbuf *m = NULL;
	263	uint8_t *data = NULL;
	264	uint32_t data_len = 0;
	265	uint32_t rand32, rand_off, rand_bit;
	266
	267	switch (pkt->pktsched_ptype) {
	268	case QP_MBUF:
	269	m = pkt->pktsched_pkt_mbuf;
	270	data = mtod(m, uint8_t *);
	271	data_len = m->m_pkthdr.len;
	272	break;
	273
	274	default:
	275	/* NOTREACHED */
	276	VERIFY(0);
	277	__builtin_unreachable();
	278	}
	279
	280	read_frandom(&rand32, sizeof(rand32));
	281	rand_bit = rand32 & 0x8;
	282	rand_off = (rand32 >> 3) % data_len;
	283	data[rand_off] ^= 1 << rand_bit;
	284	}
	285
5ba3f43e A	286	void
	287	pktsched_free_pkt(pktsched_pkt_t *pkt)
	288	{
f427ee49 A	289	uint32_t cnt = pkt->pktsched_pcnt;
	290	ASSERT(cnt != 0);
	291
5ba3f43e	292	switch (pkt->pktsched_ptype) {
f427ee49 A	293	case QP_MBUF: {
f427ee49 A	294	struct mbuf *m;
5ba3f43e	295
f427ee49 A	296	m = pkt->pktsched_pkt_mbuf;
	297	if (cnt == 1) {
	298	VERIFY(m->m_nextpkt == NULL);
	299	} else {
	300	VERIFY(m->m_nextpkt != NULL);
	301	}
	302	m_freem_list(m);
	303	break;
	304	}
5ba3f43e A	305
	306	default:
	307	VERIFY(0);
	308	/* NOTREACHED */
cb323159	309	__builtin_unreachable();
5ba3f43e	310	}
cb323159	311	pkt->pktsched_pkt = CLASSQ_PKT_INITIALIZER(pkt->pktsched_pkt);
f427ee49	312	pkt->pktsched_tail = CLASSQ_PKT_INITIALIZER(pkt->pktsched_tail);
5ba3f43e	313	pkt->pktsched_plen = 0;
f427ee49	314	pkt->pktsched_pcnt = 0;
5ba3f43e A	315	}
	316
	317	mbuf_svc_class_t
	318	pktsched_get_pkt_svc(pktsched_pkt_t *pkt)
	319	{
	320	mbuf_svc_class_t svc = MBUF_SC_UNSPEC;
	321
	322	switch (pkt->pktsched_ptype) {
	323	case QP_MBUF:
cb323159	324	svc = m_get_service_class(pkt->pktsched_pkt_mbuf);
5ba3f43e A	325	break;
	326
	327
	328	default:
	329	VERIFY(0);
	330	/* NOTREACHED */
cb323159	331	__builtin_unreachable();
5ba3f43e A	332	}
5ba3f43e A	333
0a7de745	334	return svc;
5ba3f43e A	335	}
	336
	337	void
cb323159	338	pktsched_get_pkt_vars(pktsched_pkt_t pkt, volatile uint32_t *flags,
5ba3f43e	339	uint64_t *timestamp, uint32_t flowid, uint8_t flowsrc, uint8_t proto,
f427ee49	340	uint32_t *comp_gencnt)
5ba3f43e A	341	{
	342	switch (pkt->pktsched_ptype) {
	343	case QP_MBUF: {
cb323159	344	struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
5ba3f43e	345
0a7de745	346	if (flags != NULL) {
5ba3f43e	347	*flags = &pkth->pkt_flags;
0a7de745 A	348	}
0a7de745 A	349	if (timestamp != NULL) {
5ba3f43e	350	*timestamp = &pkth->pkt_timestamp;
0a7de745 A	351	}
0a7de745 A	352	if (flowid != NULL) {
5ba3f43e	353	*flowid = pkth->pkt_flowid;
0a7de745 A	354	}
0a7de745 A	355	if (flowsrc != NULL) {
5ba3f43e	356	*flowsrc = pkth->pkt_flowsrc;
0a7de745 A	357	}
0a7de745 A	358	if (proto != NULL) {
5ba3f43e	359	*proto = pkth->pkt_proto;
0a7de745	360	}
f427ee49 A	361	if (comp_gencnt != NULL) {
f427ee49 A	362	*comp_gencnt = pkth->comp_gencnt;
0a7de745	363	}
5ba3f43e A	364
	365	break;
	366	}
	367
	368
	369	default:
	370	VERIFY(0);
	371	/* NOTREACHED */
cb323159	372	__builtin_unreachable();
5ba3f43e A	373	}
	374	}
	375
	376	struct flowadv_fcentry *
	377	pktsched_alloc_fcentry(pktsched_pkt_t pkt, struct ifnet ifp, int how)
	378	{
	379	#pragma unused(ifp)
	380	struct flowadv_fcentry *fce = NULL;
	381
	382	switch (pkt->pktsched_ptype) {
	383	case QP_MBUF: {
cb323159	384	struct mbuf *m = pkt->pktsched_pkt_mbuf;
5ba3f43e A	385
5ba3f43e A	386	fce = flowadv_alloc_entry(how);
0a7de745	387	if (fce == NULL) {
5ba3f43e	388	break;
0a7de745	389	}
5ba3f43e	390
0a7de745 A	391	_CASSERT(sizeof(m->m_pkthdr.pkt_flowid) ==
0a7de745 A	392	sizeof(fce->fce_flowid));
5ba3f43e A	393
	394	fce->fce_flowsrc_type = m->m_pkthdr.pkt_flowsrc;
	395	fce->fce_flowid = m->m_pkthdr.pkt_flowid;
	396	break;
	397	}
	398
	399
	400	default:
	401	VERIFY(0);
	402	/* NOTREACHED */
cb323159	403	__builtin_unreachable();
5ba3f43e A	404	}
5ba3f43e A	405
0a7de745	406	return fce;
5ba3f43e A	407	}
	408
	409	uint32_t *
	410	pktsched_get_pkt_sfb_vars(pktsched_pkt_t pkt, uint32_t *sfb_flags)
	411	{
	412	uint32_t *hashp = NULL;
	413
	414	switch (pkt->pktsched_ptype) {
	415	case QP_MBUF: {
cb323159	416	struct pkthdr *pkth = &(pkt->pktsched_pkt_mbuf->m_pkthdr);
5ba3f43e	417
0a7de745 A	418	_CASSERT(sizeof(pkth->pkt_mpriv_hash) == sizeof(uint32_t));
0a7de745 A	419	_CASSERT(sizeof(pkth->pkt_mpriv_flags) == sizeof(uint32_t));
5ba3f43e A	420	*sfb_flags = &pkth->pkt_mpriv_flags;
	421	hashp = &pkth->pkt_mpriv_hash;
	422	break;
	423	}
	424
	425
	426	default:
	427	VERIFY(0);
	428	/* NOTREACHED */
cb323159	429	__builtin_unreachable();
5ba3f43e A	430	}
5ba3f43e A	431
0a7de745	432	return hashp;
5ba3f43e	433	}