]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_utun.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-6153.41.3.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
CommitLineData
b0d623f7 1/*
cb323159 2 * Copyright (c) 2008-2019 Apple Inc. All rights reserved.
b0d623f7
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
b0d623f7
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
b0d623f7
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
b0d623f7
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
b0d623f7
A		 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31/* ----------------------------------------------------------------------------------
0a7de745
A		 32 * Application of kernel control for interface creation
33 *
34 * Theory of operation:
35 * utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 * This kernel control will register an interface for every client that connects.
37 * ---------------------------------------------------------------------------------- */
b0d623f7
A		 38
39#include <sys/systm.h>
40#include <sys/kern_control.h>
41#include <net/kpi_protocol.h>
42#include <net/kpi_interface.h>
43#include <sys/socket.h>
44#include <net/if.h>
45#include <net/if_types.h>
46#include <net/bpf.h>
47#include <net/if_utun.h>
0a7de745 48#include <sys/mbuf.h>
b0d623f7
A		 49#include <sys/sockio.h>
50#include <netinet/in.h>
5ba3f43e 51#include <netinet/ip.h>
b0d623f7
A		 52#include <netinet6/in6_var.h>
53#include <netinet6/in6_var.h>
54#include <sys/kauth.h>
5ba3f43e
A		 55#include <net/necp.h>
56#include <kern/zalloc.h>
cb323159 57#include <os/log.h>
5ba3f43e
A		 58
59#define UTUN_NEXUS 0
60
5ba3f43e
A		 61#if UTUN_NEXUS
62static nexus_controller_t utun_ncd;
63static int utun_ncd_refcount;
64static uuid_t utun_kpipe_uuid;
65static uuid_t utun_nx_dom_prov;
66
67typedef struct utun_nx {
68 uuid_t if_provider;
69 uuid_t if_instance;
cb323159
A		 70 uuid_t fsw_provider;
71 uuid_t fsw_instance;
72 uuid_t fsw_device;
73 uuid_t fsw_host;
74 uuid_t fsw_agent;
5ba3f43e
A		 75} *utun_nx_t;
76
77#endif // UTUN_NEXUS
78
79/* Control block allocated for each kernel control connection */
80struct utun_pcb {
0a7de745
A		 81 TAILQ_ENTRY(utun_pcb) utun_chain;
82 kern_ctl_ref utun_ctlref;
83 ifnet_t utun_ifp;
84 u_int32_t utun_unit;
85 u_int32_t utun_unique_id;
86 u_int32_t utun_flags;
87 int utun_ext_ifdata_stats;
88 u_int32_t utun_max_pending_packets;
89 char utun_if_xname[IFXNAMSIZ];
90 char utun_unique_name[IFXNAMSIZ];
5ba3f43e
A		 91 // PCB lock protects state fields and rings
92 decl_lck_rw_data(, utun_pcb_lock);
0a7de745
A		 93 struct mbuf * utun_input_chain;
94 struct mbuf * utun_input_chain_last;
cb323159 95 u_int32_t utun_input_chain_count;
5ba3f43e
A		 96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
0a7de745 98 lck_mtx_t utun_input_chain_lock;
5ba3f43e
A		 99
100#if UTUN_NEXUS
0a7de745
A		 101 struct utun_nx utun_nx;
102 int utun_kpipe_enabled;
103 uuid_t utun_kpipe_uuid;
104 void * utun_kpipe_rxring;
105 void * utun_kpipe_txring;
106 kern_pbufpool_t utun_kpipe_pp;
cb323159
A		 107 u_int32_t utun_kpipe_tx_ring_size;
108 u_int32_t utun_kpipe_rx_ring_size;
0a7de745
A		 109
110 kern_nexus_t utun_netif_nexus;
111 kern_pbufpool_t utun_netif_pp;
112 void * utun_netif_rxring;
113 void * utun_netif_txring;
114 uint64_t utun_netif_txring_size;
115
116 u_int32_t utun_slot_size;
117 u_int32_t utun_netif_ring_size;
118 u_int32_t utun_tx_fsw_ring_size;
119 u_int32_t utun_rx_fsw_ring_size;
cb323159
A		 120 // Auto attach flowswitch when netif is enabled. When set to false,
121 // it allows userspace nexus controller to attach and own flowswitch.
122 bool utun_attach_fsw;
123 bool utun_netif_connected;
0a7de745
A		 124 bool utun_use_netif;
125 bool utun_needs_netagent;
5ba3f43e
A		 126#endif // UTUN_NEXUS
127};
39037602 128
b0d623f7 129/* Kernel Control functions */
0a7de745
A		 130static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
131 void **unitinfo);
132static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
133 void **unitinfo);
134static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
135 void *unitinfo);
136static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
137 void *unitinfo, mbuf_t m, int flags);
138static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
139 int opt, void *data, size_t *len);
140static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
141 int opt, void *data, size_t len);
142static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
143 int flags);
b0d623f7
A		 144
145/* Network Interface functions */
fe8ab488 146static void utun_start(ifnet_t interface);
0a7de745
A		 147static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
148 const struct sockaddr *dest, const char *desk_linkaddr,
149 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
150static errno_t utun_output(ifnet_t interface, mbuf_t data);
151static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
152 protocol_family_t *protocol);
153static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
154 const struct ifnet_demux_desc *demux_array,
155 u_int32_t demux_count);
156static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
157static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
158static void utun_detached(ifnet_t interface);
b0d623f7
A		 159
160/* Protocol handlers */
0a7de745
A		 161static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
162static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
163 mbuf_t m, char *frame_header);
164static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
165 mbuf_t *packet, const struct sockaddr *dest, void *route,
166 char *frame_type, char *link_layer_dest);
5c9f4661 167static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
39037602 168
5ba3f43e
A		 169#if UTUN_NEXUS
170
5c9f4661 171#define UTUN_IF_DEFAULT_SLOT_SIZE 2048
5ba3f43e
A		 172#define UTUN_IF_DEFAULT_RING_SIZE 64
173#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
174#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
0a7de745 175#define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
5ba3f43e
A		 176#define UTUN_IF_HEADROOM_SIZE 32
177
cb323159 178#define UTUN_IF_MIN_RING_SIZE 8
5ba3f43e
A		 179#define UTUN_IF_MAX_RING_SIZE 1024
180
5c9f4661
A		 181#define UTUN_IF_MIN_SLOT_SIZE 1024
182#define UTUN_IF_MAX_SLOT_SIZE 4096
183
cb323159
A		 184#define UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT 512
185
186static int if_utun_max_pending_input = UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT;
187
5ba3f43e
A		 188static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
189static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
190static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
191
192static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
193static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
194static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
195
196SYSCTL_DECL(_net_utun);
197SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
198
cb323159 199SYSCTL_INT(_net_utun, OID_AUTO, max_pending_input, CTLFLAG_LOCKED | CTLFLAG_RW, &if_utun_max_pending_input, 0, "");
5ba3f43e 200SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 201 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
5ba3f43e 202SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 203 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
5ba3f43e 204SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 205 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
5ba3f43e
A		 206
207static errno_t
208utun_register_nexus(void);
209
210static errno_t
211utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
212static errno_t
213utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
214 proc_t p, kern_nexus_t nexus,
215 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
216static errno_t
217utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
218 kern_channel_t channel);
219static void
220utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
221 kern_channel_t channel);
222static void
223utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 224 kern_channel_t channel);
5ba3f43e
A		 225static void
226utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
227 kern_channel_t channel);
228static errno_t
229utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
230 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
231 void **ring_ctx);
232static void
233utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
234 kern_channel_ring_t ring);
235static errno_t
236utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
237 kern_channel_ring_t ring, uint32_t flags);
238static errno_t
239utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
240 kern_channel_ring_t ring, uint32_t flags);
241#endif // UTUN_NEXUS
39037602
A		 242
243#define UTUN_DEFAULT_MTU 1500
244#define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
b0d623f7 245
0a7de745 246static kern_ctl_ref utun_kctlref;
5ba3f43e
A		 247static lck_attr_t *utun_lck_attr;
248static lck_grp_attr_t *utun_lck_grp_attr;
249static lck_grp_t *utun_lck_grp;
250static lck_mtx_t utun_lock;
251
252TAILQ_HEAD(utun_list, utun_pcb) utun_head;
253
0a7de745
A		 254#define UTUN_PCB_ZONE_MAX 32
255#define UTUN_PCB_ZONE_NAME "net.if_utun"
5ba3f43e 256
0a7de745
A		 257static unsigned int utun_pcb_size; /* size of zone element */
258static struct zone *utun_pcb_zone; /* zone for utun_pcb */
5ba3f43e
A		 259
260#if UTUN_NEXUS
261
262static int
263sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
264{
265#pragma unused(arg1, arg2)
266 int value = if_utun_ring_size;
267
268 int error = sysctl_handle_int(oidp, &value, 0, req);
269 if (error || !req->newptr) {
0a7de745 270 return error;
5ba3f43e
A		 271 }
272
273 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 274 value > UTUN_IF_MAX_RING_SIZE) {
275 return EINVAL;
5ba3f43e
A		 276 }
277
278 if_utun_ring_size = value;
279
0a7de745 280 return 0;
5ba3f43e
A		 281}
282
283static int
284sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
285{
286#pragma unused(arg1, arg2)
287 int value = if_utun_tx_fsw_ring_size;
288
289 int error = sysctl_handle_int(oidp, &value, 0, req);
290 if (error || !req->newptr) {
0a7de745 291 return error;
5ba3f43e
A		 292 }
293
294 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 295 value > UTUN_IF_MAX_RING_SIZE) {
296 return EINVAL;
5ba3f43e
A		 297 }
298
299 if_utun_tx_fsw_ring_size = value;
300
0a7de745 301 return 0;
5ba3f43e
A		 302}
303
304static int
305sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
306{
307#pragma unused(arg1, arg2)
308 int value = if_utun_rx_fsw_ring_size;
309
310 int error = sysctl_handle_int(oidp, &value, 0, req);
311 if (error || !req->newptr) {
0a7de745 312 return error;
5ba3f43e
A		 313 }
314
315 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 316 value > UTUN_IF_MAX_RING_SIZE) {
317 return EINVAL;
5ba3f43e
A		 318 }
319
320 if_utun_rx_fsw_ring_size = value;
321
0a7de745 322 return 0;
5ba3f43e
A		 323}
324
325static errno_t
326utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 327 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
328 void **ring_ctx)
5ba3f43e
A		 329{
330#pragma unused(nxprov)
331#pragma unused(channel)
332#pragma unused(ring_ctx)
333 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
334 if (!is_tx_ring) {
335 VERIFY(pcb->utun_netif_rxring == NULL);
336 pcb->utun_netif_rxring = ring;
337 } else {
338 VERIFY(pcb->utun_netif_txring == NULL);
339 pcb->utun_netif_txring = ring;
340 }
341 return 0;
342}
343
344static void
345utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 346 kern_channel_ring_t ring)
5ba3f43e
A		 347{
348#pragma unused(nxprov)
349 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
350 if (pcb->utun_netif_rxring == ring) {
351 pcb->utun_netif_rxring = NULL;
352 } else if (pcb->utun_netif_txring == ring) {
353 pcb->utun_netif_txring = NULL;
354 }
355}
356
357static errno_t
358utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 359 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 360{
361#pragma unused(nxprov)
362#pragma unused(flags)
363 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
364
365 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
366
367 lck_rw_lock_shared(&pcb->utun_pcb_lock);
368
369 struct kern_channel_ring_stat_increment tx_ring_stats;
370 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
371 kern_channel_slot_t tx_pslot = NULL;
372 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
373
cb323159 374 STATS_INC(nifs, NETIF_STATS_TX_SYNC);
5ba3f43e
A		 375
376 if (tx_slot == NULL) {
377 // Nothing to write, don't bother signalling
378 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
379 return 0;
380 }
381
382 if (pcb->utun_kpipe_enabled) {
383 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
384 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
385
386 // Signal the kernel pipe ring to read
387 if (rx_ring != NULL) {
388 kern_channel_notify(rx_ring, 0);
389 }
390 return 0;
391 }
392
393 // If we're here, we're injecting into the utun kernel control socket
394 while (tx_slot != NULL) {
395 size_t length = 0;
396 mbuf_t data = NULL;
397
398 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
399
400 if (tx_ph == 0) {
401 // Advance TX ring
402 tx_pslot = tx_slot;
403 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
404 continue;
405 }
406 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
407
408 // Advance TX ring
409 tx_pslot = tx_slot;
410 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
411
412 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
413 VERIFY(tx_buf != NULL);
414
415 /* tx_baddr is the absolute buffer address */
416 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
417 VERIFY(tx_baddr != 0);
418
419 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
420
421 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
422 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
423
424 // The offset must be large enough for the headers
425 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
426
427 // Find family
428 uint32_t af = 0;
429 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
430 u_int ip_version = (vhl >> 4);
431 switch (ip_version) {
0a7de745
A		 432 case 4: {
433 af = AF_INET;
434 break;
435 }
436 case 6: {
437 af = AF_INET6;
438 break;
439 }
440 default: {
cb323159 441 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
0a7de745
A		 442 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
443 UTUN_HEADER_SIZE(pcb));
444 break;
445 }
5ba3f43e
A		 446 }
447
448 tx_offset -= UTUN_HEADER_SIZE(pcb);
449 tx_length += UTUN_HEADER_SIZE(pcb);
450 tx_baddr += tx_offset;
451
5c9f4661 452 length = MIN(tx_length, pcb->utun_slot_size);
5ba3f43e
A		 453
454 // Copy in family
455 memcpy(tx_baddr, &af, sizeof(af));
456 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
457 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
458 }
459
460 if (length > 0) {
461 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
462 if (error == 0) {
463 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
464 if (error == 0) {
465 error = utun_output(pcb->utun_ifp, data);
466 if (error != 0) {
cb323159 467 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
5ba3f43e
A		 468 }
469 } else {
cb323159
A		 470 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
471 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
472 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 473 mbuf_freem(data);
474 data = NULL;
475 }
476 } else {
cb323159
A		 477 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
478 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
479 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 480 }
481 } else {
cb323159
A		 482 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
483 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
484 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 485 }
486
487 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
488
489 if (data == NULL) {
490 continue;
491 }
492
cb323159
A		 493 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
494 STATS_INC(nifs, NETIF_STATS_TX_COPY_MBUF);
5ba3f43e
A		 495
496 tx_ring_stats.kcrsi_slots_transferred++;
497 tx_ring_stats.kcrsi_bytes_transferred += length;
498 }
499
500 if (tx_pslot) {
501 kern_channel_advance_slot(tx_ring, tx_pslot);
502 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
503 (void)kern_channel_reclaim(tx_ring);
504 }
505
506 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
507
508 return 0;
509}
510
511static errno_t
512utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 513 kern_channel_ring_t ring, __unused uint32_t flags)
5ba3f43e
A		 514{
515#pragma unused(nxprov)
516 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5ba3f43e
A		 517 boolean_t more = false;
518 errno_t rc = 0;
5ba3f43e 519
5c9f4661
A		 520 /*
521 * Refill and sync the ring; we may be racing against another thread doing
522 * an RX sync that also wants to do kr_enter(), and so use the blocking
523 * variant here.
524 */
525 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
526 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
cb323159 527 os_log_error(OS_LOG_DEFAULT, "%s, tx refill failed %d\n", __func__, rc);
5c9f4661
A		 528 }
529
530 (void) kr_enter(ring, TRUE);
531 lck_rw_lock_shared(&pcb->utun_pcb_lock);
532
533 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 534 uint32_t tx_available = kern_channel_available_slot_count(ring);
535 if (pcb->utun_netif_txring_size > 0 &&
0a7de745 536 tx_available >= pcb->utun_netif_txring_size - 1) {
5ba3f43e
A		 537 // No room left in tx ring, disable output for now
538 errno_t error = ifnet_disable_output(pcb->utun_ifp);
539 if (error != 0) {
cb323159 540 os_log_error(OS_LOG_DEFAULT, "utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
5ba3f43e
A		 541 }
542 }
543 }
b0d623f7 544
5c9f4661 545 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 546 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
547
548 // Unlock while calling notify
549 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
550 // Signal the kernel pipe ring to read
551 if (rx_ring != NULL) {
552 kern_channel_notify(rx_ring, 0);
553 }
554 } else {
555 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
556 }
557
5c9f4661
A		 558 kr_exit(ring);
559
0a7de745 560 return 0;
5ba3f43e
A		 561}
562
563static errno_t
564utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 565 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 566{
567#pragma unused(nxprov)
568#pragma unused(flags)
569 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
570 struct kern_channel_ring_stat_increment rx_ring_stats;
571
572 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
573
574 lck_rw_lock_shared(&pcb->utun_pcb_lock);
575
576 // Reclaim user-released slots
577 (void) kern_channel_reclaim(rx_ring);
578
cb323159 579 STATS_INC(nifs, NETIF_STATS_RX_SYNC);
5ba3f43e
A		 580
581 uint32_t avail = kern_channel_available_slot_count(rx_ring);
582 if (avail == 0) {
583 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
584 return 0;
585 }
586
587 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
588 VERIFY(rx_pp != NULL);
589 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
590 kern_channel_slot_t rx_pslot = NULL;
591 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
592
593 while (rx_slot != NULL) {
594 // Check for a waiting packet
595 lck_mtx_lock(&pcb->utun_input_chain_lock);
596 mbuf_t data = pcb->utun_input_chain;
597 if (data == NULL) {
598 lck_mtx_unlock(&pcb->utun_input_chain_lock);
599 break;
600 }
601
602 // Allocate rx packet
603 kern_packet_t rx_ph = 0;
604 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 605 if (__improbable(error != 0)) {
cb323159
A		 606 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
607 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 608 lck_mtx_unlock(&pcb->utun_input_chain_lock);
609 break;
610 }
611
612 // Advance waiting packets
cb323159
A		 613 if (pcb->utun_input_chain_count > 0) {
614 pcb->utun_input_chain_count--;
615 }
5ba3f43e
A		 616 pcb->utun_input_chain = data->m_nextpkt;
617 data->m_nextpkt = NULL;
618 if (pcb->utun_input_chain == NULL) {
619 pcb->utun_input_chain_last = NULL;
620 }
621 lck_mtx_unlock(&pcb->utun_input_chain_lock);
622
623 size_t header_offset = UTUN_HEADER_SIZE(pcb);
624 size_t length = mbuf_pkthdr_len(data);
625
626 if (length < header_offset) {
627 // mbuf is too small
628 mbuf_freem(data);
629 kern_pbufpool_free(rx_pp, rx_ph);
cb323159
A		 630 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
631 STATS_INC(nifs, NETIF_STATS_DROP);
632 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
0a7de745 633 pcb->utun_ifp->if_xname, length, header_offset);
5ba3f43e
A		 634 continue;
635 }
636
637 length -= header_offset;
638 if (length > rx_pp->pp_buflet_size) {
639 // Flush data
640 mbuf_freem(data);
641 kern_pbufpool_free(rx_pp, rx_ph);
cb323159
A		 642 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
643 STATS_INC(nifs, NETIF_STATS_DROP);
644 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
0a7de745 645 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
5ba3f43e
A		 646 continue;
647 }
648
649 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
650
651 // Fillout rx packet
652 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
653 VERIFY(rx_buf != NULL);
654 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
655 VERIFY(rx_baddr != NULL);
656
657 // Copy-in data from mbuf to buflet
658 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
0a7de745 659 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 660
661 // Finalize and attach the packet
662 error = kern_buflet_set_data_offset(rx_buf, 0);
663 VERIFY(error == 0);
664 error = kern_buflet_set_data_length(rx_buf, length);
665 VERIFY(error == 0);
cb323159 666 error = kern_packet_set_headroom(rx_ph, 0);
5ba3f43e
A		 667 VERIFY(error == 0);
668 error = kern_packet_finalize(rx_ph);
669 VERIFY(error == 0);
670 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
671 VERIFY(error == 0);
672
cb323159
A		 673 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
674 STATS_INC(nifs, NETIF_STATS_RX_COPY_MBUF);
5ba3f43e
A		 675 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
676
677 rx_ring_stats.kcrsi_slots_transferred++;
678 rx_ring_stats.kcrsi_bytes_transferred += length;
679
680 mbuf_freem(data);
681
682 // Advance ring
683 rx_pslot = rx_slot;
684 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
685 }
686
687 struct kern_channel_ring_stat_increment tx_ring_stats;
688 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
689 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
690 kern_channel_slot_t tx_pslot = NULL;
691 kern_channel_slot_t tx_slot = NULL;
692 if (tx_ring == NULL) {
693 // Net-If TX ring not set up yet, nothing to read
694 goto done;
695 }
696
697 // Unlock utun before entering ring
698 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
699
700 (void)kr_enter(tx_ring, TRUE);
701
702 // Lock again after entering and validate
703 lck_rw_lock_shared(&pcb->utun_pcb_lock);
704 if (tx_ring != pcb->utun_kpipe_txring) {
705 goto done;
706 }
707
708 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
709 if (tx_slot == NULL) {
710 // Nothing to read, don't bother signalling
711 goto done;
712 }
713
714 while (rx_slot != NULL && tx_slot != NULL) {
715 // Allocate rx packet
716 kern_packet_t rx_ph = 0;
717 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
718
719 // Advance TX ring
720 tx_pslot = tx_slot;
721 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
722
723 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
724 if (tx_ph == 0) {
725 continue;
726 }
727
a39ff7e2
A		 728 /* XXX We could try this alloc before advancing the slot to avoid
729 * dropping the packet on failure to allocate.
730 */
5ba3f43e 731 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 732 if (__improbable(error != 0)) {
cb323159
A		 733 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
734 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 735 break;
736 }
737
738 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
739 VERIFY(tx_buf != NULL);
740 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
741 VERIFY(tx_baddr != 0);
742 tx_baddr += kern_buflet_get_data_offset(tx_buf);
743
744 // Check packet length
745 size_t header_offset = UTUN_HEADER_SIZE(pcb);
746 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
747 if (tx_length < header_offset) {
748 // Packet is too small
749 kern_pbufpool_free(rx_pp, rx_ph);
cb323159
A		 750 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
751 STATS_INC(nifs, NETIF_STATS_DROP);
752 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
0a7de745 753 pcb->utun_ifp->if_xname, tx_length, header_offset);
5ba3f43e
A		 754 continue;
755 }
756
757 size_t length = MIN(tx_length - header_offset,
0a7de745 758 pcb->utun_slot_size);
5ba3f43e
A		 759
760 tx_ring_stats.kcrsi_slots_transferred++;
761 tx_ring_stats.kcrsi_bytes_transferred += length;
762
763 // Fillout rx packet
764 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
765 VERIFY(rx_buf != NULL);
766 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
767 VERIFY(rx_baddr != NULL);
768
769 // Copy-in data from tx to rx
770 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
0a7de745 771 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 772
773 // Finalize and attach the packet
774 error = kern_buflet_set_data_offset(rx_buf, 0);
775 VERIFY(error == 0);
776 error = kern_buflet_set_data_length(rx_buf, length);
777 VERIFY(error == 0);
cb323159 778 error = kern_packet_set_headroom(rx_ph, 0);
5ba3f43e
A		 779 VERIFY(error == 0);
780 error = kern_packet_finalize(rx_ph);
781 VERIFY(error == 0);
782 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
783 VERIFY(error == 0);
784
cb323159
A		 785 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
786 STATS_INC(nifs, NETIF_STATS_RX_COPY_DIRECT);
5ba3f43e
A		 787 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
788
789 rx_ring_stats.kcrsi_slots_transferred++;
790 rx_ring_stats.kcrsi_bytes_transferred += length;
791
792 rx_pslot = rx_slot;
793 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
794 }
795
796done:
797 if (rx_pslot) {
798 kern_channel_advance_slot(rx_ring, rx_pslot);
799 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
800 }
801
802 if (tx_pslot) {
803 kern_channel_advance_slot(tx_ring, tx_pslot);
804 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
805 (void)kern_channel_reclaim(tx_ring);
806 }
807
808 // Unlock first, then exit ring
809 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
810 if (tx_ring != NULL) {
811 if (tx_pslot != NULL) {
812 kern_channel_notify(tx_ring, 0);
813 }
814 kr_exit(tx_ring);
815 }
816
817 return 0;
818}
819
820static errno_t
821utun_nexus_ifattach(struct utun_pcb *pcb,
0a7de745
A		 822 struct ifnet_init_eparams *init_params,
823 struct ifnet **ifp)
5ba3f43e
A		 824{
825 errno_t err;
826 nexus_controller_t controller = kern_nexus_shared_controller();
827 struct kern_nexus_net_init net_init;
a39ff7e2 828 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 829
830 nexus_name_t provider_name;
831 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 832 "com.apple.netif.%s", pcb->utun_if_xname);
5ba3f43e
A		 833
834 struct kern_nexus_provider_init prov_init = {
835 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
836 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
837 .nxpi_pre_connect = utun_nexus_pre_connect,
838 .nxpi_connected = utun_nexus_connected,
839 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
840 .nxpi_disconnected = utun_nexus_disconnected,
841 .nxpi_ring_init = utun_netif_ring_init,
842 .nxpi_ring_fini = utun_netif_ring_fini,
843 .nxpi_slot_init = NULL,
844 .nxpi_slot_fini = NULL,
845 .nxpi_sync_tx = utun_netif_sync_tx,
846 .nxpi_sync_rx = utun_netif_sync_rx,
847 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
848 };
849
850 nexus_attr_t nxa = NULL;
851 err = kern_nexus_attr_create(&nxa);
852 if (err != 0) {
cb323159 853 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 854 __func__, err);
5ba3f43e
A		 855 goto failed;
856 }
857
5c9f4661 858 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 859 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
860 VERIFY(err == 0);
861
862 // Reset ring size for netif nexus to limit memory usage
5c9f4661 863 uint64_t ring_size = pcb->utun_netif_ring_size;
5ba3f43e
A		 864 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
865 VERIFY(err == 0);
866 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
867 VERIFY(err == 0);
868
869 pcb->utun_netif_txring_size = ring_size;
870
0a7de745 871 bzero(&pp_init, sizeof(pp_init));
a39ff7e2 872 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
cb323159 873 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
a39ff7e2
A		 874 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
875 pp_init.kbi_bufsize = pcb->utun_slot_size;
876 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
877 pp_init.kbi_max_frags = 1;
0a7de745 878 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2 879 "%s", provider_name);
cb323159
A		 880 pp_init.kbi_ctx = NULL;
881 pp_init.kbi_ctx_retain = NULL;
882 pp_init.kbi_ctx_release = NULL;
a39ff7e2 883
cb323159 884 err = kern_pbufpool_create(&pp_init, &pcb->utun_netif_pp, NULL);
a39ff7e2 885 if (err != 0) {
cb323159 886 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, err);
a39ff7e2
A		 887 goto failed;
888 }
889
5ba3f43e 890 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 891 utun_nx_dom_prov,
892 provider_name,
893 &prov_init,
894 sizeof(prov_init),
895 nxa,
896 &pcb->utun_nx.if_provider);
5ba3f43e 897 if (err != 0) {
cb323159 898 os_log_error(OS_LOG_DEFAULT, "%s register provider failed, error %d\n",
0a7de745 899 __func__, err);
5ba3f43e
A		 900 goto failed;
901 }
902
903 bzero(&net_init, sizeof(net_init));
904 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
905 net_init.nxneti_flags = 0;
906 net_init.nxneti_eparams = init_params;
907 net_init.nxneti_lladdr = NULL;
908 net_init.nxneti_prepare = utun_netif_prepare;
a39ff7e2 909 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
5ba3f43e 910 err = kern_nexus_controller_alloc_net_provider_instance(controller,
0a7de745
A		 911 pcb->utun_nx.if_provider,
912 pcb,
913 &pcb->utun_nx.if_instance,
914 &net_init,
915 ifp);
5ba3f43e 916 if (err != 0) {
cb323159 917 os_log_error(OS_LOG_DEFAULT, "%s alloc_net_provider_instance failed, %d\n",
0a7de745 918 __func__, err);
5ba3f43e 919 kern_nexus_controller_deregister_provider(controller,
0a7de745 920 pcb->utun_nx.if_provider);
5ba3f43e
A		 921 uuid_clear(pcb->utun_nx.if_provider);
922 goto failed;
923 }
924
925failed:
926 if (nxa) {
927 kern_nexus_attr_destroy(nxa);
928 }
a39ff7e2
A		 929 if (err && pcb->utun_netif_pp != NULL) {
930 kern_pbufpool_destroy(pcb->utun_netif_pp);
931 pcb->utun_netif_pp = NULL;
932 }
0a7de745 933 return err;
5ba3f43e
A		 934}
935
936static void
937utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
938{
939 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 940 errno_t err;
5ba3f43e
A		 941
942 if (!uuid_is_null(instance)) {
943 err = kern_nexus_controller_free_provider_instance(controller,
0a7de745 944 instance);
5ba3f43e 945 if (err != 0) {
cb323159 946 os_log_error(OS_LOG_DEFAULT, "%s free_provider_instance failed %d\n",
0a7de745 947 __func__, err);
5ba3f43e
A		 948 }
949 uuid_clear(instance);
950 }
951 if (!uuid_is_null(provider)) {
952 err = kern_nexus_controller_deregister_provider(controller,
0a7de745 953 provider);
5ba3f43e 954 if (err != 0) {
cb323159 955 os_log_error(OS_LOG_DEFAULT, "%s deregister_provider %d\n", __func__, err);
5ba3f43e
A		 956 }
957 uuid_clear(provider);
958 }
959 return;
960}
961
962static void
a39ff7e2 963utun_nexus_detach(struct utun_pcb *pcb)
5ba3f43e 964{
a39ff7e2 965 utun_nx_t nx = &pcb->utun_nx;
5ba3f43e 966 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 967 errno_t err;
5ba3f43e 968
cb323159 969 if (!uuid_is_null(nx->fsw_host)) {
5ba3f43e 970 err = kern_nexus_ifdetach(controller,
cb323159
A		 971 nx->fsw_instance,
972 nx->fsw_host);
5ba3f43e 973 if (err != 0) {
cb323159 974 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms host failed %d\n",
0a7de745 975 __func__, err);
5ba3f43e
A		 976 }
977 }
978
cb323159 979 if (!uuid_is_null(nx->fsw_device)) {
5ba3f43e 980 err = kern_nexus_ifdetach(controller,
cb323159
A		 981 nx->fsw_instance,
982 nx->fsw_device);
5ba3f43e 983 if (err != 0) {
cb323159 984 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms device failed %d\n",
0a7de745 985 __func__, err);
5ba3f43e
A		 986 }
987 }
988
989 utun_detach_provider_and_instance(nx->if_provider,
0a7de745 990 nx->if_instance);
cb323159
A		 991 utun_detach_provider_and_instance(nx->fsw_provider,
992 nx->fsw_instance);
5ba3f43e 993
a39ff7e2
A		 994 if (pcb->utun_netif_pp != NULL) {
995 kern_pbufpool_destroy(pcb->utun_netif_pp);
996 pcb->utun_netif_pp = NULL;
a39ff7e2 997 }
5ba3f43e
A		 998 memset(nx, 0, sizeof(*nx));
999}
1000
1001static errno_t
5c9f4661 1002utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
cb323159 1003 const char *type_name,
0a7de745
A		 1004 const char *ifname,
1005 uuid_t *provider, uuid_t *instance)
5ba3f43e
A		 1006{
1007 nexus_attr_t attr = NULL;
1008 nexus_controller_t controller = kern_nexus_shared_controller();
1009 uuid_t dom_prov;
1010 errno_t err;
1011 struct kern_nexus_init init;
0a7de745 1012 nexus_name_t provider_name;
5ba3f43e 1013
cb323159 1014 err = kern_nexus_get_default_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
0a7de745 1015 &dom_prov);
5ba3f43e 1016 if (err != 0) {
cb323159 1017 os_log_error(OS_LOG_DEFAULT, "%s can't get %s provider, error %d\n",
0a7de745 1018 __func__, type_name, err);
5ba3f43e
A		 1019 goto failed;
1020 }
1021
1022 err = kern_nexus_attr_create(&attr);
1023 if (err != 0) {
cb323159 1024 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 1025 __func__, err);
5ba3f43e
A		 1026 goto failed;
1027 }
1028
5c9f4661 1029 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 1030 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1031 VERIFY(err == 0);
1032
1033 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
5c9f4661 1034 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
5ba3f43e
A		 1035 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1036 VERIFY(err == 0);
5c9f4661 1037 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
5ba3f43e
A		 1038 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1039 VERIFY(err == 0);
1040
1041 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 1042 "com.apple.%s.%s", type_name, ifname);
5ba3f43e 1043 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 1044 dom_prov,
1045 provider_name,
1046 NULL,
1047 0,
1048 attr,
1049 provider);
5ba3f43e
A		 1050 kern_nexus_attr_destroy(attr);
1051 attr = NULL;
1052 if (err != 0) {
cb323159 1053 os_log_error(OS_LOG_DEFAULT, "%s register %s provider failed, error %d\n",
0a7de745 1054 __func__, type_name, err);
5ba3f43e
A		 1055 goto failed;
1056 }
0a7de745 1057 bzero(&init, sizeof(init));
5ba3f43e
A		 1058 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1059 err = kern_nexus_controller_alloc_provider_instance(controller,
0a7de745
A		 1060 *provider,
1061 NULL,
1062 instance, &init);
5ba3f43e 1063 if (err != 0) {
cb323159 1064 os_log_error(OS_LOG_DEFAULT, "%s alloc_provider_instance %s failed, %d\n",
0a7de745 1065 __func__, type_name, err);
5ba3f43e 1066 kern_nexus_controller_deregister_provider(controller,
0a7de745 1067 *provider);
5ba3f43e
A		 1068 uuid_clear(*provider);
1069 }
1070failed:
0a7de745 1071 return err;
5ba3f43e
A		 1072}
1073
1074static errno_t
cb323159 1075utun_flowswitch_attach(struct utun_pcb *pcb)
5ba3f43e
A		 1076{
1077 nexus_controller_t controller = kern_nexus_shared_controller();
1078 errno_t err = 0;
1079 utun_nx_t nx = &pcb->utun_nx;
1080
cb323159 1081 // Allocate flowswitch
5c9f4661 1082 err = utun_create_fs_provider_and_instance(pcb,
cb323159 1083 "flowswitch",
0a7de745 1084 pcb->utun_ifp->if_xname,
cb323159
A		 1085 &nx->fsw_provider,
1086 &nx->fsw_instance);
5ba3f43e 1087 if (err != 0) {
cb323159 1088 os_log_error(OS_LOG_DEFAULT, "%s: failed to create bridge provider and instance\n",
0a7de745 1089 __func__);
5ba3f43e
A		 1090 goto failed;
1091 }
1092
cb323159
A		 1093 // Attach flowswitch to device port
1094 err = kern_nexus_ifattach(controller, nx->fsw_instance,
0a7de745 1095 NULL, nx->if_instance,
cb323159 1096 FALSE, &nx->fsw_device);
5ba3f43e 1097 if (err != 0) {
cb323159 1098 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms device %d\n", __func__, err);
5ba3f43e
A		 1099 goto failed;
1100 }
1101
cb323159
A		 1102 // Attach flowswitch to host port
1103 err = kern_nexus_ifattach(controller, nx->fsw_instance,
0a7de745 1104 NULL, nx->if_instance,
cb323159 1105 TRUE, &nx->fsw_host);
5ba3f43e 1106 if (err != 0) {
cb323159 1107 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms host %d\n", __func__, err);
5ba3f43e
A		 1108 goto failed;
1109 }
1110
1111 // Extract the agent UUID and save for later
cb323159
A		 1112 struct kern_nexus *flowswitch_nx = nx_find(nx->fsw_instance, false);
1113 if (flowswitch_nx != NULL) {
1114 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(flowswitch_nx);
5ba3f43e
A		 1115 if (flowswitch != NULL) {
1116 FSW_RLOCK(flowswitch);
cb323159 1117 uuid_copy(nx->fsw_agent, flowswitch->fsw_agent_uuid);
5ba3f43e
A		 1118 FSW_UNLOCK(flowswitch);
1119 } else {
cb323159 1120 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - flowswitch is NULL\n");
5ba3f43e 1121 }
cb323159 1122 nx_release(flowswitch_nx);
5ba3f43e 1123 } else {
cb323159 1124 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - unable to find flowswitch nexus\n");
5ba3f43e
A		 1125 }
1126
0a7de745 1127 return 0;
5ba3f43e
A		 1128
1129failed:
a39ff7e2 1130 utun_nexus_detach(pcb);
5ba3f43e
A		 1131
1132 errno_t detach_error = 0;
1133 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
cb323159 1134 panic("utun_flowswitch_attach - ifnet_detach failed: %d\n", detach_error);
5ba3f43e
A		 1135 /* NOT REACHED */
1136 }
0a7de745
A		 1137
1138 return err;
5ba3f43e
A		 1139}
1140
1141static errno_t
cb323159 1142utun_register_kernel_pipe_nexus(struct utun_pcb *pcb)
5ba3f43e
A		 1143{
1144 nexus_attr_t nxa = NULL;
1145 errno_t result;
1146
1147 lck_mtx_lock(&utun_lock);
1148 if (utun_ncd_refcount++) {
1149 lck_mtx_unlock(&utun_lock);
1150 return 0;
1151 }
1152
1153 result = kern_nexus_controller_create(&utun_ncd);
1154 if (result) {
cb323159 1155 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_create failed: %d\n",
0a7de745 1156 __FUNCTION__, result);
5ba3f43e
A		 1157 goto done;
1158 }
1159
1160 uuid_t dom_prov;
cb323159 1161 result = kern_nexus_get_default_domain_provider(
5ba3f43e
A		 1162 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1163 if (result) {
cb323159 1164 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_get_default_domain_provider failed: %d\n",
0a7de745 1165 __FUNCTION__, result);
5ba3f43e
A		 1166 goto done;
1167 }
1168
1169 struct kern_nexus_provider_init prov_init = {
1170 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1171 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1172 .nxpi_pre_connect = utun_nexus_pre_connect,
1173 .nxpi_connected = utun_nexus_connected,
1174 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1175 .nxpi_disconnected = utun_nexus_disconnected,
1176 .nxpi_ring_init = utun_kpipe_ring_init,
1177 .nxpi_ring_fini = utun_kpipe_ring_fini,
1178 .nxpi_slot_init = NULL,
1179 .nxpi_slot_fini = NULL,
1180 .nxpi_sync_tx = utun_kpipe_sync_tx,
1181 .nxpi_sync_rx = utun_kpipe_sync_rx,
1182 .nxpi_tx_doorbell = NULL,
1183 };
1184
1185 result = kern_nexus_attr_create(&nxa);
1186 if (result) {
cb323159 1187 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 1188 __FUNCTION__, result);
5ba3f43e
A		 1189 goto done;
1190 }
1191
1192 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1193 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1194 VERIFY(result == 0);
1195
1196 // Reset ring size for kernel pipe nexus to limit memory usage
cb323159
A		 1197 uint64_t ring_size =
1198 pcb->utun_kpipe_tx_ring_size != 0 ? pcb->utun_kpipe_tx_ring_size :
1199 if_utun_ring_size;
5ba3f43e
A		 1200 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1201 VERIFY(result == 0);
cb323159
A		 1202
1203 ring_size =
1204 pcb->utun_kpipe_rx_ring_size != 0 ? pcb->utun_kpipe_rx_ring_size :
1205 if_utun_ring_size;
5ba3f43e
A		 1206 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1207 VERIFY(result == 0);
1208
1209 result = kern_nexus_controller_register_provider(utun_ncd,
0a7de745
A		 1210 dom_prov,
1211 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1212 &prov_init,
1213 sizeof(prov_init),
1214 nxa,
1215 &utun_kpipe_uuid);
5ba3f43e 1216 if (result) {
cb323159 1217 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_register_provider failed: %d\n",
0a7de745 1218 __FUNCTION__, result);
5ba3f43e
A		 1219 goto done;
1220 }
1221
1222done:
1223 if (nxa) {
1224 kern_nexus_attr_destroy(nxa);
1225 }
1226
1227 if (result) {
1228 if (utun_ncd) {
1229 kern_nexus_controller_destroy(utun_ncd);
1230 utun_ncd = NULL;
1231 }
1232 utun_ncd_refcount = 0;
1233 }
1234
1235 lck_mtx_unlock(&utun_lock);
1236
1237 return result;
1238}
1239
1240static void
1241utun_unregister_kernel_pipe_nexus(void)
1242{
1243 lck_mtx_lock(&utun_lock);
1244
1245 VERIFY(utun_ncd_refcount > 0);
1246
1247 if (--utun_ncd_refcount == 0) {
1248 kern_nexus_controller_destroy(utun_ncd);
1249 utun_ncd = NULL;
1250 }
1251
1252 lck_mtx_unlock(&utun_lock);
1253}
1254
1255// For use by socket option, not internally
1256static errno_t
1257utun_disable_channel(struct utun_pcb *pcb)
1258{
1259 errno_t result;
1260 int enabled;
1261 uuid_t uuid;
1262
1263 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1264
1265 enabled = pcb->utun_kpipe_enabled;
1266 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1267
1268 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1269
1270 pcb->utun_kpipe_enabled = 0;
1271 uuid_clear(pcb->utun_kpipe_uuid);
1272
1273 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1274
1275 if (enabled) {
1276 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1277 } else {
1278 result = ENXIO;
1279 }
1280
1281 if (!result) {
a39ff7e2
A		 1282 if (pcb->utun_kpipe_pp != NULL) {
1283 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1284 pcb->utun_kpipe_pp = NULL;
1285 }
5ba3f43e
A		 1286 utun_unregister_kernel_pipe_nexus();
1287 }
1288
1289 return result;
1290}
1291
1292static errno_t
1293utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1294{
1295 struct kern_nexus_init init;
a39ff7e2 1296 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 1297 errno_t result;
1298
d9a64523
A		 1299 kauth_cred_t cred = kauth_cred_get();
1300 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
1301 if (result) {
1302 return result;
1303 }
1304
cb323159 1305 result = utun_register_kernel_pipe_nexus(pcb);
5ba3f43e
A		 1306 if (result) {
1307 return result;
1308 }
1309
1310 VERIFY(utun_ncd);
1311
1312 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1313
1314 if (pcb->utun_kpipe_enabled) {
1315 result = EEXIST; // return success instead?
1316 goto done;
1317 }
1318
1319 /*
1320 * Make sure we can fit packets in the channel buffers and
1321 * Allow an extra 4 bytes for the protocol number header in the channel
1322 */
5c9f4661 1323 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
5ba3f43e
A		 1324 result = EOPNOTSUPP;
1325 goto done;
1326 }
1327
0a7de745 1328 bzero(&pp_init, sizeof(pp_init));
a39ff7e2 1329 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
cb323159 1330 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
a39ff7e2
A		 1331 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1332 pp_init.kbi_bufsize = pcb->utun_slot_size;
1333 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1334 pp_init.kbi_max_frags = 1;
1335 pp_init.kbi_flags |= KBIF_QUANTUM;
0a7de745 1336 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2 1337 "com.apple.kpipe.%s", pcb->utun_if_xname);
cb323159
A		 1338 pp_init.kbi_ctx = NULL;
1339 pp_init.kbi_ctx_retain = NULL;
1340 pp_init.kbi_ctx_release = NULL;
a39ff7e2 1341
cb323159 1342 result = kern_pbufpool_create(&pp_init, &pcb->utun_kpipe_pp,
a39ff7e2
A		 1343 NULL);
1344 if (result != 0) {
cb323159 1345 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, result);
a39ff7e2
A		 1346 goto done;
1347 }
1348
5ba3f43e 1349 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
0a7de745 1350 bzero(&init, sizeof(init));
5ba3f43e 1351 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
a39ff7e2 1352 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
5ba3f43e 1353 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
0a7de745 1354 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
5ba3f43e
A		 1355 if (result) {
1356 goto done;
1357 }
1358
1359 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1360 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
0a7de745
A		 1361 pcb->utun_kpipe_uuid, &port,
1362 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
5ba3f43e
A		 1363 if (result) {
1364 kern_nexus_controller_free_provider_instance(utun_ncd,
0a7de745 1365 pcb->utun_kpipe_uuid);
5ba3f43e
A		 1366 uuid_clear(pcb->utun_kpipe_uuid);
1367 goto done;
1368 }
1369
1370 pcb->utun_kpipe_enabled = 1;
1371
1372done:
1373 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1374
1375 if (result) {
a39ff7e2
A		 1376 if (pcb->utun_kpipe_pp != NULL) {
1377 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1378 pcb->utun_kpipe_pp = NULL;
1379 }
5ba3f43e
A		 1380 utun_unregister_kernel_pipe_nexus();
1381 }
1382
1383 return result;
1384}
1385
1386#endif // UTUN_NEXUS
b0d623f7
A		 1387
1388errno_t
1389utun_register_control(void)
1390{
5ba3f43e
A		 1391 struct kern_ctl_reg kern_ctl;
1392 errno_t result = 0;
0a7de745 1393
5ba3f43e
A		 1394 utun_pcb_size = sizeof(struct utun_pcb);
1395 utun_pcb_zone = zinit(utun_pcb_size,
0a7de745
A		 1396 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1397 0, UTUN_PCB_ZONE_NAME);
5ba3f43e 1398 if (utun_pcb_zone == NULL) {
cb323159 1399 os_log_error(OS_LOG_DEFAULT, "utun_register_control - zinit(utun_pcb) failed");
5ba3f43e
A		 1400 return ENOMEM;
1401 }
1402
1403#if UTUN_NEXUS
1404 utun_register_nexus();
1405#endif // UTUN_NEXUS
1406
1407 TAILQ_INIT(&utun_head);
0a7de745 1408
b0d623f7 1409 bzero(&kern_ctl, sizeof(kern_ctl));
fe8ab488 1410 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
b0d623f7 1411 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
fe8ab488 1412 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
39236c6e
A		 1413 kern_ctl.ctl_sendsize = 512 * 1024;
1414 kern_ctl.ctl_recvsize = 512 * 1024;
5c9f4661 1415 kern_ctl.ctl_bind = utun_ctl_bind;
b0d623f7
A		 1416 kern_ctl.ctl_connect = utun_ctl_connect;
1417 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1418 kern_ctl.ctl_send = utun_ctl_send;
1419 kern_ctl.ctl_setopt = utun_ctl_setopt;
1420 kern_ctl.ctl_getopt = utun_ctl_getopt;
fe8ab488 1421 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
39236c6e 1422
b0d623f7
A		 1423 result = ctl_register(&kern_ctl, &utun_kctlref);
1424 if (result != 0) {
cb323159 1425 os_log_error(OS_LOG_DEFAULT, "utun_register_control - ctl_register failed: %d\n", result);
b0d623f7
A		 1426 return result;
1427 }
0a7de745 1428
b0d623f7 1429 /* Register the protocol plumbers */
cb323159 1430 if ((result = proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN,
0a7de745 1431 utun_attach_proto, NULL)) != 0) {
cb323159
A		 1432 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN) failed: %d\n",
1433 result);
b0d623f7
A		 1434 ctl_deregister(utun_kctlref);
1435 return result;
1436 }
0a7de745 1437
b0d623f7 1438 /* Register the protocol plumbers */
cb323159 1439 if ((result = proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN,
0a7de745 1440 utun_attach_proto, NULL)) != 0) {
cb323159 1441 proto_unregister_plumber(PF_INET, IFNET_FAMILY_UTUN);
b0d623f7 1442 ctl_deregister(utun_kctlref);
cb323159
A		 1443 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN) failed: %d\n",
1444 result);
b0d623f7
A		 1445 return result;
1446 }
39037602 1447
5ba3f43e
A		 1448 utun_lck_attr = lck_attr_alloc_init();
1449 utun_lck_grp_attr = lck_grp_attr_alloc_init();
0a7de745 1450 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
5ba3f43e 1451
5ba3f43e 1452 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
5ba3f43e 1453
b0d623f7
A		 1454 return 0;
1455}
1456
1457/* Kernel control functions */
1458
5ba3f43e 1459static inline void
5c9f4661 1460utun_free_pcb(struct utun_pcb *pcb, bool in_list)
5ba3f43e
A		 1461{
1462#ifdef UTUN_NEXUS
1463 mbuf_freem_list(pcb->utun_input_chain);
cb323159 1464 pcb->utun_input_chain_count = 0;
5ba3f43e
A		 1465 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1466#endif // UTUN_NEXUS
1467 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
5c9f4661
A		 1468 if (in_list) {
1469 lck_mtx_lock(&utun_lock);
1470 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1471 lck_mtx_unlock(&utun_lock);
1472 }
5ba3f43e
A		 1473 zfree(utun_pcb_zone, pcb);
1474}
1475
b0d623f7 1476static errno_t
5c9f4661 1477utun_ctl_bind(kern_ctl_ref kctlref,
0a7de745
A		 1478 struct sockaddr_ctl *sac,
1479 void **unitinfo)
b0d623f7 1480{
5ba3f43e
A		 1481 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1482 memset(pcb, 0, sizeof(*pcb));
39037602 1483
b0d623f7 1484 *unitinfo = pcb;
d1ecb069
A		 1485 pcb->utun_ctlref = kctlref;
1486 pcb->utun_unit = sac->sc_unit;
fe8ab488 1487 pcb->utun_max_pending_packets = 1;
5ba3f43e 1488
5c9f4661
A		 1489#if UTUN_NEXUS
1490 pcb->utun_use_netif = false;
cb323159
A		 1491 pcb->utun_attach_fsw = true;
1492 pcb->utun_netif_connected = false;
5c9f4661 1493 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
cb323159
A		 1494 pcb->utun_netif_ring_size = if_utun_ring_size;
1495 pcb->utun_tx_fsw_ring_size = if_utun_tx_fsw_ring_size;
1496 pcb->utun_rx_fsw_ring_size = if_utun_rx_fsw_ring_size;
1497 pcb->utun_input_chain_count = 0;
1498 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
5c9f4661
A		 1499#endif // UTUN_NEXUS
1500
5ba3f43e
A		 1501 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1502
0a7de745 1503 return 0;
5c9f4661
A		 1504}
1505
1506static errno_t
1507utun_ctl_connect(kern_ctl_ref kctlref,
0a7de745
A		 1508 struct sockaddr_ctl *sac,
1509 void **unitinfo)
5c9f4661
A		 1510{
1511 struct ifnet_init_eparams utun_init = {};
1512 errno_t result = 0;
1513
1514 if (*unitinfo == NULL) {
1515 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1516 }
0a7de745 1517
5c9f4661
A		 1518 struct utun_pcb *pcb = *unitinfo;
1519
5ba3f43e
A		 1520 lck_mtx_lock(&utun_lock);
1521
1522 /* Find some open interface id */
1523 u_int32_t chosen_unique_id = 1;
1524 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1525 if (next_pcb != NULL) {
1526 /* List was not empty, add one to the last item */
1527 chosen_unique_id = next_pcb->utun_unique_id + 1;
1528 next_pcb = NULL;
1529
1530 /*
1531 * If this wrapped the id number, start looking at
1532 * the front of the list for an unused id.
1533 */
1534 if (chosen_unique_id == 0) {
1535 /* Find the next unused ID */
1536 chosen_unique_id = 1;
1537 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1538 if (next_pcb->utun_unique_id > chosen_unique_id) {
1539 /* We found a gap */
1540 break;
1541 }
1542
1543 chosen_unique_id = next_pcb->utun_unique_id + 1;
1544 }
1545 }
1546 }
1547
1548 pcb->utun_unique_id = chosen_unique_id;
1549
1550 if (next_pcb != NULL) {
1551 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1552 } else {
1553 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1554 }
1555 lck_mtx_unlock(&utun_lock);
1556
1557 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1558 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
cb323159 1559 os_log(OS_LOG_DEFAULT, "utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
b0d623f7
A		 1560
1561 /* Create the interface */
1562 bzero(&utun_init, sizeof(utun_init));
39236c6e 1563 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
0a7de745 1564 utun_init.len = sizeof(utun_init);
5ba3f43e
A		 1565
1566#if UTUN_NEXUS
5c9f4661
A		 1567 if (pcb->utun_use_netif) {
1568 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1569 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1570 } else
5ba3f43e 1571#endif // UTUN_NEXUS
5c9f4661
A		 1572 {
1573 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1574 utun_init.start = utun_start;
1575 utun_init.framer_extended = utun_framer;
1576 }
5ba3f43e 1577 utun_init.name = "utun";
d1ecb069 1578 utun_init.unit = pcb->utun_unit - 1;
5ba3f43e
A		 1579 utun_init.uniqueid = pcb->utun_unique_name;
1580 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
cb323159 1581 utun_init.family = IFNET_FAMILY_UTUN;
b0d623f7 1582 utun_init.type = IFT_OTHER;
b0d623f7 1583 utun_init.demux = utun_demux;
b0d623f7
A		 1584 utun_init.add_proto = utun_add_proto;
1585 utun_init.del_proto = utun_del_proto;
1586 utun_init.softc = pcb;
1587 utun_init.ioctl = utun_ioctl;
1588 utun_init.detach = utun_detached;
39037602 1589
5ba3f43e 1590#if UTUN_NEXUS
5c9f4661
A		 1591 if (pcb->utun_use_netif) {
1592 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1593 if (result != 0) {
cb323159 1594 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
5c9f4661
A		 1595 utun_free_pcb(pcb, true);
1596 *unitinfo = NULL;
1597 return result;
1598 }
5ba3f43e 1599
cb323159
A		 1600 if (pcb->utun_attach_fsw) {
1601 result = utun_flowswitch_attach(pcb);
1602 if (result != 0) {
1603 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_flowswitch_attach failed: %d\n", result);
1604 *unitinfo = NULL;
1605 return result;
1606 }
5c9f4661 1607 }
5ba3f43e 1608
5c9f4661
A		 1609 /* Attach to bpf */
1610 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1611 } else
1612#endif // UTUN_NEXUS
1613 {
1614 /*
1615 * Upon success, this holds an ifnet reference which we will
1616 * release via ifnet_release() at final detach time.
1617 */
1618 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1619 if (result != 0) {
cb323159 1620 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_allocate failed: %d\n", result);
5c9f4661
A		 1621 utun_free_pcb(pcb, true);
1622 *unitinfo = NULL;
1623 return result;
1624 }
b0d623f7 1625
5c9f4661
A		 1626 /* Set flags and additional information. */
1627 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1628 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
d1ecb069 1629
5c9f4661
A		 1630 /* The interface must generate its own IPv6 LinkLocal address,
1631 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1632 */
1633 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1634
1635 /* Reset the stats in case as the interface may have been recycled */
1636 struct ifnet_stats_param stats;
1637 bzero(&stats, sizeof(struct ifnet_stats_param));
1638 ifnet_set_stat(pcb->utun_ifp, &stats);
1639
1640 /* Attach the interface */
1641 result = ifnet_attach(pcb->utun_ifp, NULL);
1642 if (result != 0) {
cb323159 1643 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_attach failed: %d\n", result);
5c9f4661
A		 1644 /* Release reference now since attach failed */
1645 ifnet_release(pcb->utun_ifp);
1646 utun_free_pcb(pcb, true);
1647 *unitinfo = NULL;
0a7de745 1648 return result;
5c9f4661
A		 1649 }
1650
1651 /* Attach to bpf */
1652 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
39037602 1653 }
5ba3f43e 1654
5ba3f43e
A		 1655 /* The interfaces resoures allocated, mark it as running */
1656 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1657
b0d623f7
A		 1658 return result;
1659}
1660
1661static errno_t
5ba3f43e 1662utun_detach_ip(ifnet_t interface,
0a7de745
A		 1663 protocol_family_t protocol,
1664 socket_t pf_socket)
b0d623f7
A		 1665{
1666 errno_t result = EPROTONOSUPPORT;
0a7de745 1667
b0d623f7
A		 1668 /* Attempt a detach */
1669 if (protocol == PF_INET) {
0a7de745
A		 1670 struct ifreq ifr;
1671
b0d623f7
A		 1672 bzero(&ifr, sizeof(ifr));
1673 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745
A		 1674 ifnet_name(interface), ifnet_unit(interface));
1675
b0d623f7 1676 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
5ba3f43e 1677 } else if (protocol == PF_INET6) {
0a7de745
A		 1678 struct in6_ifreq ifr6;
1679
b0d623f7
A		 1680 bzero(&ifr6, sizeof(ifr6));
1681 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745
A		 1682 ifnet_name(interface), ifnet_unit(interface));
1683
b0d623f7
A		 1684 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1685 }
0a7de745 1686
b0d623f7
A		 1687 return result;
1688}
1689
1690static void
5ba3f43e 1691utun_remove_address(ifnet_t interface,
0a7de745
A		 1692 protocol_family_t protocol,
1693 ifaddr_t address,
1694 socket_t pf_socket)
b0d623f7
A		 1695{
1696 errno_t result = 0;
0a7de745 1697
b0d623f7
A		 1698 /* Attempt a detach */
1699 if (protocol == PF_INET) {
5ba3f43e 1700 struct ifreq ifr;
0a7de745 1701
b0d623f7
A		 1702 bzero(&ifr, sizeof(ifr));
1703 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745 1704 ifnet_name(interface), ifnet_unit(interface));
b0d623f7
A		 1705 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1706 if (result != 0) {
cb323159 1707 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed: %d", result);
5ba3f43e 1708 } else {
b0d623f7
A		 1709 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1710 if (result != 0) {
cb323159 1711 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR failed: %d", result);
b0d623f7
A		 1712 }
1713 }
5ba3f43e
A		 1714 } else if (protocol == PF_INET6) {
1715 struct in6_ifreq ifr6;
0a7de745 1716
b0d623f7
A		 1717 bzero(&ifr6, sizeof(ifr6));
1718 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745 1719 ifnet_name(interface), ifnet_unit(interface));
b0d623f7 1720 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
0a7de745 1721 sizeof(ifr6.ifr_addr));
b0d623f7 1722 if (result != 0) {
cb323159 1723 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed (v6): %d",
0a7de745 1724 result);
5ba3f43e 1725 } else {
b0d623f7
A		 1726 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1727 if (result != 0) {
cb323159 1728 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
0a7de745 1729 result);
b0d623f7
A		 1730 }
1731 }
1732 }
1733}
1734
1735static void
5ba3f43e 1736utun_cleanup_family(ifnet_t interface,
0a7de745 1737 protocol_family_t protocol)
5ba3f43e
A		 1738{
1739 errno_t result = 0;
1740 socket_t pf_socket = NULL;
1741 ifaddr_t *addresses = NULL;
1742 int i;
0a7de745 1743
b0d623f7 1744 if (protocol != PF_INET && protocol != PF_INET6) {
cb323159 1745 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - invalid protocol family %d\n", protocol);
b0d623f7
A		 1746 return;
1747 }
0a7de745 1748
b0d623f7
A		 1749 /* Create a socket for removing addresses and detaching the protocol */
1750 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1751 if (result != 0) {
0a7de745 1752 if (result != EAFNOSUPPORT) {
cb323159 1753 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - failed to create %s socket: %d\n",
0a7de745
A		 1754 protocol == PF_INET ? "IP" : "IPv6", result);
1755 }
b0d623f7
A		 1756 goto cleanup;
1757 }
0a7de745
A		 1758
1759 /* always set SS_PRIV, we want to close and detach regardless */
1760 sock_setpriv(pf_socket, 1);
6d2010ae 1761
b0d623f7
A		 1762 result = utun_detach_ip(interface, protocol, pf_socket);
1763 if (result == 0 || result == ENXIO) {
1764 /* We are done! We either detached or weren't attached. */
1765 goto cleanup;
5ba3f43e 1766 } else if (result != EBUSY) {
b0d623f7 1767 /* Uh, not really sure what happened here... */
cb323159 1768 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
b0d623f7
A		 1769 goto cleanup;
1770 }
0a7de745 1771
b0d623f7
A		 1772 /*
1773 * At this point, we received an EBUSY error. This means there are
1774 * addresses attached. We should detach them and then try again.
1775 */
1776 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1777 if (result != 0) {
cb323159 1778 os_log_error(OS_LOG_DEFAULT, "fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
0a7de745
A		 1779 ifnet_name(interface), ifnet_unit(interface),
1780 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
b0d623f7
A		 1781 goto cleanup;
1782 }
0a7de745 1783
b0d623f7
A		 1784 for (i = 0; addresses[i] != 0; i++) {
1785 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1786 }
1787 ifnet_free_address_list(addresses);
1788 addresses = NULL;
0a7de745 1789
b0d623f7
A		 1790 /*
1791 * The addresses should be gone, we should try the remove again.
1792 */
1793 result = utun_detach_ip(interface, protocol, pf_socket);
1794 if (result != 0 && result != ENXIO) {
cb323159 1795 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
b0d623f7 1796 }
0a7de745 1797
b0d623f7 1798cleanup:
5ba3f43e 1799 if (pf_socket != NULL) {
b0d623f7 1800 sock_close(pf_socket);
5ba3f43e 1801 }
0a7de745 1802
5ba3f43e 1803 if (addresses != NULL) {
b0d623f7 1804 ifnet_free_address_list(addresses);
5ba3f43e 1805 }
b0d623f7
A		 1806}
1807
1808static errno_t
5ba3f43e 1809utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
0a7de745
A		 1810 __unused u_int32_t unit,
1811 void *unitinfo)
b0d623f7 1812{
0a7de745 1813 struct utun_pcb *pcb = unitinfo;
5ba3f43e
A		 1814 ifnet_t ifp = NULL;
1815 errno_t result = 0;
1816
1817 if (pcb == NULL) {
1818 return EINVAL;
1819 }
1820
1821#if UTUN_NEXUS
1822 // Tell the nexus to stop all rings
cb323159 1823 if (pcb->utun_netif_nexus != NULL && pcb->utun_netif_connected) {
5ba3f43e
A		 1824 kern_nexus_stop(pcb->utun_netif_nexus);
1825 }
1826#endif // UTUN_NEXUS
316670eb 1827
5ba3f43e 1828 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
316670eb 1829
5ba3f43e
A		 1830#if UTUN_NEXUS
1831 uuid_t kpipe_uuid;
1832 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1833 uuid_clear(pcb->utun_kpipe_uuid);
1834 pcb->utun_kpipe_enabled = FALSE;
1835#endif // UTUN_NEXUS
39037602 1836
d1ecb069 1837 pcb->utun_ctlref = NULL;
5ba3f43e 1838
5c9f4661
A		 1839 ifp = pcb->utun_ifp;
1840 if (ifp != NULL) {
1841#if UTUN_NEXUS
1842 // Tell the nexus to stop all rings
1843 if (pcb->utun_netif_nexus != NULL) {
1844 /*
1845 * Quiesce the interface and flush any pending outbound packets.
1846 */
1847 if_down(ifp);
1848
1849 /* Increment refcnt, but detach interface */
1850 ifnet_incr_iorefcnt(ifp);
1851 if ((result = ifnet_detach(ifp)) != 0) {
1852 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1853 }
5ba3f43e 1854
5c9f4661
A		 1855 /*
1856 * We want to do everything in our power to ensure that the interface
1857 * really goes away when the socket is closed. We must remove IP/IPv6
1858 * addresses and detach the protocols. Finally, we can remove and
1859 * release the interface.
1860 */
1861 utun_cleanup_family(ifp, AF_INET);
1862 utun_cleanup_family(ifp, AF_INET6);
39037602 1863
5c9f4661 1864 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
39037602 1865
5c9f4661
A		 1866 if (!uuid_is_null(kpipe_uuid)) {
1867 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 1868 if (pcb->utun_kpipe_pp != NULL) {
1869 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1870 pcb->utun_kpipe_pp = NULL;
1871 }
5c9f4661
A		 1872 utun_unregister_kernel_pipe_nexus();
1873 }
1874 }
a39ff7e2 1875 utun_nexus_detach(pcb);
5c9f4661
A		 1876
1877 /* Decrement refcnt to finish detaching and freeing */
1878 ifnet_decr_iorefcnt(ifp);
1879 } else
1880#endif // UTUN_NEXUS
1881 {
1882 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
5ba3f43e
A		 1883
1884#if UTUN_NEXUS
5c9f4661
A		 1885 if (!uuid_is_null(kpipe_uuid)) {
1886 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 1887 if (pcb->utun_kpipe_pp != NULL) {
1888 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1889 pcb->utun_kpipe_pp = NULL;
1890 }
5c9f4661
A		 1891 utun_unregister_kernel_pipe_nexus();
1892 }
1893 }
5ba3f43e
A		 1894#endif // UTUN_NEXUS
1895
5c9f4661
A		 1896 /*
1897 * We want to do everything in our power to ensure that the interface
1898 * really goes away when the socket is closed. We must remove IP/IPv6
1899 * addresses and detach the protocols. Finally, we can remove and
1900 * release the interface.
1901 */
1902 utun_cleanup_family(ifp, AF_INET);
1903 utun_cleanup_family(ifp, AF_INET6);
1904
1905 /*
1906 * Detach now; utun_detach() will be called asynchronously once
1907 * the I/O reference count drops to 0. There we will invoke
1908 * ifnet_release().
1909 */
1910 if ((result = ifnet_detach(ifp)) != 0) {
cb323159 1911 os_log_error(OS_LOG_DEFAULT, "utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
5c9f4661
A		 1912 }
1913 }
1914 } else {
1915 // Bound, but not connected
1916 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1917 utun_free_pcb(pcb, false);
1918 }
0a7de745 1919
b0d623f7 1920 return 0;
39037602 1921}
b0d623f7
A		 1922
1923static errno_t
5ba3f43e 1924utun_ctl_send(__unused kern_ctl_ref kctlref,
0a7de745
A		 1925 __unused u_int32_t unit,
1926 void *unitinfo,
1927 mbuf_t m,
1928 __unused int flags)
b0d623f7 1929{
39236c6e 1930 /*
0a7de745 1931 * The userland ABI requires the first four bytes have the protocol family
39236c6e
A		 1932 * in network byte order: swap them
1933 */
39037602 1934 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
39236c6e 1935 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
39037602 1936 } else {
cb323159 1937 os_log_error(OS_LOG_DEFAULT, "%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m));
39037602 1938 }
39236c6e 1939
316670eb 1940 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
b0d623f7
A		 1941}
1942
1943static errno_t
5ba3f43e 1944utun_ctl_setopt(__unused kern_ctl_ref kctlref,
0a7de745
A		 1945 __unused u_int32_t unit,
1946 void *unitinfo,
1947 int opt,
1948 void *data,
1949 size_t len)
b0d623f7 1950{
5ba3f43e
A		 1951 struct utun_pcb *pcb = unitinfo;
1952 errno_t result = 0;
b0d623f7
A		 1953 /* check for privileges for privileged options */
1954 switch (opt) {
0a7de745
A		 1955 case UTUN_OPT_FLAGS:
1956 case UTUN_OPT_EXT_IFDATA_STATS:
1957 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1958 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1959 return EPERM;
1960 }
1961 break;
b0d623f7
A		 1962 }
1963
1964 switch (opt) {
0a7de745
A		 1965 case UTUN_OPT_FLAGS:
1966 if (len != sizeof(u_int32_t)) {
1967 result = EMSGSIZE;
1968 } else {
1969 if (pcb->utun_ifp == NULL) {
1970 // Only can set after connecting
1971 result = EINVAL;
1972 break;
1973 }
5c9f4661 1974#if UTUN_NEXUS
0a7de745
A		 1975 if (pcb->utun_use_netif) {
1976 pcb->utun_flags = *(u_int32_t *)data;
1977 } else
5c9f4661 1978#endif // UTUN_NEXUS
0a7de745
A		 1979 {
1980 u_int32_t old_flags = pcb->utun_flags;
1981 pcb->utun_flags = *(u_int32_t *)data;
1982 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1983 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1984 bpfdetach(pcb->utun_ifp);
1985 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
5c9f4661 1986 }
39037602 1987 }
0a7de745
A		 1988 }
1989 break;
1990
1991 case UTUN_OPT_EXT_IFDATA_STATS:
1992 if (len != sizeof(int)) {
1993 result = EMSGSIZE;
1994 break;
1995 }
1996 if (pcb->utun_ifp == NULL) {
1997 // Only can set after connecting
1998 result = EINVAL;
39236c6e 1999 break;
0a7de745
A		 2000 }
2001 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
2002 break;
39236c6e 2003
0a7de745
A		 2004 case UTUN_OPT_INC_IFDATA_STATS_IN:
2005 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
2006 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
2007
2008 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
2009 result = EINVAL;
d1ecb069 2010 break;
0a7de745
A		 2011 }
2012 if (pcb->utun_ifp == NULL) {
2013 // Only can set after connecting
2014 result = EINVAL;
2015 break;
2016 }
2017 if (!pcb->utun_ext_ifdata_stats) {
2018 result = EINVAL;
b0d623f7 2019 break;
d1ecb069 2020 }
0a7de745
A		 2021 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) {
2022 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2023 utsp->utsp_bytes, utsp->utsp_errors);
2024 } else {
2025 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2026 utsp->utsp_bytes, utsp->utsp_errors);
2027 }
2028 break;
2029 }
2030 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2031 ifnet_t del_ifp = NULL;
2032 char name[IFNAMSIZ];
39236c6e 2033
0a7de745
A		 2034 if (len > IFNAMSIZ - 1) {
2035 result = EMSGSIZE;
39236c6e
A		 2036 break;
2037 }
0a7de745
A		 2038 if (pcb->utun_ifp == NULL) {
2039 // Only can set after connecting
2040 result = EINVAL;
2041 break;
2042 }
2043 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2044 bcopy(data, name, len);
2045 name[len] = 0;
2046 result = ifnet_find_by_name(name, &del_ifp);
2047 }
2048 if (result == 0) {
2049 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2050 if (del_ifp) {
2051 ifnet_release(del_ifp);
fe8ab488 2052 }
0a7de745
A		 2053 }
2054 break;
2055 }
2056 case UTUN_OPT_MAX_PENDING_PACKETS: {
2057 u_int32_t max_pending_packets = 0;
2058 if (len != sizeof(u_int32_t)) {
2059 result = EMSGSIZE;
2060 break;
2061 }
2062 max_pending_packets = *(u_int32_t *)data;
2063 if (max_pending_packets == 0) {
2064 result = EINVAL;
fe8ab488
A		 2065 break;
2066 }
0a7de745
A		 2067 pcb->utun_max_pending_packets = max_pending_packets;
2068 break;
2069 }
5ba3f43e 2070#if UTUN_NEXUS
0a7de745
A		 2071 case UTUN_OPT_ENABLE_CHANNEL: {
2072 if (len != sizeof(int)) {
2073 result = EMSGSIZE;
2074 break;
2075 }
2076 if (pcb->utun_ifp == NULL) {
2077 // Only can set after connecting
2078 result = EINVAL;
2079 break;
2080 }
2081 if (*(int *)data) {
2082 result = utun_enable_channel(pcb, current_proc());
2083 } else {
2084 result = utun_disable_channel(pcb);
2085 }
2086 break;
2087 }
2088 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2089 if (len != sizeof(int)) {
2090 result = EMSGSIZE;
2091 break;
2092 }
2093 if (pcb->utun_ifp == NULL) {
2094 // Only can set after connecting
2095 result = EINVAL;
2096 break;
2097 }
cb323159 2098 if (!if_is_fsw_transport_netagent_enabled()) {
0a7de745
A		 2099 result = ENOTSUP;
2100 break;
2101 }
cb323159 2102 if (uuid_is_null(pcb->utun_nx.fsw_agent)) {
0a7de745 2103 result = ENOENT;
5ba3f43e
A		 2104 break;
2105 }
5ba3f43e 2106
cb323159
A		 2107 uint32_t flags = netagent_get_flags(pcb->utun_nx.fsw_agent);
2108
0a7de745 2109 if (*(int *)data) {
0a7de745 2110 pcb->utun_needs_netagent = true;
cb323159
A		 2111 flags |= (NETAGENT_FLAG_NEXUS_PROVIDER |
2112 NETAGENT_FLAG_NEXUS_LISTENER);
2113 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
0a7de745 2114 } else {
cb323159
A		 2115 flags &= ~(NETAGENT_FLAG_NEXUS_PROVIDER |
2116 NETAGENT_FLAG_NEXUS_LISTENER);
2117 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
0a7de745 2118 pcb->utun_needs_netagent = false;
0a7de745
A		 2119 }
2120 break;
2121 }
cb323159
A		 2122 case UTUN_OPT_ATTACH_FLOWSWITCH: {
2123 if (len != sizeof(int)) {
2124 result = EMSGSIZE;
2125 break;
2126 }
2127 if (pcb->utun_ifp != NULL) {
2128 // Only can set before connecting
2129 result = EINVAL;
2130 break;
2131 }
2132 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2133 pcb->utun_attach_fsw = !!(*(int *)data);
2134 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2135 break;
2136 }
0a7de745
A		 2137 case UTUN_OPT_ENABLE_NETIF: {
2138 if (len != sizeof(int)) {
2139 result = EMSGSIZE;
5ba3f43e
A		 2140 break;
2141 }
0a7de745
A		 2142 if (pcb->utun_ifp != NULL) {
2143 // Only can set before connecting
2144 result = EINVAL;
5c9f4661
A		 2145 break;
2146 }
0a7de745
A		 2147 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2148 pcb->utun_use_netif = !!(*(int *)data);
2149 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2150 break;
2151 }
2152 case UTUN_OPT_SLOT_SIZE: {
2153 if (len != sizeof(u_int32_t)) {
2154 result = EMSGSIZE;
5c9f4661
A		 2155 break;
2156 }
0a7de745
A		 2157 if (pcb->utun_ifp != NULL) {
2158 // Only can set before connecting
2159 result = EINVAL;
5c9f4661
A		 2160 break;
2161 }
0a7de745
A		 2162 u_int32_t slot_size = *(u_int32_t *)data;
2163 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2164 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2165 return EINVAL;
2166 }
2167 pcb->utun_slot_size = slot_size;
2168 break;
2169 }
2170 case UTUN_OPT_NETIF_RING_SIZE: {
2171 if (len != sizeof(u_int32_t)) {
2172 result = EMSGSIZE;
5c9f4661
A		 2173 break;
2174 }
0a7de745
A		 2175 if (pcb->utun_ifp != NULL) {
2176 // Only can set before connecting
2177 result = EINVAL;
5c9f4661
A		 2178 break;
2179 }
0a7de745
A		 2180 u_int32_t ring_size = *(u_int32_t *)data;
2181 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2182 ring_size > UTUN_IF_MAX_RING_SIZE) {
2183 return EINVAL;
2184 }
2185 pcb->utun_netif_ring_size = ring_size;
2186 break;
2187 }
2188 case UTUN_OPT_TX_FSW_RING_SIZE: {
2189 if (len != sizeof(u_int32_t)) {
2190 result = EMSGSIZE;
2191 break;
2192 }
2193 if (pcb->utun_ifp != NULL) {
2194 // Only can set before connecting
2195 result = EINVAL;
b0d623f7 2196 break;
fe8ab488 2197 }
0a7de745
A		 2198 u_int32_t ring_size = *(u_int32_t *)data;
2199 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2200 ring_size > UTUN_IF_MAX_RING_SIZE) {
2201 return EINVAL;
2202 }
2203 pcb->utun_tx_fsw_ring_size = ring_size;
2204 break;
2205 }
2206 case UTUN_OPT_RX_FSW_RING_SIZE: {
2207 if (len != sizeof(u_int32_t)) {
2208 result = EMSGSIZE;
2209 break;
2210 }
2211 if (pcb->utun_ifp != NULL) {
2212 // Only can set before connecting
2213 result = EINVAL;
2214 break;
2215 }
2216 u_int32_t ring_size = *(u_int32_t *)data;
2217 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2218 ring_size > UTUN_IF_MAX_RING_SIZE) {
2219 return EINVAL;
2220 }
2221 pcb->utun_rx_fsw_ring_size = ring_size;
2222 break;
2223 }
cb323159
A		 2224 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2225 if (len != sizeof(u_int32_t)) {
2226 result = EMSGSIZE;
2227 break;
2228 }
2229 if (pcb->utun_ifp != NULL) {
2230 // Only can set before connecting
2231 result = EINVAL;
2232 break;
2233 }
2234 u_int32_t ring_size = *(u_int32_t *)data;
2235 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2236 ring_size > UTUN_IF_MAX_RING_SIZE) {
2237 return EINVAL;
2238 }
2239 pcb->utun_kpipe_tx_ring_size = ring_size;
2240 break;
2241 }
2242 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2243 if (len != sizeof(u_int32_t)) {
2244 result = EMSGSIZE;
2245 break;
2246 }
2247 if (pcb->utun_ifp != NULL) {
2248 // Only can set before connecting
2249 result = EINVAL;
2250 break;
2251 }
2252 u_int32_t ring_size = *(u_int32_t *)data;
2253 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2254 ring_size > UTUN_IF_MAX_RING_SIZE) {
2255 return EINVAL;
2256 }
2257 pcb->utun_kpipe_rx_ring_size = ring_size;
2258 break;
2259 }
0a7de745
A		 2260#endif // UTUN_NEXUS
2261 default: {
2262 result = ENOPROTOOPT;
2263 break;
2264 }
b0d623f7
A		 2265 }
2266
2267 return result;
2268}
2269
2270static errno_t
5ba3f43e 2271utun_ctl_getopt(__unused kern_ctl_ref kctlref,
0a7de745
A		 2272 __unused u_int32_t unit,
2273 void *unitinfo,
2274 int opt,
2275 void *data,
2276 size_t *len)
b0d623f7 2277{
5ba3f43e
A		 2278 struct utun_pcb *pcb = unitinfo;
2279 errno_t result = 0;
0a7de745 2280
b0d623f7 2281 switch (opt) {
0a7de745
A		 2282 case UTUN_OPT_FLAGS:
2283 if (*len != sizeof(u_int32_t)) {
2284 result = EMSGSIZE;
2285 } else {
2286 *(u_int32_t *)data = pcb->utun_flags;
2287 }
2288 break;
d1ecb069 2289
0a7de745
A		 2290 case UTUN_OPT_EXT_IFDATA_STATS:
2291 if (*len != sizeof(int)) {
2292 result = EMSGSIZE;
2293 } else {
2294 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2295 }
2296 break;
d1ecb069 2297
0a7de745
A		 2298 case UTUN_OPT_IFNAME:
2299 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2300 result = EMSGSIZE;
2301 } else {
2302 if (pcb->utun_ifp == NULL) {
2303 // Only can get after connecting
2304 result = EINVAL;
2305 break;
5ba3f43e 2306 }
0a7de745 2307 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
fe8ab488 2308 }
0a7de745 2309 break;
39037602 2310
0a7de745
A		 2311 case UTUN_OPT_MAX_PENDING_PACKETS: {
2312 if (*len != sizeof(u_int32_t)) {
2313 result = EMSGSIZE;
2314 } else {
2315 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
a39ff7e2 2316 }
0a7de745
A		 2317 break;
2318 }
a39ff7e2 2319
0a7de745
A		 2320#if UTUN_NEXUS
2321 case UTUN_OPT_ENABLE_CHANNEL: {
2322 if (*len != sizeof(int)) {
2323 result = EMSGSIZE;
2324 } else {
2325 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2326 *(int *)data = pcb->utun_kpipe_enabled;
2327 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
a39ff7e2 2328 }
0a7de745
A		 2329 break;
2330 }
a39ff7e2 2331
0a7de745
A		 2332 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2333 if (*len != sizeof(int)) {
2334 result = EMSGSIZE;
2335 } else {
cb323159 2336 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.fsw_agent);
a39ff7e2 2337 }
0a7de745
A		 2338 break;
2339 }
a39ff7e2 2340
0a7de745
A		 2341 case UTUN_OPT_ENABLE_NETIF: {
2342 if (*len != sizeof(int)) {
2343 result = EMSGSIZE;
2344 } else {
5ba3f43e 2345 lck_rw_lock_shared(&pcb->utun_pcb_lock);
0a7de745 2346 *(int *)data = !!pcb->utun_use_netif;
5ba3f43e 2347 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5c9f4661 2348 }
0a7de745
A		 2349 break;
2350 }
2351
2352 case UTUN_OPT_GET_CHANNEL_UUID: {
2353 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2354 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2355 result = ENXIO;
2356 } else if (*len != sizeof(uuid_t)) {
2357 result = EMSGSIZE;
2358 } else {
2359 uuid_copy(data, pcb->utun_kpipe_uuid);
5c9f4661 2360 }
0a7de745
A		 2361 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2362 break;
2363 }
2364 case UTUN_OPT_SLOT_SIZE: {
2365 if (*len != sizeof(u_int32_t)) {
2366 result = EMSGSIZE;
2367 } else {
2368 *(u_int32_t *)data = pcb->utun_slot_size;
5c9f4661 2369 }
0a7de745
A		 2370 break;
2371 }
2372 case UTUN_OPT_NETIF_RING_SIZE: {
2373 if (*len != sizeof(u_int32_t)) {
2374 result = EMSGSIZE;
2375 } else {
2376 *(u_int32_t *)data = pcb->utun_netif_ring_size;
5c9f4661 2377 }
0a7de745
A		 2378 break;
2379 }
2380 case UTUN_OPT_TX_FSW_RING_SIZE: {
2381 if (*len != sizeof(u_int32_t)) {
2382 result = EMSGSIZE;
2383 } else {
2384 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2385 }
2386 break;
2387 }
2388 case UTUN_OPT_RX_FSW_RING_SIZE: {
2389 if (*len != sizeof(u_int32_t)) {
2390 result = EMSGSIZE;
2391 } else {
2392 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
5c9f4661 2393 }
0a7de745
A		 2394 break;
2395 }
cb323159
A		 2396 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2397 if (*len != sizeof(u_int32_t)) {
2398 result = EMSGSIZE;
2399 } else {
2400 *(u_int32_t *)data = pcb->utun_kpipe_tx_ring_size;
2401 }
2402 break;
2403 }
2404 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2405 if (*len != sizeof(u_int32_t)) {
2406 result = EMSGSIZE;
2407 } else {
2408 *(u_int32_t *)data = pcb->utun_kpipe_rx_ring_size;
2409 }
2410 break;
2411 }
5ba3f43e 2412#endif // UTUN_NEXUS
39037602 2413
0a7de745
A		 2414 default:
2415 result = ENOPROTOOPT;
2416 break;
b0d623f7 2417 }
0a7de745 2418
b0d623f7
A		 2419 return result;
2420}
2421
fe8ab488
A		 2422static void
2423utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2424{
3e170ce0 2425#pragma unused(flags)
fe8ab488
A		 2426 bool reenable_output = false;
2427 struct utun_pcb *pcb = unitinfo;
2428 if (pcb == NULL) {
2429 return;
2430 }
2431 ifnet_lock_exclusive(pcb->utun_ifp);
3e170ce0
A		 2432
2433 u_int32_t utun_packet_cnt;
2434 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2435 if (error_pc != 0) {
cb323159 2436 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
3e170ce0 2437 utun_packet_cnt = 0;
fe8ab488 2438 }
3e170ce0
A		 2439
2440 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2441 reenable_output = true;
2442 }
2443
fe8ab488
A		 2444 if (reenable_output) {
2445 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2446 if (error != 0) {
cb323159 2447 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
fe8ab488
A		 2448 }
2449 }
2450 ifnet_lock_done(pcb->utun_ifp);
2451}
2452
b0d623f7 2453/* Network Interface functions */
fe8ab488
A		 2454static void
2455utun_start(ifnet_t interface)
2456{
2457 mbuf_t data;
39037602
A		 2458 struct utun_pcb *pcb = ifnet_softc(interface);
2459
2460 VERIFY(pcb != NULL);
2461
5ba3f43e
A		 2462#if UTUN_NEXUS
2463 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2464 if (pcb->utun_kpipe_enabled) {
2465 /* It's possible to have channels enabled, but not yet have the channel opened,
2466 * in which case the rxring will not be set
2467 */
2468 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2469 if (pcb->utun_kpipe_rxring != NULL) {
2470 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2471 }
2472 return;
2473 }
2474 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2475#endif // UTUN_NEXUS
39037602 2476
fe8ab488
A		 2477 for (;;) {
2478 bool can_accept_packets = true;
2479 ifnet_lock_shared(pcb->utun_ifp);
3e170ce0
A		 2480
2481 u_int32_t utun_packet_cnt;
2482 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2483 if (error_pc != 0) {
cb323159 2484 os_log_error(OS_LOG_DEFAULT, "utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
3e170ce0
A		 2485 utun_packet_cnt = 0;
2486 }
2487
2488 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
fe8ab488
A		 2489 if (!can_accept_packets && pcb->utun_ctlref) {
2490 u_int32_t difference = 0;
2491 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2492 if (difference > 0) {
2493 // If the low-water mark has not yet been reached, we still need to enqueue data
2494 // into the buffer
2495 can_accept_packets = true;
2496 }
2497 }
2498 }
2499 if (!can_accept_packets) {
2500 errno_t error = ifnet_disable_output(interface);
2501 if (error != 0) {
cb323159 2502 os_log_error(OS_LOG_DEFAULT, "utun_start: ifnet_disable_output returned error %d\n", error);
fe8ab488
A		 2503 }
2504 ifnet_lock_done(pcb->utun_ifp);
2505 break;
2506 }
2507 ifnet_lock_done(pcb->utun_ifp);
5ba3f43e 2508 if (ifnet_dequeue(interface, &data) != 0) {
fe8ab488 2509 break;
5ba3f43e
A		 2510 }
2511 if (utun_output(interface, data) != 0) {
fe8ab488 2512 break;
5ba3f43e 2513 }
fe8ab488
A		 2514 }
2515}
2516
b0d623f7 2517static errno_t
0a7de745
A		 2518utun_output(ifnet_t interface,
2519 mbuf_t data)
b0d623f7 2520{
0a7de745 2521 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2522 errno_t result;
39037602
A		 2523
2524 VERIFY(interface == pcb->utun_ifp);
5ba3f43e 2525
5c9f4661
A		 2526#if UTUN_NEXUS
2527 if (!pcb->utun_use_netif)
2528#endif // UTUN_NEXUS
2529 {
2530 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2531 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2532 }
2533 }
2534
d1ecb069 2535 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
b0d623f7
A		 2536 /* flush data */
2537 mbuf_freem(data);
2538 return 0;
2539 }
2540
316670eb 2541 // otherwise, fall thru to ctl_enqueumbuf
d1ecb069 2542 if (pcb->utun_ctlref) {
0a7de745 2543 int length;
316670eb 2544
39236c6e
A		 2545 /*
2546 * The ABI requires the protocol in network byte order
2547 */
39037602 2548 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
39236c6e 2549 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
39037602 2550 }
39236c6e 2551
316670eb 2552 length = mbuf_pkthdr_len(data);
d1ecb069 2553 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
b0d623f7
A		 2554 if (result != 0) {
2555 mbuf_freem(data);
cb323159 2556 os_log_error(OS_LOG_DEFAULT, "utun_output - ctl_enqueuembuf failed: %d\n", result);
5c9f4661
A		 2557#if UTUN_NEXUS
2558 if (!pcb->utun_use_netif)
2559#endif // UTUN_NEXUS
2560 {
2561 ifnet_stat_increment_out(interface, 0, 0, 1);
2562 }
5ba3f43e 2563 } else {
5c9f4661
A		 2564#if UTUN_NEXUS
2565 if (!pcb->utun_use_netif)
2566#endif // UTUN_NEXUS
2567 {
2568 if (!pcb->utun_ext_ifdata_stats) {
2569 ifnet_stat_increment_out(interface, 1, length, 0);
2570 }
5ba3f43e 2571 }
b0d623f7 2572 }
5ba3f43e 2573 } else {
b0d623f7 2574 mbuf_freem(data);
5ba3f43e 2575 }
0a7de745 2576
b0d623f7
A		 2577 return 0;
2578}
2579
b0d623f7 2580static errno_t
5ba3f43e 2581utun_demux(__unused ifnet_t interface,
0a7de745
A		 2582 mbuf_t data,
2583 __unused char *frame_header,
2584 protocol_family_t *protocol)
b0d623f7 2585{
d9a64523 2586#if UTUN_NEXUS
0a7de745 2587 struct utun_pcb *pcb = ifnet_softc(interface);
d9a64523
A		 2588 struct ip *ip;
2589 u_int ip_version;
2590#endif
2591
b0d623f7
A		 2592 while (data != NULL && mbuf_len(data) < 1) {
2593 data = mbuf_next(data);
2594 }
5ba3f43e 2595
5c9f4661 2596 if (data == NULL) {
b0d623f7 2597 return ENOENT;
5c9f4661 2598 }
5ba3f43e 2599
5c9f4661
A		 2600#if UTUN_NEXUS
2601 if (pcb->utun_use_netif) {
2602 ip = mtod(data, struct ip *);
2603 ip_version = ip->ip_v;
2604
0a7de745
A		 2605 switch (ip_version) {
2606 case 4:
2607 *protocol = PF_INET;
2608 return 0;
2609 case 6:
2610 *protocol = PF_INET6;
2611 return 0;
2612 default:
2613 *protocol = 0;
2614 break;
5c9f4661
A		 2615 }
2616 } else
2617#endif // UTUN_NEXUS
2618 {
2619 *protocol = *(u_int32_t *)mbuf_data(data);
5ba3f43e
A		 2620 }
2621
b0d623f7
A		 2622 return 0;
2623}
2624
2625static errno_t
5ba3f43e 2626utun_framer(ifnet_t interface,
0a7de745
A		 2627 mbuf_t *packet,
2628 __unused const struct sockaddr *dest,
2629 __unused const char *desk_linkaddr,
2630 const char *frame_type,
2631 u_int32_t *prepend_len,
2632 u_int32_t *postpend_len)
b0d623f7 2633{
0a7de745 2634 struct utun_pcb *pcb = ifnet_softc(interface);
39037602
A		 2635 VERIFY(interface == pcb->utun_ifp);
2636
2637 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
d9a64523 2638 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
cb323159 2639 os_log_error(OS_LOG_DEFAULT, "utun_framer - ifnet_output prepend failed\n");
d1ecb069 2640
b0d623f7 2641 ifnet_stat_increment_out(interface, 0, 0, 1);
d1ecb069 2642
b0d623f7 2643 // just return, because the buffer was freed in mbuf_prepend
0a7de745 2644 return EJUSTRETURN;
d9a64523 2645 }
5ba3f43e 2646 if (prepend_len != NULL) {
39037602 2647 *prepend_len = header_length;
5ba3f43e
A		 2648 }
2649 if (postpend_len != NULL) {
39236c6e 2650 *postpend_len = 0;
5ba3f43e 2651 }
0a7de745 2652
d9a64523
A		 2653 // place protocol number at the beginning of the mbuf
2654 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
39037602
A		 2655
2656
0a7de745 2657 return 0;
b0d623f7
A		 2658}
2659
2660static errno_t
5ba3f43e 2661utun_add_proto(__unused ifnet_t interface,
0a7de745
A		 2662 protocol_family_t protocol,
2663 __unused const struct ifnet_demux_desc *demux_array,
2664 __unused u_int32_t demux_count)
b0d623f7 2665{
0a7de745
A		 2666 switch (protocol) {
2667 case PF_INET:
2668 return 0;
2669 case PF_INET6:
2670 return 0;
2671 default:
2672 break;
b0d623f7 2673 }
0a7de745 2674
b0d623f7
A		 2675 return ENOPROTOOPT;
2676}
2677
2678static errno_t
5ba3f43e 2679utun_del_proto(__unused ifnet_t interface,
0a7de745 2680 __unused protocol_family_t protocol)
b0d623f7
A		 2681{
2682 return 0;
2683}
2684
2685static errno_t
5ba3f43e 2686utun_ioctl(ifnet_t interface,
0a7de745
A		 2687 u_long command,
2688 void *data)
b0d623f7 2689{
d9a64523 2690#if UTUN_NEXUS
0a7de745 2691 struct utun_pcb *pcb = ifnet_softc(interface);
d9a64523 2692#endif
0a7de745
A		 2693 errno_t result = 0;
2694
2695 switch (command) {
2696 case SIOCSIFMTU: {
5ba3f43e 2697#if UTUN_NEXUS
0a7de745
A		 2698 if (pcb->utun_use_netif) {
2699 // Make sure we can fit packets in the channel buffers
2700 // Allow for the headroom in the slot
2701 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2702 result = EINVAL;
2703 } else {
2704 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
5c9f4661 2705 }
0a7de745
A		 2706 } else
2707#endif // UTUN_NEXUS
2708 {
2709 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
5c9f4661 2710 }
0a7de745 2711 break;
b0d623f7 2712 }
0a7de745
A		 2713
2714 case SIOCSIFFLAGS:
2715 /* ifioctl() takes care of it */
2716 break;
2717
2718 default:
2719 result = EOPNOTSUPP;
2720 }
2721
b0d623f7
A		 2722 return result;
2723}
2724
2725static void
5ba3f43e 2726utun_detached(ifnet_t interface)
b0d623f7 2727{
0a7de745 2728 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2729 (void)ifnet_release(interface);
5c9f4661 2730 utun_free_pcb(pcb, true);
b0d623f7
A		 2731}
2732
2733/* Protocol Handlers */
2734
2735static errno_t
5ba3f43e 2736utun_proto_input(__unused ifnet_t interface,
0a7de745
A		 2737 protocol_family_t protocol,
2738 mbuf_t m,
2739 __unused char *frame_header)
b0d623f7 2740{
0a7de745 2741 struct utun_pcb *pcb = ifnet_softc(interface);
5c9f4661
A		 2742#if UTUN_NEXUS
2743 if (!pcb->utun_use_netif)
2744#endif // UTUN_NEXUS
2745 {
2746 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2747 }
0a7de745 2748 int32_t pktlen = m->m_pkthdr.len;
39037602 2749 if (proto_input(protocol, m) != 0) {
6d2010ae 2750 m_freem(m);
5c9f4661
A		 2751#if UTUN_NEXUS
2752 if (!pcb->utun_use_netif)
2753#endif // UTUN_NEXUS
2754 {
2755 ifnet_stat_increment_in(interface, 0, 0, 1);
2756 }
5ba3f43e 2757 } else {
5c9f4661
A		 2758#if UTUN_NEXUS
2759 if (!pcb->utun_use_netif)
5ba3f43e 2760#endif // UTUN_NEXUS
5c9f4661 2761 {
a39ff7e2 2762 ifnet_stat_increment_in(interface, 1, pktlen, 0);
5c9f4661 2763 }
39037602 2764 }
0a7de745 2765
b0d623f7
A		 2766 return 0;
2767}
2768
2769static errno_t
5ba3f43e 2770utun_proto_pre_output(__unused ifnet_t interface,
0a7de745
A		 2771 protocol_family_t protocol,
2772 __unused mbuf_t *packet,
2773 __unused const struct sockaddr *dest,
2774 __unused void *route,
2775 char *frame_type,
2776 __unused char *link_layer_dest)
b0d623f7 2777{
6d2010ae 2778 *(protocol_family_t *)(void *)frame_type = protocol;
39037602 2779 return 0;
b0d623f7
A		 2780}
2781
2782static errno_t
5ba3f43e 2783utun_attach_proto(ifnet_t interface,
0a7de745 2784 protocol_family_t protocol)
b0d623f7 2785{
0a7de745
A		 2786 struct ifnet_attach_proto_param proto;
2787
b0d623f7
A		 2788 bzero(&proto, sizeof(proto));
2789 proto.input = utun_proto_input;
2790 proto.pre_output = utun_proto_pre_output;
2791
5ba3f43e 2792 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
b0d623f7 2793 if (result != 0 && result != EEXIST) {
cb323159 2794 os_log_error(OS_LOG_DEFAULT, "utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
0a7de745 2795 protocol, result);
b0d623f7 2796 }
0a7de745 2797
b0d623f7
A		 2798 return result;
2799}
2800
5ba3f43e
A		 2801static errno_t
2802utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2803{
5c9f4661
A		 2804#if UTUN_NEXUS
2805 if (pcb->utun_use_netif) {
2806 lck_rw_lock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2807
5c9f4661 2808 lck_mtx_lock(&pcb->utun_input_chain_lock);
cb323159
A		 2809
2810 if (pcb->utun_input_chain_count > (u_int32_t)if_utun_max_pending_input) {
2811 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2812 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2813 return ENOSPC;
2814 }
2815
5c9f4661
A		 2816 if (pcb->utun_input_chain != NULL) {
2817 pcb->utun_input_chain_last->m_nextpkt = packet;
2818 } else {
2819 pcb->utun_input_chain = packet;
2820 }
cb323159 2821 pcb->utun_input_chain_count++;
5c9f4661
A		 2822 while (packet->m_nextpkt) {
2823 VERIFY(packet != packet->m_nextpkt);
2824 packet = packet->m_nextpkt;
cb323159 2825 pcb->utun_input_chain_count++;
5c9f4661
A		 2826 }
2827 pcb->utun_input_chain_last = packet;
2828 lck_mtx_unlock(&pcb->utun_input_chain_lock);
5ba3f43e 2829
5c9f4661
A		 2830 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2831 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
316670eb 2832
5c9f4661
A		 2833 if (rx_ring != NULL) {
2834 kern_channel_notify(rx_ring, 0);
2835 }
316670eb 2836
0a7de745 2837 return 0;
5c9f4661 2838 } else
cb323159 2839#endif // UTUN_NEXUS
5c9f4661
A		 2840 {
2841 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
5ba3f43e 2842
0a7de745 2843 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
5c9f4661
A		 2844 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2845 }
2846 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2847 /* flush data */
2848 mbuf_freem(packet);
2849 return 0;
2850 }
316670eb 2851
0a7de745 2852 errno_t result = 0;
5c9f4661
A		 2853 if (!pcb->utun_ext_ifdata_stats) {
2854 struct ifnet_stat_increment_param incs = {};
2855 incs.packets_in = 1;
2856 incs.bytes_in = mbuf_pkthdr_len(packet);
2857 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2858 } else {
2859 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2860 }
2861 if (result != 0) {
2862 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
5ba3f43e 2863
cb323159 2864 os_log_error(OS_LOG_DEFAULT, "%s - ifnet_input failed: %d\n", __FUNCTION__, result);
5c9f4661
A		 2865 mbuf_freem(packet);
2866 }
5ba3f43e 2867
0a7de745 2868 return 0;
316670eb 2869 }
316670eb 2870}
5ba3f43e
A		 2871
2872#if UTUN_NEXUS
2873
2874static errno_t
2875utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2876{
2877 return 0;
2878}
2879
2880static void
2881utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2882{
2883 // Ignore
2884}
2885
2886static errno_t
2887utun_register_nexus(void)
2888{
2889 const struct kern_nexus_domain_provider_init dp_init = {
2890 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2891 .nxdpi_flags = 0,
2892 .nxdpi_init = utun_nxdp_init,
2893 .nxdpi_fini = utun_nxdp_fini
2894 };
2895 errno_t err = 0;
2896
2897 /* utun_nxdp_init() is called before this function returns */
2898 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
0a7de745
A		 2899 (const uint8_t *) "com.apple.utun",
2900 &dp_init, sizeof(dp_init),
2901 &utun_nx_dom_prov);
5ba3f43e 2902 if (err != 0) {
cb323159 2903 os_log_error(OS_LOG_DEFAULT, "%s: failed to register domain provider\n", __func__);
0a7de745 2904 return err;
5ba3f43e 2905 }
0a7de745 2906 return 0;
5ba3f43e 2907}
9d749ea3
A		 2908boolean_t
2909utun_interface_needs_netagent(ifnet_t interface)
2910{
2911 struct utun_pcb *pcb = NULL;
2912
2913 if (interface == NULL) {
0a7de745 2914 return FALSE;
9d749ea3
A		 2915 }
2916
2917 pcb = ifnet_softc(interface);
2918
2919 if (pcb == NULL) {
0a7de745 2920 return FALSE;
9d749ea3
A		 2921 }
2922
0a7de745 2923 return pcb->utun_needs_netagent == true;
9d749ea3 2924}
5ba3f43e
A		 2925
2926static errno_t
2927utun_ifnet_set_attrs(ifnet_t ifp)
2928{
2929 /* Set flags and additional information. */
2930 ifnet_set_mtu(ifp, 1500);
2931 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2932
2933 /* The interface must generate its own IPv6 LinkLocal address,
2934 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2935 */
2936 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2937
0a7de745 2938 return 0;
5ba3f43e
A		 2939}
2940
2941static errno_t
2942utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2943{
2944 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2945 pcb->utun_netif_nexus = nexus;
0a7de745 2946 return utun_ifnet_set_attrs(ifp);
5ba3f43e
A		 2947}
2948
2949static errno_t
2950utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2951 proc_t p, kern_nexus_t nexus,
2952 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2953{
2954#pragma unused(nxprov, p)
2955#pragma unused(nexus, nexus_port, channel, ch_ctx)
0a7de745 2956 return 0;
5ba3f43e
A		 2957}
2958
2959static errno_t
2960utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2961 kern_channel_t channel)
2962{
2963#pragma unused(nxprov, channel)
2964 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2965 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
cb323159
A		 2966 if (pcb->utun_netif_nexus == nexus) {
2967 pcb->utun_netif_connected = true;
2968 }
0a7de745 2969 return ok ? 0 : ENXIO;
5ba3f43e
A		 2970}
2971
2972static void
2973utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2974 kern_channel_t channel)
2975{
2976#pragma unused(nxprov, nexus, channel)
2977}
2978
2979static void
2980utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 2981 kern_channel_t channel)
5ba3f43e
A		 2982{
2983#pragma unused(nxprov, nexus, channel)
2984}
2985
2986static void
2987utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2988 kern_channel_t channel)
2989{
2990#pragma unused(nxprov, channel)
2991 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2992 if (pcb->utun_netif_nexus == nexus) {
cb323159
A		 2993 pcb->utun_netif_connected = false;
2994 if (pcb->utun_attach_fsw) {
2995 // disconnected by flowswitch that was attached by us
2996 pcb->utun_netif_nexus = NULL;
2997 }
5ba3f43e
A		 2998 }
2999 ifnet_decr_iorefcnt(pcb->utun_ifp);
3000}
3001
3002static errno_t
3003utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 3004 kern_channel_t channel, kern_channel_ring_t ring,
3005 boolean_t is_tx_ring, void **ring_ctx)
5ba3f43e
A		 3006{
3007#pragma unused(nxprov)
3008#pragma unused(channel)
3009#pragma unused(ring_ctx)
3010 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3011 if (!is_tx_ring) {
3012 VERIFY(pcb->utun_kpipe_rxring == NULL);
3013 pcb->utun_kpipe_rxring = ring;
3014 } else {
3015 VERIFY(pcb->utun_kpipe_txring == NULL);
3016 pcb->utun_kpipe_txring = ring;
3017 }
3018 return 0;
3019}
3020
3021static void
3022utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 3023 kern_channel_ring_t ring)
5ba3f43e
A		 3024{
3025#pragma unused(nxprov)
3026 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3027 if (pcb->utun_kpipe_rxring == ring) {
3028 pcb->utun_kpipe_rxring = NULL;
3029 } else if (pcb->utun_kpipe_txring == ring) {
3030 pcb->utun_kpipe_txring = NULL;
3031 }
3032}
3033
3034static errno_t
3035utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 3036 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 3037{
3038#pragma unused(nxprov)
3039#pragma unused(flags)
3040 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3041
3042 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3043 int channel_enabled = pcb->utun_kpipe_enabled;
3044 if (!channel_enabled) {
3045 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3046 return 0;
3047 }
3048
5c9f4661
A		 3049 if (pcb->utun_use_netif) {
3050 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3051 if (tx_slot == NULL) {
3052 // Nothing to write, bail
3053 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3054 return 0;
3055 }
3056
3057 // Signal the netif ring to read
3058 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
3059 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3060 if (rx_ring != NULL) {
3061 kern_channel_notify(rx_ring, 0);
3062 }
3063 } else {
5ba3f43e 3064 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3065
5c9f4661
A		 3066 struct ifnet_stat_increment_param incs = {};
3067 struct kern_channel_ring_stat_increment tx_ring_stats = {};
3068 MBUFQ_HEAD(mbufq) mbq;
3069 MBUFQ_INIT(&mbq);
3070 kern_channel_slot_t tx_pslot = NULL;
3071 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3072 while (tx_slot != NULL) {
3073 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3074
3075 // Advance TX ring
3076 tx_pslot = tx_slot;
3077 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3078
3079 if (tx_ph == 0) {
3080 continue;
3081 }
3082
3083 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3084 VERIFY(tx_buf != NULL);
3085 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3086 VERIFY(tx_baddr != 0);
3087 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3088
3089 size_t length = MIN(kern_packet_get_data_length(tx_ph),
0a7de745 3090 pcb->utun_slot_size);
5c9f4661
A		 3091
3092 mbuf_t data = NULL;
3093 if (length >= UTUN_HEADER_SIZE(pcb) &&
0a7de745 3094 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
5c9f4661
A		 3095 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
3096 VERIFY(0 == error);
3097 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
3098 VERIFY(0 == error);
3099 /*
3100 * The userland ABI requires the first four bytes have
3101 * the protocol family in network byte order: swap them
3102 */
3103 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
3104 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
3105 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3106 incs.packets_in++;
3107 incs.bytes_in += length;
3108 MBUFQ_ENQUEUE(&mbq, data);
3109 }
3110 }
3111 if (tx_pslot) {
3112 kern_channel_advance_slot(tx_ring, tx_pslot);
3113 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
3114 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3115 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3116 (void) kern_channel_reclaim(tx_ring);
3117 }
3118 if (!MBUFQ_EMPTY(&mbq)) {
3119 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
0a7de745 3120 MBUFQ_LAST(&mbq), &incs);
5c9f4661
A		 3121 MBUFQ_INIT(&mbq);
3122 }
5ba3f43e
A		 3123 }
3124
3125 return 0;
3126}
3127
3128static errno_t
3129utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 3130 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 3131{
3132#pragma unused(nxprov)
3133#pragma unused(flags)
3134 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5c9f4661 3135 struct kern_channel_ring_stat_increment rx_ring_stats = {};
5ba3f43e
A		 3136
3137 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3138
3139 int channel_enabled = pcb->utun_kpipe_enabled;
3140 if (!channel_enabled) {
3141 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3142 return 0;
3143 }
3144
3145 /* reclaim user-released slots */
3146 (void) kern_channel_reclaim(rx_ring);
3147
3148 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3149 if (avail == 0) {
3150 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3151 return 0;
3152 }
3153
5c9f4661
A		 3154 if (pcb->utun_use_netif) {
3155 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3156 if (tx_ring == NULL ||
0a7de745 3157 pcb->utun_netif_nexus == NULL) {
5c9f4661
A		 3158 // Net-If TX ring not set up yet, nothing to read
3159 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3160 return 0;
3161 }
5ba3f43e 3162
5c9f4661 3163 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
5ba3f43e 3164
5c9f4661 3165 // Unlock utun before entering ring
5ba3f43e 3166 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3167
5c9f4661 3168 (void)kr_enter(tx_ring, TRUE);
5ba3f43e 3169
5c9f4661
A		 3170 // Lock again after entering and validate
3171 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3172 if (tx_ring != pcb->utun_netif_txring) {
3173 // Ring no longer valid
3174 // Unlock first, then exit ring
3175 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3176 kr_exit(tx_ring);
3177 return 0;
5ba3f43e
A		 3178 }
3179
5c9f4661
A		 3180 struct kern_channel_ring_stat_increment tx_ring_stats;
3181 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3182 kern_channel_slot_t tx_pslot = NULL;
3183 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3184 if (tx_slot == NULL) {
3185 // Nothing to read, don't bother signalling
3186 // Unlock first, then exit ring
3187 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3188 kr_exit(tx_ring);
3189 return 0;
5ba3f43e
A		 3190 }
3191
5c9f4661
A		 3192 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3193 VERIFY(rx_pp != NULL);
3194 kern_channel_slot_t rx_pslot = NULL;
3195 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
5ba3f43e 3196
5c9f4661
A		 3197 while (rx_slot != NULL && tx_slot != NULL) {
3198 size_t length;
3199 kern_buflet_t rx_buf;
3200 void *rx_baddr;
5ba3f43e 3201
5c9f4661 3202 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
5ba3f43e 3203
5c9f4661
A		 3204 // Advance TX ring
3205 tx_pslot = tx_slot;
3206 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
5ba3f43e 3207
5c9f4661
A		 3208 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3209 if (tx_ph == 0) {
3210 continue;
3211 }
5ba3f43e 3212
5c9f4661
A		 3213 // Allocate rx packet
3214 kern_packet_t rx_ph = 0;
3215 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 3216 if (__improbable(error != 0)) {
cb323159 3217 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
0a7de745 3218 pcb->utun_ifp->if_xname);
5ba3f43e
A		 3219 break;
3220 }
5c9f4661
A		 3221
3222 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3223 VERIFY(tx_buf != NULL);
3224 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3225 VERIFY(tx_baddr != NULL);
3226 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3227
3228 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3229
3230 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
0a7de745 3231 pcb->utun_slot_size);
5c9f4661
A		 3232
3233 tx_ring_stats.kcrsi_slots_transferred++;
3234 tx_ring_stats.kcrsi_bytes_transferred += length;
3235
3236 if (length < UTUN_HEADER_SIZE(pcb) ||
0a7de745
A		 3237 length > pcb->utun_slot_size ||
3238 length > rx_pp->pp_buflet_size ||
3239 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
5c9f4661
A		 3240 /* flush data */
3241 kern_pbufpool_free(rx_pp, rx_ph);
cb323159 3242 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
0a7de745 3243 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
cb323159
A		 3244 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
3245 STATS_INC(nifs, NETIF_STATS_DROP);
5c9f4661 3246 continue;
5ba3f43e 3247 }
5c9f4661
A		 3248
3249 /* fillout packet */
3250 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3251 VERIFY(rx_buf != NULL);
3252 rx_baddr = kern_buflet_get_object_address(rx_buf);
3253 VERIFY(rx_baddr != NULL);
3254
3255 // Find family
3256 uint32_t af = 0;
3257 uint8_t vhl = *(uint8_t *)(tx_baddr);
3258 u_int ip_version = (vhl >> 4);
3259 switch (ip_version) {
0a7de745
A		 3260 case 4: {
3261 af = AF_INET;
3262 break;
3263 }
3264 case 6: {
3265 af = AF_INET6;
3266 break;
3267 }
3268 default: {
cb323159 3269 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
0a7de745
A		 3270 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3271 break;
3272 }
5ba3f43e 3273 }
5ba3f43e 3274
5c9f4661
A		 3275 // Copy header
3276 af = htonl(af);
3277 memcpy((void *)rx_baddr, &af, sizeof(af));
3278 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3279 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3280 }
5ba3f43e 3281
5c9f4661
A		 3282 // Copy data from tx to rx
3283 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3284 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
5ba3f43e 3285
5c9f4661
A		 3286 /* finalize and attach the packet */
3287 error = kern_buflet_set_data_offset(rx_buf, 0);
3288 VERIFY(error == 0);
3289 error = kern_buflet_set_data_length(rx_buf, length);
3290 VERIFY(error == 0);
3291 error = kern_packet_finalize(rx_ph);
3292 VERIFY(error == 0);
3293 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3294 VERIFY(error == 0);
5ba3f43e 3295
cb323159
A		 3296 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
3297 STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
5ba3f43e 3298
5c9f4661
A		 3299 rx_ring_stats.kcrsi_slots_transferred++;
3300 rx_ring_stats.kcrsi_bytes_transferred += length;
5ba3f43e 3301
5c9f4661
A		 3302 rx_pslot = rx_slot;
3303 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3304 }
5ba3f43e 3305
5c9f4661
A		 3306 if (rx_pslot) {
3307 kern_channel_advance_slot(rx_ring, rx_pslot);
3308 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3309 }
5ba3f43e 3310
5c9f4661
A		 3311 if (tx_pslot) {
3312 kern_channel_advance_slot(tx_ring, tx_pslot);
3313 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3314 (void)kern_channel_reclaim(tx_ring);
3315 }
5ba3f43e 3316
5c9f4661 3317 /* just like utun_ctl_rcvd(), always reenable output */
5ba3f43e
A		 3318 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3319 if (error != 0) {
cb323159 3320 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
5ba3f43e 3321 }
5ba3f43e 3322
5c9f4661
A		 3323 // Unlock first, then exit ring
3324 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3325
5c9f4661
A		 3326 if (tx_pslot != NULL) {
3327 kern_channel_notify(tx_ring, 0);
3328 }
3329 kr_exit(tx_ring);
3330 } else {
3331 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3332
3333 uint32_t mb_cnt = 0;
3334 uint32_t mb_len = 0;
3335 struct mbuf *mb_head = NULL;
3336 struct mbuf *mb_tail = NULL;
3337
3338 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
0a7de745 3339 &mb_tail, &mb_cnt, &mb_len) != 0) {
5c9f4661
A		 3340 return 0;
3341 }
3342 VERIFY(mb_cnt <= avail);
3343
3344 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3345 VERIFY(rx_pp != NULL);
3346 kern_channel_slot_t rx_pslot = NULL;
3347 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3348 while (rx_slot) {
3349 size_t length = 0;
3350 mbuf_t data = NULL;
3351 if ((data = mb_head) == NULL) {
3352 VERIFY(mb_cnt == 0);
3353 break;
3354 }
3355 mb_head = mbuf_nextpkt(mb_head);
3356 mbuf_setnextpkt(data, NULL);
3357 VERIFY(mb_cnt != 0);
3358 --mb_cnt;
3359 length = mbuf_pkthdr_len(data);
3360 if (length < UTUN_HEADER_SIZE(pcb) ||
0a7de745
A		 3361 length > pcb->utun_slot_size ||
3362 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
5c9f4661
A		 3363 /* flush data */
3364 mbuf_freem(data);
3365 continue;
3366 }
3367 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3368
3369 // Allocate rx packet
3370 kern_packet_t rx_ph = 0;
3371 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 3372 if (__improbable(error != 0)) {
cb323159 3373 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
0a7de745 3374 pcb->utun_ifp->if_xname);
5c9f4661
A		 3375 break;
3376 }
3377
3378 /*
3379 * The ABI requires the protocol in network byte order
3380 */
3381 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3382
3383 // Fillout rx packet
3384 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3385 VERIFY(rx_buf != NULL);
3386 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3387 VERIFY(rx_baddr != NULL);
3388
3389 // Copy-in data from mbuf to buflet
3390 mbuf_copydata(data, 0, length, (void *)rx_baddr);
0a7de745 3391 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5c9f4661
A		 3392
3393 // Finalize and attach the packet
3394 error = kern_buflet_set_data_offset(rx_buf, 0);
3395 VERIFY(error == 0);
3396 error = kern_buflet_set_data_length(rx_buf, length);
3397 VERIFY(error == 0);
3398 error = kern_packet_finalize(rx_ph);
3399 VERIFY(error == 0);
3400 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3401 VERIFY(error == 0);
3402
3403 rx_ring_stats.kcrsi_slots_transferred++;
3404 rx_ring_stats.kcrsi_bytes_transferred += length;
3405
3406 if (!pcb->utun_ext_ifdata_stats) {
3407 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3408 }
3409
3410 mbuf_freem(data);
3411
3412 rx_pslot = rx_slot;
3413 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3414 }
3415 if (rx_pslot) {
3416 kern_channel_advance_slot(rx_ring, rx_pslot);
3417 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3418 }
3419 if (mb_head != NULL) {
3420 VERIFY(mb_cnt != 0);
3421 mbuf_freem_list(mb_head);
3422 }
5ba3f43e 3423 }
5ba3f43e
A		 3424
3425 return 0;
3426}
39037602 3427
5ba3f43e 3428#endif // UTUN_NEXUS
39037602
A		 3429
3430
3431/*
5ba3f43e 3432 * These are place holders until coreTLS kext stops calling them
39037602 3433 */
0a7de745 3434errno_t utun_ctl_register_dtls(void *reg);
39037602
A		 3435int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3436void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3437
3438errno_t
0a7de745 3439utun_ctl_register_dtls(void *reg)
39037602
A		 3440{
3441#pragma unused(reg)
3442 return 0;
3443}
3444
3445int
3446utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3447{
3448#pragma unused(pcb)
3449#pragma unused(pkt)
3450#pragma unused(family)
3451 return 0;
3452}
3453
3454void
3455utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3456{
3457#pragma unused(pcb)
3458}
mirror of XNU