]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_utun.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4903.270.47.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
CommitLineData
b0d623f7 1/*
a39ff7e2 2 * Copyright (c) 2008-2018 Apple Inc. All rights reserved.
b0d623f7
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
b0d623f7
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
b0d623f7
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
b0d623f7
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
b0d623f7
A		 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31/* ----------------------------------------------------------------------------------
0a7de745
A		 32 * Application of kernel control for interface creation
33 *
34 * Theory of operation:
35 * utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 * This kernel control will register an interface for every client that connects.
37 * ---------------------------------------------------------------------------------- */
b0d623f7
A		 38
39#include <sys/systm.h>
40#include <sys/kern_control.h>
41#include <net/kpi_protocol.h>
42#include <net/kpi_interface.h>
43#include <sys/socket.h>
44#include <net/if.h>
45#include <net/if_types.h>
46#include <net/bpf.h>
47#include <net/if_utun.h>
0a7de745 48#include <sys/mbuf.h>
b0d623f7
A		 49#include <sys/sockio.h>
50#include <netinet/in.h>
5ba3f43e 51#include <netinet/ip.h>
b0d623f7
A		 52#include <netinet6/in6_var.h>
53#include <netinet6/in6_var.h>
54#include <sys/kauth.h>
5ba3f43e
A		 55#include <net/necp.h>
56#include <kern/zalloc.h>
57
58#define UTUN_NEXUS 0
59
5ba3f43e
A		 60#if UTUN_NEXUS
61static nexus_controller_t utun_ncd;
62static int utun_ncd_refcount;
63static uuid_t utun_kpipe_uuid;
64static uuid_t utun_nx_dom_prov;
65
66typedef struct utun_nx {
67 uuid_t if_provider;
68 uuid_t if_instance;
69 uuid_t ms_provider;
70 uuid_t ms_instance;
71 uuid_t ms_device;
72 uuid_t ms_host;
73 uuid_t ms_agent;
74} *utun_nx_t;
75
76#endif // UTUN_NEXUS
77
78/* Control block allocated for each kernel control connection */
79struct utun_pcb {
0a7de745
A		 80 TAILQ_ENTRY(utun_pcb) utun_chain;
81 kern_ctl_ref utun_ctlref;
82 ifnet_t utun_ifp;
83 u_int32_t utun_unit;
84 u_int32_t utun_unique_id;
85 u_int32_t utun_flags;
86 int utun_ext_ifdata_stats;
87 u_int32_t utun_max_pending_packets;
88 char utun_if_xname[IFXNAMSIZ];
89 char utun_unique_name[IFXNAMSIZ];
5ba3f43e
A		 90 // PCB lock protects state fields and rings
91 decl_lck_rw_data(, utun_pcb_lock);
0a7de745
A		 92 struct mbuf * utun_input_chain;
93 struct mbuf * utun_input_chain_last;
5ba3f43e
A		 94 // Input chain lock protects the list of input mbufs
95 // The input chain lock must be taken AFTER the PCB lock if both are held
0a7de745 96 lck_mtx_t utun_input_chain_lock;
5ba3f43e
A		 97
98#if UTUN_NEXUS
0a7de745
A		 99 struct utun_nx utun_nx;
100 int utun_kpipe_enabled;
101 uuid_t utun_kpipe_uuid;
102 void * utun_kpipe_rxring;
103 void * utun_kpipe_txring;
104 kern_pbufpool_t utun_kpipe_pp;
105
106 kern_nexus_t utun_netif_nexus;
107 kern_pbufpool_t utun_netif_pp;
108 void * utun_netif_rxring;
109 void * utun_netif_txring;
110 uint64_t utun_netif_txring_size;
111
112 u_int32_t utun_slot_size;
113 u_int32_t utun_netif_ring_size;
114 u_int32_t utun_tx_fsw_ring_size;
115 u_int32_t utun_rx_fsw_ring_size;
116 bool utun_use_netif;
117 bool utun_needs_netagent;
5ba3f43e
A		 118#endif // UTUN_NEXUS
119};
39037602 120
b0d623f7 121/* Kernel Control functions */
0a7de745
A		 122static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
123 void **unitinfo);
124static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
125 void **unitinfo);
126static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
127 void *unitinfo);
128static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
129 void *unitinfo, mbuf_t m, int flags);
130static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
131 int opt, void *data, size_t *len);
132static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
133 int opt, void *data, size_t len);
134static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
135 int flags);
b0d623f7
A		 136
137/* Network Interface functions */
fe8ab488 138static void utun_start(ifnet_t interface);
0a7de745
A		 139static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
140 const struct sockaddr *dest, const char *desk_linkaddr,
141 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
142static errno_t utun_output(ifnet_t interface, mbuf_t data);
143static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
144 protocol_family_t *protocol);
145static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
146 const struct ifnet_demux_desc *demux_array,
147 u_int32_t demux_count);
148static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
149static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
150static void utun_detached(ifnet_t interface);
b0d623f7
A		 151
152/* Protocol handlers */
0a7de745
A		 153static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
154static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
155 mbuf_t m, char *frame_header);
156static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
157 mbuf_t *packet, const struct sockaddr *dest, void *route,
158 char *frame_type, char *link_layer_dest);
5c9f4661 159static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
39037602 160
5ba3f43e
A		 161#if UTUN_NEXUS
162
5c9f4661 163#define UTUN_IF_DEFAULT_SLOT_SIZE 2048
5ba3f43e
A		 164#define UTUN_IF_DEFAULT_RING_SIZE 64
165#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
166#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
0a7de745 167#define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
5ba3f43e
A		 168#define UTUN_IF_HEADROOM_SIZE 32
169
170#define UTUN_IF_MIN_RING_SIZE 16
171#define UTUN_IF_MAX_RING_SIZE 1024
172
5c9f4661
A		 173#define UTUN_IF_MIN_SLOT_SIZE 1024
174#define UTUN_IF_MAX_SLOT_SIZE 4096
175
5ba3f43e
A		 176static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
177static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
178static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
179
180static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
181static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
182static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
183
184SYSCTL_DECL(_net_utun);
185SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
186
187SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 188 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
5ba3f43e 189SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 190 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
5ba3f43e 191SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 192 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
5ba3f43e
A		 193
194static errno_t
195utun_register_nexus(void);
196
197static errno_t
198utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
199static errno_t
200utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
201 proc_t p, kern_nexus_t nexus,
202 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
203static errno_t
204utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
205 kern_channel_t channel);
206static void
207utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
208 kern_channel_t channel);
209static void
210utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 211 kern_channel_t channel);
5ba3f43e
A		 212static void
213utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
214 kern_channel_t channel);
215static errno_t
216utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
217 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
218 void **ring_ctx);
219static void
220utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
221 kern_channel_ring_t ring);
222static errno_t
223utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
224 kern_channel_ring_t ring, uint32_t flags);
225static errno_t
226utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
227 kern_channel_ring_t ring, uint32_t flags);
228#endif // UTUN_NEXUS
39037602
A		 229
230#define UTUN_DEFAULT_MTU 1500
231#define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
b0d623f7 232
0a7de745
A		 233static kern_ctl_ref utun_kctlref;
234static u_int32_t utun_family;
5ba3f43e
A		 235static lck_attr_t *utun_lck_attr;
236static lck_grp_attr_t *utun_lck_grp_attr;
237static lck_grp_t *utun_lck_grp;
238static lck_mtx_t utun_lock;
239
240TAILQ_HEAD(utun_list, utun_pcb) utun_head;
241
0a7de745
A		 242#define UTUN_PCB_ZONE_MAX 32
243#define UTUN_PCB_ZONE_NAME "net.if_utun"
5ba3f43e 244
0a7de745
A		 245static unsigned int utun_pcb_size; /* size of zone element */
246static struct zone *utun_pcb_zone; /* zone for utun_pcb */
5ba3f43e
A		 247
248#if UTUN_NEXUS
249
250static int
251sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
252{
253#pragma unused(arg1, arg2)
254 int value = if_utun_ring_size;
255
256 int error = sysctl_handle_int(oidp, &value, 0, req);
257 if (error || !req->newptr) {
0a7de745 258 return error;
5ba3f43e
A		 259 }
260
261 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 262 value > UTUN_IF_MAX_RING_SIZE) {
263 return EINVAL;
5ba3f43e
A		 264 }
265
266 if_utun_ring_size = value;
267
0a7de745 268 return 0;
5ba3f43e
A		 269}
270
271static int
272sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
273{
274#pragma unused(arg1, arg2)
275 int value = if_utun_tx_fsw_ring_size;
276
277 int error = sysctl_handle_int(oidp, &value, 0, req);
278 if (error || !req->newptr) {
0a7de745 279 return error;
5ba3f43e
A		 280 }
281
282 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 283 value > UTUN_IF_MAX_RING_SIZE) {
284 return EINVAL;
5ba3f43e
A		 285 }
286
287 if_utun_tx_fsw_ring_size = value;
288
0a7de745 289 return 0;
5ba3f43e
A		 290}
291
292static int
293sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
294{
295#pragma unused(arg1, arg2)
296 int value = if_utun_rx_fsw_ring_size;
297
298 int error = sysctl_handle_int(oidp, &value, 0, req);
299 if (error || !req->newptr) {
0a7de745 300 return error;
5ba3f43e
A		 301 }
302
303 if (value < UTUN_IF_MIN_RING_SIZE ||
0a7de745
A		 304 value > UTUN_IF_MAX_RING_SIZE) {
305 return EINVAL;
5ba3f43e
A		 306 }
307
308 if_utun_rx_fsw_ring_size = value;
309
0a7de745 310 return 0;
5ba3f43e
A		 311}
312
313static errno_t
314utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 315 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
316 void **ring_ctx)
5ba3f43e
A		 317{
318#pragma unused(nxprov)
319#pragma unused(channel)
320#pragma unused(ring_ctx)
321 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
322 if (!is_tx_ring) {
323 VERIFY(pcb->utun_netif_rxring == NULL);
324 pcb->utun_netif_rxring = ring;
325 } else {
326 VERIFY(pcb->utun_netif_txring == NULL);
327 pcb->utun_netif_txring = ring;
328 }
329 return 0;
330}
331
332static void
333utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 334 kern_channel_ring_t ring)
5ba3f43e
A		 335{
336#pragma unused(nxprov)
337 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
338 if (pcb->utun_netif_rxring == ring) {
339 pcb->utun_netif_rxring = NULL;
340 } else if (pcb->utun_netif_txring == ring) {
341 pcb->utun_netif_txring = NULL;
342 }
343}
344
345static errno_t
346utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 347 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 348{
349#pragma unused(nxprov)
350#pragma unused(flags)
351 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
352
353 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
354
355 lck_rw_lock_shared(&pcb->utun_pcb_lock);
356
357 struct kern_channel_ring_stat_increment tx_ring_stats;
358 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
359 kern_channel_slot_t tx_pslot = NULL;
360 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
361
362 STATS_INC(nifs, NETIF_STATS_TXSYNC);
363
364 if (tx_slot == NULL) {
365 // Nothing to write, don't bother signalling
366 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
367 return 0;
368 }
369
370 if (pcb->utun_kpipe_enabled) {
371 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
372 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
373
374 // Signal the kernel pipe ring to read
375 if (rx_ring != NULL) {
376 kern_channel_notify(rx_ring, 0);
377 }
378 return 0;
379 }
380
381 // If we're here, we're injecting into the utun kernel control socket
382 while (tx_slot != NULL) {
383 size_t length = 0;
384 mbuf_t data = NULL;
385
386 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
387
388 if (tx_ph == 0) {
389 // Advance TX ring
390 tx_pslot = tx_slot;
391 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
392 continue;
393 }
394 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
395
396 // Advance TX ring
397 tx_pslot = tx_slot;
398 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
399
400 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
401 VERIFY(tx_buf != NULL);
402
403 /* tx_baddr is the absolute buffer address */
404 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
405 VERIFY(tx_baddr != 0);
406
407 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
408
409 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
410 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
411
412 // The offset must be large enough for the headers
413 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
414
415 // Find family
416 uint32_t af = 0;
417 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
418 u_int ip_version = (vhl >> 4);
419 switch (ip_version) {
0a7de745
A		 420 case 4: {
421 af = AF_INET;
422 break;
423 }
424 case 6: {
425 af = AF_INET6;
426 break;
427 }
428 default: {
429 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
430 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
431 UTUN_HEADER_SIZE(pcb));
432 break;
433 }
5ba3f43e
A		 434 }
435
436 tx_offset -= UTUN_HEADER_SIZE(pcb);
437 tx_length += UTUN_HEADER_SIZE(pcb);
438 tx_baddr += tx_offset;
439
5c9f4661 440 length = MIN(tx_length, pcb->utun_slot_size);
5ba3f43e
A		 441
442 // Copy in family
443 memcpy(tx_baddr, &af, sizeof(af));
444 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
445 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
446 }
447
448 if (length > 0) {
449 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
450 if (error == 0) {
451 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
452 if (error == 0) {
453 error = utun_output(pcb->utun_ifp, data);
454 if (error != 0) {
455 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
456 }
457 } else {
458 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
459 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
460 STATS_INC(nifs, NETIF_STATS_DROPPED);
461 mbuf_freem(data);
462 data = NULL;
463 }
464 } else {
465 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
466 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
467 STATS_INC(nifs, NETIF_STATS_DROPPED);
468 }
469 } else {
470 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
471 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
472 STATS_INC(nifs, NETIF_STATS_DROPPED);
473 }
474
475 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
476
477 if (data == NULL) {
478 continue;
479 }
480
481 STATS_INC(nifs, NETIF_STATS_TXPKTS);
482 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
483
484 tx_ring_stats.kcrsi_slots_transferred++;
485 tx_ring_stats.kcrsi_bytes_transferred += length;
486 }
487
488 if (tx_pslot) {
489 kern_channel_advance_slot(tx_ring, tx_pslot);
490 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
491 (void)kern_channel_reclaim(tx_ring);
492 }
493
494 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
495
496 return 0;
497}
498
499static errno_t
500utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 501 kern_channel_ring_t ring, __unused uint32_t flags)
5ba3f43e
A		 502{
503#pragma unused(nxprov)
504 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5ba3f43e
A		 505 boolean_t more = false;
506 errno_t rc = 0;
5ba3f43e 507
5c9f4661
A		 508 /*
509 * Refill and sync the ring; we may be racing against another thread doing
510 * an RX sync that also wants to do kr_enter(), and so use the blocking
511 * variant here.
512 */
513 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
514 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
515 printf("%s, tx refill failed %d\n", __func__, rc);
516 }
517
518 (void) kr_enter(ring, TRUE);
519 lck_rw_lock_shared(&pcb->utun_pcb_lock);
520
521 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 522 uint32_t tx_available = kern_channel_available_slot_count(ring);
523 if (pcb->utun_netif_txring_size > 0 &&
0a7de745 524 tx_available >= pcb->utun_netif_txring_size - 1) {
5ba3f43e
A		 525 // No room left in tx ring, disable output for now
526 errno_t error = ifnet_disable_output(pcb->utun_ifp);
527 if (error != 0) {
528 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
5ba3f43e
A		 529 }
530 }
531 }
b0d623f7 532
5c9f4661 533 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 534 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
535
536 // Unlock while calling notify
537 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
538 // Signal the kernel pipe ring to read
539 if (rx_ring != NULL) {
540 kern_channel_notify(rx_ring, 0);
541 }
542 } else {
543 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
544 }
545
5c9f4661
A		 546 kr_exit(ring);
547
0a7de745 548 return 0;
5ba3f43e
A		 549}
550
551static errno_t
552utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 553 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 554{
555#pragma unused(nxprov)
556#pragma unused(flags)
557 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
558 struct kern_channel_ring_stat_increment rx_ring_stats;
559
560 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
561
562 lck_rw_lock_shared(&pcb->utun_pcb_lock);
563
564 // Reclaim user-released slots
565 (void) kern_channel_reclaim(rx_ring);
566
567 STATS_INC(nifs, NETIF_STATS_RXSYNC);
568
569 uint32_t avail = kern_channel_available_slot_count(rx_ring);
570 if (avail == 0) {
571 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
572 return 0;
573 }
574
575 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
576 VERIFY(rx_pp != NULL);
577 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
578 kern_channel_slot_t rx_pslot = NULL;
579 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
580
581 while (rx_slot != NULL) {
582 // Check for a waiting packet
583 lck_mtx_lock(&pcb->utun_input_chain_lock);
584 mbuf_t data = pcb->utun_input_chain;
585 if (data == NULL) {
586 lck_mtx_unlock(&pcb->utun_input_chain_lock);
587 break;
588 }
589
590 // Allocate rx packet
591 kern_packet_t rx_ph = 0;
592 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 593 if (__improbable(error != 0)) {
5ba3f43e
A		 594 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
595 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 596 lck_mtx_unlock(&pcb->utun_input_chain_lock);
597 break;
598 }
599
600 // Advance waiting packets
601 pcb->utun_input_chain = data->m_nextpkt;
602 data->m_nextpkt = NULL;
603 if (pcb->utun_input_chain == NULL) {
604 pcb->utun_input_chain_last = NULL;
605 }
606 lck_mtx_unlock(&pcb->utun_input_chain_lock);
607
608 size_t header_offset = UTUN_HEADER_SIZE(pcb);
609 size_t length = mbuf_pkthdr_len(data);
610
611 if (length < header_offset) {
612 // mbuf is too small
613 mbuf_freem(data);
614 kern_pbufpool_free(rx_pp, rx_ph);
615 STATS_INC(nifs, NETIF_STATS_BADLEN);
616 STATS_INC(nifs, NETIF_STATS_DROPPED);
617 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
0a7de745 618 pcb->utun_ifp->if_xname, length, header_offset);
5ba3f43e
A		 619 continue;
620 }
621
622 length -= header_offset;
623 if (length > rx_pp->pp_buflet_size) {
624 // Flush data
625 mbuf_freem(data);
626 kern_pbufpool_free(rx_pp, rx_ph);
627 STATS_INC(nifs, NETIF_STATS_BADLEN);
628 STATS_INC(nifs, NETIF_STATS_DROPPED);
629 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
0a7de745 630 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
5ba3f43e
A		 631 continue;
632 }
633
634 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
635
636 // Fillout rx packet
637 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
638 VERIFY(rx_buf != NULL);
639 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
640 VERIFY(rx_baddr != NULL);
641
642 // Copy-in data from mbuf to buflet
643 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
0a7de745 644 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 645
646 // Finalize and attach the packet
647 error = kern_buflet_set_data_offset(rx_buf, 0);
648 VERIFY(error == 0);
649 error = kern_buflet_set_data_length(rx_buf, length);
650 VERIFY(error == 0);
651 error = kern_packet_set_link_header_offset(rx_ph, 0);
652 VERIFY(error == 0);
653 error = kern_packet_set_network_header_offset(rx_ph, 0);
654 VERIFY(error == 0);
655 error = kern_packet_finalize(rx_ph);
656 VERIFY(error == 0);
657 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
658 VERIFY(error == 0);
659
660 STATS_INC(nifs, NETIF_STATS_RXPKTS);
661 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
662 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
663
664 rx_ring_stats.kcrsi_slots_transferred++;
665 rx_ring_stats.kcrsi_bytes_transferred += length;
666
667 mbuf_freem(data);
668
669 // Advance ring
670 rx_pslot = rx_slot;
671 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
672 }
673
674 struct kern_channel_ring_stat_increment tx_ring_stats;
675 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
676 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
677 kern_channel_slot_t tx_pslot = NULL;
678 kern_channel_slot_t tx_slot = NULL;
679 if (tx_ring == NULL) {
680 // Net-If TX ring not set up yet, nothing to read
681 goto done;
682 }
683
684 // Unlock utun before entering ring
685 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
686
687 (void)kr_enter(tx_ring, TRUE);
688
689 // Lock again after entering and validate
690 lck_rw_lock_shared(&pcb->utun_pcb_lock);
691 if (tx_ring != pcb->utun_kpipe_txring) {
692 goto done;
693 }
694
695 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
696 if (tx_slot == NULL) {
697 // Nothing to read, don't bother signalling
698 goto done;
699 }
700
701 while (rx_slot != NULL && tx_slot != NULL) {
702 // Allocate rx packet
703 kern_packet_t rx_ph = 0;
704 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
705
706 // Advance TX ring
707 tx_pslot = tx_slot;
708 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
709
710 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
711 if (tx_ph == 0) {
712 continue;
713 }
714
a39ff7e2
A		 715 /* XXX We could try this alloc before advancing the slot to avoid
716 * dropping the packet on failure to allocate.
717 */
5ba3f43e 718 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 719 if (__improbable(error != 0)) {
5ba3f43e
A		 720 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
721 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 722 break;
723 }
724
725 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
726 VERIFY(tx_buf != NULL);
727 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
728 VERIFY(tx_baddr != 0);
729 tx_baddr += kern_buflet_get_data_offset(tx_buf);
730
731 // Check packet length
732 size_t header_offset = UTUN_HEADER_SIZE(pcb);
733 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
734 if (tx_length < header_offset) {
735 // Packet is too small
736 kern_pbufpool_free(rx_pp, rx_ph);
737 STATS_INC(nifs, NETIF_STATS_BADLEN);
738 STATS_INC(nifs, NETIF_STATS_DROPPED);
739 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
0a7de745 740 pcb->utun_ifp->if_xname, tx_length, header_offset);
5ba3f43e
A		 741 continue;
742 }
743
744 size_t length = MIN(tx_length - header_offset,
0a7de745 745 pcb->utun_slot_size);
5ba3f43e
A		 746
747 tx_ring_stats.kcrsi_slots_transferred++;
748 tx_ring_stats.kcrsi_bytes_transferred += length;
749
750 // Fillout rx packet
751 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
752 VERIFY(rx_buf != NULL);
753 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
754 VERIFY(rx_baddr != NULL);
755
756 // Copy-in data from tx to rx
757 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
0a7de745 758 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 759
760 // Finalize and attach the packet
761 error = kern_buflet_set_data_offset(rx_buf, 0);
762 VERIFY(error == 0);
763 error = kern_buflet_set_data_length(rx_buf, length);
764 VERIFY(error == 0);
765 error = kern_packet_set_link_header_offset(rx_ph, 0);
766 VERIFY(error == 0);
767 error = kern_packet_set_network_header_offset(rx_ph, 0);
768 VERIFY(error == 0);
769 error = kern_packet_finalize(rx_ph);
770 VERIFY(error == 0);
771 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
772 VERIFY(error == 0);
773
774 STATS_INC(nifs, NETIF_STATS_RXPKTS);
775 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
776 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
777
778 rx_ring_stats.kcrsi_slots_transferred++;
779 rx_ring_stats.kcrsi_bytes_transferred += length;
780
781 rx_pslot = rx_slot;
782 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
783 }
784
785done:
786 if (rx_pslot) {
787 kern_channel_advance_slot(rx_ring, rx_pslot);
788 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
789 }
790
791 if (tx_pslot) {
792 kern_channel_advance_slot(tx_ring, tx_pslot);
793 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
794 (void)kern_channel_reclaim(tx_ring);
795 }
796
797 // Unlock first, then exit ring
798 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
799 if (tx_ring != NULL) {
800 if (tx_pslot != NULL) {
801 kern_channel_notify(tx_ring, 0);
802 }
803 kr_exit(tx_ring);
804 }
805
806 return 0;
807}
808
809static errno_t
810utun_nexus_ifattach(struct utun_pcb *pcb,
0a7de745
A		 811 struct ifnet_init_eparams *init_params,
812 struct ifnet **ifp)
5ba3f43e
A		 813{
814 errno_t err;
815 nexus_controller_t controller = kern_nexus_shared_controller();
816 struct kern_nexus_net_init net_init;
a39ff7e2 817 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 818
819 nexus_name_t provider_name;
820 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 821 "com.apple.netif.%s", pcb->utun_if_xname);
5ba3f43e
A		 822
823 struct kern_nexus_provider_init prov_init = {
824 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
825 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
826 .nxpi_pre_connect = utun_nexus_pre_connect,
827 .nxpi_connected = utun_nexus_connected,
828 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
829 .nxpi_disconnected = utun_nexus_disconnected,
830 .nxpi_ring_init = utun_netif_ring_init,
831 .nxpi_ring_fini = utun_netif_ring_fini,
832 .nxpi_slot_init = NULL,
833 .nxpi_slot_fini = NULL,
834 .nxpi_sync_tx = utun_netif_sync_tx,
835 .nxpi_sync_rx = utun_netif_sync_rx,
836 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
837 };
838
839 nexus_attr_t nxa = NULL;
840 err = kern_nexus_attr_create(&nxa);
841 if (err != 0) {
842 printf("%s: kern_nexus_attr_create failed: %d\n",
0a7de745 843 __func__, err);
5ba3f43e
A		 844 goto failed;
845 }
846
5c9f4661 847 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 848 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
849 VERIFY(err == 0);
850
851 // Reset ring size for netif nexus to limit memory usage
5c9f4661 852 uint64_t ring_size = pcb->utun_netif_ring_size;
5ba3f43e
A		 853 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
854 VERIFY(err == 0);
855 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
856 VERIFY(err == 0);
857
858 pcb->utun_netif_txring_size = ring_size;
859
0a7de745 860 bzero(&pp_init, sizeof(pp_init));
a39ff7e2
A		 861 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
862 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
863 pp_init.kbi_bufsize = pcb->utun_slot_size;
864 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
865 pp_init.kbi_max_frags = 1;
0a7de745 866 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2
A		 867 "%s", provider_name);
868
869 err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_netif_pp, NULL);
870 if (err != 0) {
871 printf("%s pbufbool create failed, error %d\n", __func__, err);
872 goto failed;
873 }
874
5ba3f43e 875 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 876 utun_nx_dom_prov,
877 provider_name,
878 &prov_init,
879 sizeof(prov_init),
880 nxa,
881 &pcb->utun_nx.if_provider);
5ba3f43e
A		 882 if (err != 0) {
883 printf("%s register provider failed, error %d\n",
0a7de745 884 __func__, err);
5ba3f43e
A		 885 goto failed;
886 }
887
888 bzero(&net_init, sizeof(net_init));
889 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
890 net_init.nxneti_flags = 0;
891 net_init.nxneti_eparams = init_params;
892 net_init.nxneti_lladdr = NULL;
893 net_init.nxneti_prepare = utun_netif_prepare;
a39ff7e2 894 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
5ba3f43e 895 err = kern_nexus_controller_alloc_net_provider_instance(controller,
0a7de745
A		 896 pcb->utun_nx.if_provider,
897 pcb,
898 &pcb->utun_nx.if_instance,
899 &net_init,
900 ifp);
5ba3f43e
A		 901 if (err != 0) {
902 printf("%s alloc_net_provider_instance failed, %d\n",
0a7de745 903 __func__, err);
5ba3f43e 904 kern_nexus_controller_deregister_provider(controller,
0a7de745 905 pcb->utun_nx.if_provider);
5ba3f43e
A		 906 uuid_clear(pcb->utun_nx.if_provider);
907 goto failed;
908 }
909
910failed:
911 if (nxa) {
912 kern_nexus_attr_destroy(nxa);
913 }
a39ff7e2
A		 914 if (err && pcb->utun_netif_pp != NULL) {
915 kern_pbufpool_destroy(pcb->utun_netif_pp);
916 pcb->utun_netif_pp = NULL;
917 }
0a7de745 918 return err;
5ba3f43e
A		 919}
920
921static void
922utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
923{
924 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 925 errno_t err;
5ba3f43e
A		 926
927 if (!uuid_is_null(instance)) {
928 err = kern_nexus_controller_free_provider_instance(controller,
0a7de745 929 instance);
5ba3f43e
A		 930 if (err != 0) {
931 printf("%s free_provider_instance failed %d\n",
0a7de745 932 __func__, err);
5ba3f43e
A		 933 }
934 uuid_clear(instance);
935 }
936 if (!uuid_is_null(provider)) {
937 err = kern_nexus_controller_deregister_provider(controller,
0a7de745 938 provider);
5ba3f43e
A		 939 if (err != 0) {
940 printf("%s deregister_provider %d\n", __func__, err);
941 }
942 uuid_clear(provider);
943 }
944 return;
945}
946
947static void
a39ff7e2 948utun_nexus_detach(struct utun_pcb *pcb)
5ba3f43e 949{
a39ff7e2 950 utun_nx_t nx = &pcb->utun_nx;
5ba3f43e 951 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 952 errno_t err;
5ba3f43e
A		 953
954 if (!uuid_is_null(nx->ms_host)) {
955 err = kern_nexus_ifdetach(controller,
0a7de745
A		 956 nx->ms_instance,
957 nx->ms_host);
5ba3f43e
A		 958 if (err != 0) {
959 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
0a7de745 960 __func__, err);
5ba3f43e
A		 961 }
962 }
963
964 if (!uuid_is_null(nx->ms_device)) {
965 err = kern_nexus_ifdetach(controller,
0a7de745
A		 966 nx->ms_instance,
967 nx->ms_device);
5ba3f43e
A		 968 if (err != 0) {
969 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
0a7de745 970 __func__, err);
5ba3f43e
A		 971 }
972 }
973
974 utun_detach_provider_and_instance(nx->if_provider,
0a7de745 975 nx->if_instance);
5ba3f43e 976 utun_detach_provider_and_instance(nx->ms_provider,
0a7de745 977 nx->ms_instance);
5ba3f43e 978
a39ff7e2
A		 979 if (pcb->utun_netif_pp != NULL) {
980 kern_pbufpool_destroy(pcb->utun_netif_pp);
981 pcb->utun_netif_pp = NULL;
a39ff7e2 982 }
5ba3f43e
A		 983 memset(nx, 0, sizeof(*nx));
984}
985
986static errno_t
5c9f4661 987utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
0a7de745
A		 988 uint32_t subtype, const char *type_name,
989 const char *ifname,
990 uuid_t *provider, uuid_t *instance)
5ba3f43e
A		 991{
992 nexus_attr_t attr = NULL;
993 nexus_controller_t controller = kern_nexus_shared_controller();
994 uuid_t dom_prov;
995 errno_t err;
996 struct kern_nexus_init init;
0a7de745 997 nexus_name_t provider_name;
5ba3f43e
A		 998
999 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
0a7de745 1000 &dom_prov);
5ba3f43e
A		 1001 if (err != 0) {
1002 printf("%s can't get %s provider, error %d\n",
0a7de745 1003 __func__, type_name, err);
5ba3f43e
A		 1004 goto failed;
1005 }
1006
1007 err = kern_nexus_attr_create(&attr);
1008 if (err != 0) {
1009 printf("%s: kern_nexus_attr_create failed: %d\n",
0a7de745 1010 __func__, err);
5ba3f43e
A		 1011 goto failed;
1012 }
1013
1014 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
1015 VERIFY(err == 0);
1016
5c9f4661 1017 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 1018 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1019 VERIFY(err == 0);
1020
1021 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
5c9f4661 1022 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
5ba3f43e
A		 1023 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1024 VERIFY(err == 0);
5c9f4661 1025 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
5ba3f43e
A		 1026 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1027 VERIFY(err == 0);
1028
1029 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 1030 "com.apple.%s.%s", type_name, ifname);
5ba3f43e 1031 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 1032 dom_prov,
1033 provider_name,
1034 NULL,
1035 0,
1036 attr,
1037 provider);
5ba3f43e
A		 1038 kern_nexus_attr_destroy(attr);
1039 attr = NULL;
1040 if (err != 0) {
1041 printf("%s register %s provider failed, error %d\n",
0a7de745 1042 __func__, type_name, err);
5ba3f43e
A		 1043 goto failed;
1044 }
0a7de745 1045 bzero(&init, sizeof(init));
5ba3f43e
A		 1046 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1047 err = kern_nexus_controller_alloc_provider_instance(controller,
0a7de745
A		 1048 *provider,
1049 NULL,
1050 instance, &init);
5ba3f43e
A		 1051 if (err != 0) {
1052 printf("%s alloc_provider_instance %s failed, %d\n",
0a7de745 1053 __func__, type_name, err);
5ba3f43e 1054 kern_nexus_controller_deregister_provider(controller,
0a7de745 1055 *provider);
5ba3f43e
A		 1056 uuid_clear(*provider);
1057 }
1058failed:
0a7de745 1059 return err;
5ba3f43e
A		 1060}
1061
1062static errno_t
1063utun_multistack_attach(struct utun_pcb *pcb)
1064{
1065 nexus_controller_t controller = kern_nexus_shared_controller();
1066 errno_t err = 0;
1067 utun_nx_t nx = &pcb->utun_nx;
1068
1069 // Allocate multistack flowswitch
5c9f4661 1070 err = utun_create_fs_provider_and_instance(pcb,
0a7de745
A		 1071 NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
1072 "multistack",
1073 pcb->utun_ifp->if_xname,
1074 &nx->ms_provider,
1075 &nx->ms_instance);
5ba3f43e
A		 1076 if (err != 0) {
1077 printf("%s: failed to create bridge provider and instance\n",
0a7de745 1078 __func__);
5ba3f43e
A		 1079 goto failed;
1080 }
1081
1082 // Attach multistack to device port
1083 err = kern_nexus_ifattach(controller, nx->ms_instance,
0a7de745
A		 1084 NULL, nx->if_instance,
1085 FALSE, &nx->ms_device);
5ba3f43e
A		 1086 if (err != 0) {
1087 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1088 goto failed;
1089 }
1090
1091 // Attach multistack to host port
1092 err = kern_nexus_ifattach(controller, nx->ms_instance,
0a7de745
A		 1093 NULL, nx->if_instance,
1094 TRUE, &nx->ms_host);
5ba3f43e
A		 1095 if (err != 0) {
1096 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1097 goto failed;
1098 }
1099
1100 // Extract the agent UUID and save for later
1101 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1102 if (multistack_nx != NULL) {
1103 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1104 if (flowswitch != NULL) {
1105 FSW_RLOCK(flowswitch);
1106 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1107 if (ms_context != NULL) {
1108 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1109 } else {
1110 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1111 }
1112 FSW_UNLOCK(flowswitch);
1113 } else {
1114 printf("utun_multistack_attach - flowswitch is NULL\n");
1115 }
1116 nx_release(multistack_nx);
1117 } else {
1118 printf("utun_multistack_attach - unable to find multistack nexus\n");
1119 }
1120
0a7de745 1121 return 0;
5ba3f43e
A		 1122
1123failed:
a39ff7e2 1124 utun_nexus_detach(pcb);
5ba3f43e
A		 1125
1126 errno_t detach_error = 0;
1127 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1128 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1129 /* NOT REACHED */
1130 }
0a7de745
A		 1131
1132 return err;
5ba3f43e
A		 1133}
1134
1135static errno_t
1136utun_register_kernel_pipe_nexus(void)
1137{
1138 nexus_attr_t nxa = NULL;
1139 errno_t result;
1140
1141 lck_mtx_lock(&utun_lock);
1142 if (utun_ncd_refcount++) {
1143 lck_mtx_unlock(&utun_lock);
1144 return 0;
1145 }
1146
1147 result = kern_nexus_controller_create(&utun_ncd);
1148 if (result) {
1149 printf("%s: kern_nexus_controller_create failed: %d\n",
0a7de745 1150 __FUNCTION__, result);
5ba3f43e
A		 1151 goto done;
1152 }
1153
1154 uuid_t dom_prov;
1155 result = kern_nexus_get_builtin_domain_provider(
1156 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1157 if (result) {
1158 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
0a7de745 1159 __FUNCTION__, result);
5ba3f43e
A		 1160 goto done;
1161 }
1162
1163 struct kern_nexus_provider_init prov_init = {
1164 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1165 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1166 .nxpi_pre_connect = utun_nexus_pre_connect,
1167 .nxpi_connected = utun_nexus_connected,
1168 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1169 .nxpi_disconnected = utun_nexus_disconnected,
1170 .nxpi_ring_init = utun_kpipe_ring_init,
1171 .nxpi_ring_fini = utun_kpipe_ring_fini,
1172 .nxpi_slot_init = NULL,
1173 .nxpi_slot_fini = NULL,
1174 .nxpi_sync_tx = utun_kpipe_sync_tx,
1175 .nxpi_sync_rx = utun_kpipe_sync_rx,
1176 .nxpi_tx_doorbell = NULL,
1177 };
1178
1179 result = kern_nexus_attr_create(&nxa);
1180 if (result) {
1181 printf("%s: kern_nexus_attr_create failed: %d\n",
0a7de745 1182 __FUNCTION__, result);
5ba3f43e
A		 1183 goto done;
1184 }
1185
1186 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1187 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1188 VERIFY(result == 0);
1189
1190 // Reset ring size for kernel pipe nexus to limit memory usage
1191 uint64_t ring_size = if_utun_ring_size;
1192 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1193 VERIFY(result == 0);
1194 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1195 VERIFY(result == 0);
1196
1197 result = kern_nexus_controller_register_provider(utun_ncd,
0a7de745
A		 1198 dom_prov,
1199 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1200 &prov_init,
1201 sizeof(prov_init),
1202 nxa,
1203 &utun_kpipe_uuid);
5ba3f43e
A		 1204 if (result) {
1205 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
0a7de745 1206 __FUNCTION__, result);
5ba3f43e
A		 1207 goto done;
1208 }
1209
1210done:
1211 if (nxa) {
1212 kern_nexus_attr_destroy(nxa);
1213 }
1214
1215 if (result) {
1216 if (utun_ncd) {
1217 kern_nexus_controller_destroy(utun_ncd);
1218 utun_ncd = NULL;
1219 }
1220 utun_ncd_refcount = 0;
1221 }
1222
1223 lck_mtx_unlock(&utun_lock);
1224
1225 return result;
1226}
1227
1228static void
1229utun_unregister_kernel_pipe_nexus(void)
1230{
1231 lck_mtx_lock(&utun_lock);
1232
1233 VERIFY(utun_ncd_refcount > 0);
1234
1235 if (--utun_ncd_refcount == 0) {
1236 kern_nexus_controller_destroy(utun_ncd);
1237 utun_ncd = NULL;
1238 }
1239
1240 lck_mtx_unlock(&utun_lock);
1241}
1242
1243// For use by socket option, not internally
1244static errno_t
1245utun_disable_channel(struct utun_pcb *pcb)
1246{
1247 errno_t result;
1248 int enabled;
1249 uuid_t uuid;
1250
1251 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1252
1253 enabled = pcb->utun_kpipe_enabled;
1254 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1255
1256 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1257
1258 pcb->utun_kpipe_enabled = 0;
1259 uuid_clear(pcb->utun_kpipe_uuid);
1260
1261 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1262
1263 if (enabled) {
1264 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1265 } else {
1266 result = ENXIO;
1267 }
1268
1269 if (!result) {
a39ff7e2
A		 1270 if (pcb->utun_kpipe_pp != NULL) {
1271 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1272 pcb->utun_kpipe_pp = NULL;
1273 }
5ba3f43e
A		 1274 utun_unregister_kernel_pipe_nexus();
1275 }
1276
1277 return result;
1278}
1279
1280static errno_t
1281utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1282{
1283 struct kern_nexus_init init;
a39ff7e2 1284 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 1285 errno_t result;
1286
d9a64523
A		 1287 kauth_cred_t cred = kauth_cred_get();
1288 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
1289 if (result) {
1290 return result;
1291 }
1292
5ba3f43e
A		 1293 result = utun_register_kernel_pipe_nexus();
1294 if (result) {
1295 return result;
1296 }
1297
1298 VERIFY(utun_ncd);
1299
1300 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1301
1302 if (pcb->utun_kpipe_enabled) {
1303 result = EEXIST; // return success instead?
1304 goto done;
1305 }
1306
1307 /*
1308 * Make sure we can fit packets in the channel buffers and
1309 * Allow an extra 4 bytes for the protocol number header in the channel
1310 */
5c9f4661 1311 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
5ba3f43e
A		 1312 result = EOPNOTSUPP;
1313 goto done;
1314 }
1315
0a7de745 1316 bzero(&pp_init, sizeof(pp_init));
a39ff7e2
A		 1317 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1318 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1319 pp_init.kbi_bufsize = pcb->utun_slot_size;
1320 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1321 pp_init.kbi_max_frags = 1;
1322 pp_init.kbi_flags |= KBIF_QUANTUM;
0a7de745 1323 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2
A		 1324 "com.apple.kpipe.%s", pcb->utun_if_xname);
1325
1326 result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_kpipe_pp,
1327 NULL);
1328 if (result != 0) {
1329 printf("%s pbufbool create failed, error %d\n", __func__, result);
1330 goto done;
1331 }
1332
5ba3f43e 1333 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
0a7de745 1334 bzero(&init, sizeof(init));
5ba3f43e 1335 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
a39ff7e2 1336 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
5ba3f43e 1337 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
0a7de745 1338 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
5ba3f43e
A		 1339 if (result) {
1340 goto done;
1341 }
1342
1343 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1344 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
0a7de745
A		 1345 pcb->utun_kpipe_uuid, &port,
1346 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
5ba3f43e
A		 1347 if (result) {
1348 kern_nexus_controller_free_provider_instance(utun_ncd,
0a7de745 1349 pcb->utun_kpipe_uuid);
5ba3f43e
A		 1350 uuid_clear(pcb->utun_kpipe_uuid);
1351 goto done;
1352 }
1353
1354 pcb->utun_kpipe_enabled = 1;
1355
1356done:
1357 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1358
1359 if (result) {
a39ff7e2
A		 1360 if (pcb->utun_kpipe_pp != NULL) {
1361 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1362 pcb->utun_kpipe_pp = NULL;
1363 }
5ba3f43e
A		 1364 utun_unregister_kernel_pipe_nexus();
1365 }
1366
1367 return result;
1368}
1369
1370#endif // UTUN_NEXUS
b0d623f7
A		 1371
1372errno_t
1373utun_register_control(void)
1374{
5ba3f43e
A		 1375 struct kern_ctl_reg kern_ctl;
1376 errno_t result = 0;
0a7de745 1377
b0d623f7
A		 1378 /* Find a unique value for our interface family */
1379 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1380 if (result != 0) {
1381 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1382 return result;
1383 }
5ba3f43e
A		 1384
1385 utun_pcb_size = sizeof(struct utun_pcb);
1386 utun_pcb_zone = zinit(utun_pcb_size,
0a7de745
A		 1387 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1388 0, UTUN_PCB_ZONE_NAME);
5ba3f43e
A		 1389 if (utun_pcb_zone == NULL) {
1390 printf("utun_register_control - zinit(utun_pcb) failed");
1391 return ENOMEM;
1392 }
1393
1394#if UTUN_NEXUS
1395 utun_register_nexus();
1396#endif // UTUN_NEXUS
1397
1398 TAILQ_INIT(&utun_head);
0a7de745 1399
b0d623f7 1400 bzero(&kern_ctl, sizeof(kern_ctl));
fe8ab488 1401 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
b0d623f7 1402 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
fe8ab488 1403 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
39236c6e
A		 1404 kern_ctl.ctl_sendsize = 512 * 1024;
1405 kern_ctl.ctl_recvsize = 512 * 1024;
5c9f4661 1406 kern_ctl.ctl_bind = utun_ctl_bind;
b0d623f7
A		 1407 kern_ctl.ctl_connect = utun_ctl_connect;
1408 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1409 kern_ctl.ctl_send = utun_ctl_send;
1410 kern_ctl.ctl_setopt = utun_ctl_setopt;
1411 kern_ctl.ctl_getopt = utun_ctl_getopt;
fe8ab488 1412 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
39236c6e 1413
b0d623f7
A		 1414 result = ctl_register(&kern_ctl, &utun_kctlref);
1415 if (result != 0) {
1416 printf("utun_register_control - ctl_register failed: %d\n", result);
1417 return result;
1418 }
0a7de745 1419
b0d623f7
A		 1420 /* Register the protocol plumbers */
1421 if ((result = proto_register_plumber(PF_INET, utun_family,
0a7de745 1422 utun_attach_proto, NULL)) != 0) {
b0d623f7 1423 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
0a7de745 1424 utun_family, result);
b0d623f7
A		 1425 ctl_deregister(utun_kctlref);
1426 return result;
1427 }
0a7de745 1428
b0d623f7
A		 1429 /* Register the protocol plumbers */
1430 if ((result = proto_register_plumber(PF_INET6, utun_family,
0a7de745 1431 utun_attach_proto, NULL)) != 0) {
b0d623f7
A		 1432 proto_unregister_plumber(PF_INET, utun_family);
1433 ctl_deregister(utun_kctlref);
1434 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
0a7de745 1435 utun_family, result);
b0d623f7
A		 1436 return result;
1437 }
39037602 1438
5ba3f43e
A		 1439 utun_lck_attr = lck_attr_alloc_init();
1440 utun_lck_grp_attr = lck_grp_attr_alloc_init();
0a7de745 1441 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
5ba3f43e 1442
5ba3f43e 1443 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
5ba3f43e 1444
b0d623f7
A		 1445 return 0;
1446}
1447
1448/* Kernel control functions */
1449
5ba3f43e 1450static inline void
5c9f4661 1451utun_free_pcb(struct utun_pcb *pcb, bool in_list)
5ba3f43e
A		 1452{
1453#ifdef UTUN_NEXUS
1454 mbuf_freem_list(pcb->utun_input_chain);
1455 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1456#endif // UTUN_NEXUS
1457 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
5c9f4661
A		 1458 if (in_list) {
1459 lck_mtx_lock(&utun_lock);
1460 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1461 lck_mtx_unlock(&utun_lock);
1462 }
5ba3f43e
A		 1463 zfree(utun_pcb_zone, pcb);
1464}
1465
b0d623f7 1466static errno_t
5c9f4661 1467utun_ctl_bind(kern_ctl_ref kctlref,
0a7de745
A		 1468 struct sockaddr_ctl *sac,
1469 void **unitinfo)
b0d623f7 1470{
5ba3f43e
A		 1471 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1472 memset(pcb, 0, sizeof(*pcb));
39037602 1473
b0d623f7 1474 *unitinfo = pcb;
d1ecb069
A		 1475 pcb->utun_ctlref = kctlref;
1476 pcb->utun_unit = sac->sc_unit;
fe8ab488 1477 pcb->utun_max_pending_packets = 1;
5ba3f43e 1478
5c9f4661
A		 1479#if UTUN_NEXUS
1480 pcb->utun_use_netif = false;
1481 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1482 pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
1483 pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
1484 pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
1485#endif // UTUN_NEXUS
1486
5ba3f43e
A		 1487 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1488 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1489
0a7de745 1490 return 0;
5c9f4661
A		 1491}
1492
1493static errno_t
1494utun_ctl_connect(kern_ctl_ref kctlref,
0a7de745
A		 1495 struct sockaddr_ctl *sac,
1496 void **unitinfo)
5c9f4661
A		 1497{
1498 struct ifnet_init_eparams utun_init = {};
1499 errno_t result = 0;
1500
1501 if (*unitinfo == NULL) {
1502 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1503 }
0a7de745 1504
5c9f4661
A		 1505 struct utun_pcb *pcb = *unitinfo;
1506
5ba3f43e
A		 1507 lck_mtx_lock(&utun_lock);
1508
1509 /* Find some open interface id */
1510 u_int32_t chosen_unique_id = 1;
1511 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1512 if (next_pcb != NULL) {
1513 /* List was not empty, add one to the last item */
1514 chosen_unique_id = next_pcb->utun_unique_id + 1;
1515 next_pcb = NULL;
1516
1517 /*
1518 * If this wrapped the id number, start looking at
1519 * the front of the list for an unused id.
1520 */
1521 if (chosen_unique_id == 0) {
1522 /* Find the next unused ID */
1523 chosen_unique_id = 1;
1524 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1525 if (next_pcb->utun_unique_id > chosen_unique_id) {
1526 /* We found a gap */
1527 break;
1528 }
1529
1530 chosen_unique_id = next_pcb->utun_unique_id + 1;
1531 }
1532 }
1533 }
1534
1535 pcb->utun_unique_id = chosen_unique_id;
1536
1537 if (next_pcb != NULL) {
1538 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1539 } else {
1540 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1541 }
1542 lck_mtx_unlock(&utun_lock);
1543
1544 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1545 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1546 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
b0d623f7
A		 1547
1548 /* Create the interface */
1549 bzero(&utun_init, sizeof(utun_init));
39236c6e 1550 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
0a7de745 1551 utun_init.len = sizeof(utun_init);
5ba3f43e
A		 1552
1553#if UTUN_NEXUS
5c9f4661
A		 1554 if (pcb->utun_use_netif) {
1555 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1556 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1557 } else
5ba3f43e 1558#endif // UTUN_NEXUS
5c9f4661
A		 1559 {
1560 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1561 utun_init.start = utun_start;
1562 utun_init.framer_extended = utun_framer;
1563 }
5ba3f43e 1564 utun_init.name = "utun";
d1ecb069 1565 utun_init.unit = pcb->utun_unit - 1;
5ba3f43e
A		 1566 utun_init.uniqueid = pcb->utun_unique_name;
1567 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
b0d623f7 1568 utun_init.family = utun_family;
39037602 1569 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
b0d623f7 1570 utun_init.type = IFT_OTHER;
b0d623f7 1571 utun_init.demux = utun_demux;
b0d623f7
A		 1572 utun_init.add_proto = utun_add_proto;
1573 utun_init.del_proto = utun_del_proto;
1574 utun_init.softc = pcb;
1575 utun_init.ioctl = utun_ioctl;
1576 utun_init.detach = utun_detached;
39037602 1577
5ba3f43e 1578#if UTUN_NEXUS
5c9f4661
A		 1579 if (pcb->utun_use_netif) {
1580 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1581 if (result != 0) {
1582 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1583 utun_free_pcb(pcb, true);
1584 *unitinfo = NULL;
1585 return result;
1586 }
5ba3f43e 1587
5c9f4661
A		 1588 result = utun_multistack_attach(pcb);
1589 if (result != 0) {
1590 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1591 *unitinfo = NULL;
1592 return result;
1593 }
5ba3f43e 1594
5c9f4661
A		 1595 /* Attach to bpf */
1596 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1597 } else
1598#endif // UTUN_NEXUS
1599 {
1600 /*
1601 * Upon success, this holds an ifnet reference which we will
1602 * release via ifnet_release() at final detach time.
1603 */
1604 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1605 if (result != 0) {
1606 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1607 utun_free_pcb(pcb, true);
1608 *unitinfo = NULL;
1609 return result;
1610 }
b0d623f7 1611
5c9f4661
A		 1612 /* Set flags and additional information. */
1613 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1614 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
d1ecb069 1615
5c9f4661
A		 1616 /* The interface must generate its own IPv6 LinkLocal address,
1617 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1618 */
1619 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1620
1621 /* Reset the stats in case as the interface may have been recycled */
1622 struct ifnet_stats_param stats;
1623 bzero(&stats, sizeof(struct ifnet_stats_param));
1624 ifnet_set_stat(pcb->utun_ifp, &stats);
1625
1626 /* Attach the interface */
1627 result = ifnet_attach(pcb->utun_ifp, NULL);
1628 if (result != 0) {
1629 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
1630 /* Release reference now since attach failed */
1631 ifnet_release(pcb->utun_ifp);
1632 utun_free_pcb(pcb, true);
1633 *unitinfo = NULL;
0a7de745 1634 return result;
5c9f4661
A		 1635 }
1636
1637 /* Attach to bpf */
1638 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
39037602 1639 }
5ba3f43e 1640
5ba3f43e
A		 1641 /* The interfaces resoures allocated, mark it as running */
1642 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1643
b0d623f7
A		 1644 return result;
1645}
1646
1647static errno_t
5ba3f43e 1648utun_detach_ip(ifnet_t interface,
0a7de745
A		 1649 protocol_family_t protocol,
1650 socket_t pf_socket)
b0d623f7
A		 1651{
1652 errno_t result = EPROTONOSUPPORT;
0a7de745 1653
b0d623f7
A		 1654 /* Attempt a detach */
1655 if (protocol == PF_INET) {
0a7de745
A		 1656 struct ifreq ifr;
1657
b0d623f7
A		 1658 bzero(&ifr, sizeof(ifr));
1659 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745
A		 1660 ifnet_name(interface), ifnet_unit(interface));
1661
b0d623f7 1662 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
5ba3f43e 1663 } else if (protocol == PF_INET6) {
0a7de745
A		 1664 struct in6_ifreq ifr6;
1665
b0d623f7
A		 1666 bzero(&ifr6, sizeof(ifr6));
1667 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745
A		 1668 ifnet_name(interface), ifnet_unit(interface));
1669
b0d623f7
A		 1670 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1671 }
0a7de745 1672
b0d623f7
A		 1673 return result;
1674}
1675
1676static void
5ba3f43e 1677utun_remove_address(ifnet_t interface,
0a7de745
A		 1678 protocol_family_t protocol,
1679 ifaddr_t address,
1680 socket_t pf_socket)
b0d623f7
A		 1681{
1682 errno_t result = 0;
0a7de745 1683
b0d623f7
A		 1684 /* Attempt a detach */
1685 if (protocol == PF_INET) {
5ba3f43e 1686 struct ifreq ifr;
0a7de745 1687
b0d623f7
A		 1688 bzero(&ifr, sizeof(ifr));
1689 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745 1690 ifnet_name(interface), ifnet_unit(interface));
b0d623f7
A		 1691 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1692 if (result != 0) {
1693 printf("utun_remove_address - ifaddr_address failed: %d", result);
5ba3f43e 1694 } else {
b0d623f7
A		 1695 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1696 if (result != 0) {
1697 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1698 }
1699 }
5ba3f43e
A		 1700 } else if (protocol == PF_INET6) {
1701 struct in6_ifreq ifr6;
0a7de745 1702
b0d623f7
A		 1703 bzero(&ifr6, sizeof(ifr6));
1704 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745 1705 ifnet_name(interface), ifnet_unit(interface));
b0d623f7 1706 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
0a7de745 1707 sizeof(ifr6.ifr_addr));
b0d623f7
A		 1708 if (result != 0) {
1709 printf("utun_remove_address - ifaddr_address failed (v6): %d",
0a7de745 1710 result);
5ba3f43e 1711 } else {
b0d623f7
A		 1712 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1713 if (result != 0) {
1714 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
0a7de745 1715 result);
b0d623f7
A		 1716 }
1717 }
1718 }
1719}
1720
1721static void
5ba3f43e 1722utun_cleanup_family(ifnet_t interface,
0a7de745 1723 protocol_family_t protocol)
5ba3f43e
A		 1724{
1725 errno_t result = 0;
1726 socket_t pf_socket = NULL;
1727 ifaddr_t *addresses = NULL;
1728 int i;
0a7de745 1729
b0d623f7
A		 1730 if (protocol != PF_INET && protocol != PF_INET6) {
1731 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1732 return;
1733 }
0a7de745 1734
b0d623f7
A		 1735 /* Create a socket for removing addresses and detaching the protocol */
1736 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1737 if (result != 0) {
0a7de745 1738 if (result != EAFNOSUPPORT) {
b0d623f7 1739 printf("utun_cleanup_family - failed to create %s socket: %d\n",
0a7de745
A		 1740 protocol == PF_INET ? "IP" : "IPv6", result);
1741 }
b0d623f7
A		 1742 goto cleanup;
1743 }
0a7de745
A		 1744
1745 /* always set SS_PRIV, we want to close and detach regardless */
1746 sock_setpriv(pf_socket, 1);
6d2010ae 1747
b0d623f7
A		 1748 result = utun_detach_ip(interface, protocol, pf_socket);
1749 if (result == 0 || result == ENXIO) {
1750 /* We are done! We either detached or weren't attached. */
1751 goto cleanup;
5ba3f43e 1752 } else if (result != EBUSY) {
b0d623f7
A		 1753 /* Uh, not really sure what happened here... */
1754 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1755 goto cleanup;
1756 }
0a7de745 1757
b0d623f7
A		 1758 /*
1759 * At this point, we received an EBUSY error. This means there are
1760 * addresses attached. We should detach them and then try again.
1761 */
1762 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1763 if (result != 0) {
1764 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
0a7de745
A		 1765 ifnet_name(interface), ifnet_unit(interface),
1766 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
b0d623f7
A		 1767 goto cleanup;
1768 }
0a7de745 1769
b0d623f7
A		 1770 for (i = 0; addresses[i] != 0; i++) {
1771 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1772 }
1773 ifnet_free_address_list(addresses);
1774 addresses = NULL;
0a7de745 1775
b0d623f7
A		 1776 /*
1777 * The addresses should be gone, we should try the remove again.
1778 */
1779 result = utun_detach_ip(interface, protocol, pf_socket);
1780 if (result != 0 && result != ENXIO) {
1781 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1782 }
0a7de745 1783
b0d623f7 1784cleanup:
5ba3f43e 1785 if (pf_socket != NULL) {
b0d623f7 1786 sock_close(pf_socket);
5ba3f43e 1787 }
0a7de745 1788
5ba3f43e 1789 if (addresses != NULL) {
b0d623f7 1790 ifnet_free_address_list(addresses);
5ba3f43e 1791 }
b0d623f7
A		 1792}
1793
1794static errno_t
5ba3f43e 1795utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
0a7de745
A		 1796 __unused u_int32_t unit,
1797 void *unitinfo)
b0d623f7 1798{
0a7de745 1799 struct utun_pcb *pcb = unitinfo;
5ba3f43e
A		 1800 ifnet_t ifp = NULL;
1801 errno_t result = 0;
1802
1803 if (pcb == NULL) {
1804 return EINVAL;
1805 }
1806
1807#if UTUN_NEXUS
1808 // Tell the nexus to stop all rings
1809 if (pcb->utun_netif_nexus != NULL) {
1810 kern_nexus_stop(pcb->utun_netif_nexus);
1811 }
1812#endif // UTUN_NEXUS
316670eb 1813
5ba3f43e 1814 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
316670eb 1815
5ba3f43e
A		 1816#if UTUN_NEXUS
1817 uuid_t kpipe_uuid;
1818 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1819 uuid_clear(pcb->utun_kpipe_uuid);
1820 pcb->utun_kpipe_enabled = FALSE;
1821#endif // UTUN_NEXUS
39037602 1822
d1ecb069 1823 pcb->utun_ctlref = NULL;
5ba3f43e 1824
5c9f4661
A		 1825 ifp = pcb->utun_ifp;
1826 if (ifp != NULL) {
1827#if UTUN_NEXUS
1828 // Tell the nexus to stop all rings
1829 if (pcb->utun_netif_nexus != NULL) {
1830 /*
1831 * Quiesce the interface and flush any pending outbound packets.
1832 */
1833 if_down(ifp);
1834
1835 /* Increment refcnt, but detach interface */
1836 ifnet_incr_iorefcnt(ifp);
1837 if ((result = ifnet_detach(ifp)) != 0) {
1838 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1839 }
5ba3f43e 1840
5c9f4661
A		 1841 /*
1842 * We want to do everything in our power to ensure that the interface
1843 * really goes away when the socket is closed. We must remove IP/IPv6
1844 * addresses and detach the protocols. Finally, we can remove and
1845 * release the interface.
1846 */
1847 utun_cleanup_family(ifp, AF_INET);
1848 utun_cleanup_family(ifp, AF_INET6);
39037602 1849
5c9f4661 1850 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
39037602 1851
5c9f4661
A		 1852 if (!uuid_is_null(kpipe_uuid)) {
1853 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 1854 if (pcb->utun_kpipe_pp != NULL) {
1855 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1856 pcb->utun_kpipe_pp = NULL;
1857 }
5c9f4661
A		 1858 utun_unregister_kernel_pipe_nexus();
1859 }
1860 }
a39ff7e2 1861 utun_nexus_detach(pcb);
5c9f4661
A		 1862
1863 /* Decrement refcnt to finish detaching and freeing */
1864 ifnet_decr_iorefcnt(ifp);
1865 } else
1866#endif // UTUN_NEXUS
1867 {
1868 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
5ba3f43e
A		 1869
1870#if UTUN_NEXUS
5c9f4661
A		 1871 if (!uuid_is_null(kpipe_uuid)) {
1872 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 1873 if (pcb->utun_kpipe_pp != NULL) {
1874 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1875 pcb->utun_kpipe_pp = NULL;
1876 }
5c9f4661
A		 1877 utun_unregister_kernel_pipe_nexus();
1878 }
1879 }
5ba3f43e
A		 1880#endif // UTUN_NEXUS
1881
5c9f4661
A		 1882 /*
1883 * We want to do everything in our power to ensure that the interface
1884 * really goes away when the socket is closed. We must remove IP/IPv6
1885 * addresses and detach the protocols. Finally, we can remove and
1886 * release the interface.
1887 */
1888 utun_cleanup_family(ifp, AF_INET);
1889 utun_cleanup_family(ifp, AF_INET6);
1890
1891 /*
1892 * Detach now; utun_detach() will be called asynchronously once
1893 * the I/O reference count drops to 0. There we will invoke
1894 * ifnet_release().
1895 */
1896 if ((result = ifnet_detach(ifp)) != 0) {
1897 printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1898 }
1899 }
1900 } else {
1901 // Bound, but not connected
1902 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1903 utun_free_pcb(pcb, false);
1904 }
0a7de745 1905
b0d623f7 1906 return 0;
39037602 1907}
b0d623f7
A		 1908
1909static errno_t
5ba3f43e 1910utun_ctl_send(__unused kern_ctl_ref kctlref,
0a7de745
A		 1911 __unused u_int32_t unit,
1912 void *unitinfo,
1913 mbuf_t m,
1914 __unused int flags)
b0d623f7 1915{
39236c6e 1916 /*
0a7de745 1917 * The userland ABI requires the first four bytes have the protocol family
39236c6e
A		 1918 * in network byte order: swap them
1919 */
39037602 1920 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
39236c6e 1921 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
39037602 1922 } else {
0a7de745 1923 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m));
39037602 1924 }
39236c6e 1925
316670eb 1926 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
b0d623f7
A		 1927}
1928
1929static errno_t
5ba3f43e 1930utun_ctl_setopt(__unused kern_ctl_ref kctlref,
0a7de745
A		 1931 __unused u_int32_t unit,
1932 void *unitinfo,
1933 int opt,
1934 void *data,
1935 size_t len)
b0d623f7 1936{
5ba3f43e
A		 1937 struct utun_pcb *pcb = unitinfo;
1938 errno_t result = 0;
b0d623f7
A		 1939 /* check for privileges for privileged options */
1940 switch (opt) {
0a7de745
A		 1941 case UTUN_OPT_FLAGS:
1942 case UTUN_OPT_EXT_IFDATA_STATS:
1943 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1944 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1945 return EPERM;
1946 }
1947 break;
b0d623f7
A		 1948 }
1949
1950 switch (opt) {
0a7de745
A		 1951 case UTUN_OPT_FLAGS:
1952 if (len != sizeof(u_int32_t)) {
1953 result = EMSGSIZE;
1954 } else {
1955 if (pcb->utun_ifp == NULL) {
1956 // Only can set after connecting
1957 result = EINVAL;
1958 break;
1959 }
5c9f4661 1960#if UTUN_NEXUS
0a7de745
A		 1961 if (pcb->utun_use_netif) {
1962 pcb->utun_flags = *(u_int32_t *)data;
1963 } else
5c9f4661 1964#endif // UTUN_NEXUS
0a7de745
A		 1965 {
1966 u_int32_t old_flags = pcb->utun_flags;
1967 pcb->utun_flags = *(u_int32_t *)data;
1968 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1969 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1970 bpfdetach(pcb->utun_ifp);
1971 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
5c9f4661 1972 }
39037602 1973 }
0a7de745
A		 1974 }
1975 break;
1976
1977 case UTUN_OPT_EXT_IFDATA_STATS:
1978 if (len != sizeof(int)) {
1979 result = EMSGSIZE;
1980 break;
1981 }
1982 if (pcb->utun_ifp == NULL) {
1983 // Only can set after connecting
1984 result = EINVAL;
39236c6e 1985 break;
0a7de745
A		 1986 }
1987 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1988 break;
39236c6e 1989
0a7de745
A		 1990 case UTUN_OPT_INC_IFDATA_STATS_IN:
1991 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1992 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1993
1994 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1995 result = EINVAL;
d1ecb069 1996 break;
0a7de745
A		 1997 }
1998 if (pcb->utun_ifp == NULL) {
1999 // Only can set after connecting
2000 result = EINVAL;
2001 break;
2002 }
2003 if (!pcb->utun_ext_ifdata_stats) {
2004 result = EINVAL;
b0d623f7 2005 break;
d1ecb069 2006 }
0a7de745
A		 2007 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) {
2008 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2009 utsp->utsp_bytes, utsp->utsp_errors);
2010 } else {
2011 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2012 utsp->utsp_bytes, utsp->utsp_errors);
2013 }
2014 break;
2015 }
2016 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2017 ifnet_t del_ifp = NULL;
2018 char name[IFNAMSIZ];
39236c6e 2019
0a7de745
A		 2020 if (len > IFNAMSIZ - 1) {
2021 result = EMSGSIZE;
39236c6e
A		 2022 break;
2023 }
0a7de745
A		 2024 if (pcb->utun_ifp == NULL) {
2025 // Only can set after connecting
2026 result = EINVAL;
2027 break;
2028 }
2029 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2030 bcopy(data, name, len);
2031 name[len] = 0;
2032 result = ifnet_find_by_name(name, &del_ifp);
2033 }
2034 if (result == 0) {
2035 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2036 if (del_ifp) {
2037 ifnet_release(del_ifp);
fe8ab488 2038 }
0a7de745
A		 2039 }
2040 break;
2041 }
2042 case UTUN_OPT_MAX_PENDING_PACKETS: {
2043 u_int32_t max_pending_packets = 0;
2044 if (len != sizeof(u_int32_t)) {
2045 result = EMSGSIZE;
2046 break;
2047 }
2048 max_pending_packets = *(u_int32_t *)data;
2049 if (max_pending_packets == 0) {
2050 result = EINVAL;
fe8ab488
A		 2051 break;
2052 }
0a7de745
A		 2053 pcb->utun_max_pending_packets = max_pending_packets;
2054 break;
2055 }
5ba3f43e 2056#if UTUN_NEXUS
0a7de745
A		 2057 case UTUN_OPT_ENABLE_CHANNEL: {
2058 if (len != sizeof(int)) {
2059 result = EMSGSIZE;
2060 break;
2061 }
2062 if (pcb->utun_ifp == NULL) {
2063 // Only can set after connecting
2064 result = EINVAL;
2065 break;
2066 }
2067 if (*(int *)data) {
2068 result = utun_enable_channel(pcb, current_proc());
2069 } else {
2070 result = utun_disable_channel(pcb);
2071 }
2072 break;
2073 }
2074 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2075 if (len != sizeof(int)) {
2076 result = EMSGSIZE;
2077 break;
2078 }
2079 if (pcb->utun_ifp == NULL) {
2080 // Only can set after connecting
2081 result = EINVAL;
2082 break;
2083 }
2084 if (!if_is_netagent_enabled()) {
2085 result = ENOTSUP;
2086 break;
2087 }
2088 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
2089 result = ENOENT;
5ba3f43e
A		 2090 break;
2091 }
5ba3f43e 2092
0a7de745
A		 2093 if (*(int *)data) {
2094 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2095 pcb->utun_needs_netagent = true;
2096 } else {
2097 pcb->utun_needs_netagent = false;
2098 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2099 }
2100 break;
2101 }
2102 case UTUN_OPT_ENABLE_NETIF: {
2103 if (len != sizeof(int)) {
2104 result = EMSGSIZE;
5ba3f43e
A		 2105 break;
2106 }
0a7de745
A		 2107 if (pcb->utun_ifp != NULL) {
2108 // Only can set before connecting
2109 result = EINVAL;
5c9f4661
A		 2110 break;
2111 }
0a7de745
A		 2112 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2113 pcb->utun_use_netif = !!(*(int *)data);
2114 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2115 break;
2116 }
2117 case UTUN_OPT_SLOT_SIZE: {
2118 if (len != sizeof(u_int32_t)) {
2119 result = EMSGSIZE;
5c9f4661
A		 2120 break;
2121 }
0a7de745
A		 2122 if (pcb->utun_ifp != NULL) {
2123 // Only can set before connecting
2124 result = EINVAL;
5c9f4661
A		 2125 break;
2126 }
0a7de745
A		 2127 u_int32_t slot_size = *(u_int32_t *)data;
2128 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2129 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2130 return EINVAL;
2131 }
2132 pcb->utun_slot_size = slot_size;
2133 break;
2134 }
2135 case UTUN_OPT_NETIF_RING_SIZE: {
2136 if (len != sizeof(u_int32_t)) {
2137 result = EMSGSIZE;
5c9f4661
A		 2138 break;
2139 }
0a7de745
A		 2140 if (pcb->utun_ifp != NULL) {
2141 // Only can set before connecting
2142 result = EINVAL;
5c9f4661
A		 2143 break;
2144 }
0a7de745
A		 2145 u_int32_t ring_size = *(u_int32_t *)data;
2146 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2147 ring_size > UTUN_IF_MAX_RING_SIZE) {
2148 return EINVAL;
2149 }
2150 pcb->utun_netif_ring_size = ring_size;
2151 break;
2152 }
2153 case UTUN_OPT_TX_FSW_RING_SIZE: {
2154 if (len != sizeof(u_int32_t)) {
2155 result = EMSGSIZE;
2156 break;
2157 }
2158 if (pcb->utun_ifp != NULL) {
2159 // Only can set before connecting
2160 result = EINVAL;
b0d623f7 2161 break;
fe8ab488 2162 }
0a7de745
A		 2163 u_int32_t ring_size = *(u_int32_t *)data;
2164 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2165 ring_size > UTUN_IF_MAX_RING_SIZE) {
2166 return EINVAL;
2167 }
2168 pcb->utun_tx_fsw_ring_size = ring_size;
2169 break;
2170 }
2171 case UTUN_OPT_RX_FSW_RING_SIZE: {
2172 if (len != sizeof(u_int32_t)) {
2173 result = EMSGSIZE;
2174 break;
2175 }
2176 if (pcb->utun_ifp != NULL) {
2177 // Only can set before connecting
2178 result = EINVAL;
2179 break;
2180 }
2181 u_int32_t ring_size = *(u_int32_t *)data;
2182 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2183 ring_size > UTUN_IF_MAX_RING_SIZE) {
2184 return EINVAL;
2185 }
2186 pcb->utun_rx_fsw_ring_size = ring_size;
2187 break;
2188 }
2189#endif // UTUN_NEXUS
2190 default: {
2191 result = ENOPROTOOPT;
2192 break;
2193 }
b0d623f7
A		 2194 }
2195
2196 return result;
2197}
2198
2199static errno_t
5ba3f43e 2200utun_ctl_getopt(__unused kern_ctl_ref kctlref,
0a7de745
A		 2201 __unused u_int32_t unit,
2202 void *unitinfo,
2203 int opt,
2204 void *data,
2205 size_t *len)
b0d623f7 2206{
5ba3f43e
A		 2207 struct utun_pcb *pcb = unitinfo;
2208 errno_t result = 0;
0a7de745 2209
b0d623f7 2210 switch (opt) {
0a7de745
A		 2211 case UTUN_OPT_FLAGS:
2212 if (*len != sizeof(u_int32_t)) {
2213 result = EMSGSIZE;
2214 } else {
2215 *(u_int32_t *)data = pcb->utun_flags;
2216 }
2217 break;
d1ecb069 2218
0a7de745
A		 2219 case UTUN_OPT_EXT_IFDATA_STATS:
2220 if (*len != sizeof(int)) {
2221 result = EMSGSIZE;
2222 } else {
2223 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2224 }
2225 break;
d1ecb069 2226
0a7de745
A		 2227 case UTUN_OPT_IFNAME:
2228 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2229 result = EMSGSIZE;
2230 } else {
2231 if (pcb->utun_ifp == NULL) {
2232 // Only can get after connecting
2233 result = EINVAL;
2234 break;
5ba3f43e 2235 }
0a7de745 2236 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
fe8ab488 2237 }
0a7de745 2238 break;
39037602 2239
0a7de745
A		 2240 case UTUN_OPT_MAX_PENDING_PACKETS: {
2241 if (*len != sizeof(u_int32_t)) {
2242 result = EMSGSIZE;
2243 } else {
2244 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
a39ff7e2 2245 }
0a7de745
A		 2246 break;
2247 }
a39ff7e2 2248
0a7de745
A		 2249#if UTUN_NEXUS
2250 case UTUN_OPT_ENABLE_CHANNEL: {
2251 if (*len != sizeof(int)) {
2252 result = EMSGSIZE;
2253 } else {
2254 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2255 *(int *)data = pcb->utun_kpipe_enabled;
2256 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
a39ff7e2 2257 }
0a7de745
A		 2258 break;
2259 }
a39ff7e2 2260
0a7de745
A		 2261 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2262 if (*len != sizeof(int)) {
2263 result = EMSGSIZE;
2264 } else {
2265 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
a39ff7e2 2266 }
0a7de745
A		 2267 break;
2268 }
a39ff7e2 2269
0a7de745
A		 2270 case UTUN_OPT_ENABLE_NETIF: {
2271 if (*len != sizeof(int)) {
2272 result = EMSGSIZE;
2273 } else {
5ba3f43e 2274 lck_rw_lock_shared(&pcb->utun_pcb_lock);
0a7de745 2275 *(int *)data = !!pcb->utun_use_netif;
5ba3f43e 2276 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5c9f4661 2277 }
0a7de745
A		 2278 break;
2279 }
2280
2281 case UTUN_OPT_GET_CHANNEL_UUID: {
2282 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2283 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2284 result = ENXIO;
2285 } else if (*len != sizeof(uuid_t)) {
2286 result = EMSGSIZE;
2287 } else {
2288 uuid_copy(data, pcb->utun_kpipe_uuid);
5c9f4661 2289 }
0a7de745
A		 2290 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2291 break;
2292 }
2293 case UTUN_OPT_SLOT_SIZE: {
2294 if (*len != sizeof(u_int32_t)) {
2295 result = EMSGSIZE;
2296 } else {
2297 *(u_int32_t *)data = pcb->utun_slot_size;
5c9f4661 2298 }
0a7de745
A		 2299 break;
2300 }
2301 case UTUN_OPT_NETIF_RING_SIZE: {
2302 if (*len != sizeof(u_int32_t)) {
2303 result = EMSGSIZE;
2304 } else {
2305 *(u_int32_t *)data = pcb->utun_netif_ring_size;
5c9f4661 2306 }
0a7de745
A		 2307 break;
2308 }
2309 case UTUN_OPT_TX_FSW_RING_SIZE: {
2310 if (*len != sizeof(u_int32_t)) {
2311 result = EMSGSIZE;
2312 } else {
2313 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2314 }
2315 break;
2316 }
2317 case UTUN_OPT_RX_FSW_RING_SIZE: {
2318 if (*len != sizeof(u_int32_t)) {
2319 result = EMSGSIZE;
2320 } else {
2321 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
5c9f4661 2322 }
0a7de745
A		 2323 break;
2324 }
5ba3f43e 2325#endif // UTUN_NEXUS
39037602 2326
0a7de745
A		 2327 default:
2328 result = ENOPROTOOPT;
2329 break;
b0d623f7 2330 }
0a7de745 2331
b0d623f7
A		 2332 return result;
2333}
2334
fe8ab488
A		 2335static void
2336utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2337{
3e170ce0 2338#pragma unused(flags)
fe8ab488
A		 2339 bool reenable_output = false;
2340 struct utun_pcb *pcb = unitinfo;
2341 if (pcb == NULL) {
2342 return;
2343 }
2344 ifnet_lock_exclusive(pcb->utun_ifp);
3e170ce0
A		 2345
2346 u_int32_t utun_packet_cnt;
2347 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2348 if (error_pc != 0) {
2349 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2350 utun_packet_cnt = 0;
fe8ab488 2351 }
3e170ce0
A		 2352
2353 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2354 reenable_output = true;
2355 }
2356
fe8ab488
A		 2357 if (reenable_output) {
2358 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2359 if (error != 0) {
2360 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2361 }
2362 }
2363 ifnet_lock_done(pcb->utun_ifp);
2364}
2365
b0d623f7 2366/* Network Interface functions */
fe8ab488
A		 2367static void
2368utun_start(ifnet_t interface)
2369{
2370 mbuf_t data;
39037602
A		 2371 struct utun_pcb *pcb = ifnet_softc(interface);
2372
2373 VERIFY(pcb != NULL);
2374
5ba3f43e
A		 2375#if UTUN_NEXUS
2376 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2377 if (pcb->utun_kpipe_enabled) {
2378 /* It's possible to have channels enabled, but not yet have the channel opened,
2379 * in which case the rxring will not be set
2380 */
2381 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2382 if (pcb->utun_kpipe_rxring != NULL) {
2383 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2384 }
2385 return;
2386 }
2387 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2388#endif // UTUN_NEXUS
39037602 2389
fe8ab488
A		 2390 for (;;) {
2391 bool can_accept_packets = true;
2392 ifnet_lock_shared(pcb->utun_ifp);
3e170ce0
A		 2393
2394 u_int32_t utun_packet_cnt;
2395 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2396 if (error_pc != 0) {
2397 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2398 utun_packet_cnt = 0;
2399 }
2400
2401 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
fe8ab488
A		 2402 if (!can_accept_packets && pcb->utun_ctlref) {
2403 u_int32_t difference = 0;
2404 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2405 if (difference > 0) {
2406 // If the low-water mark has not yet been reached, we still need to enqueue data
2407 // into the buffer
2408 can_accept_packets = true;
2409 }
2410 }
2411 }
2412 if (!can_accept_packets) {
2413 errno_t error = ifnet_disable_output(interface);
2414 if (error != 0) {
2415 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2416 }
2417 ifnet_lock_done(pcb->utun_ifp);
2418 break;
2419 }
2420 ifnet_lock_done(pcb->utun_ifp);
5ba3f43e 2421 if (ifnet_dequeue(interface, &data) != 0) {
fe8ab488 2422 break;
5ba3f43e
A		 2423 }
2424 if (utun_output(interface, data) != 0) {
fe8ab488 2425 break;
5ba3f43e 2426 }
fe8ab488
A		 2427 }
2428}
2429
b0d623f7 2430static errno_t
0a7de745
A		 2431utun_output(ifnet_t interface,
2432 mbuf_t data)
b0d623f7 2433{
0a7de745 2434 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2435 errno_t result;
39037602
A		 2436
2437 VERIFY(interface == pcb->utun_ifp);
5ba3f43e 2438
5c9f4661
A		 2439#if UTUN_NEXUS
2440 if (!pcb->utun_use_netif)
2441#endif // UTUN_NEXUS
2442 {
2443 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2444 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2445 }
2446 }
2447
d1ecb069 2448 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
b0d623f7
A		 2449 /* flush data */
2450 mbuf_freem(data);
2451 return 0;
2452 }
2453
316670eb 2454 // otherwise, fall thru to ctl_enqueumbuf
d1ecb069 2455 if (pcb->utun_ctlref) {
0a7de745 2456 int length;
316670eb 2457
39236c6e
A		 2458 /*
2459 * The ABI requires the protocol in network byte order
2460 */
39037602 2461 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
39236c6e 2462 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
39037602 2463 }
39236c6e 2464
316670eb 2465 length = mbuf_pkthdr_len(data);
d1ecb069 2466 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
b0d623f7
A		 2467 if (result != 0) {
2468 mbuf_freem(data);
2469 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
5c9f4661
A		 2470#if UTUN_NEXUS
2471 if (!pcb->utun_use_netif)
2472#endif // UTUN_NEXUS
2473 {
2474 ifnet_stat_increment_out(interface, 0, 0, 1);
2475 }
5ba3f43e 2476 } else {
5c9f4661
A		 2477#if UTUN_NEXUS
2478 if (!pcb->utun_use_netif)
2479#endif // UTUN_NEXUS
2480 {
2481 if (!pcb->utun_ext_ifdata_stats) {
2482 ifnet_stat_increment_out(interface, 1, length, 0);
2483 }
5ba3f43e 2484 }
b0d623f7 2485 }
5ba3f43e 2486 } else {
b0d623f7 2487 mbuf_freem(data);
5ba3f43e 2488 }
0a7de745 2489
b0d623f7
A		 2490 return 0;
2491}
2492
b0d623f7 2493static errno_t
5ba3f43e 2494utun_demux(__unused ifnet_t interface,
0a7de745
A		 2495 mbuf_t data,
2496 __unused char *frame_header,
2497 protocol_family_t *protocol)
b0d623f7 2498{
d9a64523 2499#if UTUN_NEXUS
0a7de745 2500 struct utun_pcb *pcb = ifnet_softc(interface);
d9a64523
A		 2501 struct ip *ip;
2502 u_int ip_version;
2503#endif
2504
b0d623f7
A		 2505 while (data != NULL && mbuf_len(data) < 1) {
2506 data = mbuf_next(data);
2507 }
5ba3f43e 2508
5c9f4661 2509 if (data == NULL) {
b0d623f7 2510 return ENOENT;
5c9f4661 2511 }
5ba3f43e 2512
5c9f4661
A		 2513#if UTUN_NEXUS
2514 if (pcb->utun_use_netif) {
2515 ip = mtod(data, struct ip *);
2516 ip_version = ip->ip_v;
2517
0a7de745
A		 2518 switch (ip_version) {
2519 case 4:
2520 *protocol = PF_INET;
2521 return 0;
2522 case 6:
2523 *protocol = PF_INET6;
2524 return 0;
2525 default:
2526 *protocol = 0;
2527 break;
5c9f4661
A		 2528 }
2529 } else
2530#endif // UTUN_NEXUS
2531 {
2532 *protocol = *(u_int32_t *)mbuf_data(data);
5ba3f43e
A		 2533 }
2534
b0d623f7
A		 2535 return 0;
2536}
2537
2538static errno_t
5ba3f43e 2539utun_framer(ifnet_t interface,
0a7de745
A		 2540 mbuf_t *packet,
2541 __unused const struct sockaddr *dest,
2542 __unused const char *desk_linkaddr,
2543 const char *frame_type,
2544 u_int32_t *prepend_len,
2545 u_int32_t *postpend_len)
b0d623f7 2546{
0a7de745 2547 struct utun_pcb *pcb = ifnet_softc(interface);
39037602
A		 2548 VERIFY(interface == pcb->utun_ifp);
2549
2550 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
d9a64523 2551 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
b0d623f7 2552 printf("utun_framer - ifnet_output prepend failed\n");
d1ecb069 2553
b0d623f7 2554 ifnet_stat_increment_out(interface, 0, 0, 1);
d1ecb069 2555
b0d623f7 2556 // just return, because the buffer was freed in mbuf_prepend
0a7de745 2557 return EJUSTRETURN;
d9a64523 2558 }
5ba3f43e 2559 if (prepend_len != NULL) {
39037602 2560 *prepend_len = header_length;
5ba3f43e
A		 2561 }
2562 if (postpend_len != NULL) {
39236c6e 2563 *postpend_len = 0;
5ba3f43e 2564 }
0a7de745 2565
d9a64523
A		 2566 // place protocol number at the beginning of the mbuf
2567 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
39037602
A		 2568
2569
0a7de745 2570 return 0;
b0d623f7
A		 2571}
2572
2573static errno_t
5ba3f43e 2574utun_add_proto(__unused ifnet_t interface,
0a7de745
A		 2575 protocol_family_t protocol,
2576 __unused const struct ifnet_demux_desc *demux_array,
2577 __unused u_int32_t demux_count)
b0d623f7 2578{
0a7de745
A		 2579 switch (protocol) {
2580 case PF_INET:
2581 return 0;
2582 case PF_INET6:
2583 return 0;
2584 default:
2585 break;
b0d623f7 2586 }
0a7de745 2587
b0d623f7
A		 2588 return ENOPROTOOPT;
2589}
2590
2591static errno_t
5ba3f43e 2592utun_del_proto(__unused ifnet_t interface,
0a7de745 2593 __unused protocol_family_t protocol)
b0d623f7
A		 2594{
2595 return 0;
2596}
2597
2598static errno_t
5ba3f43e 2599utun_ioctl(ifnet_t interface,
0a7de745
A		 2600 u_long command,
2601 void *data)
b0d623f7 2602{
d9a64523 2603#if UTUN_NEXUS
0a7de745 2604 struct utun_pcb *pcb = ifnet_softc(interface);
d9a64523 2605#endif
0a7de745
A		 2606 errno_t result = 0;
2607
2608 switch (command) {
2609 case SIOCSIFMTU: {
5ba3f43e 2610#if UTUN_NEXUS
0a7de745
A		 2611 if (pcb->utun_use_netif) {
2612 // Make sure we can fit packets in the channel buffers
2613 // Allow for the headroom in the slot
2614 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2615 result = EINVAL;
2616 } else {
2617 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
5c9f4661 2618 }
0a7de745
A		 2619 } else
2620#endif // UTUN_NEXUS
2621 {
2622 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
5c9f4661 2623 }
0a7de745 2624 break;
b0d623f7 2625 }
0a7de745
A		 2626
2627 case SIOCSIFFLAGS:
2628 /* ifioctl() takes care of it */
2629 break;
2630
2631 default:
2632 result = EOPNOTSUPP;
2633 }
2634
b0d623f7
A		 2635 return result;
2636}
2637
2638static void
5ba3f43e 2639utun_detached(ifnet_t interface)
b0d623f7 2640{
0a7de745 2641 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2642 (void)ifnet_release(interface);
5c9f4661 2643 utun_free_pcb(pcb, true);
b0d623f7
A		 2644}
2645
2646/* Protocol Handlers */
2647
2648static errno_t
5ba3f43e 2649utun_proto_input(__unused ifnet_t interface,
0a7de745
A		 2650 protocol_family_t protocol,
2651 mbuf_t m,
2652 __unused char *frame_header)
b0d623f7 2653{
0a7de745 2654 struct utun_pcb *pcb = ifnet_softc(interface);
5c9f4661
A		 2655#if UTUN_NEXUS
2656 if (!pcb->utun_use_netif)
2657#endif // UTUN_NEXUS
2658 {
2659 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2660 }
0a7de745 2661 int32_t pktlen = m->m_pkthdr.len;
39037602 2662 if (proto_input(protocol, m) != 0) {
6d2010ae 2663 m_freem(m);
5c9f4661
A		 2664#if UTUN_NEXUS
2665 if (!pcb->utun_use_netif)
2666#endif // UTUN_NEXUS
2667 {
2668 ifnet_stat_increment_in(interface, 0, 0, 1);
2669 }
5ba3f43e 2670 } else {
5c9f4661
A		 2671#if UTUN_NEXUS
2672 if (!pcb->utun_use_netif)
5ba3f43e 2673#endif // UTUN_NEXUS
5c9f4661 2674 {
a39ff7e2 2675 ifnet_stat_increment_in(interface, 1, pktlen, 0);
5c9f4661 2676 }
39037602 2677 }
0a7de745 2678
b0d623f7
A		 2679 return 0;
2680}
2681
2682static errno_t
5ba3f43e 2683utun_proto_pre_output(__unused ifnet_t interface,
0a7de745
A		 2684 protocol_family_t protocol,
2685 __unused mbuf_t *packet,
2686 __unused const struct sockaddr *dest,
2687 __unused void *route,
2688 char *frame_type,
2689 __unused char *link_layer_dest)
b0d623f7 2690{
6d2010ae 2691 *(protocol_family_t *)(void *)frame_type = protocol;
39037602 2692 return 0;
b0d623f7
A		 2693}
2694
2695static errno_t
5ba3f43e 2696utun_attach_proto(ifnet_t interface,
0a7de745 2697 protocol_family_t protocol)
b0d623f7 2698{
0a7de745
A		 2699 struct ifnet_attach_proto_param proto;
2700
b0d623f7
A		 2701 bzero(&proto, sizeof(proto));
2702 proto.input = utun_proto_input;
2703 proto.pre_output = utun_proto_pre_output;
2704
5ba3f43e 2705 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
b0d623f7
A		 2706 if (result != 0 && result != EEXIST) {
2707 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
0a7de745 2708 protocol, result);
b0d623f7 2709 }
0a7de745 2710
b0d623f7
A		 2711 return result;
2712}
2713
5ba3f43e
A		 2714static errno_t
2715utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2716{
5c9f4661
A		 2717#if UTUN_NEXUS
2718 if (pcb->utun_use_netif) {
2719 lck_rw_lock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2720
5c9f4661
A		 2721 lck_mtx_lock(&pcb->utun_input_chain_lock);
2722 if (pcb->utun_input_chain != NULL) {
2723 pcb->utun_input_chain_last->m_nextpkt = packet;
2724 } else {
2725 pcb->utun_input_chain = packet;
2726 }
2727 while (packet->m_nextpkt) {
2728 VERIFY(packet != packet->m_nextpkt);
2729 packet = packet->m_nextpkt;
2730 }
2731 pcb->utun_input_chain_last = packet;
2732 lck_mtx_unlock(&pcb->utun_input_chain_lock);
5ba3f43e 2733
5c9f4661
A		 2734 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2735 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
316670eb 2736
5c9f4661
A		 2737 if (rx_ring != NULL) {
2738 kern_channel_notify(rx_ring, 0);
2739 }
316670eb 2740
0a7de745 2741 return 0;
5c9f4661
A		 2742 } else
2743#endif // IPSEC_NEXUS
2744 {
2745 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
5ba3f43e 2746
0a7de745 2747 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
5c9f4661
A		 2748 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2749 }
2750 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2751 /* flush data */
2752 mbuf_freem(packet);
2753 return 0;
2754 }
316670eb 2755
0a7de745 2756 errno_t result = 0;
5c9f4661
A		 2757 if (!pcb->utun_ext_ifdata_stats) {
2758 struct ifnet_stat_increment_param incs = {};
2759 incs.packets_in = 1;
2760 incs.bytes_in = mbuf_pkthdr_len(packet);
2761 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2762 } else {
2763 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2764 }
2765 if (result != 0) {
2766 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
5ba3f43e 2767
5c9f4661
A		 2768 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2769 mbuf_freem(packet);
2770 }
5ba3f43e 2771
0a7de745 2772 return 0;
316670eb 2773 }
316670eb 2774}
5ba3f43e
A		 2775
2776#if UTUN_NEXUS
2777
2778static errno_t
2779utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2780{
2781 return 0;
2782}
2783
2784static void
2785utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2786{
2787 // Ignore
2788}
2789
2790static errno_t
2791utun_register_nexus(void)
2792{
2793 const struct kern_nexus_domain_provider_init dp_init = {
2794 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2795 .nxdpi_flags = 0,
2796 .nxdpi_init = utun_nxdp_init,
2797 .nxdpi_fini = utun_nxdp_fini
2798 };
2799 errno_t err = 0;
2800
2801 /* utun_nxdp_init() is called before this function returns */
2802 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
0a7de745
A		 2803 (const uint8_t *) "com.apple.utun",
2804 &dp_init, sizeof(dp_init),
2805 &utun_nx_dom_prov);
5ba3f43e
A		 2806 if (err != 0) {
2807 printf("%s: failed to register domain provider\n", __func__);
0a7de745 2808 return err;
5ba3f43e 2809 }
0a7de745 2810 return 0;
5ba3f43e 2811}
9d749ea3
A		 2812boolean_t
2813utun_interface_needs_netagent(ifnet_t interface)
2814{
2815 struct utun_pcb *pcb = NULL;
2816
2817 if (interface == NULL) {
0a7de745 2818 return FALSE;
9d749ea3
A		 2819 }
2820
2821 pcb = ifnet_softc(interface);
2822
2823 if (pcb == NULL) {
0a7de745 2824 return FALSE;
9d749ea3
A		 2825 }
2826
0a7de745 2827 return pcb->utun_needs_netagent == true;
9d749ea3 2828}
5ba3f43e
A		 2829
2830static errno_t
2831utun_ifnet_set_attrs(ifnet_t ifp)
2832{
2833 /* Set flags and additional information. */
2834 ifnet_set_mtu(ifp, 1500);
2835 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2836
2837 /* The interface must generate its own IPv6 LinkLocal address,
2838 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2839 */
2840 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2841
0a7de745 2842 return 0;
5ba3f43e
A		 2843}
2844
2845static errno_t
2846utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2847{
2848 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2849 pcb->utun_netif_nexus = nexus;
0a7de745 2850 return utun_ifnet_set_attrs(ifp);
5ba3f43e
A		 2851}
2852
2853static errno_t
2854utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2855 proc_t p, kern_nexus_t nexus,
2856 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2857{
2858#pragma unused(nxprov, p)
2859#pragma unused(nexus, nexus_port, channel, ch_ctx)
0a7de745 2860 return 0;
5ba3f43e
A		 2861}
2862
2863static errno_t
2864utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2865 kern_channel_t channel)
2866{
2867#pragma unused(nxprov, channel)
2868 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2869 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
0a7de745 2870 return ok ? 0 : ENXIO;
5ba3f43e
A		 2871}
2872
2873static void
2874utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2875 kern_channel_t channel)
2876{
2877#pragma unused(nxprov, nexus, channel)
2878}
2879
2880static void
2881utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 2882 kern_channel_t channel)
5ba3f43e
A		 2883{
2884#pragma unused(nxprov, nexus, channel)
2885}
2886
2887static void
2888utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2889 kern_channel_t channel)
2890{
2891#pragma unused(nxprov, channel)
2892 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2893 if (pcb->utun_netif_nexus == nexus) {
2894 pcb->utun_netif_nexus = NULL;
2895 }
2896 ifnet_decr_iorefcnt(pcb->utun_ifp);
2897}
2898
2899static errno_t
2900utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 2901 kern_channel_t channel, kern_channel_ring_t ring,
2902 boolean_t is_tx_ring, void **ring_ctx)
5ba3f43e
A		 2903{
2904#pragma unused(nxprov)
2905#pragma unused(channel)
2906#pragma unused(ring_ctx)
2907 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2908 if (!is_tx_ring) {
2909 VERIFY(pcb->utun_kpipe_rxring == NULL);
2910 pcb->utun_kpipe_rxring = ring;
2911 } else {
2912 VERIFY(pcb->utun_kpipe_txring == NULL);
2913 pcb->utun_kpipe_txring = ring;
2914 }
2915 return 0;
2916}
2917
2918static void
2919utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 2920 kern_channel_ring_t ring)
5ba3f43e
A		 2921{
2922#pragma unused(nxprov)
2923 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2924 if (pcb->utun_kpipe_rxring == ring) {
2925 pcb->utun_kpipe_rxring = NULL;
2926 } else if (pcb->utun_kpipe_txring == ring) {
2927 pcb->utun_kpipe_txring = NULL;
2928 }
2929}
2930
2931static errno_t
2932utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 2933 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 2934{
2935#pragma unused(nxprov)
2936#pragma unused(flags)
2937 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2938
2939 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2940 int channel_enabled = pcb->utun_kpipe_enabled;
2941 if (!channel_enabled) {
2942 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2943 return 0;
2944 }
2945
5c9f4661
A		 2946 if (pcb->utun_use_netif) {
2947 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2948 if (tx_slot == NULL) {
2949 // Nothing to write, bail
2950 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2951 return 0;
2952 }
2953
2954 // Signal the netif ring to read
2955 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2956 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2957 if (rx_ring != NULL) {
2958 kern_channel_notify(rx_ring, 0);
2959 }
2960 } else {
5ba3f43e 2961 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2962
5c9f4661
A		 2963 struct ifnet_stat_increment_param incs = {};
2964 struct kern_channel_ring_stat_increment tx_ring_stats = {};
2965 MBUFQ_HEAD(mbufq) mbq;
2966 MBUFQ_INIT(&mbq);
2967 kern_channel_slot_t tx_pslot = NULL;
2968 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2969 while (tx_slot != NULL) {
2970 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2971
2972 // Advance TX ring
2973 tx_pslot = tx_slot;
2974 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2975
2976 if (tx_ph == 0) {
2977 continue;
2978 }
2979
2980 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2981 VERIFY(tx_buf != NULL);
2982 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2983 VERIFY(tx_baddr != 0);
2984 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2985
2986 size_t length = MIN(kern_packet_get_data_length(tx_ph),
0a7de745 2987 pcb->utun_slot_size);
5c9f4661
A		 2988
2989 mbuf_t data = NULL;
2990 if (length >= UTUN_HEADER_SIZE(pcb) &&
0a7de745 2991 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
5c9f4661
A		 2992 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
2993 VERIFY(0 == error);
2994 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
2995 VERIFY(0 == error);
2996 /*
2997 * The userland ABI requires the first four bytes have
2998 * the protocol family in network byte order: swap them
2999 */
3000 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
3001 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
3002 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3003 incs.packets_in++;
3004 incs.bytes_in += length;
3005 MBUFQ_ENQUEUE(&mbq, data);
3006 }
3007 }
3008 if (tx_pslot) {
3009 kern_channel_advance_slot(tx_ring, tx_pslot);
3010 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
3011 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3012 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3013 (void) kern_channel_reclaim(tx_ring);
3014 }
3015 if (!MBUFQ_EMPTY(&mbq)) {
3016 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
0a7de745 3017 MBUFQ_LAST(&mbq), &incs);
5c9f4661
A		 3018 MBUFQ_INIT(&mbq);
3019 }
5ba3f43e
A		 3020 }
3021
3022 return 0;
3023}
3024
3025static errno_t
3026utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 3027 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 3028{
3029#pragma unused(nxprov)
3030#pragma unused(flags)
3031 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5c9f4661 3032 struct kern_channel_ring_stat_increment rx_ring_stats = {};
5ba3f43e
A		 3033
3034 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3035
3036 int channel_enabled = pcb->utun_kpipe_enabled;
3037 if (!channel_enabled) {
3038 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3039 return 0;
3040 }
3041
3042 /* reclaim user-released slots */
3043 (void) kern_channel_reclaim(rx_ring);
3044
3045 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3046 if (avail == 0) {
3047 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3048 return 0;
3049 }
3050
5c9f4661
A		 3051 if (pcb->utun_use_netif) {
3052 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3053 if (tx_ring == NULL ||
0a7de745 3054 pcb->utun_netif_nexus == NULL) {
5c9f4661
A		 3055 // Net-If TX ring not set up yet, nothing to read
3056 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3057 return 0;
3058 }
5ba3f43e 3059
5c9f4661 3060 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
5ba3f43e 3061
5c9f4661 3062 // Unlock utun before entering ring
5ba3f43e 3063 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3064
5c9f4661 3065 (void)kr_enter(tx_ring, TRUE);
5ba3f43e 3066
5c9f4661
A		 3067 // Lock again after entering and validate
3068 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3069 if (tx_ring != pcb->utun_netif_txring) {
3070 // Ring no longer valid
3071 // Unlock first, then exit ring
3072 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3073 kr_exit(tx_ring);
3074 return 0;
5ba3f43e
A		 3075 }
3076
5c9f4661
A		 3077 struct kern_channel_ring_stat_increment tx_ring_stats;
3078 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3079 kern_channel_slot_t tx_pslot = NULL;
3080 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3081 if (tx_slot == NULL) {
3082 // Nothing to read, don't bother signalling
3083 // Unlock first, then exit ring
3084 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3085 kr_exit(tx_ring);
3086 return 0;
5ba3f43e
A		 3087 }
3088
5c9f4661
A		 3089 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3090 VERIFY(rx_pp != NULL);
3091 kern_channel_slot_t rx_pslot = NULL;
3092 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
5ba3f43e 3093
5c9f4661
A		 3094 while (rx_slot != NULL && tx_slot != NULL) {
3095 size_t length;
3096 kern_buflet_t rx_buf;
3097 void *rx_baddr;
5ba3f43e 3098
5c9f4661 3099 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
5ba3f43e 3100
5c9f4661
A		 3101 // Advance TX ring
3102 tx_pslot = tx_slot;
3103 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
5ba3f43e 3104
5c9f4661
A		 3105 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3106 if (tx_ph == 0) {
3107 continue;
3108 }
5ba3f43e 3109
5c9f4661
A		 3110 // Allocate rx packet
3111 kern_packet_t rx_ph = 0;
3112 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 3113 if (__improbable(error != 0)) {
5c9f4661 3114 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
0a7de745 3115 pcb->utun_ifp->if_xname);
5ba3f43e
A		 3116 break;
3117 }
5c9f4661
A		 3118
3119 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3120 VERIFY(tx_buf != NULL);
3121 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3122 VERIFY(tx_baddr != NULL);
3123 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3124
3125 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3126
3127 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
0a7de745 3128 pcb->utun_slot_size);
5c9f4661
A		 3129
3130 tx_ring_stats.kcrsi_slots_transferred++;
3131 tx_ring_stats.kcrsi_bytes_transferred += length;
3132
3133 if (length < UTUN_HEADER_SIZE(pcb) ||
0a7de745
A		 3134 length > pcb->utun_slot_size ||
3135 length > rx_pp->pp_buflet_size ||
3136 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
5c9f4661
A		 3137 /* flush data */
3138 kern_pbufpool_free(rx_pp, rx_ph);
3139 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
0a7de745 3140 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
5c9f4661
A		 3141 STATS_INC(nifs, NETIF_STATS_BADLEN);
3142 STATS_INC(nifs, NETIF_STATS_DROPPED);
3143 continue;
5ba3f43e 3144 }
5c9f4661
A		 3145
3146 /* fillout packet */
3147 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3148 VERIFY(rx_buf != NULL);
3149 rx_baddr = kern_buflet_get_object_address(rx_buf);
3150 VERIFY(rx_baddr != NULL);
3151
3152 // Find family
3153 uint32_t af = 0;
3154 uint8_t vhl = *(uint8_t *)(tx_baddr);
3155 u_int ip_version = (vhl >> 4);
3156 switch (ip_version) {
0a7de745
A		 3157 case 4: {
3158 af = AF_INET;
3159 break;
3160 }
3161 case 6: {
3162 af = AF_INET6;
3163 break;
3164 }
3165 default: {
3166 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3167 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3168 break;
3169 }
5ba3f43e 3170 }
5ba3f43e 3171
5c9f4661
A		 3172 // Copy header
3173 af = htonl(af);
3174 memcpy((void *)rx_baddr, &af, sizeof(af));
3175 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3176 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3177 }
5ba3f43e 3178
5c9f4661
A		 3179 // Copy data from tx to rx
3180 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3181 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
5ba3f43e 3182
5c9f4661
A		 3183 /* finalize and attach the packet */
3184 error = kern_buflet_set_data_offset(rx_buf, 0);
3185 VERIFY(error == 0);
3186 error = kern_buflet_set_data_length(rx_buf, length);
3187 VERIFY(error == 0);
3188 error = kern_packet_finalize(rx_ph);
3189 VERIFY(error == 0);
3190 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3191 VERIFY(error == 0);
5ba3f43e 3192
5c9f4661
A		 3193 STATS_INC(nifs, NETIF_STATS_TXPKTS);
3194 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
5ba3f43e 3195
5c9f4661
A		 3196 rx_ring_stats.kcrsi_slots_transferred++;
3197 rx_ring_stats.kcrsi_bytes_transferred += length;
5ba3f43e 3198
5c9f4661
A		 3199 rx_pslot = rx_slot;
3200 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3201 }
5ba3f43e 3202
5c9f4661
A		 3203 if (rx_pslot) {
3204 kern_channel_advance_slot(rx_ring, rx_pslot);
3205 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3206 }
5ba3f43e 3207
5c9f4661
A		 3208 if (tx_pslot) {
3209 kern_channel_advance_slot(tx_ring, tx_pslot);
3210 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3211 (void)kern_channel_reclaim(tx_ring);
3212 }
5ba3f43e 3213
5c9f4661 3214 /* just like utun_ctl_rcvd(), always reenable output */
5ba3f43e
A		 3215 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3216 if (error != 0) {
3217 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
5ba3f43e 3218 }
5ba3f43e 3219
5c9f4661
A		 3220 // Unlock first, then exit ring
3221 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3222
5c9f4661
A		 3223 if (tx_pslot != NULL) {
3224 kern_channel_notify(tx_ring, 0);
3225 }
3226 kr_exit(tx_ring);
3227 } else {
3228 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3229
3230 uint32_t mb_cnt = 0;
3231 uint32_t mb_len = 0;
3232 struct mbuf *mb_head = NULL;
3233 struct mbuf *mb_tail = NULL;
3234
3235 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
0a7de745 3236 &mb_tail, &mb_cnt, &mb_len) != 0) {
5c9f4661
A		 3237 return 0;
3238 }
3239 VERIFY(mb_cnt <= avail);
3240
3241 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3242 VERIFY(rx_pp != NULL);
3243 kern_channel_slot_t rx_pslot = NULL;
3244 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3245 while (rx_slot) {
3246 size_t length = 0;
3247 mbuf_t data = NULL;
3248 if ((data = mb_head) == NULL) {
3249 VERIFY(mb_cnt == 0);
3250 break;
3251 }
3252 mb_head = mbuf_nextpkt(mb_head);
3253 mbuf_setnextpkt(data, NULL);
3254 VERIFY(mb_cnt != 0);
3255 --mb_cnt;
3256 length = mbuf_pkthdr_len(data);
3257 if (length < UTUN_HEADER_SIZE(pcb) ||
0a7de745
A		 3258 length > pcb->utun_slot_size ||
3259 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
5c9f4661
A		 3260 /* flush data */
3261 mbuf_freem(data);
3262 continue;
3263 }
3264 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3265
3266 // Allocate rx packet
3267 kern_packet_t rx_ph = 0;
3268 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 3269 if (__improbable(error != 0)) {
5c9f4661 3270 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
0a7de745 3271 pcb->utun_ifp->if_xname);
5c9f4661
A		 3272 break;
3273 }
3274
3275 /*
3276 * The ABI requires the protocol in network byte order
3277 */
3278 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3279
3280 // Fillout rx packet
3281 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3282 VERIFY(rx_buf != NULL);
3283 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3284 VERIFY(rx_baddr != NULL);
3285
3286 // Copy-in data from mbuf to buflet
3287 mbuf_copydata(data, 0, length, (void *)rx_baddr);
0a7de745 3288 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5c9f4661
A		 3289
3290 // Finalize and attach the packet
3291 error = kern_buflet_set_data_offset(rx_buf, 0);
3292 VERIFY(error == 0);
3293 error = kern_buflet_set_data_length(rx_buf, length);
3294 VERIFY(error == 0);
3295 error = kern_packet_finalize(rx_ph);
3296 VERIFY(error == 0);
3297 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3298 VERIFY(error == 0);
3299
3300 rx_ring_stats.kcrsi_slots_transferred++;
3301 rx_ring_stats.kcrsi_bytes_transferred += length;
3302
3303 if (!pcb->utun_ext_ifdata_stats) {
3304 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3305 }
3306
3307 mbuf_freem(data);
3308
3309 rx_pslot = rx_slot;
3310 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3311 }
3312 if (rx_pslot) {
3313 kern_channel_advance_slot(rx_ring, rx_pslot);
3314 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3315 }
3316 if (mb_head != NULL) {
3317 VERIFY(mb_cnt != 0);
3318 mbuf_freem_list(mb_head);
3319 }
5ba3f43e 3320 }
5ba3f43e
A		 3321
3322 return 0;
3323}
39037602 3324
5ba3f43e 3325#endif // UTUN_NEXUS
39037602
A		 3326
3327
3328/*
5ba3f43e 3329 * These are place holders until coreTLS kext stops calling them
39037602 3330 */
0a7de745 3331errno_t utun_ctl_register_dtls(void *reg);
39037602
A		 3332int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3333void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3334
3335errno_t
0a7de745 3336utun_ctl_register_dtls(void *reg)
39037602
A		 3337{
3338#pragma unused(reg)
3339 return 0;
3340}
3341
3342int
3343utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3344{
3345#pragma unused(pcb)
3346#pragma unused(pkt)
3347#pragma unused(family)
3348 return 0;
3349}
3350
3351void
3352utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3353{
3354#pragma unused(pcb)
3355}
mirror of XNU