]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_utun.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.31.3.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
CommitLineData
b0d623f7 1/*
5ba3f43e 2 * Copyright (c) 2008-2017 Apple Inc. All rights reserved.
b0d623f7
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31/* ----------------------------------------------------------------------------------
32Application of kernel control for interface creation
33
34Theory of operation:
35utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36This kernel control will register an interface for every client that connects.
37---------------------------------------------------------------------------------- */
38
39#include <sys/systm.h>
40#include <sys/kern_control.h>
41#include <net/kpi_protocol.h>
42#include <net/kpi_interface.h>
43#include <sys/socket.h>
44#include <net/if.h>
45#include <net/if_types.h>
46#include <net/bpf.h>
47#include <net/if_utun.h>
b0d623f7
A		 48#include <sys/mbuf.h>
49#include <sys/sockio.h>
50#include <netinet/in.h>
5ba3f43e 51#include <netinet/ip.h>
b0d623f7
A		 52#include <netinet6/in6_var.h>
53#include <netinet6/in6_var.h>
54#include <sys/kauth.h>
5ba3f43e
A		 55#include <net/necp.h>
56#include <kern/zalloc.h>
57
58#define UTUN_NEXUS 0
59
60extern unsigned int if_enable_netagent;
61
62#if UTUN_NEXUS
63static nexus_controller_t utun_ncd;
64static int utun_ncd_refcount;
65static uuid_t utun_kpipe_uuid;
66static uuid_t utun_nx_dom_prov;
67
68typedef struct utun_nx {
69 uuid_t if_provider;
70 uuid_t if_instance;
71 uuid_t ms_provider;
72 uuid_t ms_instance;
73 uuid_t ms_device;
74 uuid_t ms_host;
75 uuid_t ms_agent;
76} *utun_nx_t;
77
78#endif // UTUN_NEXUS
79
80/* Control block allocated for each kernel control connection */
81struct utun_pcb {
82 TAILQ_ENTRY(utun_pcb) utun_chain;
83 kern_ctl_ref utun_ctlref;
84 ifnet_t utun_ifp;
85 u_int32_t utun_unit;
86 u_int32_t utun_unique_id;
87 u_int32_t utun_flags;
88 int utun_ext_ifdata_stats;
89 u_int32_t utun_max_pending_packets;
90 char utun_if_xname[IFXNAMSIZ];
91 char utun_unique_name[IFXNAMSIZ];
92 // PCB lock protects state fields and rings
93 decl_lck_rw_data(, utun_pcb_lock);
94 struct mbuf * utun_input_chain;
95 struct mbuf * utun_input_chain_last;
96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
98 lck_mtx_t utun_input_chain_lock;
5ba3f43e
A		 99
100#if UTUN_NEXUS
101 struct utun_nx utun_nx;
102 int utun_kpipe_enabled;
103 uuid_t utun_kpipe_uuid;
104 void * utun_kpipe_rxring;
105 void * utun_kpipe_txring;
106
107 kern_nexus_t utun_netif_nexus;
108 void * utun_netif_rxring;
109 void * utun_netif_txring;
110 uint64_t utun_netif_txring_size;
5c9f4661
A		 111
112 u_int32_t utun_slot_size;
113 u_int32_t utun_netif_ring_size;
114 u_int32_t utun_tx_fsw_ring_size;
115 u_int32_t utun_rx_fsw_ring_size;
116 bool utun_use_netif;
5ba3f43e
A		 117#endif // UTUN_NEXUS
118};
39037602 119
b0d623f7 120/* Kernel Control functions */
5c9f4661
A		 121static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
122 void **unitinfo);
b0d623f7
A		 123static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
124 void **unitinfo);
125static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
126 void *unitinfo);
127static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
128 void *unitinfo, mbuf_t m, int flags);
129static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
130 int opt, void *data, size_t *len);
131static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
132 int opt, void *data, size_t len);
fe8ab488
A		 133static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
134 int flags);
b0d623f7
A		 135
136/* Network Interface functions */
fe8ab488 137static void utun_start(ifnet_t interface);
5ba3f43e
A		 138static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
139 const struct sockaddr *dest, const char *desk_linkaddr,
140 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
b0d623f7
A		 141static errno_t utun_output(ifnet_t interface, mbuf_t data);
142static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
143 protocol_family_t *protocol);
b0d623f7
A		 144static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
145 const struct ifnet_demux_desc *demux_array,
146 u_int32_t demux_count);
147static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
148static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
149static void utun_detached(ifnet_t interface);
150
151/* Protocol handlers */
152static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
153static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
154 mbuf_t m, char *frame_header);
155static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
156 mbuf_t *packet, const struct sockaddr *dest, void *route,
157 char *frame_type, char *link_layer_dest);
5c9f4661 158static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
39037602 159
5ba3f43e
A		 160#if UTUN_NEXUS
161
5c9f4661 162#define UTUN_IF_DEFAULT_SLOT_SIZE 2048
5ba3f43e
A		 163#define UTUN_IF_DEFAULT_RING_SIZE 64
164#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
165#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
166#define UTUN_IF_HEADROOM_SIZE 32
167
168#define UTUN_IF_MIN_RING_SIZE 16
169#define UTUN_IF_MAX_RING_SIZE 1024
170
5c9f4661
A		 171#define UTUN_IF_MIN_SLOT_SIZE 1024
172#define UTUN_IF_MAX_SLOT_SIZE 4096
173
5ba3f43e
A		 174static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
175static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
176static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
177
178static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
179static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
180static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
181
182SYSCTL_DECL(_net_utun);
183SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
184
185SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
186 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
187SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
188 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
189SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
190 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
191
192static errno_t
193utun_register_nexus(void);
194
195static errno_t
196utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
197static errno_t
198utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
199 proc_t p, kern_nexus_t nexus,
200 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
201static errno_t
202utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
203 kern_channel_t channel);
204static void
205utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
206 kern_channel_t channel);
207static void
208utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
209 kern_channel_t channel);
210static void
211utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
212 kern_channel_t channel);
213static errno_t
214utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
215 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
216 void **ring_ctx);
217static void
218utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
219 kern_channel_ring_t ring);
220static errno_t
221utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
222 kern_channel_ring_t ring, uint32_t flags);
223static errno_t
224utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
225 kern_channel_ring_t ring, uint32_t flags);
226#endif // UTUN_NEXUS
39037602
A		 227
228#define UTUN_DEFAULT_MTU 1500
229#define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
b0d623f7
A		 230
231static kern_ctl_ref utun_kctlref;
232static u_int32_t utun_family;
5ba3f43e
A		 233static lck_attr_t *utun_lck_attr;
234static lck_grp_attr_t *utun_lck_grp_attr;
235static lck_grp_t *utun_lck_grp;
236static lck_mtx_t utun_lock;
237
238TAILQ_HEAD(utun_list, utun_pcb) utun_head;
239
240#define UTUN_PCB_ZONE_MAX 32
241#define UTUN_PCB_ZONE_NAME "net.if_utun"
242
243static unsigned int utun_pcb_size; /* size of zone element */
244static struct zone *utun_pcb_zone; /* zone for utun_pcb */
245
246#if UTUN_NEXUS
247
248static int
249sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
250{
251#pragma unused(arg1, arg2)
252 int value = if_utun_ring_size;
253
254 int error = sysctl_handle_int(oidp, &value, 0, req);
255 if (error || !req->newptr) {
256 return (error);
257 }
258
259 if (value < UTUN_IF_MIN_RING_SIZE ||
260 value > UTUN_IF_MAX_RING_SIZE) {
261 return (EINVAL);
262 }
263
264 if_utun_ring_size = value;
265
266 return (0);
267}
268
269static int
270sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
271{
272#pragma unused(arg1, arg2)
273 int value = if_utun_tx_fsw_ring_size;
274
275 int error = sysctl_handle_int(oidp, &value, 0, req);
276 if (error || !req->newptr) {
277 return (error);
278 }
279
280 if (value < UTUN_IF_MIN_RING_SIZE ||
281 value > UTUN_IF_MAX_RING_SIZE) {
282 return (EINVAL);
283 }
284
285 if_utun_tx_fsw_ring_size = value;
286
287 return (0);
288}
289
290static int
291sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
292{
293#pragma unused(arg1, arg2)
294 int value = if_utun_rx_fsw_ring_size;
295
296 int error = sysctl_handle_int(oidp, &value, 0, req);
297 if (error || !req->newptr) {
298 return (error);
299 }
300
301 if (value < UTUN_IF_MIN_RING_SIZE ||
302 value > UTUN_IF_MAX_RING_SIZE) {
303 return (EINVAL);
304 }
305
306 if_utun_rx_fsw_ring_size = value;
307
308 return (0);
309}
310
311static errno_t
312utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
313 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
314 void **ring_ctx)
315{
316#pragma unused(nxprov)
317#pragma unused(channel)
318#pragma unused(ring_ctx)
319 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
320 if (!is_tx_ring) {
321 VERIFY(pcb->utun_netif_rxring == NULL);
322 pcb->utun_netif_rxring = ring;
323 } else {
324 VERIFY(pcb->utun_netif_txring == NULL);
325 pcb->utun_netif_txring = ring;
326 }
327 return 0;
328}
329
330static void
331utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
332 kern_channel_ring_t ring)
333{
334#pragma unused(nxprov)
335 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
336 if (pcb->utun_netif_rxring == ring) {
337 pcb->utun_netif_rxring = NULL;
338 } else if (pcb->utun_netif_txring == ring) {
339 pcb->utun_netif_txring = NULL;
340 }
341}
342
343static errno_t
344utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
345 kern_channel_ring_t tx_ring, uint32_t flags)
346{
347#pragma unused(nxprov)
348#pragma unused(flags)
349 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
350
351 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
352
353 lck_rw_lock_shared(&pcb->utun_pcb_lock);
354
355 struct kern_channel_ring_stat_increment tx_ring_stats;
356 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
357 kern_channel_slot_t tx_pslot = NULL;
358 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
359
360 STATS_INC(nifs, NETIF_STATS_TXSYNC);
361
362 if (tx_slot == NULL) {
363 // Nothing to write, don't bother signalling
364 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
365 return 0;
366 }
367
368 if (pcb->utun_kpipe_enabled) {
369 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
370 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
371
372 // Signal the kernel pipe ring to read
373 if (rx_ring != NULL) {
374 kern_channel_notify(rx_ring, 0);
375 }
376 return 0;
377 }
378
379 // If we're here, we're injecting into the utun kernel control socket
380 while (tx_slot != NULL) {
381 size_t length = 0;
382 mbuf_t data = NULL;
383
384 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
385
386 if (tx_ph == 0) {
387 // Advance TX ring
388 tx_pslot = tx_slot;
389 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
390 continue;
391 }
392 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
393
394 // Advance TX ring
395 tx_pslot = tx_slot;
396 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
397
398 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
399 VERIFY(tx_buf != NULL);
400
401 /* tx_baddr is the absolute buffer address */
402 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
403 VERIFY(tx_baddr != 0);
404
405 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
406
407 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
408 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
409
410 // The offset must be large enough for the headers
411 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
412
413 // Find family
414 uint32_t af = 0;
415 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
416 u_int ip_version = (vhl >> 4);
417 switch (ip_version) {
418 case 4: {
419 af = AF_INET;
420 break;
421 }
422 case 6: {
423 af = AF_INET6;
424 break;
425 }
426 default: {
427 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
428 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
429 UTUN_HEADER_SIZE(pcb));
430 break;
431 }
432 }
433
434 tx_offset -= UTUN_HEADER_SIZE(pcb);
435 tx_length += UTUN_HEADER_SIZE(pcb);
436 tx_baddr += tx_offset;
437
5c9f4661 438 length = MIN(tx_length, pcb->utun_slot_size);
5ba3f43e
A		 439
440 // Copy in family
441 memcpy(tx_baddr, &af, sizeof(af));
442 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
443 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
444 }
445
446 if (length > 0) {
447 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
448 if (error == 0) {
449 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
450 if (error == 0) {
451 error = utun_output(pcb->utun_ifp, data);
452 if (error != 0) {
453 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
454 }
455 } else {
456 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
457 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
458 STATS_INC(nifs, NETIF_STATS_DROPPED);
459 mbuf_freem(data);
460 data = NULL;
461 }
462 } else {
463 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
464 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
465 STATS_INC(nifs, NETIF_STATS_DROPPED);
466 }
467 } else {
468 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
469 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
470 STATS_INC(nifs, NETIF_STATS_DROPPED);
471 }
472
473 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
474
475 if (data == NULL) {
476 continue;
477 }
478
479 STATS_INC(nifs, NETIF_STATS_TXPKTS);
480 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
481
482 tx_ring_stats.kcrsi_slots_transferred++;
483 tx_ring_stats.kcrsi_bytes_transferred += length;
484 }
485
486 if (tx_pslot) {
487 kern_channel_advance_slot(tx_ring, tx_pslot);
488 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
489 (void)kern_channel_reclaim(tx_ring);
490 }
491
492 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
493
494 return 0;
495}
496
497static errno_t
498utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
499 kern_channel_ring_t ring, __unused uint32_t flags)
500{
501#pragma unused(nxprov)
502 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5ba3f43e
A		 503 boolean_t more = false;
504 errno_t rc = 0;
5ba3f43e 505
5c9f4661
A		 506 /*
507 * Refill and sync the ring; we may be racing against another thread doing
508 * an RX sync that also wants to do kr_enter(), and so use the blocking
509 * variant here.
510 */
511 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
512 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
513 printf("%s, tx refill failed %d\n", __func__, rc);
514 }
515
516 (void) kr_enter(ring, TRUE);
517 lck_rw_lock_shared(&pcb->utun_pcb_lock);
518
519 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 520 uint32_t tx_available = kern_channel_available_slot_count(ring);
521 if (pcb->utun_netif_txring_size > 0 &&
522 tx_available >= pcb->utun_netif_txring_size - 1) {
523 // No room left in tx ring, disable output for now
524 errno_t error = ifnet_disable_output(pcb->utun_ifp);
525 if (error != 0) {
526 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
5ba3f43e
A		 527 }
528 }
529 }
b0d623f7 530
5c9f4661 531 if (pcb->utun_kpipe_enabled) {
5ba3f43e
A		 532 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
533
534 // Unlock while calling notify
535 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
536 // Signal the kernel pipe ring to read
537 if (rx_ring != NULL) {
538 kern_channel_notify(rx_ring, 0);
539 }
540 } else {
541 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
542 }
543
5c9f4661
A		 544 kr_exit(ring);
545
5ba3f43e
A		 546 return (0);
547}
548
549static errno_t
550utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
551 kern_channel_ring_t rx_ring, uint32_t flags)
552{
553#pragma unused(nxprov)
554#pragma unused(flags)
555 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
556 struct kern_channel_ring_stat_increment rx_ring_stats;
557
558 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
559
560 lck_rw_lock_shared(&pcb->utun_pcb_lock);
561
562 // Reclaim user-released slots
563 (void) kern_channel_reclaim(rx_ring);
564
565 STATS_INC(nifs, NETIF_STATS_RXSYNC);
566
567 uint32_t avail = kern_channel_available_slot_count(rx_ring);
568 if (avail == 0) {
569 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
570 return 0;
571 }
572
573 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
574 VERIFY(rx_pp != NULL);
575 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
576 kern_channel_slot_t rx_pslot = NULL;
577 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
578
579 while (rx_slot != NULL) {
580 // Check for a waiting packet
581 lck_mtx_lock(&pcb->utun_input_chain_lock);
582 mbuf_t data = pcb->utun_input_chain;
583 if (data == NULL) {
584 lck_mtx_unlock(&pcb->utun_input_chain_lock);
585 break;
586 }
587
588 // Allocate rx packet
589 kern_packet_t rx_ph = 0;
590 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
591 if (unlikely(error != 0)) {
592 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
593 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 594 lck_mtx_unlock(&pcb->utun_input_chain_lock);
595 break;
596 }
597
598 // Advance waiting packets
599 pcb->utun_input_chain = data->m_nextpkt;
600 data->m_nextpkt = NULL;
601 if (pcb->utun_input_chain == NULL) {
602 pcb->utun_input_chain_last = NULL;
603 }
604 lck_mtx_unlock(&pcb->utun_input_chain_lock);
605
606 size_t header_offset = UTUN_HEADER_SIZE(pcb);
607 size_t length = mbuf_pkthdr_len(data);
608
609 if (length < header_offset) {
610 // mbuf is too small
611 mbuf_freem(data);
612 kern_pbufpool_free(rx_pp, rx_ph);
613 STATS_INC(nifs, NETIF_STATS_BADLEN);
614 STATS_INC(nifs, NETIF_STATS_DROPPED);
615 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
616 pcb->utun_ifp->if_xname, length, header_offset);
617 continue;
618 }
619
620 length -= header_offset;
621 if (length > rx_pp->pp_buflet_size) {
622 // Flush data
623 mbuf_freem(data);
624 kern_pbufpool_free(rx_pp, rx_ph);
625 STATS_INC(nifs, NETIF_STATS_BADLEN);
626 STATS_INC(nifs, NETIF_STATS_DROPPED);
627 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
628 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
629 continue;
630 }
631
632 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
633
634 // Fillout rx packet
635 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
636 VERIFY(rx_buf != NULL);
637 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
638 VERIFY(rx_baddr != NULL);
639
640 // Copy-in data from mbuf to buflet
641 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
642 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
643
644 // Finalize and attach the packet
645 error = kern_buflet_set_data_offset(rx_buf, 0);
646 VERIFY(error == 0);
647 error = kern_buflet_set_data_length(rx_buf, length);
648 VERIFY(error == 0);
649 error = kern_packet_set_link_header_offset(rx_ph, 0);
650 VERIFY(error == 0);
651 error = kern_packet_set_network_header_offset(rx_ph, 0);
652 VERIFY(error == 0);
653 error = kern_packet_finalize(rx_ph);
654 VERIFY(error == 0);
655 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
656 VERIFY(error == 0);
657
658 STATS_INC(nifs, NETIF_STATS_RXPKTS);
659 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
660 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
661
662 rx_ring_stats.kcrsi_slots_transferred++;
663 rx_ring_stats.kcrsi_bytes_transferred += length;
664
665 mbuf_freem(data);
666
667 // Advance ring
668 rx_pslot = rx_slot;
669 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
670 }
671
672 struct kern_channel_ring_stat_increment tx_ring_stats;
673 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
674 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
675 kern_channel_slot_t tx_pslot = NULL;
676 kern_channel_slot_t tx_slot = NULL;
677 if (tx_ring == NULL) {
678 // Net-If TX ring not set up yet, nothing to read
679 goto done;
680 }
681
682 // Unlock utun before entering ring
683 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
684
685 (void)kr_enter(tx_ring, TRUE);
686
687 // Lock again after entering and validate
688 lck_rw_lock_shared(&pcb->utun_pcb_lock);
689 if (tx_ring != pcb->utun_kpipe_txring) {
690 goto done;
691 }
692
693 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
694 if (tx_slot == NULL) {
695 // Nothing to read, don't bother signalling
696 goto done;
697 }
698
699 while (rx_slot != NULL && tx_slot != NULL) {
700 // Allocate rx packet
701 kern_packet_t rx_ph = 0;
702 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
703
704 // Advance TX ring
705 tx_pslot = tx_slot;
706 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
707
708 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
709 if (tx_ph == 0) {
710 continue;
711 }
712
713 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
714 if (unlikely(error != 0)) {
715 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
716 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 717 break;
718 }
719
720 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
721 VERIFY(tx_buf != NULL);
722 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
723 VERIFY(tx_baddr != 0);
724 tx_baddr += kern_buflet_get_data_offset(tx_buf);
725
726 // Check packet length
727 size_t header_offset = UTUN_HEADER_SIZE(pcb);
728 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
729 if (tx_length < header_offset) {
730 // Packet is too small
731 kern_pbufpool_free(rx_pp, rx_ph);
732 STATS_INC(nifs, NETIF_STATS_BADLEN);
733 STATS_INC(nifs, NETIF_STATS_DROPPED);
734 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
735 pcb->utun_ifp->if_xname, tx_length, header_offset);
736 continue;
737 }
738
739 size_t length = MIN(tx_length - header_offset,
5c9f4661 740 pcb->utun_slot_size);
5ba3f43e
A		 741
742 tx_ring_stats.kcrsi_slots_transferred++;
743 tx_ring_stats.kcrsi_bytes_transferred += length;
744
745 // Fillout rx packet
746 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
747 VERIFY(rx_buf != NULL);
748 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
749 VERIFY(rx_baddr != NULL);
750
751 // Copy-in data from tx to rx
752 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
753 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
754
755 // Finalize and attach the packet
756 error = kern_buflet_set_data_offset(rx_buf, 0);
757 VERIFY(error == 0);
758 error = kern_buflet_set_data_length(rx_buf, length);
759 VERIFY(error == 0);
760 error = kern_packet_set_link_header_offset(rx_ph, 0);
761 VERIFY(error == 0);
762 error = kern_packet_set_network_header_offset(rx_ph, 0);
763 VERIFY(error == 0);
764 error = kern_packet_finalize(rx_ph);
765 VERIFY(error == 0);
766 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
767 VERIFY(error == 0);
768
769 STATS_INC(nifs, NETIF_STATS_RXPKTS);
770 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
771 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
772
773 rx_ring_stats.kcrsi_slots_transferred++;
774 rx_ring_stats.kcrsi_bytes_transferred += length;
775
776 rx_pslot = rx_slot;
777 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
778 }
779
780done:
781 if (rx_pslot) {
782 kern_channel_advance_slot(rx_ring, rx_pslot);
783 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
784 }
785
786 if (tx_pslot) {
787 kern_channel_advance_slot(tx_ring, tx_pslot);
788 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
789 (void)kern_channel_reclaim(tx_ring);
790 }
791
792 // Unlock first, then exit ring
793 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
794 if (tx_ring != NULL) {
795 if (tx_pslot != NULL) {
796 kern_channel_notify(tx_ring, 0);
797 }
798 kr_exit(tx_ring);
799 }
800
801 return 0;
802}
803
804static errno_t
805utun_nexus_ifattach(struct utun_pcb *pcb,
806 struct ifnet_init_eparams *init_params,
807 struct ifnet **ifp)
808{
809 errno_t err;
810 nexus_controller_t controller = kern_nexus_shared_controller();
811 struct kern_nexus_net_init net_init;
812
813 nexus_name_t provider_name;
814 snprintf((char *)provider_name, sizeof(provider_name),
815 "com.apple.netif.utun%d", pcb->utun_unit);
816
817 struct kern_nexus_provider_init prov_init = {
818 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
819 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
820 .nxpi_pre_connect = utun_nexus_pre_connect,
821 .nxpi_connected = utun_nexus_connected,
822 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
823 .nxpi_disconnected = utun_nexus_disconnected,
824 .nxpi_ring_init = utun_netif_ring_init,
825 .nxpi_ring_fini = utun_netif_ring_fini,
826 .nxpi_slot_init = NULL,
827 .nxpi_slot_fini = NULL,
828 .nxpi_sync_tx = utun_netif_sync_tx,
829 .nxpi_sync_rx = utun_netif_sync_rx,
830 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
831 };
832
833 nexus_attr_t nxa = NULL;
834 err = kern_nexus_attr_create(&nxa);
835 if (err != 0) {
836 printf("%s: kern_nexus_attr_create failed: %d\n",
837 __func__, err);
838 goto failed;
839 }
840
5c9f4661 841 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 842 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
843 VERIFY(err == 0);
844
845 // Reset ring size for netif nexus to limit memory usage
5c9f4661 846 uint64_t ring_size = pcb->utun_netif_ring_size;
5ba3f43e
A		 847 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
848 VERIFY(err == 0);
849 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
850 VERIFY(err == 0);
851
852 pcb->utun_netif_txring_size = ring_size;
853
854 err = kern_nexus_controller_register_provider(controller,
855 utun_nx_dom_prov,
856 provider_name,
857 &prov_init,
858 sizeof(prov_init),
859 nxa,
860 &pcb->utun_nx.if_provider);
861 if (err != 0) {
862 printf("%s register provider failed, error %d\n",
863 __func__, err);
864 goto failed;
865 }
866
867 bzero(&net_init, sizeof(net_init));
868 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
869 net_init.nxneti_flags = 0;
870 net_init.nxneti_eparams = init_params;
871 net_init.nxneti_lladdr = NULL;
872 net_init.nxneti_prepare = utun_netif_prepare;
873 err = kern_nexus_controller_alloc_net_provider_instance(controller,
874 pcb->utun_nx.if_provider,
875 pcb,
876 &pcb->utun_nx.if_instance,
877 &net_init,
878 ifp);
879 if (err != 0) {
880 printf("%s alloc_net_provider_instance failed, %d\n",
881 __func__, err);
882 kern_nexus_controller_deregister_provider(controller,
883 pcb->utun_nx.if_provider);
884 uuid_clear(pcb->utun_nx.if_provider);
885 goto failed;
886 }
887
888failed:
889 if (nxa) {
890 kern_nexus_attr_destroy(nxa);
891 }
892 return (err);
893}
894
895static void
896utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
897{
898 nexus_controller_t controller = kern_nexus_shared_controller();
899 errno_t err;
900
901 if (!uuid_is_null(instance)) {
902 err = kern_nexus_controller_free_provider_instance(controller,
903 instance);
904 if (err != 0) {
905 printf("%s free_provider_instance failed %d\n",
906 __func__, err);
907 }
908 uuid_clear(instance);
909 }
910 if (!uuid_is_null(provider)) {
911 err = kern_nexus_controller_deregister_provider(controller,
912 provider);
913 if (err != 0) {
914 printf("%s deregister_provider %d\n", __func__, err);
915 }
916 uuid_clear(provider);
917 }
918 return;
919}
920
921static void
922utun_nexus_detach(utun_nx_t nx)
923{
924 nexus_controller_t controller = kern_nexus_shared_controller();
925 errno_t err;
926
927 if (!uuid_is_null(nx->ms_host)) {
928 err = kern_nexus_ifdetach(controller,
929 nx->ms_instance,
930 nx->ms_host);
931 if (err != 0) {
932 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
933 __func__, err);
934 }
935 }
936
937 if (!uuid_is_null(nx->ms_device)) {
938 err = kern_nexus_ifdetach(controller,
939 nx->ms_instance,
940 nx->ms_device);
941 if (err != 0) {
942 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
943 __func__, err);
944 }
945 }
946
947 utun_detach_provider_and_instance(nx->if_provider,
948 nx->if_instance);
949 utun_detach_provider_and_instance(nx->ms_provider,
950 nx->ms_instance);
951
952 memset(nx, 0, sizeof(*nx));
953}
954
955static errno_t
5c9f4661
A		 956utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
957 uint32_t subtype, const char *type_name,
5ba3f43e
A		 958 const char *ifname,
959 uuid_t *provider, uuid_t *instance)
960{
961 nexus_attr_t attr = NULL;
962 nexus_controller_t controller = kern_nexus_shared_controller();
963 uuid_t dom_prov;
964 errno_t err;
965 struct kern_nexus_init init;
966 nexus_name_t provider_name;
967
968 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
969 &dom_prov);
970 if (err != 0) {
971 printf("%s can't get %s provider, error %d\n",
972 __func__, type_name, err);
973 goto failed;
974 }
975
976 err = kern_nexus_attr_create(&attr);
977 if (err != 0) {
978 printf("%s: kern_nexus_attr_create failed: %d\n",
979 __func__, err);
980 goto failed;
981 }
982
983 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
984 VERIFY(err == 0);
985
5c9f4661 986 uint64_t slot_buffer_size = pcb->utun_slot_size;
5ba3f43e
A		 987 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
988 VERIFY(err == 0);
989
990 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
5c9f4661 991 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
5ba3f43e
A		 992 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
993 VERIFY(err == 0);
5c9f4661 994 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
5ba3f43e
A		 995 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
996 VERIFY(err == 0);
997
998 snprintf((char *)provider_name, sizeof(provider_name),
999 "com.apple.%s.%s", type_name, ifname);
1000 err = kern_nexus_controller_register_provider(controller,
1001 dom_prov,
1002 provider_name,
1003 NULL,
1004 0,
1005 attr,
1006 provider);
1007 kern_nexus_attr_destroy(attr);
1008 attr = NULL;
1009 if (err != 0) {
1010 printf("%s register %s provider failed, error %d\n",
1011 __func__, type_name, err);
1012 goto failed;
1013 }
1014 bzero(&init, sizeof (init));
1015 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1016 err = kern_nexus_controller_alloc_provider_instance(controller,
1017 *provider,
1018 NULL,
1019 instance, &init);
1020 if (err != 0) {
1021 printf("%s alloc_provider_instance %s failed, %d\n",
1022 __func__, type_name, err);
1023 kern_nexus_controller_deregister_provider(controller,
1024 *provider);
1025 uuid_clear(*provider);
1026 }
1027failed:
1028 return (err);
1029}
1030
1031static errno_t
1032utun_multistack_attach(struct utun_pcb *pcb)
1033{
1034 nexus_controller_t controller = kern_nexus_shared_controller();
1035 errno_t err = 0;
1036 utun_nx_t nx = &pcb->utun_nx;
1037
1038 // Allocate multistack flowswitch
5c9f4661
A		 1039 err = utun_create_fs_provider_and_instance(pcb,
1040 NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
5ba3f43e
A		 1041 "multistack",
1042 pcb->utun_ifp->if_xname,
1043 &nx->ms_provider,
1044 &nx->ms_instance);
1045 if (err != 0) {
1046 printf("%s: failed to create bridge provider and instance\n",
1047 __func__);
1048 goto failed;
1049 }
1050
1051 // Attach multistack to device port
1052 err = kern_nexus_ifattach(controller, nx->ms_instance,
1053 NULL, nx->if_instance,
1054 FALSE, &nx->ms_device);
1055 if (err != 0) {
1056 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1057 goto failed;
1058 }
1059
1060 // Attach multistack to host port
1061 err = kern_nexus_ifattach(controller, nx->ms_instance,
1062 NULL, nx->if_instance,
1063 TRUE, &nx->ms_host);
1064 if (err != 0) {
1065 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1066 goto failed;
1067 }
1068
1069 // Extract the agent UUID and save for later
1070 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1071 if (multistack_nx != NULL) {
1072 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1073 if (flowswitch != NULL) {
1074 FSW_RLOCK(flowswitch);
1075 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1076 if (ms_context != NULL) {
1077 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1078 } else {
1079 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1080 }
1081 FSW_UNLOCK(flowswitch);
1082 } else {
1083 printf("utun_multistack_attach - flowswitch is NULL\n");
1084 }
1085 nx_release(multistack_nx);
1086 } else {
1087 printf("utun_multistack_attach - unable to find multistack nexus\n");
1088 }
1089
1090 return (0);
1091
1092failed:
1093 utun_nexus_detach(nx);
1094
1095 errno_t detach_error = 0;
1096 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1097 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1098 /* NOT REACHED */
1099 }
1100
1101 return (err);
1102}
1103
1104static errno_t
1105utun_register_kernel_pipe_nexus(void)
1106{
1107 nexus_attr_t nxa = NULL;
1108 errno_t result;
1109
1110 lck_mtx_lock(&utun_lock);
1111 if (utun_ncd_refcount++) {
1112 lck_mtx_unlock(&utun_lock);
1113 return 0;
1114 }
1115
1116 result = kern_nexus_controller_create(&utun_ncd);
1117 if (result) {
1118 printf("%s: kern_nexus_controller_create failed: %d\n",
1119 __FUNCTION__, result);
1120 goto done;
1121 }
1122
1123 uuid_t dom_prov;
1124 result = kern_nexus_get_builtin_domain_provider(
1125 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1126 if (result) {
1127 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1128 __FUNCTION__, result);
1129 goto done;
1130 }
1131
1132 struct kern_nexus_provider_init prov_init = {
1133 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1134 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1135 .nxpi_pre_connect = utun_nexus_pre_connect,
1136 .nxpi_connected = utun_nexus_connected,
1137 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1138 .nxpi_disconnected = utun_nexus_disconnected,
1139 .nxpi_ring_init = utun_kpipe_ring_init,
1140 .nxpi_ring_fini = utun_kpipe_ring_fini,
1141 .nxpi_slot_init = NULL,
1142 .nxpi_slot_fini = NULL,
1143 .nxpi_sync_tx = utun_kpipe_sync_tx,
1144 .nxpi_sync_rx = utun_kpipe_sync_rx,
1145 .nxpi_tx_doorbell = NULL,
1146 };
1147
1148 result = kern_nexus_attr_create(&nxa);
1149 if (result) {
1150 printf("%s: kern_nexus_attr_create failed: %d\n",
1151 __FUNCTION__, result);
1152 goto done;
1153 }
1154
1155 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1156 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1157 VERIFY(result == 0);
1158
1159 // Reset ring size for kernel pipe nexus to limit memory usage
1160 uint64_t ring_size = if_utun_ring_size;
1161 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1162 VERIFY(result == 0);
1163 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1164 VERIFY(result == 0);
1165
1166 result = kern_nexus_controller_register_provider(utun_ncd,
1167 dom_prov,
1168 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1169 &prov_init,
1170 sizeof(prov_init),
1171 nxa,
1172 &utun_kpipe_uuid);
1173 if (result) {
1174 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1175 __FUNCTION__, result);
1176 goto done;
1177 }
1178
1179done:
1180 if (nxa) {
1181 kern_nexus_attr_destroy(nxa);
1182 }
1183
1184 if (result) {
1185 if (utun_ncd) {
1186 kern_nexus_controller_destroy(utun_ncd);
1187 utun_ncd = NULL;
1188 }
1189 utun_ncd_refcount = 0;
1190 }
1191
1192 lck_mtx_unlock(&utun_lock);
1193
1194 return result;
1195}
1196
1197static void
1198utun_unregister_kernel_pipe_nexus(void)
1199{
1200 lck_mtx_lock(&utun_lock);
1201
1202 VERIFY(utun_ncd_refcount > 0);
1203
1204 if (--utun_ncd_refcount == 0) {
1205 kern_nexus_controller_destroy(utun_ncd);
1206 utun_ncd = NULL;
1207 }
1208
1209 lck_mtx_unlock(&utun_lock);
1210}
1211
1212// For use by socket option, not internally
1213static errno_t
1214utun_disable_channel(struct utun_pcb *pcb)
1215{
1216 errno_t result;
1217 int enabled;
1218 uuid_t uuid;
1219
1220 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1221
1222 enabled = pcb->utun_kpipe_enabled;
1223 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1224
1225 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1226
1227 pcb->utun_kpipe_enabled = 0;
1228 uuid_clear(pcb->utun_kpipe_uuid);
1229
1230 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1231
1232 if (enabled) {
1233 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1234 } else {
1235 result = ENXIO;
1236 }
1237
1238 if (!result) {
1239 utun_unregister_kernel_pipe_nexus();
1240 }
1241
1242 return result;
1243}
1244
1245static errno_t
1246utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1247{
1248 struct kern_nexus_init init;
1249 errno_t result;
1250
1251 result = utun_register_kernel_pipe_nexus();
1252 if (result) {
1253 return result;
1254 }
1255
1256 VERIFY(utun_ncd);
1257
1258 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1259
1260 if (pcb->utun_kpipe_enabled) {
1261 result = EEXIST; // return success instead?
1262 goto done;
1263 }
1264
1265 /*
1266 * Make sure we can fit packets in the channel buffers and
1267 * Allow an extra 4 bytes for the protocol number header in the channel
1268 */
5c9f4661 1269 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
5ba3f43e
A		 1270 result = EOPNOTSUPP;
1271 goto done;
1272 }
1273
1274 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1275 bzero(&init, sizeof (init));
1276 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1277 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1278 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1279 if (result) {
1280 goto done;
1281 }
1282
1283 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1284 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1285 pcb->utun_kpipe_uuid, &port,
1286 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1287 if (result) {
1288 kern_nexus_controller_free_provider_instance(utun_ncd,
1289 pcb->utun_kpipe_uuid);
1290 uuid_clear(pcb->utun_kpipe_uuid);
1291 goto done;
1292 }
1293
1294 pcb->utun_kpipe_enabled = 1;
1295
1296done:
1297 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1298
1299 if (result) {
1300 utun_unregister_kernel_pipe_nexus();
1301 }
1302
1303 return result;
1304}
1305
1306#endif // UTUN_NEXUS
b0d623f7
A		 1307
1308errno_t
1309utun_register_control(void)
1310{
5ba3f43e
A		 1311 struct kern_ctl_reg kern_ctl;
1312 errno_t result = 0;
b0d623f7 1313
b0d623f7
A		 1314 /* Find a unique value for our interface family */
1315 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1316 if (result != 0) {
1317 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1318 return result;
1319 }
5ba3f43e
A		 1320
1321 utun_pcb_size = sizeof(struct utun_pcb);
1322 utun_pcb_zone = zinit(utun_pcb_size,
1323 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1324 0, UTUN_PCB_ZONE_NAME);
1325 if (utun_pcb_zone == NULL) {
1326 printf("utun_register_control - zinit(utun_pcb) failed");
1327 return ENOMEM;
1328 }
1329
1330#if UTUN_NEXUS
1331 utun_register_nexus();
1332#endif // UTUN_NEXUS
1333
1334 TAILQ_INIT(&utun_head);
b0d623f7
A		 1335
1336 bzero(&kern_ctl, sizeof(kern_ctl));
fe8ab488 1337 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
b0d623f7 1338 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
fe8ab488 1339 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
39236c6e
A		 1340 kern_ctl.ctl_sendsize = 512 * 1024;
1341 kern_ctl.ctl_recvsize = 512 * 1024;
5c9f4661 1342 kern_ctl.ctl_bind = utun_ctl_bind;
b0d623f7
A		 1343 kern_ctl.ctl_connect = utun_ctl_connect;
1344 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1345 kern_ctl.ctl_send = utun_ctl_send;
1346 kern_ctl.ctl_setopt = utun_ctl_setopt;
1347 kern_ctl.ctl_getopt = utun_ctl_getopt;
fe8ab488 1348 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
39236c6e 1349
b0d623f7
A		 1350 result = ctl_register(&kern_ctl, &utun_kctlref);
1351 if (result != 0) {
1352 printf("utun_register_control - ctl_register failed: %d\n", result);
1353 return result;
1354 }
1355
1356 /* Register the protocol plumbers */
1357 if ((result = proto_register_plumber(PF_INET, utun_family,
1358 utun_attach_proto, NULL)) != 0) {
1359 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
1360 utun_family, result);
1361 ctl_deregister(utun_kctlref);
1362 return result;
1363 }
1364
1365 /* Register the protocol plumbers */
1366 if ((result = proto_register_plumber(PF_INET6, utun_family,
1367 utun_attach_proto, NULL)) != 0) {
1368 proto_unregister_plumber(PF_INET, utun_family);
1369 ctl_deregister(utun_kctlref);
1370 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
1371 utun_family, result);
1372 return result;
1373 }
39037602 1374
5ba3f43e
A		 1375 utun_lck_attr = lck_attr_alloc_init();
1376 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1377 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1378
5ba3f43e 1379 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
5ba3f43e 1380
b0d623f7
A		 1381 return 0;
1382}
1383
1384/* Kernel control functions */
1385
5ba3f43e 1386static inline void
5c9f4661 1387utun_free_pcb(struct utun_pcb *pcb, bool in_list)
5ba3f43e
A		 1388{
1389#ifdef UTUN_NEXUS
1390 mbuf_freem_list(pcb->utun_input_chain);
1391 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1392#endif // UTUN_NEXUS
1393 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
5c9f4661
A		 1394 if (in_list) {
1395 lck_mtx_lock(&utun_lock);
1396 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1397 lck_mtx_unlock(&utun_lock);
1398 }
5ba3f43e
A		 1399 zfree(utun_pcb_zone, pcb);
1400}
1401
b0d623f7 1402static errno_t
5c9f4661
A		 1403utun_ctl_bind(kern_ctl_ref kctlref,
1404 struct sockaddr_ctl *sac,
1405 void **unitinfo)
b0d623f7 1406{
5ba3f43e
A		 1407 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1408 memset(pcb, 0, sizeof(*pcb));
39037602 1409
b0d623f7 1410 *unitinfo = pcb;
d1ecb069
A		 1411 pcb->utun_ctlref = kctlref;
1412 pcb->utun_unit = sac->sc_unit;
fe8ab488 1413 pcb->utun_max_pending_packets = 1;
5ba3f43e 1414
5c9f4661
A		 1415#if UTUN_NEXUS
1416 pcb->utun_use_netif = false;
1417 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1418 pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
1419 pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
1420 pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
1421#endif // UTUN_NEXUS
1422
5ba3f43e
A		 1423 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1424 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1425
5c9f4661
A		 1426 return (0);
1427}
1428
1429static errno_t
1430utun_ctl_connect(kern_ctl_ref kctlref,
1431 struct sockaddr_ctl *sac,
1432 void **unitinfo)
1433{
1434 struct ifnet_init_eparams utun_init = {};
1435 errno_t result = 0;
1436
1437 if (*unitinfo == NULL) {
1438 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1439 }
1440
1441 struct utun_pcb *pcb = *unitinfo;
1442
5ba3f43e
A		 1443 lck_mtx_lock(&utun_lock);
1444
1445 /* Find some open interface id */
1446 u_int32_t chosen_unique_id = 1;
1447 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1448 if (next_pcb != NULL) {
1449 /* List was not empty, add one to the last item */
1450 chosen_unique_id = next_pcb->utun_unique_id + 1;
1451 next_pcb = NULL;
1452
1453 /*
1454 * If this wrapped the id number, start looking at
1455 * the front of the list for an unused id.
1456 */
1457 if (chosen_unique_id == 0) {
1458 /* Find the next unused ID */
1459 chosen_unique_id = 1;
1460 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1461 if (next_pcb->utun_unique_id > chosen_unique_id) {
1462 /* We found a gap */
1463 break;
1464 }
1465
1466 chosen_unique_id = next_pcb->utun_unique_id + 1;
1467 }
1468 }
1469 }
1470
1471 pcb->utun_unique_id = chosen_unique_id;
1472
1473 if (next_pcb != NULL) {
1474 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1475 } else {
1476 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1477 }
1478 lck_mtx_unlock(&utun_lock);
1479
1480 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1481 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1482 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
b0d623f7
A		 1483
1484 /* Create the interface */
1485 bzero(&utun_init, sizeof(utun_init));
39236c6e
A		 1486 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1487 utun_init.len = sizeof (utun_init);
5ba3f43e
A		 1488
1489#if UTUN_NEXUS
5c9f4661
A		 1490 if (pcb->utun_use_netif) {
1491 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1492 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1493 } else
5ba3f43e 1494#endif // UTUN_NEXUS
5c9f4661
A		 1495 {
1496 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1497 utun_init.start = utun_start;
1498 utun_init.framer_extended = utun_framer;
1499 }
5ba3f43e 1500 utun_init.name = "utun";
d1ecb069 1501 utun_init.unit = pcb->utun_unit - 1;
5ba3f43e
A		 1502 utun_init.uniqueid = pcb->utun_unique_name;
1503 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
b0d623f7 1504 utun_init.family = utun_family;
39037602 1505 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
b0d623f7 1506 utun_init.type = IFT_OTHER;
b0d623f7 1507 utun_init.demux = utun_demux;
b0d623f7
A		 1508 utun_init.add_proto = utun_add_proto;
1509 utun_init.del_proto = utun_del_proto;
1510 utun_init.softc = pcb;
1511 utun_init.ioctl = utun_ioctl;
1512 utun_init.detach = utun_detached;
39037602 1513
5ba3f43e 1514#if UTUN_NEXUS
5c9f4661
A		 1515 if (pcb->utun_use_netif) {
1516 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1517 if (result != 0) {
1518 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1519 utun_free_pcb(pcb, true);
1520 *unitinfo = NULL;
1521 return result;
1522 }
5ba3f43e 1523
5c9f4661
A		 1524 result = utun_multistack_attach(pcb);
1525 if (result != 0) {
1526 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1527 *unitinfo = NULL;
1528 return result;
1529 }
5ba3f43e 1530
5c9f4661
A		 1531 /* Attach to bpf */
1532 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1533 } else
1534#endif // UTUN_NEXUS
1535 {
1536 /*
1537 * Upon success, this holds an ifnet reference which we will
1538 * release via ifnet_release() at final detach time.
1539 */
1540 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1541 if (result != 0) {
1542 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1543 utun_free_pcb(pcb, true);
1544 *unitinfo = NULL;
1545 return result;
1546 }
b0d623f7 1547
5c9f4661
A		 1548 /* Set flags and additional information. */
1549 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1550 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
d1ecb069 1551
5c9f4661
A		 1552 /* The interface must generate its own IPv6 LinkLocal address,
1553 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1554 */
1555 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1556
1557 /* Reset the stats in case as the interface may have been recycled */
1558 struct ifnet_stats_param stats;
1559 bzero(&stats, sizeof(struct ifnet_stats_param));
1560 ifnet_set_stat(pcb->utun_ifp, &stats);
1561
1562 /* Attach the interface */
1563 result = ifnet_attach(pcb->utun_ifp, NULL);
1564 if (result != 0) {
1565 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
1566 /* Release reference now since attach failed */
1567 ifnet_release(pcb->utun_ifp);
1568 utun_free_pcb(pcb, true);
1569 *unitinfo = NULL;
1570 return (result);
1571 }
1572
1573 /* Attach to bpf */
1574 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
39037602 1575 }
5ba3f43e 1576
5ba3f43e
A		 1577 /* The interfaces resoures allocated, mark it as running */
1578 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1579
b0d623f7
A		 1580 return result;
1581}
1582
1583static errno_t
5ba3f43e
A		 1584utun_detach_ip(ifnet_t interface,
1585 protocol_family_t protocol,
1586 socket_t pf_socket)
b0d623f7
A		 1587{
1588 errno_t result = EPROTONOSUPPORT;
1589
1590 /* Attempt a detach */
1591 if (protocol == PF_INET) {
1592 struct ifreq ifr;
1593
1594 bzero(&ifr, sizeof(ifr));
1595 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1596 ifnet_name(interface), ifnet_unit(interface));
1597
1598 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
5ba3f43e 1599 } else if (protocol == PF_INET6) {
b0d623f7
A		 1600 struct in6_ifreq ifr6;
1601
1602 bzero(&ifr6, sizeof(ifr6));
1603 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1604 ifnet_name(interface), ifnet_unit(interface));
1605
1606 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1607 }
1608
1609 return result;
1610}
1611
1612static void
5ba3f43e
A		 1613utun_remove_address(ifnet_t interface,
1614 protocol_family_t protocol,
1615 ifaddr_t address,
1616 socket_t pf_socket)
b0d623f7
A		 1617{
1618 errno_t result = 0;
1619
1620 /* Attempt a detach */
1621 if (protocol == PF_INET) {
5ba3f43e 1622 struct ifreq ifr;
b0d623f7
A		 1623
1624 bzero(&ifr, sizeof(ifr));
1625 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1626 ifnet_name(interface), ifnet_unit(interface));
1627 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1628 if (result != 0) {
1629 printf("utun_remove_address - ifaddr_address failed: %d", result);
5ba3f43e 1630 } else {
b0d623f7
A		 1631 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1632 if (result != 0) {
1633 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1634 }
1635 }
5ba3f43e
A		 1636 } else if (protocol == PF_INET6) {
1637 struct in6_ifreq ifr6;
b0d623f7
A		 1638
1639 bzero(&ifr6, sizeof(ifr6));
1640 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1641 ifnet_name(interface), ifnet_unit(interface));
1642 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1643 sizeof(ifr6.ifr_addr));
1644 if (result != 0) {
1645 printf("utun_remove_address - ifaddr_address failed (v6): %d",
1646 result);
5ba3f43e 1647 } else {
b0d623f7
A		 1648 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1649 if (result != 0) {
1650 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1651 result);
1652 }
1653 }
1654 }
1655}
1656
1657static void
5ba3f43e
A		 1658utun_cleanup_family(ifnet_t interface,
1659 protocol_family_t protocol)
1660{
1661 errno_t result = 0;
1662 socket_t pf_socket = NULL;
1663 ifaddr_t *addresses = NULL;
1664 int i;
b0d623f7
A		 1665
1666 if (protocol != PF_INET && protocol != PF_INET6) {
1667 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1668 return;
1669 }
1670
1671 /* Create a socket for removing addresses and detaching the protocol */
1672 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1673 if (result != 0) {
1674 if (result != EAFNOSUPPORT)
1675 printf("utun_cleanup_family - failed to create %s socket: %d\n",
1676 protocol == PF_INET ? "IP" : "IPv6", result);
1677 goto cleanup;
1678 }
1679
6d2010ae
A		 1680 /* always set SS_PRIV, we want to close and detach regardless */
1681 sock_setpriv(pf_socket, 1);
1682
b0d623f7
A		 1683 result = utun_detach_ip(interface, protocol, pf_socket);
1684 if (result == 0 || result == ENXIO) {
1685 /* We are done! We either detached or weren't attached. */
1686 goto cleanup;
5ba3f43e 1687 } else if (result != EBUSY) {
b0d623f7
A		 1688 /* Uh, not really sure what happened here... */
1689 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1690 goto cleanup;
1691 }
1692
1693 /*
1694 * At this point, we received an EBUSY error. This means there are
1695 * addresses attached. We should detach them and then try again.
1696 */
1697 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1698 if (result != 0) {
1699 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1700 ifnet_name(interface), ifnet_unit(interface),
1701 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1702 goto cleanup;
1703 }
1704
1705 for (i = 0; addresses[i] != 0; i++) {
1706 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1707 }
1708 ifnet_free_address_list(addresses);
1709 addresses = NULL;
1710
1711 /*
1712 * The addresses should be gone, we should try the remove again.
1713 */
1714 result = utun_detach_ip(interface, protocol, pf_socket);
1715 if (result != 0 && result != ENXIO) {
1716 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1717 }
1718
1719cleanup:
5ba3f43e 1720 if (pf_socket != NULL) {
b0d623f7 1721 sock_close(pf_socket);
5ba3f43e 1722 }
b0d623f7 1723
5ba3f43e 1724 if (addresses != NULL) {
b0d623f7 1725 ifnet_free_address_list(addresses);
5ba3f43e 1726 }
b0d623f7
A		 1727}
1728
1729static errno_t
5ba3f43e
A		 1730utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1731 __unused u_int32_t unit,
1732 void *unitinfo)
b0d623f7
A		 1733{
1734 struct utun_pcb *pcb = unitinfo;
5ba3f43e
A		 1735 ifnet_t ifp = NULL;
1736 errno_t result = 0;
1737
1738 if (pcb == NULL) {
1739 return EINVAL;
1740 }
1741
1742#if UTUN_NEXUS
1743 // Tell the nexus to stop all rings
1744 if (pcb->utun_netif_nexus != NULL) {
1745 kern_nexus_stop(pcb->utun_netif_nexus);
1746 }
1747#endif // UTUN_NEXUS
316670eb 1748
5ba3f43e 1749 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
316670eb 1750
5ba3f43e
A		 1751#if UTUN_NEXUS
1752 uuid_t kpipe_uuid;
1753 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1754 uuid_clear(pcb->utun_kpipe_uuid);
1755 pcb->utun_kpipe_enabled = FALSE;
1756#endif // UTUN_NEXUS
39037602 1757
d1ecb069 1758 pcb->utun_ctlref = NULL;
5ba3f43e 1759
5c9f4661
A		 1760 ifp = pcb->utun_ifp;
1761 if (ifp != NULL) {
1762#if UTUN_NEXUS
1763 // Tell the nexus to stop all rings
1764 if (pcb->utun_netif_nexus != NULL) {
1765 /*
1766 * Quiesce the interface and flush any pending outbound packets.
1767 */
1768 if_down(ifp);
1769
1770 /* Increment refcnt, but detach interface */
1771 ifnet_incr_iorefcnt(ifp);
1772 if ((result = ifnet_detach(ifp)) != 0) {
1773 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1774 }
5ba3f43e 1775
5c9f4661
A		 1776 /*
1777 * We want to do everything in our power to ensure that the interface
1778 * really goes away when the socket is closed. We must remove IP/IPv6
1779 * addresses and detach the protocols. Finally, we can remove and
1780 * release the interface.
1781 */
1782 utun_cleanup_family(ifp, AF_INET);
1783 utun_cleanup_family(ifp, AF_INET6);
39037602 1784
5c9f4661 1785 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
39037602 1786
5c9f4661
A		 1787 if (!uuid_is_null(kpipe_uuid)) {
1788 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1789 utun_unregister_kernel_pipe_nexus();
1790 }
1791 }
1792 utun_nexus_detach(&pcb->utun_nx);
1793
1794 /* Decrement refcnt to finish detaching and freeing */
1795 ifnet_decr_iorefcnt(ifp);
1796 } else
1797#endif // UTUN_NEXUS
1798 {
1799 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
5ba3f43e
A		 1800
1801#if UTUN_NEXUS
5c9f4661
A		 1802 if (!uuid_is_null(kpipe_uuid)) {
1803 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1804 utun_unregister_kernel_pipe_nexus();
1805 }
1806 }
5ba3f43e
A		 1807#endif // UTUN_NEXUS
1808
5c9f4661
A		 1809 /*
1810 * We want to do everything in our power to ensure that the interface
1811 * really goes away when the socket is closed. We must remove IP/IPv6
1812 * addresses and detach the protocols. Finally, we can remove and
1813 * release the interface.
1814 */
1815 utun_cleanup_family(ifp, AF_INET);
1816 utun_cleanup_family(ifp, AF_INET6);
1817
1818 /*
1819 * Detach now; utun_detach() will be called asynchronously once
1820 * the I/O reference count drops to 0. There we will invoke
1821 * ifnet_release().
1822 */
1823 if ((result = ifnet_detach(ifp)) != 0) {
1824 printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1825 }
1826 }
1827 } else {
1828 // Bound, but not connected
1829 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1830 utun_free_pcb(pcb, false);
1831 }
b0d623f7 1832
b0d623f7 1833 return 0;
39037602 1834}
b0d623f7
A		 1835
1836static errno_t
5ba3f43e
A		 1837utun_ctl_send(__unused kern_ctl_ref kctlref,
1838 __unused u_int32_t unit,
1839 void *unitinfo,
1840 mbuf_t m,
1841 __unused int flags)
b0d623f7 1842{
39236c6e
A		 1843 /*
1844 * The userland ABI requires the first four bytes have the protocol family
1845 * in network byte order: swap them
1846 */
39037602 1847 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
39236c6e 1848 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
39037602 1849 } else {
39236c6e 1850 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
39037602 1851 }
39236c6e 1852
316670eb 1853 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
b0d623f7
A		 1854}
1855
1856static errno_t
5ba3f43e
A		 1857utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1858 __unused u_int32_t unit,
1859 void *unitinfo,
1860 int opt,
1861 void *data,
1862 size_t len)
b0d623f7 1863{
5ba3f43e
A		 1864 struct utun_pcb *pcb = unitinfo;
1865 errno_t result = 0;
b0d623f7
A		 1866 /* check for privileges for privileged options */
1867 switch (opt) {
1868 case UTUN_OPT_FLAGS:
d1ecb069 1869 case UTUN_OPT_EXT_IFDATA_STATS:
39236c6e 1870 case UTUN_OPT_SET_DELEGATE_INTERFACE:
b0d623f7
A		 1871 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1872 return EPERM;
1873 }
1874 break;
1875 }
1876
1877 switch (opt) {
1878 case UTUN_OPT_FLAGS:
39037602 1879 if (len != sizeof(u_int32_t)) {
b0d623f7 1880 result = EMSGSIZE;
39037602 1881 } else {
5c9f4661
A		 1882 if (pcb->utun_ifp == NULL) {
1883 // Only can set after connecting
1884 result = EINVAL;
1885 break;
1886 }
1887#if UTUN_NEXUS
1888 if (pcb->utun_use_netif) {
1889 pcb->utun_flags = *(u_int32_t *)data;
1890 } else
1891#endif // UTUN_NEXUS
1892 {
1893 u_int32_t old_flags = pcb->utun_flags;
1894 pcb->utun_flags = *(u_int32_t *)data;
1895 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1896 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1897 bpfdetach(pcb->utun_ifp);
1898 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1899 }
1900 }
39037602 1901 }
39236c6e
A		 1902 break;
1903
d1ecb069
A		 1904 case UTUN_OPT_EXT_IFDATA_STATS:
1905 if (len != sizeof(int)) {
1906 result = EMSGSIZE;
1907 break;
1908 }
5c9f4661
A		 1909 if (pcb->utun_ifp == NULL) {
1910 // Only can set after connecting
1911 result = EINVAL;
1912 break;
1913 }
d1ecb069
A		 1914 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1915 break;
1916
1917 case UTUN_OPT_INC_IFDATA_STATS_IN:
1918 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1919 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1920
1921 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1922 result = EINVAL;
1923 break;
1924 }
5c9f4661
A		 1925 if (pcb->utun_ifp == NULL) {
1926 // Only can set after connecting
1927 result = EINVAL;
1928 break;
1929 }
d1ecb069
A		 1930 if (!pcb->utun_ext_ifdata_stats) {
1931 result = EINVAL;
1932 break;
1933 }
1934 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN)
1935 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
1936 utsp->utsp_bytes, utsp->utsp_errors);
1937 else
1938 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
1939 utsp->utsp_bytes, utsp->utsp_errors);
b0d623f7 1940 break;
d1ecb069 1941 }
fe8ab488 1942 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
39236c6e
A		 1943 ifnet_t del_ifp = NULL;
1944 char name[IFNAMSIZ];
1945
1946 if (len > IFNAMSIZ - 1) {
1947 result = EMSGSIZE;
1948 break;
1949 }
5c9f4661
A		 1950 if (pcb->utun_ifp == NULL) {
1951 // Only can set after connecting
1952 result = EINVAL;
1953 break;
1954 }
39236c6e
A		 1955 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
1956 bcopy(data, name, len);
1957 name[len] = 0;
1958 result = ifnet_find_by_name(name, &del_ifp);
1959 }
1960 if (result == 0) {
1961 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
1962 if (del_ifp)
1963 ifnet_release(del_ifp);
1964 }
1965 break;
1966 }
fe8ab488
A		 1967 case UTUN_OPT_MAX_PENDING_PACKETS: {
1968 u_int32_t max_pending_packets = 0;
1969 if (len != sizeof(u_int32_t)) {
1970 result = EMSGSIZE;
1971 break;
1972 }
1973 max_pending_packets = *(u_int32_t *)data;
1974 if (max_pending_packets == 0) {
1975 result = EINVAL;
1976 break;
1977 }
1978 pcb->utun_max_pending_packets = max_pending_packets;
1979 break;
1980 }
5ba3f43e
A		 1981#if UTUN_NEXUS
1982 case UTUN_OPT_ENABLE_CHANNEL: {
1983 if (len != sizeof(int)) {
1984 result = EMSGSIZE;
1985 break;
1986 }
5c9f4661
A		 1987 if (pcb->utun_ifp == NULL) {
1988 // Only can set after connecting
1989 result = EINVAL;
1990 break;
1991 }
5ba3f43e
A		 1992 if (*(int *)data) {
1993 result = utun_enable_channel(pcb, current_proc());
1994 } else {
1995 result = utun_disable_channel(pcb);
1996 }
1997 break;
1998 }
1999 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2000 if (len != sizeof(int)) {
2001 result = EMSGSIZE;
2002 break;
2003 }
5c9f4661
A		 2004 if (pcb->utun_ifp == NULL) {
2005 // Only can set after connecting
2006 result = EINVAL;
2007 break;
2008 }
5ba3f43e
A		 2009 if (!if_enable_netagent) {
2010 result = ENOTSUP;
2011 break;
2012 }
2013 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
2014 result = ENOENT;
2015 break;
2016 }
2017
2018 if (*(int *)data) {
2019 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2020 } else {
2021 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2022 }
2023 break;
2024 }
5c9f4661
A		 2025 case UTUN_OPT_ENABLE_NETIF: {
2026 if (len != sizeof(int)) {
2027 result = EMSGSIZE;
2028 break;
2029 }
2030 if (pcb->utun_ifp != NULL) {
2031 // Only can set before connecting
2032 result = EINVAL;
2033 break;
2034 }
2035 pcb->utun_use_netif = true;
2036 break;
2037 }
2038 case UTUN_OPT_SLOT_SIZE: {
2039 if (len != sizeof(u_int32_t)) {
2040 result = EMSGSIZE;
2041 break;
2042 }
2043 if (pcb->utun_ifp != NULL) {
2044 // Only can set before connecting
2045 result = EINVAL;
2046 break;
2047 }
2048 u_int32_t slot_size = *(u_int32_t *)data;
2049 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2050 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2051 return (EINVAL);
2052 }
2053 pcb->utun_slot_size = slot_size;
2054 break;
2055 }
2056 case UTUN_OPT_NETIF_RING_SIZE: {
2057 if (len != sizeof(u_int32_t)) {
2058 result = EMSGSIZE;
2059 break;
2060 }
2061 if (pcb->utun_ifp != NULL) {
2062 // Only can set before connecting
2063 result = EINVAL;
2064 break;
2065 }
2066 u_int32_t ring_size = *(u_int32_t *)data;
2067 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2068 ring_size > UTUN_IF_MAX_RING_SIZE) {
2069 return (EINVAL);
2070 }
2071 pcb->utun_netif_ring_size = ring_size;
2072 break;
2073 }
2074 case UTUN_OPT_TX_FSW_RING_SIZE: {
2075 if (len != sizeof(u_int32_t)) {
2076 result = EMSGSIZE;
2077 break;
2078 }
2079 if (pcb->utun_ifp != NULL) {
2080 // Only can set before connecting
2081 result = EINVAL;
2082 break;
2083 }
2084 u_int32_t ring_size = *(u_int32_t *)data;
2085 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2086 ring_size > UTUN_IF_MAX_RING_SIZE) {
2087 return (EINVAL);
2088 }
2089 pcb->utun_tx_fsw_ring_size = ring_size;
2090 break;
2091 }
2092 case UTUN_OPT_RX_FSW_RING_SIZE: {
2093 if (len != sizeof(u_int32_t)) {
2094 result = EMSGSIZE;
2095 break;
2096 }
2097 if (pcb->utun_ifp != NULL) {
2098 // Only can set before connecting
2099 result = EINVAL;
2100 break;
2101 }
2102 u_int32_t ring_size = *(u_int32_t *)data;
2103 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2104 ring_size > UTUN_IF_MAX_RING_SIZE) {
2105 return (EINVAL);
2106 }
2107 pcb->utun_rx_fsw_ring_size = ring_size;
2108 break;
2109 }
5ba3f43e 2110#endif // UTUN_NEXUS
fe8ab488 2111 default: {
b0d623f7
A		 2112 result = ENOPROTOOPT;
2113 break;
fe8ab488 2114 }
b0d623f7
A		 2115 }
2116
2117 return result;
2118}
2119
2120static errno_t
5ba3f43e
A		 2121utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2122 __unused u_int32_t unit,
2123 void *unitinfo,
2124 int opt,
2125 void *data,
2126 size_t *len)
b0d623f7 2127{
5ba3f43e
A		 2128 struct utun_pcb *pcb = unitinfo;
2129 errno_t result = 0;
b0d623f7
A		 2130
2131 switch (opt) {
2132 case UTUN_OPT_FLAGS:
5ba3f43e 2133 if (*len != sizeof(u_int32_t)) {
b0d623f7 2134 result = EMSGSIZE;
5ba3f43e 2135 } else {
d1ecb069 2136 *(u_int32_t *)data = pcb->utun_flags;
5ba3f43e 2137 }
b0d623f7 2138 break;
d1ecb069
A		 2139
2140 case UTUN_OPT_EXT_IFDATA_STATS:
5ba3f43e 2141 if (*len != sizeof(int)) {
d1ecb069 2142 result = EMSGSIZE;
5ba3f43e 2143 } else {
d1ecb069 2144 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
5ba3f43e 2145 }
d1ecb069
A		 2146 break;
2147
b0d623f7 2148 case UTUN_OPT_IFNAME:
5ba3f43e
A		 2149 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2150 result = EMSGSIZE;
2151 } else {
5c9f4661
A		 2152 if (pcb->utun_ifp == NULL) {
2153 // Only can get after connecting
2154 result = EINVAL;
2155 break;
2156 }
5ba3f43e
A		 2157 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2158 }
b0d623f7 2159 break;
d1ecb069 2160
fe8ab488 2161 case UTUN_OPT_MAX_PENDING_PACKETS: {
5ba3f43e
A		 2162 if (*len != sizeof(u_int32_t)) {
2163 result = EMSGSIZE;
2164 } else {
2165 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2166 }
fe8ab488
A		 2167 break;
2168 }
39037602 2169
5ba3f43e 2170#if UTUN_NEXUS
5c9f4661 2171 case UTUN_OPT_GET_CHANNEL_UUID: {
5ba3f43e
A		 2172 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2173 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2174 result = ENXIO;
2175 } else if (*len != sizeof(uuid_t)) {
2176 result = EMSGSIZE;
2177 } else {
2178 uuid_copy(data, pcb->utun_kpipe_uuid);
2179 }
2180 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2181 break;
5c9f4661
A		 2182 }
2183 case UTUN_OPT_SLOT_SIZE: {
2184 if (*len != sizeof(u_int32_t)) {
2185 result = EMSGSIZE;
2186 } else {
2187 *(u_int32_t *)data = pcb->utun_slot_size;
2188 }
2189 break;
2190 }
2191 case UTUN_OPT_NETIF_RING_SIZE: {
2192 if (*len != sizeof(u_int32_t)) {
2193 result = EMSGSIZE;
2194 } else {
2195 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2196 }
2197 break;
2198 }
2199 case UTUN_OPT_TX_FSW_RING_SIZE: {
2200 if (*len != sizeof(u_int32_t)) {
2201 result = EMSGSIZE;
2202 } else {
2203 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2204 }
2205 break;
2206 }
2207 case UTUN_OPT_RX_FSW_RING_SIZE: {
2208 if (*len != sizeof(u_int32_t)) {
2209 result = EMSGSIZE;
2210 } else {
2211 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2212 }
2213 break;
2214 }
5ba3f43e 2215#endif // UTUN_NEXUS
39037602 2216
b0d623f7
A		 2217 default:
2218 result = ENOPROTOOPT;
2219 break;
2220 }
2221
2222 return result;
2223}
2224
fe8ab488
A		 2225static void
2226utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2227{
3e170ce0 2228#pragma unused(flags)
fe8ab488
A		 2229 bool reenable_output = false;
2230 struct utun_pcb *pcb = unitinfo;
2231 if (pcb == NULL) {
2232 return;
2233 }
2234 ifnet_lock_exclusive(pcb->utun_ifp);
3e170ce0
A		 2235
2236 u_int32_t utun_packet_cnt;
2237 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2238 if (error_pc != 0) {
2239 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2240 utun_packet_cnt = 0;
fe8ab488 2241 }
3e170ce0
A		 2242
2243 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2244 reenable_output = true;
2245 }
2246
fe8ab488
A		 2247 if (reenable_output) {
2248 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2249 if (error != 0) {
2250 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2251 }
2252 }
2253 ifnet_lock_done(pcb->utun_ifp);
2254}
2255
b0d623f7 2256/* Network Interface functions */
fe8ab488
A		 2257static void
2258utun_start(ifnet_t interface)
2259{
2260 mbuf_t data;
39037602
A		 2261 struct utun_pcb *pcb = ifnet_softc(interface);
2262
2263 VERIFY(pcb != NULL);
2264
5ba3f43e
A		 2265#if UTUN_NEXUS
2266 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2267 if (pcb->utun_kpipe_enabled) {
2268 /* It's possible to have channels enabled, but not yet have the channel opened,
2269 * in which case the rxring will not be set
2270 */
2271 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2272 if (pcb->utun_kpipe_rxring != NULL) {
2273 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2274 }
2275 return;
2276 }
2277 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2278#endif // UTUN_NEXUS
39037602 2279
fe8ab488
A		 2280 for (;;) {
2281 bool can_accept_packets = true;
2282 ifnet_lock_shared(pcb->utun_ifp);
3e170ce0
A		 2283
2284 u_int32_t utun_packet_cnt;
2285 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2286 if (error_pc != 0) {
2287 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2288 utun_packet_cnt = 0;
2289 }
2290
2291 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
fe8ab488
A		 2292 if (!can_accept_packets && pcb->utun_ctlref) {
2293 u_int32_t difference = 0;
2294 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2295 if (difference > 0) {
2296 // If the low-water mark has not yet been reached, we still need to enqueue data
2297 // into the buffer
2298 can_accept_packets = true;
2299 }
2300 }
2301 }
2302 if (!can_accept_packets) {
2303 errno_t error = ifnet_disable_output(interface);
2304 if (error != 0) {
2305 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2306 }
2307 ifnet_lock_done(pcb->utun_ifp);
2308 break;
2309 }
2310 ifnet_lock_done(pcb->utun_ifp);
5ba3f43e 2311 if (ifnet_dequeue(interface, &data) != 0) {
fe8ab488 2312 break;
5ba3f43e
A		 2313 }
2314 if (utun_output(interface, data) != 0) {
fe8ab488 2315 break;
5ba3f43e 2316 }
fe8ab488
A		 2317 }
2318}
2319
b0d623f7 2320static errno_t
39037602
A		 2321utun_output(ifnet_t interface,
2322 mbuf_t data)
b0d623f7
A		 2323{
2324 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2325 errno_t result;
39037602
A		 2326
2327 VERIFY(interface == pcb->utun_ifp);
5ba3f43e 2328
5c9f4661
A		 2329#if UTUN_NEXUS
2330 if (!pcb->utun_use_netif)
2331#endif // UTUN_NEXUS
2332 {
2333 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2334 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2335 }
2336 }
2337
d1ecb069 2338 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
b0d623f7
A		 2339 /* flush data */
2340 mbuf_freem(data);
2341 return 0;
2342 }
2343
316670eb 2344 // otherwise, fall thru to ctl_enqueumbuf
d1ecb069 2345 if (pcb->utun_ctlref) {
316670eb
A		 2346 int length;
2347
39236c6e
A		 2348 /*
2349 * The ABI requires the protocol in network byte order
2350 */
39037602 2351 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
39236c6e 2352 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
39037602 2353 }
39236c6e 2354
316670eb 2355 length = mbuf_pkthdr_len(data);
d1ecb069 2356 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
b0d623f7
A		 2357 if (result != 0) {
2358 mbuf_freem(data);
2359 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
5c9f4661
A		 2360#if UTUN_NEXUS
2361 if (!pcb->utun_use_netif)
2362#endif // UTUN_NEXUS
2363 {
2364 ifnet_stat_increment_out(interface, 0, 0, 1);
2365 }
5ba3f43e 2366 } else {
5c9f4661
A		 2367#if UTUN_NEXUS
2368 if (!pcb->utun_use_netif)
2369#endif // UTUN_NEXUS
2370 {
2371 if (!pcb->utun_ext_ifdata_stats) {
2372 ifnet_stat_increment_out(interface, 1, length, 0);
2373 }
5ba3f43e 2374 }
b0d623f7 2375 }
5ba3f43e 2376 } else {
b0d623f7 2377 mbuf_freem(data);
5ba3f43e 2378 }
b0d623f7
A		 2379
2380 return 0;
2381}
2382
b0d623f7 2383static errno_t
5ba3f43e
A		 2384utun_demux(__unused ifnet_t interface,
2385 mbuf_t data,
2386 __unused char *frame_header,
2387 protocol_family_t *protocol)
b0d623f7 2388{
5c9f4661 2389 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e
A		 2390 struct ip *ip;
2391 u_int ip_version;
2392
b0d623f7
A		 2393 while (data != NULL && mbuf_len(data) < 1) {
2394 data = mbuf_next(data);
2395 }
5ba3f43e 2396
5c9f4661 2397 if (data == NULL) {
b0d623f7 2398 return ENOENT;
5c9f4661 2399 }
5ba3f43e 2400
5c9f4661
A		 2401#if UTUN_NEXUS
2402 if (pcb->utun_use_netif) {
2403 ip = mtod(data, struct ip *);
2404 ip_version = ip->ip_v;
2405
2406 switch(ip_version) {
2407 case 4:
2408 *protocol = PF_INET;
2409 return 0;
2410 case 6:
2411 *protocol = PF_INET6;
2412 return 0;
2413 default:
2414 *protocol = 0;
2415 break;
2416 }
2417 } else
2418#endif // UTUN_NEXUS
2419 {
2420 *protocol = *(u_int32_t *)mbuf_data(data);
5ba3f43e
A		 2421 }
2422
b0d623f7
A		 2423 return 0;
2424}
2425
2426static errno_t
5ba3f43e
A		 2427utun_framer(ifnet_t interface,
2428 mbuf_t *packet,
2429 __unused const struct sockaddr *dest,
b0d623f7 2430 __unused const char *desk_linkaddr,
39236c6e 2431 const char *frame_type,
5ba3f43e 2432 u_int32_t *prepend_len,
39236c6e 2433 u_int32_t *postpend_len)
b0d623f7 2434{
39037602
A		 2435 struct utun_pcb *pcb = ifnet_softc(interface);
2436 VERIFY(interface == pcb->utun_ifp);
2437
2438 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2439 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
b0d623f7 2440 printf("utun_framer - ifnet_output prepend failed\n");
d1ecb069 2441
b0d623f7 2442 ifnet_stat_increment_out(interface, 0, 0, 1);
d1ecb069 2443
b0d623f7
A		 2444 // just return, because the buffer was freed in mbuf_prepend
2445 return EJUSTRETURN;
2446 }
5ba3f43e 2447 if (prepend_len != NULL) {
39037602 2448 *prepend_len = header_length;
5ba3f43e
A		 2449 }
2450 if (postpend_len != NULL) {
39236c6e 2451 *postpend_len = 0;
5ba3f43e 2452 }
b0d623f7
A		 2453
2454 // place protocol number at the beginning of the mbuf
39236c6e 2455 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
39037602
A		 2456
2457
b0d623f7
A		 2458 return 0;
2459}
2460
2461static errno_t
5ba3f43e
A		 2462utun_add_proto(__unused ifnet_t interface,
2463 protocol_family_t protocol,
2464 __unused const struct ifnet_demux_desc *demux_array,
2465 __unused u_int32_t demux_count)
b0d623f7
A		 2466{
2467 switch(protocol) {
2468 case PF_INET:
2469 return 0;
2470 case PF_INET6:
2471 return 0;
2472 default:
2473 break;
2474 }
2475
2476 return ENOPROTOOPT;
2477}
2478
2479static errno_t
5ba3f43e
A		 2480utun_del_proto(__unused ifnet_t interface,
2481 __unused protocol_family_t protocol)
b0d623f7
A		 2482{
2483 return 0;
2484}
2485
2486static errno_t
5ba3f43e
A		 2487utun_ioctl(ifnet_t interface,
2488 u_long command,
2489 void *data)
b0d623f7 2490{
5c9f4661 2491 struct utun_pcb *pcb = ifnet_softc(interface);
b0d623f7 2492 errno_t result = 0;
b0d623f7
A		 2493
2494 switch(command) {
5c9f4661 2495 case SIOCSIFMTU: {
5ba3f43e 2496#if UTUN_NEXUS
5c9f4661
A		 2497 if (pcb->utun_use_netif) {
2498 // Make sure we can fit packets in the channel buffers
2499 // Allow for the headroom in the slot
2500 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2501 result = EINVAL;
2502 } else {
2503 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2504 }
2505 } else
5ba3f43e 2506#endif // UTUN_NEXUS
5c9f4661
A		 2507 {
2508 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2509 }
b0d623f7 2510 break;
5c9f4661 2511 }
d1ecb069
A		 2512
2513 case SIOCSIFFLAGS:
2514 /* ifioctl() takes care of it */
2515 break;
2516
b0d623f7
A		 2517 default:
2518 result = EOPNOTSUPP;
2519 }
2520
2521 return result;
2522}
2523
2524static void
5ba3f43e 2525utun_detached(ifnet_t interface)
b0d623f7
A		 2526{
2527 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2528 (void)ifnet_release(interface);
5c9f4661 2529 utun_free_pcb(pcb, true);
b0d623f7
A		 2530}
2531
2532/* Protocol Handlers */
2533
2534static errno_t
5ba3f43e
A		 2535utun_proto_input(__unused ifnet_t interface,
2536 protocol_family_t protocol,
2537 mbuf_t m,
2538 __unused char *frame_header)
b0d623f7 2539{
5c9f4661
A		 2540 struct utun_pcb *pcb = ifnet_softc(interface);
2541#if UTUN_NEXUS
2542 if (!pcb->utun_use_netif)
2543#endif // UTUN_NEXUS
2544 {
2545 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2546 }
39037602 2547 if (proto_input(protocol, m) != 0) {
6d2010ae 2548 m_freem(m);
5c9f4661
A		 2549#if UTUN_NEXUS
2550 if (!pcb->utun_use_netif)
2551#endif // UTUN_NEXUS
2552 {
2553 ifnet_stat_increment_in(interface, 0, 0, 1);
2554 }
5ba3f43e 2555 } else {
5c9f4661
A		 2556#if UTUN_NEXUS
2557 if (!pcb->utun_use_netif)
5ba3f43e 2558#endif // UTUN_NEXUS
5c9f4661
A		 2559 {
2560 ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0);
2561 }
39037602 2562 }
b0d623f7
A		 2563
2564 return 0;
2565}
2566
2567static errno_t
5ba3f43e
A		 2568utun_proto_pre_output(__unused ifnet_t interface,
2569 protocol_family_t protocol,
2570 __unused mbuf_t *packet,
2571 __unused const struct sockaddr *dest,
2572 __unused void *route,
2573 char *frame_type,
2574 __unused char *link_layer_dest)
b0d623f7 2575{
6d2010ae 2576 *(protocol_family_t *)(void *)frame_type = protocol;
39037602 2577 return 0;
b0d623f7
A		 2578}
2579
2580static errno_t
5ba3f43e
A		 2581utun_attach_proto(ifnet_t interface,
2582 protocol_family_t protocol)
b0d623f7
A		 2583{
2584 struct ifnet_attach_proto_param proto;
b0d623f7
A		 2585
2586 bzero(&proto, sizeof(proto));
2587 proto.input = utun_proto_input;
2588 proto.pre_output = utun_proto_pre_output;
2589
5ba3f43e 2590 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
b0d623f7
A		 2591 if (result != 0 && result != EEXIST) {
2592 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2593 protocol, result);
2594 }
2595
2596 return result;
2597}
2598
5ba3f43e
A		 2599static errno_t
2600utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2601{
5c9f4661
A		 2602#if UTUN_NEXUS
2603 if (pcb->utun_use_netif) {
2604 lck_rw_lock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2605
5c9f4661
A		 2606 lck_mtx_lock(&pcb->utun_input_chain_lock);
2607 if (pcb->utun_input_chain != NULL) {
2608 pcb->utun_input_chain_last->m_nextpkt = packet;
2609 } else {
2610 pcb->utun_input_chain = packet;
2611 }
2612 while (packet->m_nextpkt) {
2613 VERIFY(packet != packet->m_nextpkt);
2614 packet = packet->m_nextpkt;
2615 }
2616 pcb->utun_input_chain_last = packet;
2617 lck_mtx_unlock(&pcb->utun_input_chain_lock);
5ba3f43e 2618
5c9f4661
A		 2619 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2620 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
316670eb 2621
5c9f4661
A		 2622 if (rx_ring != NULL) {
2623 kern_channel_notify(rx_ring, 0);
2624 }
316670eb 2625
5c9f4661
A		 2626 return (0);
2627 } else
2628#endif // IPSEC_NEXUS
2629 {
2630 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
5ba3f43e 2631
5c9f4661
A		 2632 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2633 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2634 }
2635 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2636 /* flush data */
2637 mbuf_freem(packet);
2638 return 0;
2639 }
316670eb 2640
5c9f4661
A		 2641 errno_t result = 0;
2642 if (!pcb->utun_ext_ifdata_stats) {
2643 struct ifnet_stat_increment_param incs = {};
2644 incs.packets_in = 1;
2645 incs.bytes_in = mbuf_pkthdr_len(packet);
2646 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2647 } else {
2648 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2649 }
2650 if (result != 0) {
2651 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
5ba3f43e 2652
5c9f4661
A		 2653 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2654 mbuf_freem(packet);
2655 }
5ba3f43e 2656
5c9f4661 2657 return (0);
316670eb 2658 }
316670eb 2659}
5ba3f43e
A		 2660
2661#if UTUN_NEXUS
2662
2663static errno_t
2664utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2665{
2666 return 0;
2667}
2668
2669static void
2670utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2671{
2672 // Ignore
2673}
2674
2675static errno_t
2676utun_register_nexus(void)
2677{
2678 const struct kern_nexus_domain_provider_init dp_init = {
2679 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2680 .nxdpi_flags = 0,
2681 .nxdpi_init = utun_nxdp_init,
2682 .nxdpi_fini = utun_nxdp_fini
2683 };
2684 errno_t err = 0;
2685
2686 /* utun_nxdp_init() is called before this function returns */
2687 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2688 (const uint8_t *) "com.apple.utun",
2689 &dp_init, sizeof(dp_init),
2690 &utun_nx_dom_prov);
2691 if (err != 0) {
2692 printf("%s: failed to register domain provider\n", __func__);
2693 return (err);
2694 }
2695 return (0);
2696}
2697
2698static errno_t
2699utun_ifnet_set_attrs(ifnet_t ifp)
2700{
2701 /* Set flags and additional information. */
2702 ifnet_set_mtu(ifp, 1500);
2703 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2704
2705 /* The interface must generate its own IPv6 LinkLocal address,
2706 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2707 */
2708 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2709
2710 return (0);
2711}
2712
2713static errno_t
2714utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2715{
2716 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2717 pcb->utun_netif_nexus = nexus;
2718 return (utun_ifnet_set_attrs(ifp));
2719}
2720
2721static errno_t
2722utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2723 proc_t p, kern_nexus_t nexus,
2724 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2725{
2726#pragma unused(nxprov, p)
2727#pragma unused(nexus, nexus_port, channel, ch_ctx)
2728 return (0);
2729}
2730
2731static errno_t
2732utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2733 kern_channel_t channel)
2734{
2735#pragma unused(nxprov, channel)
2736 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2737 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2738 return (ok ? 0 : ENXIO);
2739}
2740
2741static void
2742utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2743 kern_channel_t channel)
2744{
2745#pragma unused(nxprov, nexus, channel)
2746}
2747
2748static void
2749utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2750 kern_channel_t channel)
2751{
2752#pragma unused(nxprov, nexus, channel)
2753}
2754
2755static void
2756utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2757 kern_channel_t channel)
2758{
2759#pragma unused(nxprov, channel)
2760 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2761 if (pcb->utun_netif_nexus == nexus) {
2762 pcb->utun_netif_nexus = NULL;
2763 }
2764 ifnet_decr_iorefcnt(pcb->utun_ifp);
2765}
2766
2767static errno_t
2768utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2769 kern_channel_t channel, kern_channel_ring_t ring,
2770 boolean_t is_tx_ring, void **ring_ctx)
2771{
2772#pragma unused(nxprov)
2773#pragma unused(channel)
2774#pragma unused(ring_ctx)
2775 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2776 if (!is_tx_ring) {
2777 VERIFY(pcb->utun_kpipe_rxring == NULL);
2778 pcb->utun_kpipe_rxring = ring;
2779 } else {
2780 VERIFY(pcb->utun_kpipe_txring == NULL);
2781 pcb->utun_kpipe_txring = ring;
2782 }
2783 return 0;
2784}
2785
2786static void
2787utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2788 kern_channel_ring_t ring)
2789{
2790#pragma unused(nxprov)
2791 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2792 if (pcb->utun_kpipe_rxring == ring) {
2793 pcb->utun_kpipe_rxring = NULL;
2794 } else if (pcb->utun_kpipe_txring == ring) {
2795 pcb->utun_kpipe_txring = NULL;
2796 }
2797}
2798
2799static errno_t
2800utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2801 kern_channel_ring_t tx_ring, uint32_t flags)
2802{
2803#pragma unused(nxprov)
2804#pragma unused(flags)
2805 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2806
2807 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2808 int channel_enabled = pcb->utun_kpipe_enabled;
2809 if (!channel_enabled) {
2810 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2811 return 0;
2812 }
2813
5c9f4661
A		 2814 if (pcb->utun_use_netif) {
2815 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2816 if (tx_slot == NULL) {
2817 // Nothing to write, bail
2818 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2819 return 0;
2820 }
2821
2822 // Signal the netif ring to read
2823 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2824 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2825 if (rx_ring != NULL) {
2826 kern_channel_notify(rx_ring, 0);
2827 }
2828 } else {
5ba3f43e 2829 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2830
5c9f4661
A		 2831 struct ifnet_stat_increment_param incs = {};
2832 struct kern_channel_ring_stat_increment tx_ring_stats = {};
2833 MBUFQ_HEAD(mbufq) mbq;
2834 MBUFQ_INIT(&mbq);
2835 kern_channel_slot_t tx_pslot = NULL;
2836 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2837 while (tx_slot != NULL) {
2838 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2839
2840 // Advance TX ring
2841 tx_pslot = tx_slot;
2842 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2843
2844 if (tx_ph == 0) {
2845 continue;
2846 }
2847
2848 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2849 VERIFY(tx_buf != NULL);
2850 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2851 VERIFY(tx_baddr != 0);
2852 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2853
2854 size_t length = MIN(kern_packet_get_data_length(tx_ph),
2855 pcb->utun_slot_size);
2856
2857 mbuf_t data = NULL;
2858 if (length >= UTUN_HEADER_SIZE(pcb) &&
2859 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
2860 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
2861 VERIFY(0 == error);
2862 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
2863 VERIFY(0 == error);
2864 /*
2865 * The userland ABI requires the first four bytes have
2866 * the protocol family in network byte order: swap them
2867 */
2868 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
2869 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
2870 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2871 incs.packets_in++;
2872 incs.bytes_in += length;
2873 MBUFQ_ENQUEUE(&mbq, data);
2874 }
2875 }
2876 if (tx_pslot) {
2877 kern_channel_advance_slot(tx_ring, tx_pslot);
2878 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
2879 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
2880 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
2881 (void) kern_channel_reclaim(tx_ring);
2882 }
2883 if (!MBUFQ_EMPTY(&mbq)) {
2884 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
2885 MBUFQ_LAST(&mbq), &incs);
2886 MBUFQ_INIT(&mbq);
2887 }
5ba3f43e
A		 2888 }
2889
2890 return 0;
2891}
2892
2893static errno_t
2894utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2895 kern_channel_ring_t rx_ring, uint32_t flags)
2896{
2897#pragma unused(nxprov)
2898#pragma unused(flags)
2899 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
5c9f4661 2900 struct kern_channel_ring_stat_increment rx_ring_stats = {};
5ba3f43e
A		 2901
2902 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2903
2904 int channel_enabled = pcb->utun_kpipe_enabled;
2905 if (!channel_enabled) {
2906 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2907 return 0;
2908 }
2909
2910 /* reclaim user-released slots */
2911 (void) kern_channel_reclaim(rx_ring);
2912
2913 uint32_t avail = kern_channel_available_slot_count(rx_ring);
2914 if (avail == 0) {
2915 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2916 return 0;
2917 }
2918
5c9f4661
A		 2919 if (pcb->utun_use_netif) {
2920 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
2921 if (tx_ring == NULL ||
2922 pcb->utun_netif_nexus == NULL) {
2923 // Net-If TX ring not set up yet, nothing to read
2924 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2925 return 0;
2926 }
5ba3f43e 2927
5c9f4661 2928 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
5ba3f43e 2929
5c9f4661 2930 // Unlock utun before entering ring
5ba3f43e 2931 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 2932
5c9f4661 2933 (void)kr_enter(tx_ring, TRUE);
5ba3f43e 2934
5c9f4661
A		 2935 // Lock again after entering and validate
2936 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2937 if (tx_ring != pcb->utun_netif_txring) {
2938 // Ring no longer valid
2939 // Unlock first, then exit ring
2940 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2941 kr_exit(tx_ring);
2942 return 0;
5ba3f43e
A		 2943 }
2944
5c9f4661
A		 2945 struct kern_channel_ring_stat_increment tx_ring_stats;
2946 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
2947 kern_channel_slot_t tx_pslot = NULL;
2948 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2949 if (tx_slot == NULL) {
2950 // Nothing to read, don't bother signalling
2951 // Unlock first, then exit ring
2952 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2953 kr_exit(tx_ring);
2954 return 0;
5ba3f43e
A		 2955 }
2956
5c9f4661
A		 2957 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
2958 VERIFY(rx_pp != NULL);
2959 kern_channel_slot_t rx_pslot = NULL;
2960 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
5ba3f43e 2961
5c9f4661
A		 2962 while (rx_slot != NULL && tx_slot != NULL) {
2963 size_t length;
2964 kern_buflet_t rx_buf;
2965 void *rx_baddr;
5ba3f43e 2966
5c9f4661 2967 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
5ba3f43e 2968
5c9f4661
A		 2969 // Advance TX ring
2970 tx_pslot = tx_slot;
2971 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
5ba3f43e 2972
5c9f4661
A		 2973 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
2974 if (tx_ph == 0) {
2975 continue;
2976 }
5ba3f43e 2977
5c9f4661
A		 2978 // Allocate rx packet
2979 kern_packet_t rx_ph = 0;
2980 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
2981 if (unlikely(error != 0)) {
2982 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
2983 pcb->utun_ifp->if_xname);
5ba3f43e
A		 2984 break;
2985 }
5c9f4661
A		 2986
2987 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2988 VERIFY(tx_buf != NULL);
2989 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2990 VERIFY(tx_baddr != NULL);
2991 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2992
2993 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
2994
2995 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
2996 pcb->utun_slot_size);
2997
2998 tx_ring_stats.kcrsi_slots_transferred++;
2999 tx_ring_stats.kcrsi_bytes_transferred += length;
3000
3001 if (length < UTUN_HEADER_SIZE(pcb) ||
3002 length > pcb->utun_slot_size ||
3003 length > rx_pp->pp_buflet_size ||
3004 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3005 /* flush data */
3006 kern_pbufpool_free(rx_pp, rx_ph);
3007 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3008 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3009 STATS_INC(nifs, NETIF_STATS_BADLEN);
3010 STATS_INC(nifs, NETIF_STATS_DROPPED);
3011 continue;
5ba3f43e 3012 }
5c9f4661
A		 3013
3014 /* fillout packet */
3015 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3016 VERIFY(rx_buf != NULL);
3017 rx_baddr = kern_buflet_get_object_address(rx_buf);
3018 VERIFY(rx_baddr != NULL);
3019
3020 // Find family
3021 uint32_t af = 0;
3022 uint8_t vhl = *(uint8_t *)(tx_baddr);
3023 u_int ip_version = (vhl >> 4);
3024 switch (ip_version) {
3025 case 4: {
3026 af = AF_INET;
3027 break;
3028 }
3029 case 6: {
3030 af = AF_INET6;
3031 break;
3032 }
3033 default: {
3034 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3035 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3036 break;
3037 }
5ba3f43e 3038 }
5ba3f43e 3039
5c9f4661
A		 3040 // Copy header
3041 af = htonl(af);
3042 memcpy((void *)rx_baddr, &af, sizeof(af));
3043 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3044 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3045 }
5ba3f43e 3046
5c9f4661
A		 3047 // Copy data from tx to rx
3048 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3049 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
5ba3f43e 3050
5c9f4661
A		 3051 /* finalize and attach the packet */
3052 error = kern_buflet_set_data_offset(rx_buf, 0);
3053 VERIFY(error == 0);
3054 error = kern_buflet_set_data_length(rx_buf, length);
3055 VERIFY(error == 0);
3056 error = kern_packet_finalize(rx_ph);
3057 VERIFY(error == 0);
3058 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3059 VERIFY(error == 0);
5ba3f43e 3060
5c9f4661
A		 3061 STATS_INC(nifs, NETIF_STATS_TXPKTS);
3062 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
5ba3f43e 3063
5c9f4661
A		 3064 rx_ring_stats.kcrsi_slots_transferred++;
3065 rx_ring_stats.kcrsi_bytes_transferred += length;
5ba3f43e 3066
5c9f4661
A		 3067 rx_pslot = rx_slot;
3068 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3069 }
5ba3f43e 3070
5c9f4661
A		 3071 if (rx_pslot) {
3072 kern_channel_advance_slot(rx_ring, rx_pslot);
3073 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3074 }
5ba3f43e 3075
5c9f4661
A		 3076 if (tx_pslot) {
3077 kern_channel_advance_slot(tx_ring, tx_pslot);
3078 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3079 (void)kern_channel_reclaim(tx_ring);
3080 }
5ba3f43e 3081
5c9f4661 3082 /* just like utun_ctl_rcvd(), always reenable output */
5ba3f43e
A		 3083 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3084 if (error != 0) {
3085 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
5ba3f43e 3086 }
5ba3f43e 3087
5c9f4661
A		 3088 // Unlock first, then exit ring
3089 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
5ba3f43e 3090
5c9f4661
A		 3091 if (tx_pslot != NULL) {
3092 kern_channel_notify(tx_ring, 0);
3093 }
3094 kr_exit(tx_ring);
3095 } else {
3096 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3097
3098 uint32_t mb_cnt = 0;
3099 uint32_t mb_len = 0;
3100 struct mbuf *mb_head = NULL;
3101 struct mbuf *mb_tail = NULL;
3102
3103 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3104 &mb_tail, &mb_cnt, &mb_len) != 0) {
3105 return 0;
3106 }
3107 VERIFY(mb_cnt <= avail);
3108
3109 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3110 VERIFY(rx_pp != NULL);
3111 kern_channel_slot_t rx_pslot = NULL;
3112 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3113 while (rx_slot) {
3114 size_t length = 0;
3115 mbuf_t data = NULL;
3116 if ((data = mb_head) == NULL) {
3117 VERIFY(mb_cnt == 0);
3118 break;
3119 }
3120 mb_head = mbuf_nextpkt(mb_head);
3121 mbuf_setnextpkt(data, NULL);
3122 VERIFY(mb_cnt != 0);
3123 --mb_cnt;
3124 length = mbuf_pkthdr_len(data);
3125 if (length < UTUN_HEADER_SIZE(pcb) ||
3126 length > pcb->utun_slot_size ||
3127 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3128 /* flush data */
3129 mbuf_freem(data);
3130 continue;
3131 }
3132 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3133
3134 // Allocate rx packet
3135 kern_packet_t rx_ph = 0;
3136 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3137 if (unlikely(error != 0)) {
3138 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
3139 pcb->utun_ifp->if_xname);
3140 break;
3141 }
3142
3143 /*
3144 * The ABI requires the protocol in network byte order
3145 */
3146 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3147
3148 // Fillout rx packet
3149 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3150 VERIFY(rx_buf != NULL);
3151 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3152 VERIFY(rx_baddr != NULL);
3153
3154 // Copy-in data from mbuf to buflet
3155 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3156 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3157
3158 // Finalize and attach the packet
3159 error = kern_buflet_set_data_offset(rx_buf, 0);
3160 VERIFY(error == 0);
3161 error = kern_buflet_set_data_length(rx_buf, length);
3162 VERIFY(error == 0);
3163 error = kern_packet_finalize(rx_ph);
3164 VERIFY(error == 0);
3165 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3166 VERIFY(error == 0);
3167
3168 rx_ring_stats.kcrsi_slots_transferred++;
3169 rx_ring_stats.kcrsi_bytes_transferred += length;
3170
3171 if (!pcb->utun_ext_ifdata_stats) {
3172 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3173 }
3174
3175 mbuf_freem(data);
3176
3177 rx_pslot = rx_slot;
3178 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3179 }
3180 if (rx_pslot) {
3181 kern_channel_advance_slot(rx_ring, rx_pslot);
3182 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3183 }
3184 if (mb_head != NULL) {
3185 VERIFY(mb_cnt != 0);
3186 mbuf_freem_list(mb_head);
3187 }
5ba3f43e 3188 }
5ba3f43e
A		 3189
3190 return 0;
3191}
39037602 3192
5ba3f43e 3193#endif // UTUN_NEXUS
39037602
A		 3194
3195
3196/*
5ba3f43e 3197 * These are place holders until coreTLS kext stops calling them
39037602
A		 3198 */
3199errno_t utun_ctl_register_dtls (void *reg);
3200int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3201void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3202
3203errno_t
3204utun_ctl_register_dtls (void *reg)
3205{
3206#pragma unused(reg)
3207 return 0;
3208}
3209
3210int
3211utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3212{
3213#pragma unused(pcb)
3214#pragma unused(pkt)
3215#pragma unused(family)
3216 return 0;
3217}
3218
3219void
3220utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3221{
3222#pragma unused(pcb)
3223}
mirror of XNU