]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_utun.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.20.62.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
CommitLineData
b0d623f7 1/*
5ba3f43e 2 * Copyright (c) 2008-2017 Apple Inc. All rights reserved.
b0d623f7
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31/* ----------------------------------------------------------------------------------
32Application of kernel control for interface creation
33
34Theory of operation:
35utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36This kernel control will register an interface for every client that connects.
37---------------------------------------------------------------------------------- */
38
39#include <sys/systm.h>
40#include <sys/kern_control.h>
41#include <net/kpi_protocol.h>
42#include <net/kpi_interface.h>
43#include <sys/socket.h>
44#include <net/if.h>
45#include <net/if_types.h>
46#include <net/bpf.h>
47#include <net/if_utun.h>
b0d623f7
A		 48#include <sys/mbuf.h>
49#include <sys/sockio.h>
50#include <netinet/in.h>
5ba3f43e 51#include <netinet/ip.h>
b0d623f7
A		 52#include <netinet6/in6_var.h>
53#include <netinet6/in6_var.h>
54#include <sys/kauth.h>
5ba3f43e
A		 55#include <net/necp.h>
56#include <kern/zalloc.h>
57
58#define UTUN_NEXUS 0
59
60extern unsigned int if_enable_netagent;
61
62#if UTUN_NEXUS
63static nexus_controller_t utun_ncd;
64static int utun_ncd_refcount;
65static uuid_t utun_kpipe_uuid;
66static uuid_t utun_nx_dom_prov;
67
68typedef struct utun_nx {
69 uuid_t if_provider;
70 uuid_t if_instance;
71 uuid_t ms_provider;
72 uuid_t ms_instance;
73 uuid_t ms_device;
74 uuid_t ms_host;
75 uuid_t ms_agent;
76} *utun_nx_t;
77
78#endif // UTUN_NEXUS
79
80/* Control block allocated for each kernel control connection */
81struct utun_pcb {
82 TAILQ_ENTRY(utun_pcb) utun_chain;
83 kern_ctl_ref utun_ctlref;
84 ifnet_t utun_ifp;
85 u_int32_t utun_unit;
86 u_int32_t utun_unique_id;
87 u_int32_t utun_flags;
88 int utun_ext_ifdata_stats;
89 u_int32_t utun_max_pending_packets;
90 char utun_if_xname[IFXNAMSIZ];
91 char utun_unique_name[IFXNAMSIZ];
92 // PCB lock protects state fields and rings
93 decl_lck_rw_data(, utun_pcb_lock);
94 struct mbuf * utun_input_chain;
95 struct mbuf * utun_input_chain_last;
96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
98 lck_mtx_t utun_input_chain_lock;
99 bool utun_output_disabled;
100
101#if UTUN_NEXUS
102 struct utun_nx utun_nx;
103 int utun_kpipe_enabled;
104 uuid_t utun_kpipe_uuid;
105 void * utun_kpipe_rxring;
106 void * utun_kpipe_txring;
107
108 kern_nexus_t utun_netif_nexus;
109 void * utun_netif_rxring;
110 void * utun_netif_txring;
111 uint64_t utun_netif_txring_size;
112#endif // UTUN_NEXUS
113};
39037602 114
b0d623f7
A		 115/* Kernel Control functions */
116static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
117 void **unitinfo);
118static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
119 void *unitinfo);
120static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
121 void *unitinfo, mbuf_t m, int flags);
122static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
123 int opt, void *data, size_t *len);
124static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
125 int opt, void *data, size_t len);
fe8ab488
A		 126static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
127 int flags);
b0d623f7
A		 128
129/* Network Interface functions */
5ba3f43e 130#if !UTUN_NEXUS
fe8ab488 131static void utun_start(ifnet_t interface);
5ba3f43e
A		 132static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
133 const struct sockaddr *dest, const char *desk_linkaddr,
134 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
135#endif // !UTUN_NEXUS
b0d623f7
A		 136static errno_t utun_output(ifnet_t interface, mbuf_t data);
137static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
138 protocol_family_t *protocol);
b0d623f7
A		 139static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
140 const struct ifnet_demux_desc *demux_array,
141 u_int32_t demux_count);
142static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
143static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
144static void utun_detached(ifnet_t interface);
145
146/* Protocol handlers */
147static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
148static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
149 mbuf_t m, char *frame_header);
150static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
151 mbuf_t *packet, const struct sockaddr *dest, void *route,
152 char *frame_type, char *link_layer_dest);
39037602
A		 153static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m);
154
5ba3f43e
A		 155#if UTUN_NEXUS
156
157#define UTUN_IF_DEFAULT_SLOT_SIZE 4096
158#define UTUN_IF_DEFAULT_RING_SIZE 64
159#define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
160#define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
161#define UTUN_IF_HEADROOM_SIZE 32
162
163#define UTUN_IF_MIN_RING_SIZE 16
164#define UTUN_IF_MAX_RING_SIZE 1024
165
166static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
167static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
168static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
169
170static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
171static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
172static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
173
174SYSCTL_DECL(_net_utun);
175SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
176
177SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
178 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
179SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
180 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
181SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
182 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
183
184static errno_t
185utun_register_nexus(void);
186
187static errno_t
188utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
189static errno_t
190utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
191 proc_t p, kern_nexus_t nexus,
192 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
193static errno_t
194utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
195 kern_channel_t channel);
196static void
197utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
198 kern_channel_t channel);
199static void
200utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
201 kern_channel_t channel);
202static void
203utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
204 kern_channel_t channel);
205static errno_t
206utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
207 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
208 void **ring_ctx);
209static void
210utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
211 kern_channel_ring_t ring);
212static errno_t
213utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
214 kern_channel_ring_t ring, uint32_t flags);
215static errno_t
216utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
217 kern_channel_ring_t ring, uint32_t flags);
218#endif // UTUN_NEXUS
39037602
A		 219
220#define UTUN_DEFAULT_MTU 1500
221#define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
b0d623f7
A		 222
223static kern_ctl_ref utun_kctlref;
224static u_int32_t utun_family;
5ba3f43e
A		 225static lck_attr_t *utun_lck_attr;
226static lck_grp_attr_t *utun_lck_grp_attr;
227static lck_grp_t *utun_lck_grp;
228static lck_mtx_t utun_lock;
229
230TAILQ_HEAD(utun_list, utun_pcb) utun_head;
231
232#define UTUN_PCB_ZONE_MAX 32
233#define UTUN_PCB_ZONE_NAME "net.if_utun"
234
235static unsigned int utun_pcb_size; /* size of zone element */
236static struct zone *utun_pcb_zone; /* zone for utun_pcb */
237
238#if UTUN_NEXUS
239
240static int
241sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
242{
243#pragma unused(arg1, arg2)
244 int value = if_utun_ring_size;
245
246 int error = sysctl_handle_int(oidp, &value, 0, req);
247 if (error || !req->newptr) {
248 return (error);
249 }
250
251 if (value < UTUN_IF_MIN_RING_SIZE ||
252 value > UTUN_IF_MAX_RING_SIZE) {
253 return (EINVAL);
254 }
255
256 if_utun_ring_size = value;
257
258 return (0);
259}
260
261static int
262sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
263{
264#pragma unused(arg1, arg2)
265 int value = if_utun_tx_fsw_ring_size;
266
267 int error = sysctl_handle_int(oidp, &value, 0, req);
268 if (error || !req->newptr) {
269 return (error);
270 }
271
272 if (value < UTUN_IF_MIN_RING_SIZE ||
273 value > UTUN_IF_MAX_RING_SIZE) {
274 return (EINVAL);
275 }
276
277 if_utun_tx_fsw_ring_size = value;
278
279 return (0);
280}
281
282static int
283sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
284{
285#pragma unused(arg1, arg2)
286 int value = if_utun_rx_fsw_ring_size;
287
288 int error = sysctl_handle_int(oidp, &value, 0, req);
289 if (error || !req->newptr) {
290 return (error);
291 }
292
293 if (value < UTUN_IF_MIN_RING_SIZE ||
294 value > UTUN_IF_MAX_RING_SIZE) {
295 return (EINVAL);
296 }
297
298 if_utun_rx_fsw_ring_size = value;
299
300 return (0);
301}
302
303static errno_t
304utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
305 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
306 void **ring_ctx)
307{
308#pragma unused(nxprov)
309#pragma unused(channel)
310#pragma unused(ring_ctx)
311 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
312 if (!is_tx_ring) {
313 VERIFY(pcb->utun_netif_rxring == NULL);
314 pcb->utun_netif_rxring = ring;
315 } else {
316 VERIFY(pcb->utun_netif_txring == NULL);
317 pcb->utun_netif_txring = ring;
318 }
319 return 0;
320}
321
322static void
323utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
324 kern_channel_ring_t ring)
325{
326#pragma unused(nxprov)
327 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
328 if (pcb->utun_netif_rxring == ring) {
329 pcb->utun_netif_rxring = NULL;
330 } else if (pcb->utun_netif_txring == ring) {
331 pcb->utun_netif_txring = NULL;
332 }
333}
334
335static errno_t
336utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
337 kern_channel_ring_t tx_ring, uint32_t flags)
338{
339#pragma unused(nxprov)
340#pragma unused(flags)
341 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
342
343 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
344
345 lck_rw_lock_shared(&pcb->utun_pcb_lock);
346
347 struct kern_channel_ring_stat_increment tx_ring_stats;
348 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
349 kern_channel_slot_t tx_pslot = NULL;
350 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
351
352 STATS_INC(nifs, NETIF_STATS_TXSYNC);
353
354 if (tx_slot == NULL) {
355 // Nothing to write, don't bother signalling
356 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
357 return 0;
358 }
359
360 if (pcb->utun_kpipe_enabled) {
361 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
362 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
363
364 // Signal the kernel pipe ring to read
365 if (rx_ring != NULL) {
366 kern_channel_notify(rx_ring, 0);
367 }
368 return 0;
369 }
370
371 // If we're here, we're injecting into the utun kernel control socket
372 while (tx_slot != NULL) {
373 size_t length = 0;
374 mbuf_t data = NULL;
375
376 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
377
378 if (tx_ph == 0) {
379 // Advance TX ring
380 tx_pslot = tx_slot;
381 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
382 continue;
383 }
384 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
385
386 // Advance TX ring
387 tx_pslot = tx_slot;
388 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
389
390 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
391 VERIFY(tx_buf != NULL);
392
393 /* tx_baddr is the absolute buffer address */
394 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
395 VERIFY(tx_baddr != 0);
396
397 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
398
399 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
400 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
401
402 // The offset must be large enough for the headers
403 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
404
405 // Find family
406 uint32_t af = 0;
407 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
408 u_int ip_version = (vhl >> 4);
409 switch (ip_version) {
410 case 4: {
411 af = AF_INET;
412 break;
413 }
414 case 6: {
415 af = AF_INET6;
416 break;
417 }
418 default: {
419 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
420 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
421 UTUN_HEADER_SIZE(pcb));
422 break;
423 }
424 }
425
426 tx_offset -= UTUN_HEADER_SIZE(pcb);
427 tx_length += UTUN_HEADER_SIZE(pcb);
428 tx_baddr += tx_offset;
429
430 length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE);
431
432 // Copy in family
433 memcpy(tx_baddr, &af, sizeof(af));
434 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
435 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
436 }
437
438 if (length > 0) {
439 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
440 if (error == 0) {
441 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
442 if (error == 0) {
443 error = utun_output(pcb->utun_ifp, data);
444 if (error != 0) {
445 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
446 }
447 } else {
448 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
449 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
450 STATS_INC(nifs, NETIF_STATS_DROPPED);
451 mbuf_freem(data);
452 data = NULL;
453 }
454 } else {
455 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
456 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
457 STATS_INC(nifs, NETIF_STATS_DROPPED);
458 }
459 } else {
460 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
461 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
462 STATS_INC(nifs, NETIF_STATS_DROPPED);
463 }
464
465 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
466
467 if (data == NULL) {
468 continue;
469 }
470
471 STATS_INC(nifs, NETIF_STATS_TXPKTS);
472 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
473
474 tx_ring_stats.kcrsi_slots_transferred++;
475 tx_ring_stats.kcrsi_bytes_transferred += length;
476 }
477
478 if (tx_pslot) {
479 kern_channel_advance_slot(tx_ring, tx_pslot);
480 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
481 (void)kern_channel_reclaim(tx_ring);
482 }
483
484 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
485
486 return 0;
487}
488
489static errno_t
490utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
491 kern_channel_ring_t ring, __unused uint32_t flags)
492{
493#pragma unused(nxprov)
494 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
495
496 lck_rw_lock_shared(&pcb->utun_pcb_lock);
497
498 boolean_t more = false;
499 errno_t rc = 0;
500 do {
501 /* Refill and sync the ring */
502 rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more);
503 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
504 printf("%s, tx refill failed %d\n", __func__, rc);
505 }
506 } while ((rc == 0) && more);
507
508 if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) {
509 uint32_t tx_available = kern_channel_available_slot_count(ring);
510 if (pcb->utun_netif_txring_size > 0 &&
511 tx_available >= pcb->utun_netif_txring_size - 1) {
512 // No room left in tx ring, disable output for now
513 errno_t error = ifnet_disable_output(pcb->utun_ifp);
514 if (error != 0) {
515 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
516 } else {
517 pcb->utun_output_disabled = true;
518 }
519 }
520 }
b0d623f7 521
5ba3f43e
A		 522 if (pcb->utun_kpipe_enabled &&
523 (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) {
524 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
525
526 // Unlock while calling notify
527 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
528 // Signal the kernel pipe ring to read
529 if (rx_ring != NULL) {
530 kern_channel_notify(rx_ring, 0);
531 }
532 } else {
533 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
534 }
535
536 return (0);
537}
538
539static errno_t
540utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
541 kern_channel_ring_t rx_ring, uint32_t flags)
542{
543#pragma unused(nxprov)
544#pragma unused(flags)
545 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
546 struct kern_channel_ring_stat_increment rx_ring_stats;
547
548 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
549
550 lck_rw_lock_shared(&pcb->utun_pcb_lock);
551
552 // Reclaim user-released slots
553 (void) kern_channel_reclaim(rx_ring);
554
555 STATS_INC(nifs, NETIF_STATS_RXSYNC);
556
557 uint32_t avail = kern_channel_available_slot_count(rx_ring);
558 if (avail == 0) {
559 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
560 return 0;
561 }
562
563 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
564 VERIFY(rx_pp != NULL);
565 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
566 kern_channel_slot_t rx_pslot = NULL;
567 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
568
569 while (rx_slot != NULL) {
570 // Check for a waiting packet
571 lck_mtx_lock(&pcb->utun_input_chain_lock);
572 mbuf_t data = pcb->utun_input_chain;
573 if (data == NULL) {
574 lck_mtx_unlock(&pcb->utun_input_chain_lock);
575 break;
576 }
577
578 // Allocate rx packet
579 kern_packet_t rx_ph = 0;
580 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
581 if (unlikely(error != 0)) {
582 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
583 STATS_INC(nifs, NETIF_STATS_DROPPED);
584 printf("utun_netif_sync_rx %s: failed to allocate packet\n",
585 pcb->utun_ifp->if_xname);
586 lck_mtx_unlock(&pcb->utun_input_chain_lock);
587 break;
588 }
589
590 // Advance waiting packets
591 pcb->utun_input_chain = data->m_nextpkt;
592 data->m_nextpkt = NULL;
593 if (pcb->utun_input_chain == NULL) {
594 pcb->utun_input_chain_last = NULL;
595 }
596 lck_mtx_unlock(&pcb->utun_input_chain_lock);
597
598 size_t header_offset = UTUN_HEADER_SIZE(pcb);
599 size_t length = mbuf_pkthdr_len(data);
600
601 if (length < header_offset) {
602 // mbuf is too small
603 mbuf_freem(data);
604 kern_pbufpool_free(rx_pp, rx_ph);
605 STATS_INC(nifs, NETIF_STATS_BADLEN);
606 STATS_INC(nifs, NETIF_STATS_DROPPED);
607 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
608 pcb->utun_ifp->if_xname, length, header_offset);
609 continue;
610 }
611
612 length -= header_offset;
613 if (length > rx_pp->pp_buflet_size) {
614 // Flush data
615 mbuf_freem(data);
616 kern_pbufpool_free(rx_pp, rx_ph);
617 STATS_INC(nifs, NETIF_STATS_BADLEN);
618 STATS_INC(nifs, NETIF_STATS_DROPPED);
619 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
620 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
621 continue;
622 }
623
624 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
625
626 // Fillout rx packet
627 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
628 VERIFY(rx_buf != NULL);
629 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
630 VERIFY(rx_baddr != NULL);
631
632 // Copy-in data from mbuf to buflet
633 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
634 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
635
636 // Finalize and attach the packet
637 error = kern_buflet_set_data_offset(rx_buf, 0);
638 VERIFY(error == 0);
639 error = kern_buflet_set_data_length(rx_buf, length);
640 VERIFY(error == 0);
641 error = kern_packet_set_link_header_offset(rx_ph, 0);
642 VERIFY(error == 0);
643 error = kern_packet_set_network_header_offset(rx_ph, 0);
644 VERIFY(error == 0);
645 error = kern_packet_finalize(rx_ph);
646 VERIFY(error == 0);
647 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
648 VERIFY(error == 0);
649
650 STATS_INC(nifs, NETIF_STATS_RXPKTS);
651 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
652 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
653
654 rx_ring_stats.kcrsi_slots_transferred++;
655 rx_ring_stats.kcrsi_bytes_transferred += length;
656
657 mbuf_freem(data);
658
659 // Advance ring
660 rx_pslot = rx_slot;
661 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
662 }
663
664 struct kern_channel_ring_stat_increment tx_ring_stats;
665 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
666 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
667 kern_channel_slot_t tx_pslot = NULL;
668 kern_channel_slot_t tx_slot = NULL;
669 if (tx_ring == NULL) {
670 // Net-If TX ring not set up yet, nothing to read
671 goto done;
672 }
673
674 // Unlock utun before entering ring
675 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
676
677 (void)kr_enter(tx_ring, TRUE);
678
679 // Lock again after entering and validate
680 lck_rw_lock_shared(&pcb->utun_pcb_lock);
681 if (tx_ring != pcb->utun_kpipe_txring) {
682 goto done;
683 }
684
685 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
686 if (tx_slot == NULL) {
687 // Nothing to read, don't bother signalling
688 goto done;
689 }
690
691 while (rx_slot != NULL && tx_slot != NULL) {
692 // Allocate rx packet
693 kern_packet_t rx_ph = 0;
694 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
695
696 // Advance TX ring
697 tx_pslot = tx_slot;
698 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
699
700 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
701 if (tx_ph == 0) {
702 continue;
703 }
704
705 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
706 if (unlikely(error != 0)) {
707 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
708 STATS_INC(nifs, NETIF_STATS_DROPPED);
709 printf("utun_netif_sync_rx %s: failed to allocate packet\n",
710 pcb->utun_ifp->if_xname);
711 break;
712 }
713
714 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
715 VERIFY(tx_buf != NULL);
716 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
717 VERIFY(tx_baddr != 0);
718 tx_baddr += kern_buflet_get_data_offset(tx_buf);
719
720 // Check packet length
721 size_t header_offset = UTUN_HEADER_SIZE(pcb);
722 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
723 if (tx_length < header_offset) {
724 // Packet is too small
725 kern_pbufpool_free(rx_pp, rx_ph);
726 STATS_INC(nifs, NETIF_STATS_BADLEN);
727 STATS_INC(nifs, NETIF_STATS_DROPPED);
728 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
729 pcb->utun_ifp->if_xname, tx_length, header_offset);
730 continue;
731 }
732
733 size_t length = MIN(tx_length - header_offset,
734 UTUN_IF_DEFAULT_SLOT_SIZE);
735
736 tx_ring_stats.kcrsi_slots_transferred++;
737 tx_ring_stats.kcrsi_bytes_transferred += length;
738
739 // Fillout rx packet
740 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
741 VERIFY(rx_buf != NULL);
742 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
743 VERIFY(rx_baddr != NULL);
744
745 // Copy-in data from tx to rx
746 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
747 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
748
749 // Finalize and attach the packet
750 error = kern_buflet_set_data_offset(rx_buf, 0);
751 VERIFY(error == 0);
752 error = kern_buflet_set_data_length(rx_buf, length);
753 VERIFY(error == 0);
754 error = kern_packet_set_link_header_offset(rx_ph, 0);
755 VERIFY(error == 0);
756 error = kern_packet_set_network_header_offset(rx_ph, 0);
757 VERIFY(error == 0);
758 error = kern_packet_finalize(rx_ph);
759 VERIFY(error == 0);
760 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
761 VERIFY(error == 0);
762
763 STATS_INC(nifs, NETIF_STATS_RXPKTS);
764 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
765 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
766
767 rx_ring_stats.kcrsi_slots_transferred++;
768 rx_ring_stats.kcrsi_bytes_transferred += length;
769
770 rx_pslot = rx_slot;
771 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
772 }
773
774done:
775 if (rx_pslot) {
776 kern_channel_advance_slot(rx_ring, rx_pslot);
777 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
778 }
779
780 if (tx_pslot) {
781 kern_channel_advance_slot(tx_ring, tx_pslot);
782 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
783 (void)kern_channel_reclaim(tx_ring);
784 }
785
786 // Unlock first, then exit ring
787 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
788 if (tx_ring != NULL) {
789 if (tx_pslot != NULL) {
790 kern_channel_notify(tx_ring, 0);
791 }
792 kr_exit(tx_ring);
793 }
794
795 return 0;
796}
797
798static errno_t
799utun_nexus_ifattach(struct utun_pcb *pcb,
800 struct ifnet_init_eparams *init_params,
801 struct ifnet **ifp)
802{
803 errno_t err;
804 nexus_controller_t controller = kern_nexus_shared_controller();
805 struct kern_nexus_net_init net_init;
806
807 nexus_name_t provider_name;
808 snprintf((char *)provider_name, sizeof(provider_name),
809 "com.apple.netif.utun%d", pcb->utun_unit);
810
811 struct kern_nexus_provider_init prov_init = {
812 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
813 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
814 .nxpi_pre_connect = utun_nexus_pre_connect,
815 .nxpi_connected = utun_nexus_connected,
816 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
817 .nxpi_disconnected = utun_nexus_disconnected,
818 .nxpi_ring_init = utun_netif_ring_init,
819 .nxpi_ring_fini = utun_netif_ring_fini,
820 .nxpi_slot_init = NULL,
821 .nxpi_slot_fini = NULL,
822 .nxpi_sync_tx = utun_netif_sync_tx,
823 .nxpi_sync_rx = utun_netif_sync_rx,
824 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
825 };
826
827 nexus_attr_t nxa = NULL;
828 err = kern_nexus_attr_create(&nxa);
829 if (err != 0) {
830 printf("%s: kern_nexus_attr_create failed: %d\n",
831 __func__, err);
832 goto failed;
833 }
834
835 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
836 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
837 VERIFY(err == 0);
838
839 // Reset ring size for netif nexus to limit memory usage
840 uint64_t ring_size = if_utun_ring_size;
841 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
842 VERIFY(err == 0);
843 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
844 VERIFY(err == 0);
845
846 pcb->utun_netif_txring_size = ring_size;
847
848 err = kern_nexus_controller_register_provider(controller,
849 utun_nx_dom_prov,
850 provider_name,
851 &prov_init,
852 sizeof(prov_init),
853 nxa,
854 &pcb->utun_nx.if_provider);
855 if (err != 0) {
856 printf("%s register provider failed, error %d\n",
857 __func__, err);
858 goto failed;
859 }
860
861 bzero(&net_init, sizeof(net_init));
862 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
863 net_init.nxneti_flags = 0;
864 net_init.nxneti_eparams = init_params;
865 net_init.nxneti_lladdr = NULL;
866 net_init.nxneti_prepare = utun_netif_prepare;
867 err = kern_nexus_controller_alloc_net_provider_instance(controller,
868 pcb->utun_nx.if_provider,
869 pcb,
870 &pcb->utun_nx.if_instance,
871 &net_init,
872 ifp);
873 if (err != 0) {
874 printf("%s alloc_net_provider_instance failed, %d\n",
875 __func__, err);
876 kern_nexus_controller_deregister_provider(controller,
877 pcb->utun_nx.if_provider);
878 uuid_clear(pcb->utun_nx.if_provider);
879 goto failed;
880 }
881
882failed:
883 if (nxa) {
884 kern_nexus_attr_destroy(nxa);
885 }
886 return (err);
887}
888
889static void
890utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
891{
892 nexus_controller_t controller = kern_nexus_shared_controller();
893 errno_t err;
894
895 if (!uuid_is_null(instance)) {
896 err = kern_nexus_controller_free_provider_instance(controller,
897 instance);
898 if (err != 0) {
899 printf("%s free_provider_instance failed %d\n",
900 __func__, err);
901 }
902 uuid_clear(instance);
903 }
904 if (!uuid_is_null(provider)) {
905 err = kern_nexus_controller_deregister_provider(controller,
906 provider);
907 if (err != 0) {
908 printf("%s deregister_provider %d\n", __func__, err);
909 }
910 uuid_clear(provider);
911 }
912 return;
913}
914
915static void
916utun_nexus_detach(utun_nx_t nx)
917{
918 nexus_controller_t controller = kern_nexus_shared_controller();
919 errno_t err;
920
921 if (!uuid_is_null(nx->ms_host)) {
922 err = kern_nexus_ifdetach(controller,
923 nx->ms_instance,
924 nx->ms_host);
925 if (err != 0) {
926 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
927 __func__, err);
928 }
929 }
930
931 if (!uuid_is_null(nx->ms_device)) {
932 err = kern_nexus_ifdetach(controller,
933 nx->ms_instance,
934 nx->ms_device);
935 if (err != 0) {
936 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
937 __func__, err);
938 }
939 }
940
941 utun_detach_provider_and_instance(nx->if_provider,
942 nx->if_instance);
943 utun_detach_provider_and_instance(nx->ms_provider,
944 nx->ms_instance);
945
946 memset(nx, 0, sizeof(*nx));
947}
948
949static errno_t
950utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
951 const char *ifname,
952 uuid_t *provider, uuid_t *instance)
953{
954 nexus_attr_t attr = NULL;
955 nexus_controller_t controller = kern_nexus_shared_controller();
956 uuid_t dom_prov;
957 errno_t err;
958 struct kern_nexus_init init;
959 nexus_name_t provider_name;
960
961 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
962 &dom_prov);
963 if (err != 0) {
964 printf("%s can't get %s provider, error %d\n",
965 __func__, type_name, err);
966 goto failed;
967 }
968
969 err = kern_nexus_attr_create(&attr);
970 if (err != 0) {
971 printf("%s: kern_nexus_attr_create failed: %d\n",
972 __func__, err);
973 goto failed;
974 }
975
976 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
977 VERIFY(err == 0);
978
979 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
980 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
981 VERIFY(err == 0);
982
983 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
984 uint64_t tx_ring_size = if_utun_tx_fsw_ring_size;
985 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
986 VERIFY(err == 0);
987 uint64_t rx_ring_size = if_utun_rx_fsw_ring_size;
988 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
989 VERIFY(err == 0);
990
991 snprintf((char *)provider_name, sizeof(provider_name),
992 "com.apple.%s.%s", type_name, ifname);
993 err = kern_nexus_controller_register_provider(controller,
994 dom_prov,
995 provider_name,
996 NULL,
997 0,
998 attr,
999 provider);
1000 kern_nexus_attr_destroy(attr);
1001 attr = NULL;
1002 if (err != 0) {
1003 printf("%s register %s provider failed, error %d\n",
1004 __func__, type_name, err);
1005 goto failed;
1006 }
1007 bzero(&init, sizeof (init));
1008 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1009 err = kern_nexus_controller_alloc_provider_instance(controller,
1010 *provider,
1011 NULL,
1012 instance, &init);
1013 if (err != 0) {
1014 printf("%s alloc_provider_instance %s failed, %d\n",
1015 __func__, type_name, err);
1016 kern_nexus_controller_deregister_provider(controller,
1017 *provider);
1018 uuid_clear(*provider);
1019 }
1020failed:
1021 return (err);
1022}
1023
1024static errno_t
1025utun_multistack_attach(struct utun_pcb *pcb)
1026{
1027 nexus_controller_t controller = kern_nexus_shared_controller();
1028 errno_t err = 0;
1029 utun_nx_t nx = &pcb->utun_nx;
1030
1031 // Allocate multistack flowswitch
1032 err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
1033 "multistack",
1034 pcb->utun_ifp->if_xname,
1035 &nx->ms_provider,
1036 &nx->ms_instance);
1037 if (err != 0) {
1038 printf("%s: failed to create bridge provider and instance\n",
1039 __func__);
1040 goto failed;
1041 }
1042
1043 // Attach multistack to device port
1044 err = kern_nexus_ifattach(controller, nx->ms_instance,
1045 NULL, nx->if_instance,
1046 FALSE, &nx->ms_device);
1047 if (err != 0) {
1048 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1049 goto failed;
1050 }
1051
1052 // Attach multistack to host port
1053 err = kern_nexus_ifattach(controller, nx->ms_instance,
1054 NULL, nx->if_instance,
1055 TRUE, &nx->ms_host);
1056 if (err != 0) {
1057 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1058 goto failed;
1059 }
1060
1061 // Extract the agent UUID and save for later
1062 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1063 if (multistack_nx != NULL) {
1064 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1065 if (flowswitch != NULL) {
1066 FSW_RLOCK(flowswitch);
1067 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1068 if (ms_context != NULL) {
1069 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1070 } else {
1071 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1072 }
1073 FSW_UNLOCK(flowswitch);
1074 } else {
1075 printf("utun_multistack_attach - flowswitch is NULL\n");
1076 }
1077 nx_release(multistack_nx);
1078 } else {
1079 printf("utun_multistack_attach - unable to find multistack nexus\n");
1080 }
1081
1082 return (0);
1083
1084failed:
1085 utun_nexus_detach(nx);
1086
1087 errno_t detach_error = 0;
1088 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1089 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1090 /* NOT REACHED */
1091 }
1092
1093 return (err);
1094}
1095
1096static errno_t
1097utun_register_kernel_pipe_nexus(void)
1098{
1099 nexus_attr_t nxa = NULL;
1100 errno_t result;
1101
1102 lck_mtx_lock(&utun_lock);
1103 if (utun_ncd_refcount++) {
1104 lck_mtx_unlock(&utun_lock);
1105 return 0;
1106 }
1107
1108 result = kern_nexus_controller_create(&utun_ncd);
1109 if (result) {
1110 printf("%s: kern_nexus_controller_create failed: %d\n",
1111 __FUNCTION__, result);
1112 goto done;
1113 }
1114
1115 uuid_t dom_prov;
1116 result = kern_nexus_get_builtin_domain_provider(
1117 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1118 if (result) {
1119 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1120 __FUNCTION__, result);
1121 goto done;
1122 }
1123
1124 struct kern_nexus_provider_init prov_init = {
1125 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1126 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1127 .nxpi_pre_connect = utun_nexus_pre_connect,
1128 .nxpi_connected = utun_nexus_connected,
1129 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1130 .nxpi_disconnected = utun_nexus_disconnected,
1131 .nxpi_ring_init = utun_kpipe_ring_init,
1132 .nxpi_ring_fini = utun_kpipe_ring_fini,
1133 .nxpi_slot_init = NULL,
1134 .nxpi_slot_fini = NULL,
1135 .nxpi_sync_tx = utun_kpipe_sync_tx,
1136 .nxpi_sync_rx = utun_kpipe_sync_rx,
1137 .nxpi_tx_doorbell = NULL,
1138 };
1139
1140 result = kern_nexus_attr_create(&nxa);
1141 if (result) {
1142 printf("%s: kern_nexus_attr_create failed: %d\n",
1143 __FUNCTION__, result);
1144 goto done;
1145 }
1146
1147 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1148 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1149 VERIFY(result == 0);
1150
1151 // Reset ring size for kernel pipe nexus to limit memory usage
1152 uint64_t ring_size = if_utun_ring_size;
1153 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1154 VERIFY(result == 0);
1155 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1156 VERIFY(result == 0);
1157
1158 result = kern_nexus_controller_register_provider(utun_ncd,
1159 dom_prov,
1160 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1161 &prov_init,
1162 sizeof(prov_init),
1163 nxa,
1164 &utun_kpipe_uuid);
1165 if (result) {
1166 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1167 __FUNCTION__, result);
1168 goto done;
1169 }
1170
1171done:
1172 if (nxa) {
1173 kern_nexus_attr_destroy(nxa);
1174 }
1175
1176 if (result) {
1177 if (utun_ncd) {
1178 kern_nexus_controller_destroy(utun_ncd);
1179 utun_ncd = NULL;
1180 }
1181 utun_ncd_refcount = 0;
1182 }
1183
1184 lck_mtx_unlock(&utun_lock);
1185
1186 return result;
1187}
1188
1189static void
1190utun_unregister_kernel_pipe_nexus(void)
1191{
1192 lck_mtx_lock(&utun_lock);
1193
1194 VERIFY(utun_ncd_refcount > 0);
1195
1196 if (--utun_ncd_refcount == 0) {
1197 kern_nexus_controller_destroy(utun_ncd);
1198 utun_ncd = NULL;
1199 }
1200
1201 lck_mtx_unlock(&utun_lock);
1202}
1203
1204// For use by socket option, not internally
1205static errno_t
1206utun_disable_channel(struct utun_pcb *pcb)
1207{
1208 errno_t result;
1209 int enabled;
1210 uuid_t uuid;
1211
1212 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1213
1214 enabled = pcb->utun_kpipe_enabled;
1215 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1216
1217 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1218
1219 pcb->utun_kpipe_enabled = 0;
1220 uuid_clear(pcb->utun_kpipe_uuid);
1221
1222 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1223
1224 if (enabled) {
1225 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1226 } else {
1227 result = ENXIO;
1228 }
1229
1230 if (!result) {
1231 utun_unregister_kernel_pipe_nexus();
1232 }
1233
1234 return result;
1235}
1236
1237static errno_t
1238utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1239{
1240 struct kern_nexus_init init;
1241 errno_t result;
1242
1243 result = utun_register_kernel_pipe_nexus();
1244 if (result) {
1245 return result;
1246 }
1247
1248 VERIFY(utun_ncd);
1249
1250 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1251
1252 if (pcb->utun_kpipe_enabled) {
1253 result = EEXIST; // return success instead?
1254 goto done;
1255 }
1256
1257 /*
1258 * Make sure we can fit packets in the channel buffers and
1259 * Allow an extra 4 bytes for the protocol number header in the channel
1260 */
1261 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) {
1262 result = EOPNOTSUPP;
1263 goto done;
1264 }
1265
1266 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1267 bzero(&init, sizeof (init));
1268 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1269 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1270 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1271 if (result) {
1272 goto done;
1273 }
1274
1275 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1276 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1277 pcb->utun_kpipe_uuid, &port,
1278 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1279 if (result) {
1280 kern_nexus_controller_free_provider_instance(utun_ncd,
1281 pcb->utun_kpipe_uuid);
1282 uuid_clear(pcb->utun_kpipe_uuid);
1283 goto done;
1284 }
1285
1286 pcb->utun_kpipe_enabled = 1;
1287
1288done:
1289 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1290
1291 if (result) {
1292 utun_unregister_kernel_pipe_nexus();
1293 }
1294
1295 return result;
1296}
1297
1298#endif // UTUN_NEXUS
b0d623f7
A		 1299
1300errno_t
1301utun_register_control(void)
1302{
5ba3f43e
A		 1303 struct kern_ctl_reg kern_ctl;
1304 errno_t result = 0;
b0d623f7 1305
b0d623f7
A		 1306 /* Find a unique value for our interface family */
1307 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1308 if (result != 0) {
1309 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1310 return result;
1311 }
5ba3f43e
A		 1312
1313 utun_pcb_size = sizeof(struct utun_pcb);
1314 utun_pcb_zone = zinit(utun_pcb_size,
1315 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1316 0, UTUN_PCB_ZONE_NAME);
1317 if (utun_pcb_zone == NULL) {
1318 printf("utun_register_control - zinit(utun_pcb) failed");
1319 return ENOMEM;
1320 }
1321
1322#if UTUN_NEXUS
1323 utun_register_nexus();
1324#endif // UTUN_NEXUS
1325
1326 TAILQ_INIT(&utun_head);
b0d623f7
A		 1327
1328 bzero(&kern_ctl, sizeof(kern_ctl));
fe8ab488 1329 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
b0d623f7 1330 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
fe8ab488 1331 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
39236c6e
A		 1332 kern_ctl.ctl_sendsize = 512 * 1024;
1333 kern_ctl.ctl_recvsize = 512 * 1024;
b0d623f7
A		 1334 kern_ctl.ctl_connect = utun_ctl_connect;
1335 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1336 kern_ctl.ctl_send = utun_ctl_send;
1337 kern_ctl.ctl_setopt = utun_ctl_setopt;
1338 kern_ctl.ctl_getopt = utun_ctl_getopt;
fe8ab488 1339 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
39236c6e 1340
b0d623f7
A		 1341 result = ctl_register(&kern_ctl, &utun_kctlref);
1342 if (result != 0) {
1343 printf("utun_register_control - ctl_register failed: %d\n", result);
1344 return result;
1345 }
1346
1347 /* Register the protocol plumbers */
1348 if ((result = proto_register_plumber(PF_INET, utun_family,
1349 utun_attach_proto, NULL)) != 0) {
1350 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
1351 utun_family, result);
1352 ctl_deregister(utun_kctlref);
1353 return result;
1354 }
1355
1356 /* Register the protocol plumbers */
1357 if ((result = proto_register_plumber(PF_INET6, utun_family,
1358 utun_attach_proto, NULL)) != 0) {
1359 proto_unregister_plumber(PF_INET, utun_family);
1360 ctl_deregister(utun_kctlref);
1361 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
1362 utun_family, result);
1363 return result;
1364 }
39037602 1365
5ba3f43e
A		 1366 utun_lck_attr = lck_attr_alloc_init();
1367 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1368 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1369
1370#if UTUN_NEXUS
1371 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1372#endif // UTUN_NEXUS
1373
b0d623f7
A		 1374 return 0;
1375}
1376
1377/* Kernel control functions */
1378
5ba3f43e
A		 1379static inline void
1380utun_free_pcb(struct utun_pcb *pcb)
1381{
1382#ifdef UTUN_NEXUS
1383 mbuf_freem_list(pcb->utun_input_chain);
1384 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1385#endif // UTUN_NEXUS
1386 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1387 lck_mtx_lock(&utun_lock);
1388 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1389 lck_mtx_unlock(&utun_lock);
1390 zfree(utun_pcb_zone, pcb);
1391}
1392
b0d623f7 1393static errno_t
5ba3f43e
A		 1394utun_ctl_connect(kern_ctl_ref kctlref,
1395 struct sockaddr_ctl *sac,
1396 void **unitinfo)
b0d623f7 1397{
5ba3f43e
A		 1398 struct ifnet_init_eparams utun_init = {};
1399 errno_t result = 0;
b0d623f7 1400
5ba3f43e
A		 1401 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1402 memset(pcb, 0, sizeof(*pcb));
39037602 1403
b0d623f7 1404 *unitinfo = pcb;
d1ecb069
A		 1405 pcb->utun_ctlref = kctlref;
1406 pcb->utun_unit = sac->sc_unit;
fe8ab488 1407 pcb->utun_max_pending_packets = 1;
5ba3f43e
A		 1408
1409 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1410 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1411
1412 lck_mtx_lock(&utun_lock);
1413
1414 /* Find some open interface id */
1415 u_int32_t chosen_unique_id = 1;
1416 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1417 if (next_pcb != NULL) {
1418 /* List was not empty, add one to the last item */
1419 chosen_unique_id = next_pcb->utun_unique_id + 1;
1420 next_pcb = NULL;
1421
1422 /*
1423 * If this wrapped the id number, start looking at
1424 * the front of the list for an unused id.
1425 */
1426 if (chosen_unique_id == 0) {
1427 /* Find the next unused ID */
1428 chosen_unique_id = 1;
1429 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1430 if (next_pcb->utun_unique_id > chosen_unique_id) {
1431 /* We found a gap */
1432 break;
1433 }
1434
1435 chosen_unique_id = next_pcb->utun_unique_id + 1;
1436 }
1437 }
1438 }
1439
1440 pcb->utun_unique_id = chosen_unique_id;
1441
1442 if (next_pcb != NULL) {
1443 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1444 } else {
1445 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1446 }
1447 lck_mtx_unlock(&utun_lock);
1448
1449 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1450 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1451 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
b0d623f7
A		 1452
1453 /* Create the interface */
1454 bzero(&utun_init, sizeof(utun_init));
39236c6e
A		 1455 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1456 utun_init.len = sizeof (utun_init);
5ba3f43e
A		 1457
1458#if UTUN_NEXUS
1459 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1460 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1461#else // UTUN_NEXUS
1462 utun_init.flags = IFNET_INIT_NX_NOAUTO;
fe8ab488 1463 utun_init.start = utun_start;
5ba3f43e
A		 1464 utun_init.framer_extended = utun_framer;
1465#endif // UTUN_NEXUS
1466 utun_init.name = "utun";
d1ecb069 1467 utun_init.unit = pcb->utun_unit - 1;
5ba3f43e
A		 1468 utun_init.uniqueid = pcb->utun_unique_name;
1469 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
b0d623f7 1470 utun_init.family = utun_family;
39037602 1471 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
b0d623f7 1472 utun_init.type = IFT_OTHER;
b0d623f7 1473 utun_init.demux = utun_demux;
b0d623f7
A		 1474 utun_init.add_proto = utun_add_proto;
1475 utun_init.del_proto = utun_del_proto;
1476 utun_init.softc = pcb;
1477 utun_init.ioctl = utun_ioctl;
1478 utun_init.detach = utun_detached;
39037602 1479
5ba3f43e
A		 1480#if UTUN_NEXUS
1481 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1482 if (result != 0) {
1483 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1484 utun_free_pcb(pcb);
1485 *unitinfo = NULL;
1486 return result;
1487 }
1488
1489 result = utun_multistack_attach(pcb);
1490 if (result != 0) {
1491 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1492 *unitinfo = NULL;
1493 return result;
1494 }
1495
1496#else // UTUN_NEXUS
39037602
A		 1497 /*
1498 * Upon success, this holds an ifnet reference which we will
1499 * release via ifnet_release() at final detach time.
1500 */
39236c6e 1501 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
b0d623f7
A		 1502 if (result != 0) {
1503 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
5ba3f43e 1504 utun_free_pcb(pcb);
39037602 1505 *unitinfo = NULL;
b0d623f7
A		 1506 return result;
1507 }
b0d623f7
A		 1508
1509 /* Set flags and additional information. */
39037602 1510 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
d1ecb069 1511 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
b0d623f7
A		 1512
1513 /* The interface must generate its own IPv6 LinkLocal address,
1514 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1515 */
d1ecb069 1516 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
b0d623f7 1517
d1ecb069 1518 /* Reset the stats in case as the interface may have been recycled */
5ba3f43e 1519 struct ifnet_stats_param stats;
d1ecb069
A		 1520 bzero(&stats, sizeof(struct ifnet_stats_param));
1521 ifnet_set_stat(pcb->utun_ifp, &stats);
1522
b0d623f7 1523 /* Attach the interface */
d1ecb069 1524 result = ifnet_attach(pcb->utun_ifp, NULL);
b0d623f7 1525 if (result != 0) {
5ba3f43e 1526 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
39037602 1527 /* Release reference now since attach failed */
d1ecb069 1528 ifnet_release(pcb->utun_ifp);
5ba3f43e 1529 utun_free_pcb(pcb);
39037602 1530 *unitinfo = NULL;
5ba3f43e 1531 return (result);
39037602 1532 }
5ba3f43e
A		 1533#endif // UTUN_NEXUS
1534
1535 /* Attach to bpf */
1536 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1537 /* The interfaces resoures allocated, mark it as running */
1538 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1539
b0d623f7
A		 1540 return result;
1541}
1542
1543static errno_t
5ba3f43e
A		 1544utun_detach_ip(ifnet_t interface,
1545 protocol_family_t protocol,
1546 socket_t pf_socket)
b0d623f7
A		 1547{
1548 errno_t result = EPROTONOSUPPORT;
1549
1550 /* Attempt a detach */
1551 if (protocol == PF_INET) {
1552 struct ifreq ifr;
1553
1554 bzero(&ifr, sizeof(ifr));
1555 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1556 ifnet_name(interface), ifnet_unit(interface));
1557
1558 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
5ba3f43e 1559 } else if (protocol == PF_INET6) {
b0d623f7
A		 1560 struct in6_ifreq ifr6;
1561
1562 bzero(&ifr6, sizeof(ifr6));
1563 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1564 ifnet_name(interface), ifnet_unit(interface));
1565
1566 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1567 }
1568
1569 return result;
1570}
1571
1572static void
5ba3f43e
A		 1573utun_remove_address(ifnet_t interface,
1574 protocol_family_t protocol,
1575 ifaddr_t address,
1576 socket_t pf_socket)
b0d623f7
A		 1577{
1578 errno_t result = 0;
1579
1580 /* Attempt a detach */
1581 if (protocol == PF_INET) {
5ba3f43e 1582 struct ifreq ifr;
b0d623f7
A		 1583
1584 bzero(&ifr, sizeof(ifr));
1585 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1586 ifnet_name(interface), ifnet_unit(interface));
1587 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1588 if (result != 0) {
1589 printf("utun_remove_address - ifaddr_address failed: %d", result);
5ba3f43e 1590 } else {
b0d623f7
A		 1591 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1592 if (result != 0) {
1593 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1594 }
1595 }
5ba3f43e
A		 1596 } else if (protocol == PF_INET6) {
1597 struct in6_ifreq ifr6;
b0d623f7
A		 1598
1599 bzero(&ifr6, sizeof(ifr6));
1600 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1601 ifnet_name(interface), ifnet_unit(interface));
1602 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1603 sizeof(ifr6.ifr_addr));
1604 if (result != 0) {
1605 printf("utun_remove_address - ifaddr_address failed (v6): %d",
1606 result);
5ba3f43e 1607 } else {
b0d623f7
A		 1608 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1609 if (result != 0) {
1610 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1611 result);
1612 }
1613 }
1614 }
1615}
1616
1617static void
5ba3f43e
A		 1618utun_cleanup_family(ifnet_t interface,
1619 protocol_family_t protocol)
1620{
1621 errno_t result = 0;
1622 socket_t pf_socket = NULL;
1623 ifaddr_t *addresses = NULL;
1624 int i;
b0d623f7
A		 1625
1626 if (protocol != PF_INET && protocol != PF_INET6) {
1627 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1628 return;
1629 }
1630
1631 /* Create a socket for removing addresses and detaching the protocol */
1632 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1633 if (result != 0) {
1634 if (result != EAFNOSUPPORT)
1635 printf("utun_cleanup_family - failed to create %s socket: %d\n",
1636 protocol == PF_INET ? "IP" : "IPv6", result);
1637 goto cleanup;
1638 }
1639
6d2010ae
A		 1640 /* always set SS_PRIV, we want to close and detach regardless */
1641 sock_setpriv(pf_socket, 1);
1642
b0d623f7
A		 1643 result = utun_detach_ip(interface, protocol, pf_socket);
1644 if (result == 0 || result == ENXIO) {
1645 /* We are done! We either detached or weren't attached. */
1646 goto cleanup;
5ba3f43e 1647 } else if (result != EBUSY) {
b0d623f7
A		 1648 /* Uh, not really sure what happened here... */
1649 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1650 goto cleanup;
1651 }
1652
1653 /*
1654 * At this point, we received an EBUSY error. This means there are
1655 * addresses attached. We should detach them and then try again.
1656 */
1657 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1658 if (result != 0) {
1659 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1660 ifnet_name(interface), ifnet_unit(interface),
1661 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1662 goto cleanup;
1663 }
1664
1665 for (i = 0; addresses[i] != 0; i++) {
1666 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1667 }
1668 ifnet_free_address_list(addresses);
1669 addresses = NULL;
1670
1671 /*
1672 * The addresses should be gone, we should try the remove again.
1673 */
1674 result = utun_detach_ip(interface, protocol, pf_socket);
1675 if (result != 0 && result != ENXIO) {
1676 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1677 }
1678
1679cleanup:
5ba3f43e 1680 if (pf_socket != NULL) {
b0d623f7 1681 sock_close(pf_socket);
5ba3f43e 1682 }
b0d623f7 1683
5ba3f43e 1684 if (addresses != NULL) {
b0d623f7 1685 ifnet_free_address_list(addresses);
5ba3f43e 1686 }
b0d623f7
A		 1687}
1688
1689static errno_t
5ba3f43e
A		 1690utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1691 __unused u_int32_t unit,
1692 void *unitinfo)
b0d623f7
A		 1693{
1694 struct utun_pcb *pcb = unitinfo;
5ba3f43e
A		 1695 ifnet_t ifp = NULL;
1696 errno_t result = 0;
1697
1698 if (pcb == NULL) {
1699 return EINVAL;
1700 }
1701
1702#if UTUN_NEXUS
1703 // Tell the nexus to stop all rings
1704 if (pcb->utun_netif_nexus != NULL) {
1705 kern_nexus_stop(pcb->utun_netif_nexus);
1706 }
1707#endif // UTUN_NEXUS
316670eb 1708
5ba3f43e 1709 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
316670eb 1710
5ba3f43e
A		 1711#if UTUN_NEXUS
1712 uuid_t kpipe_uuid;
1713 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1714 uuid_clear(pcb->utun_kpipe_uuid);
1715 pcb->utun_kpipe_enabled = FALSE;
1716#endif // UTUN_NEXUS
39037602
A		 1717
1718 ifp = pcb->utun_ifp;
1719 VERIFY(ifp != NULL);
d1ecb069 1720 pcb->utun_ctlref = NULL;
5ba3f43e
A		 1721
1722 /*
1723 * Quiesce the interface and flush any pending outbound packets.
1724 */
1725 if_down(ifp);
1726
1727 /* Increment refcnt, but detach interface */
1728 ifnet_incr_iorefcnt(ifp);
1729 if ((result = ifnet_detach(ifp)) != 0) {
1730 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1731 }
39037602 1732
b0d623f7
A		 1733 /*
1734 * We want to do everything in our power to ensure that the interface
1735 * really goes away when the socket is closed. We must remove IP/IPv6
1736 * addresses and detach the protocols. Finally, we can remove and
1737 * release the interface.
1738 */
1739 utun_cleanup_family(ifp, AF_INET);
1740 utun_cleanup_family(ifp, AF_INET6);
39037602 1741
5ba3f43e
A		 1742 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1743
1744#if UTUN_NEXUS
1745 if (!uuid_is_null(kpipe_uuid)) {
1746 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1747 utun_unregister_kernel_pipe_nexus();
1748 }
b0d623f7 1749 }
5ba3f43e
A		 1750 utun_nexus_detach(&pcb->utun_nx);
1751#endif // UTUN_NEXUS
1752
1753 /* Decrement refcnt to finish detaching and freeing */
1754 ifnet_decr_iorefcnt(ifp);
b0d623f7 1755
b0d623f7 1756 return 0;
39037602 1757}
b0d623f7
A		 1758
1759static errno_t
5ba3f43e
A		 1760utun_ctl_send(__unused kern_ctl_ref kctlref,
1761 __unused u_int32_t unit,
1762 void *unitinfo,
1763 mbuf_t m,
1764 __unused int flags)
b0d623f7 1765{
39236c6e
A		 1766 /*
1767 * The userland ABI requires the first four bytes have the protocol family
1768 * in network byte order: swap them
1769 */
39037602 1770 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
39236c6e 1771 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
39037602 1772 } else {
39236c6e 1773 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
39037602 1774 }
39236c6e 1775
316670eb 1776 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
b0d623f7
A		 1777}
1778
1779static errno_t
5ba3f43e
A		 1780utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1781 __unused u_int32_t unit,
1782 void *unitinfo,
1783 int opt,
1784 void *data,
1785 size_t len)
b0d623f7 1786{
5ba3f43e
A		 1787 struct utun_pcb *pcb = unitinfo;
1788 errno_t result = 0;
b0d623f7
A		 1789 /* check for privileges for privileged options */
1790 switch (opt) {
1791 case UTUN_OPT_FLAGS:
d1ecb069 1792 case UTUN_OPT_EXT_IFDATA_STATS:
39236c6e 1793 case UTUN_OPT_SET_DELEGATE_INTERFACE:
b0d623f7
A		 1794 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1795 return EPERM;
1796 }
1797 break;
1798 }
1799
1800 switch (opt) {
1801 case UTUN_OPT_FLAGS:
39037602 1802 if (len != sizeof(u_int32_t)) {
b0d623f7 1803 result = EMSGSIZE;
39037602 1804 } else {
d1ecb069 1805 pcb->utun_flags = *(u_int32_t *)data;
39037602 1806 }
39236c6e
A		 1807 break;
1808
d1ecb069
A		 1809 case UTUN_OPT_EXT_IFDATA_STATS:
1810 if (len != sizeof(int)) {
1811 result = EMSGSIZE;
1812 break;
1813 }
1814 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1815 break;
1816
1817 case UTUN_OPT_INC_IFDATA_STATS_IN:
1818 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1819 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1820
1821 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1822 result = EINVAL;
1823 break;
1824 }
1825 if (!pcb->utun_ext_ifdata_stats) {
1826 result = EINVAL;
1827 break;
1828 }
1829 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN)
1830 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
1831 utsp->utsp_bytes, utsp->utsp_errors);
1832 else
1833 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
1834 utsp->utsp_bytes, utsp->utsp_errors);
b0d623f7 1835 break;
d1ecb069 1836 }
fe8ab488 1837 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
39236c6e
A		 1838 ifnet_t del_ifp = NULL;
1839 char name[IFNAMSIZ];
1840
1841 if (len > IFNAMSIZ - 1) {
1842 result = EMSGSIZE;
1843 break;
1844 }
1845 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
1846 bcopy(data, name, len);
1847 name[len] = 0;
1848 result = ifnet_find_by_name(name, &del_ifp);
1849 }
1850 if (result == 0) {
1851 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
1852 if (del_ifp)
1853 ifnet_release(del_ifp);
1854 }
1855 break;
1856 }
fe8ab488
A		 1857 case UTUN_OPT_MAX_PENDING_PACKETS: {
1858 u_int32_t max_pending_packets = 0;
1859 if (len != sizeof(u_int32_t)) {
1860 result = EMSGSIZE;
1861 break;
1862 }
1863 max_pending_packets = *(u_int32_t *)data;
1864 if (max_pending_packets == 0) {
1865 result = EINVAL;
1866 break;
1867 }
1868 pcb->utun_max_pending_packets = max_pending_packets;
1869 break;
1870 }
5ba3f43e
A		 1871#if UTUN_NEXUS
1872 case UTUN_OPT_ENABLE_CHANNEL: {
1873 if (len != sizeof(int)) {
1874 result = EMSGSIZE;
1875 break;
1876 }
1877 if (*(int *)data) {
1878 result = utun_enable_channel(pcb, current_proc());
1879 } else {
1880 result = utun_disable_channel(pcb);
1881 }
1882 break;
1883 }
1884 case UTUN_OPT_ENABLE_FLOWSWITCH: {
1885 if (len != sizeof(int)) {
1886 result = EMSGSIZE;
1887 break;
1888 }
1889 if (!if_enable_netagent) {
1890 result = ENOTSUP;
1891 break;
1892 }
1893 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
1894 result = ENOENT;
1895 break;
1896 }
1897
1898 if (*(int *)data) {
1899 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
1900 } else {
1901 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
1902 }
1903 break;
1904 }
1905#endif // UTUN_NEXUS
fe8ab488 1906 default: {
b0d623f7
A		 1907 result = ENOPROTOOPT;
1908 break;
fe8ab488 1909 }
b0d623f7
A		 1910 }
1911
1912 return result;
1913}
1914
1915static errno_t
5ba3f43e
A		 1916utun_ctl_getopt(__unused kern_ctl_ref kctlref,
1917 __unused u_int32_t unit,
1918 void *unitinfo,
1919 int opt,
1920 void *data,
1921 size_t *len)
b0d623f7 1922{
5ba3f43e
A		 1923 struct utun_pcb *pcb = unitinfo;
1924 errno_t result = 0;
b0d623f7
A		 1925
1926 switch (opt) {
1927 case UTUN_OPT_FLAGS:
5ba3f43e 1928 if (*len != sizeof(u_int32_t)) {
b0d623f7 1929 result = EMSGSIZE;
5ba3f43e 1930 } else {
d1ecb069 1931 *(u_int32_t *)data = pcb->utun_flags;
5ba3f43e 1932 }
b0d623f7 1933 break;
d1ecb069
A		 1934
1935 case UTUN_OPT_EXT_IFDATA_STATS:
5ba3f43e 1936 if (*len != sizeof(int)) {
d1ecb069 1937 result = EMSGSIZE;
5ba3f43e 1938 } else {
d1ecb069 1939 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
5ba3f43e 1940 }
d1ecb069
A		 1941 break;
1942
b0d623f7 1943 case UTUN_OPT_IFNAME:
5ba3f43e
A		 1944 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
1945 result = EMSGSIZE;
1946 } else {
1947 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
1948 }
b0d623f7 1949 break;
d1ecb069 1950
fe8ab488 1951 case UTUN_OPT_MAX_PENDING_PACKETS: {
5ba3f43e
A		 1952 if (*len != sizeof(u_int32_t)) {
1953 result = EMSGSIZE;
1954 } else {
1955 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
1956 }
fe8ab488
A		 1957 break;
1958 }
39037602 1959
5ba3f43e
A		 1960#if UTUN_NEXUS
1961 case UTUN_OPT_GET_CHANNEL_UUID:
1962 lck_rw_lock_shared(&pcb->utun_pcb_lock);
1963 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
1964 result = ENXIO;
1965 } else if (*len != sizeof(uuid_t)) {
1966 result = EMSGSIZE;
1967 } else {
1968 uuid_copy(data, pcb->utun_kpipe_uuid);
1969 }
1970 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
1971 break;
1972#endif // UTUN_NEXUS
39037602 1973
b0d623f7
A		 1974 default:
1975 result = ENOPROTOOPT;
1976 break;
1977 }
1978
1979 return result;
1980}
1981
fe8ab488
A		 1982static void
1983utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
1984{
3e170ce0 1985#pragma unused(flags)
fe8ab488
A		 1986 bool reenable_output = false;
1987 struct utun_pcb *pcb = unitinfo;
1988 if (pcb == NULL) {
1989 return;
1990 }
1991 ifnet_lock_exclusive(pcb->utun_ifp);
3e170ce0
A		 1992
1993 u_int32_t utun_packet_cnt;
1994 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
1995 if (error_pc != 0) {
1996 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
1997 utun_packet_cnt = 0;
fe8ab488 1998 }
3e170ce0
A		 1999
2000 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2001 reenable_output = true;
2002 }
2003
fe8ab488
A		 2004 if (reenable_output) {
2005 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2006 if (error != 0) {
2007 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2008 }
2009 }
2010 ifnet_lock_done(pcb->utun_ifp);
2011}
2012
b0d623f7 2013/* Network Interface functions */
5ba3f43e 2014#if !UTUN_NEXUS
fe8ab488
A		 2015static void
2016utun_start(ifnet_t interface)
2017{
2018 mbuf_t data;
39037602
A		 2019 struct utun_pcb *pcb = ifnet_softc(interface);
2020
2021 VERIFY(pcb != NULL);
2022
5ba3f43e
A		 2023#if UTUN_NEXUS
2024 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2025 if (pcb->utun_kpipe_enabled) {
2026 /* It's possible to have channels enabled, but not yet have the channel opened,
2027 * in which case the rxring will not be set
2028 */
2029 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2030 if (pcb->utun_kpipe_rxring != NULL) {
2031 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2032 }
2033 return;
2034 }
2035 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2036#endif // UTUN_NEXUS
39037602 2037
fe8ab488
A		 2038 for (;;) {
2039 bool can_accept_packets = true;
2040 ifnet_lock_shared(pcb->utun_ifp);
3e170ce0
A		 2041
2042 u_int32_t utun_packet_cnt;
2043 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2044 if (error_pc != 0) {
2045 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2046 utun_packet_cnt = 0;
2047 }
2048
2049 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
fe8ab488
A		 2050 if (!can_accept_packets && pcb->utun_ctlref) {
2051 u_int32_t difference = 0;
2052 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2053 if (difference > 0) {
2054 // If the low-water mark has not yet been reached, we still need to enqueue data
2055 // into the buffer
2056 can_accept_packets = true;
2057 }
2058 }
2059 }
2060 if (!can_accept_packets) {
2061 errno_t error = ifnet_disable_output(interface);
2062 if (error != 0) {
2063 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2064 }
2065 ifnet_lock_done(pcb->utun_ifp);
2066 break;
2067 }
2068 ifnet_lock_done(pcb->utun_ifp);
5ba3f43e 2069 if (ifnet_dequeue(interface, &data) != 0) {
fe8ab488 2070 break;
5ba3f43e
A		 2071 }
2072 if (utun_output(interface, data) != 0) {
fe8ab488 2073 break;
5ba3f43e 2074 }
fe8ab488
A		 2075 }
2076}
5ba3f43e 2077#endif // !UTUN_NEXUS
fe8ab488 2078
b0d623f7 2079static errno_t
39037602
A		 2080utun_output(ifnet_t interface,
2081 mbuf_t data)
b0d623f7
A		 2082{
2083 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e 2084 errno_t result;
39037602
A		 2085
2086 VERIFY(interface == pcb->utun_ifp);
5ba3f43e 2087
d1ecb069 2088 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
b0d623f7
A		 2089 /* flush data */
2090 mbuf_freem(data);
2091 return 0;
2092 }
2093
316670eb 2094 // otherwise, fall thru to ctl_enqueumbuf
d1ecb069 2095 if (pcb->utun_ctlref) {
316670eb
A		 2096 int length;
2097
39236c6e
A		 2098 /*
2099 * The ABI requires the protocol in network byte order
2100 */
39037602 2101 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
39236c6e 2102 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
39037602 2103 }
39236c6e 2104
316670eb 2105 length = mbuf_pkthdr_len(data);
d1ecb069 2106 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
b0d623f7
A		 2107 if (result != 0) {
2108 mbuf_freem(data);
2109 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
5ba3f43e 2110#if !UTUN_NEXUS
b0d623f7 2111 ifnet_stat_increment_out(interface, 0, 0, 1);
5ba3f43e
A		 2112 } else {
2113 if (!pcb->utun_ext_ifdata_stats) {
d1ecb069 2114 ifnet_stat_increment_out(interface, 1, length, 0);
5ba3f43e
A		 2115 }
2116#endif // !UTUN_NEXUS
b0d623f7 2117 }
5ba3f43e 2118 } else {
b0d623f7 2119 mbuf_freem(data);
5ba3f43e 2120 }
b0d623f7
A		 2121
2122 return 0;
2123}
2124
b0d623f7 2125static errno_t
5ba3f43e
A		 2126utun_demux(__unused ifnet_t interface,
2127 mbuf_t data,
2128 __unused char *frame_header,
2129 protocol_family_t *protocol)
b0d623f7
A		 2130{
2131
5ba3f43e
A		 2132 struct ip *ip;
2133 u_int ip_version;
2134
b0d623f7
A		 2135 while (data != NULL && mbuf_len(data) < 1) {
2136 data = mbuf_next(data);
2137 }
5ba3f43e 2138
b0d623f7
A		 2139 if (data == NULL)
2140 return ENOENT;
5ba3f43e
A		 2141
2142 ip = mtod(data, struct ip *);
2143 ip_version = ip->ip_v;
2144
2145 switch(ip_version) {
2146 case 4:
2147 *protocol = PF_INET;
2148 return 0;
2149 case 6:
2150 *protocol = PF_INET6;
2151 return 0;
2152 default:
2153 *protocol = 0;
2154 break;
2155 }
2156
b0d623f7
A		 2157 return 0;
2158}
2159
5ba3f43e 2160#if !UTUN_NEXUS
b0d623f7 2161static errno_t
5ba3f43e
A		 2162utun_framer(ifnet_t interface,
2163 mbuf_t *packet,
2164 __unused const struct sockaddr *dest,
b0d623f7 2165 __unused const char *desk_linkaddr,
39236c6e 2166 const char *frame_type,
5ba3f43e 2167 u_int32_t *prepend_len,
39236c6e 2168 u_int32_t *postpend_len)
b0d623f7 2169{
39037602
A		 2170 struct utun_pcb *pcb = ifnet_softc(interface);
2171 VERIFY(interface == pcb->utun_ifp);
2172
2173 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2174 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
b0d623f7 2175 printf("utun_framer - ifnet_output prepend failed\n");
d1ecb069 2176
b0d623f7 2177 ifnet_stat_increment_out(interface, 0, 0, 1);
d1ecb069 2178
b0d623f7
A		 2179 // just return, because the buffer was freed in mbuf_prepend
2180 return EJUSTRETURN;
2181 }
5ba3f43e 2182 if (prepend_len != NULL) {
39037602 2183 *prepend_len = header_length;
5ba3f43e
A		 2184 }
2185 if (postpend_len != NULL) {
39236c6e 2186 *postpend_len = 0;
5ba3f43e 2187 }
b0d623f7
A		 2188
2189 // place protocol number at the beginning of the mbuf
39236c6e 2190 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
39037602
A		 2191
2192
b0d623f7
A		 2193 return 0;
2194}
5ba3f43e 2195#endif // !UTUN_NEXUS
b0d623f7
A		 2196
2197static errno_t
5ba3f43e
A		 2198utun_add_proto(__unused ifnet_t interface,
2199 protocol_family_t protocol,
2200 __unused const struct ifnet_demux_desc *demux_array,
2201 __unused u_int32_t demux_count)
b0d623f7
A		 2202{
2203 switch(protocol) {
2204 case PF_INET:
2205 return 0;
2206 case PF_INET6:
2207 return 0;
2208 default:
2209 break;
2210 }
2211
2212 return ENOPROTOOPT;
2213}
2214
2215static errno_t
5ba3f43e
A		 2216utun_del_proto(__unused ifnet_t interface,
2217 __unused protocol_family_t protocol)
b0d623f7
A		 2218{
2219 return 0;
2220}
2221
2222static errno_t
5ba3f43e
A		 2223utun_ioctl(ifnet_t interface,
2224 u_long command,
2225 void *data)
b0d623f7
A		 2226{
2227 errno_t result = 0;
b0d623f7
A		 2228
2229 switch(command) {
2230 case SIOCSIFMTU:
5ba3f43e
A		 2231#if UTUN_NEXUS
2232 {
2233 // Make sure we can fit packets in the channel buffers
2234 // Allow for the headroom in the slot
2235 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) {
2236 ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE);
2237 break;
2238 }
2239 }
2240#endif // UTUN_NEXUS
b0d623f7
A		 2241 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2242 break;
d1ecb069
A		 2243
2244 case SIOCSIFFLAGS:
2245 /* ifioctl() takes care of it */
2246 break;
2247
b0d623f7
A		 2248 default:
2249 result = EOPNOTSUPP;
2250 }
2251
2252 return result;
2253}
2254
2255static void
5ba3f43e 2256utun_detached(ifnet_t interface)
b0d623f7
A		 2257{
2258 struct utun_pcb *pcb = ifnet_softc(interface);
5ba3f43e
A		 2259 (void)ifnet_release(interface);
2260 utun_free_pcb(pcb);
b0d623f7
A		 2261}
2262
2263/* Protocol Handlers */
2264
2265static errno_t
5ba3f43e
A		 2266utun_proto_input(__unused ifnet_t interface,
2267 protocol_family_t protocol,
2268 mbuf_t m,
2269 __unused char *frame_header)
b0d623f7 2270{
39037602 2271 if (proto_input(protocol, m) != 0) {
6d2010ae 2272 m_freem(m);
5ba3f43e
A		 2273#if !UTUN_NEXUS
2274 ifnet_stat_increment_in(interface, 0, 0, 1);
2275 } else {
2276 ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0);
2277#endif // UTUN_NEXUS
39037602 2278 }
b0d623f7
A		 2279
2280 return 0;
2281}
2282
2283static errno_t
5ba3f43e
A		 2284utun_proto_pre_output(__unused ifnet_t interface,
2285 protocol_family_t protocol,
2286 __unused mbuf_t *packet,
2287 __unused const struct sockaddr *dest,
2288 __unused void *route,
2289 char *frame_type,
2290 __unused char *link_layer_dest)
b0d623f7 2291{
6d2010ae 2292 *(protocol_family_t *)(void *)frame_type = protocol;
39037602 2293 return 0;
b0d623f7
A		 2294}
2295
2296static errno_t
5ba3f43e
A		 2297utun_attach_proto(ifnet_t interface,
2298 protocol_family_t protocol)
b0d623f7
A		 2299{
2300 struct ifnet_attach_proto_param proto;
b0d623f7
A		 2301
2302 bzero(&proto, sizeof(proto));
2303 proto.input = utun_proto_input;
2304 proto.pre_output = utun_proto_pre_output;
2305
5ba3f43e 2306 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
b0d623f7
A		 2307 if (result != 0 && result != EEXIST) {
2308 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2309 protocol, result);
2310 }
2311
2312 return result;
2313}
2314
5ba3f43e
A		 2315#if UTUN_NEXUS
2316static errno_t
2317utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2318{
2319 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2320
2321 lck_mtx_lock(&pcb->utun_input_chain_lock);
2322 if (pcb->utun_input_chain != NULL) {
2323 pcb->utun_input_chain_last->m_nextpkt = packet;
2324 } else {
2325 pcb->utun_input_chain = packet;
2326 }
2327 while (packet->m_nextpkt) {
2328 VERIFY(packet != packet->m_nextpkt);
2329 packet = packet->m_nextpkt;
2330 }
2331 pcb->utun_input_chain_last = packet;
2332 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2333
2334 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2335 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2336
2337 if (rx_ring != NULL) {
2338 kern_channel_notify(rx_ring, 0);
2339 }
2340
2341 return (0);
2342}
2343#else
39037602 2344static errno_t
316670eb
A		 2345utun_pkt_input (struct utun_pcb *pcb, mbuf_t m)
2346{
2347 errno_t result;
39236c6e 2348 protocol_family_t protocol = 0;
316670eb
A		 2349
2350 mbuf_pkthdr_setrcvif(m, pcb->utun_ifp);
2351
39037602 2352 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
39236c6e 2353 protocol = *(u_int32_t *)mbuf_data(m);
5ba3f43e 2354
39236c6e
A		 2355 bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0);
2356 }
316670eb
A		 2357 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2358 /* flush data */
2359 mbuf_freem(m);
2360 return 0;
2361 }
316670eb 2362
316670eb
A		 2363 if (!pcb->utun_ext_ifdata_stats) {
2364 struct ifnet_stat_increment_param incs;
5ba3f43e 2365
316670eb
A		 2366 bzero(&incs, sizeof(incs));
2367 incs.packets_in = 1;
2368 incs.bytes_in = mbuf_pkthdr_len(m);
2369 result = ifnet_input(pcb->utun_ifp, m, &incs);
2370 } else {
2371 result = ifnet_input(pcb->utun_ifp, m, NULL);
2372 }
2373 if (result != 0) {
2374 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
5ba3f43e 2375
316670eb
A		 2376 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2377 mbuf_freem(m);
2378 }
2379
2380 return 0;
2381}
5ba3f43e
A		 2382#endif // UTUN_NEXUS
2383
2384
2385#if UTUN_NEXUS
2386
2387static errno_t
2388utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2389{
2390 return 0;
2391}
2392
2393static void
2394utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2395{
2396 // Ignore
2397}
2398
2399static errno_t
2400utun_register_nexus(void)
2401{
2402 const struct kern_nexus_domain_provider_init dp_init = {
2403 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2404 .nxdpi_flags = 0,
2405 .nxdpi_init = utun_nxdp_init,
2406 .nxdpi_fini = utun_nxdp_fini
2407 };
2408 errno_t err = 0;
2409
2410 /* utun_nxdp_init() is called before this function returns */
2411 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2412 (const uint8_t *) "com.apple.utun",
2413 &dp_init, sizeof(dp_init),
2414 &utun_nx_dom_prov);
2415 if (err != 0) {
2416 printf("%s: failed to register domain provider\n", __func__);
2417 return (err);
2418 }
2419 return (0);
2420}
2421
2422static errno_t
2423utun_ifnet_set_attrs(ifnet_t ifp)
2424{
2425 /* Set flags and additional information. */
2426 ifnet_set_mtu(ifp, 1500);
2427 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2428
2429 /* The interface must generate its own IPv6 LinkLocal address,
2430 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2431 */
2432 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2433
2434 return (0);
2435}
2436
2437static errno_t
2438utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2439{
2440 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2441 pcb->utun_netif_nexus = nexus;
2442 return (utun_ifnet_set_attrs(ifp));
2443}
2444
2445static errno_t
2446utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2447 proc_t p, kern_nexus_t nexus,
2448 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2449{
2450#pragma unused(nxprov, p)
2451#pragma unused(nexus, nexus_port, channel, ch_ctx)
2452 return (0);
2453}
2454
2455static errno_t
2456utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2457 kern_channel_t channel)
2458{
2459#pragma unused(nxprov, channel)
2460 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2461 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2462 return (ok ? 0 : ENXIO);
2463}
2464
2465static void
2466utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2467 kern_channel_t channel)
2468{
2469#pragma unused(nxprov, nexus, channel)
2470}
2471
2472static void
2473utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2474 kern_channel_t channel)
2475{
2476#pragma unused(nxprov, nexus, channel)
2477}
2478
2479static void
2480utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2481 kern_channel_t channel)
2482{
2483#pragma unused(nxprov, channel)
2484 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2485 if (pcb->utun_netif_nexus == nexus) {
2486 pcb->utun_netif_nexus = NULL;
2487 }
2488 ifnet_decr_iorefcnt(pcb->utun_ifp);
2489}
2490
2491static errno_t
2492utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2493 kern_channel_t channel, kern_channel_ring_t ring,
2494 boolean_t is_tx_ring, void **ring_ctx)
2495{
2496#pragma unused(nxprov)
2497#pragma unused(channel)
2498#pragma unused(ring_ctx)
2499 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2500 if (!is_tx_ring) {
2501 VERIFY(pcb->utun_kpipe_rxring == NULL);
2502 pcb->utun_kpipe_rxring = ring;
2503 } else {
2504 VERIFY(pcb->utun_kpipe_txring == NULL);
2505 pcb->utun_kpipe_txring = ring;
2506 }
2507 return 0;
2508}
2509
2510static void
2511utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2512 kern_channel_ring_t ring)
2513{
2514#pragma unused(nxprov)
2515 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2516 if (pcb->utun_kpipe_rxring == ring) {
2517 pcb->utun_kpipe_rxring = NULL;
2518 } else if (pcb->utun_kpipe_txring == ring) {
2519 pcb->utun_kpipe_txring = NULL;
2520 }
2521}
2522
2523static errno_t
2524utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2525 kern_channel_ring_t tx_ring, uint32_t flags)
2526{
2527#pragma unused(nxprov)
2528#pragma unused(flags)
2529 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2530
2531 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2532 int channel_enabled = pcb->utun_kpipe_enabled;
2533 if (!channel_enabled) {
2534 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2535 return 0;
2536 }
2537
2538 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2539 if (tx_slot == NULL) {
2540 // Nothing to write, bail
2541 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2542 return 0;
2543 }
2544
2545 // Signal the netif ring to read
2546 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2547 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2548 if (rx_ring != NULL) {
2549 kern_channel_notify(rx_ring, 0);
2550 }
2551
2552 return 0;
2553}
2554
2555static errno_t
2556utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2557 kern_channel_ring_t rx_ring, uint32_t flags)
2558{
2559#pragma unused(nxprov)
2560#pragma unused(flags)
2561 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2562 struct kern_channel_ring_stat_increment rx_ring_stats;
2563
2564 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2565
2566 int channel_enabled = pcb->utun_kpipe_enabled;
2567 if (!channel_enabled) {
2568 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2569 return 0;
2570 }
2571
2572 /* reclaim user-released slots */
2573 (void) kern_channel_reclaim(rx_ring);
2574
2575 uint32_t avail = kern_channel_available_slot_count(rx_ring);
2576 if (avail == 0) {
2577 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2578 return 0;
2579 }
2580
2581 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
2582 if (tx_ring == NULL ||
2583 pcb->utun_netif_nexus == NULL) {
2584 // Net-If TX ring not set up yet, nothing to read
2585 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2586 return 0;
2587 }
2588
2589 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
2590
2591 // Unlock utun before entering ring
2592 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2593
2594 (void)kr_enter(tx_ring, TRUE);
2595
2596 // Lock again after entering and validate
2597 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2598 if (tx_ring != pcb->utun_netif_txring) {
2599 // Ring no longer valid
2600 // Unlock first, then exit ring
2601 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2602 kr_exit(tx_ring);
2603 return 0;
2604 }
2605
2606 struct kern_channel_ring_stat_increment tx_ring_stats;
2607 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
2608 kern_channel_slot_t tx_pslot = NULL;
2609 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2610 if (tx_slot == NULL) {
2611 // Nothing to read, don't bother signalling
2612 // Unlock first, then exit ring
2613 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2614 kr_exit(tx_ring);
2615 return 0;
2616 }
2617
2618 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
2619 VERIFY(rx_pp != NULL);
2620 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
2621 kern_channel_slot_t rx_pslot = NULL;
2622 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
2623
2624 while (rx_slot != NULL && tx_slot != NULL) {
2625 size_t length;
2626 kern_buflet_t rx_buf;
2627 void *rx_baddr;
2628
2629 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2630
2631 // Advance TX ring
2632 tx_pslot = tx_slot;
2633 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2634
2635 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
2636 if (tx_ph == 0) {
2637 continue;
2638 }
2639
2640 // Allocate rx packet
2641 kern_packet_t rx_ph = 0;
2642 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
2643 if (unlikely(error != 0)) {
2644 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
2645 pcb->utun_ifp->if_xname);
2646 break;
2647 }
2648
2649 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2650 VERIFY(tx_buf != NULL);
2651 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2652 VERIFY(tx_baddr != NULL);
2653 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2654
2655 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
2656
2657 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
2658 UTUN_IF_DEFAULT_SLOT_SIZE);
2659
2660 tx_ring_stats.kcrsi_slots_transferred++;
2661 tx_ring_stats.kcrsi_bytes_transferred += length;
2662
2663 if (length < UTUN_HEADER_SIZE(pcb) ||
2664 length > UTUN_IF_DEFAULT_SLOT_SIZE ||
2665 length > rx_pp->pp_buflet_size ||
2666 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
2667 /* flush data */
2668 kern_pbufpool_free(rx_pp, rx_ph);
2669 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
2670 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
2671 STATS_INC(nifs, NETIF_STATS_BADLEN);
2672 STATS_INC(nifs, NETIF_STATS_DROPPED);
2673 continue;
2674 }
2675
2676 /* fillout packet */
2677 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
2678 VERIFY(rx_buf != NULL);
2679 rx_baddr = kern_buflet_get_object_address(rx_buf);
2680 VERIFY(rx_baddr != NULL);
2681
2682 // Find family
2683 uint32_t af = 0;
2684 uint8_t vhl = *(uint8_t *)(tx_baddr);
2685 u_int ip_version = (vhl >> 4);
2686 switch (ip_version) {
2687 case 4: {
2688 af = AF_INET;
2689 break;
2690 }
2691 case 6: {
2692 af = AF_INET6;
2693 break;
2694 }
2695 default: {
2696 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
2697 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
2698 break;
2699 }
2700 }
2701
2702 // Copy header
2703 af = htonl(af);
2704 memcpy((void *)rx_baddr, &af, sizeof(af));
2705 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
2706 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
2707 }
2708
2709 // Copy data from tx to rx
2710 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
2711 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
2712
2713 /* finalize and attach the packet */
2714 error = kern_buflet_set_data_offset(rx_buf, 0);
2715 VERIFY(error == 0);
2716 error = kern_buflet_set_data_length(rx_buf, length);
2717 VERIFY(error == 0);
2718 error = kern_packet_finalize(rx_ph);
2719 VERIFY(error == 0);
2720 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
2721 VERIFY(error == 0);
2722
2723 STATS_INC(nifs, NETIF_STATS_TXPKTS);
2724 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
2725
2726 rx_ring_stats.kcrsi_slots_transferred++;
2727 rx_ring_stats.kcrsi_bytes_transferred += length;
2728
2729 rx_pslot = rx_slot;
2730 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
2731 }
2732
2733 if (rx_pslot) {
2734 kern_channel_advance_slot(rx_ring, rx_pslot);
2735 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
2736 }
2737
2738 if (tx_pslot) {
2739 kern_channel_advance_slot(tx_ring, tx_pslot);
2740 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
2741 (void)kern_channel_reclaim(tx_ring);
2742 }
2743
2744 if (pcb->utun_output_disabled) {
2745 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2746 if (error != 0) {
2747 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
2748 } else {
2749 pcb->utun_output_disabled = false;
2750 }
2751 }
2752
2753 // Unlock first, then exit ring
2754 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2755
2756 if (tx_pslot != NULL) {
2757 kern_channel_notify(tx_ring, 0);
2758 }
2759 kr_exit(tx_ring);
2760
2761 return 0;
2762}
39037602 2763
5ba3f43e 2764#endif // UTUN_NEXUS
39037602
A		 2765
2766
2767/*
5ba3f43e 2768 * These are place holders until coreTLS kext stops calling them
39037602
A		 2769 */
2770errno_t utun_ctl_register_dtls (void *reg);
2771int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
2772void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
2773
2774errno_t
2775utun_ctl_register_dtls (void *reg)
2776{
2777#pragma unused(reg)
2778 return 0;
2779}
2780
2781int
2782utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
2783{
2784#pragma unused(pcb)
2785#pragma unused(pkt)
2786#pragma unused(family)
2787 return 0;
2788}
2789
2790void
2791utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
2792{
2793#pragma unused(pcb)
2794}
mirror of XNU