]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/if_utun.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.11.26.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
1 /*
2 * Copyright (c) 2008-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 * Application of kernel control for interface creation
33 *
34 * Theory of operation:
35 * utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 * This kernel control will register an interface for every client that connects.
37 * ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57 #include <os/log.h>
58
59 #define UTUN_NEXUS 0
60
61 #if UTUN_NEXUS
62 static nexus_controller_t utun_ncd;
63 static int utun_ncd_refcount;
64 static uuid_t utun_kpipe_uuid;
65 static uuid_t utun_nx_dom_prov;
66
67 typedef struct utun_nx {
68 uuid_t if_provider;
69 uuid_t if_instance;
70 uuid_t fsw_provider;
71 uuid_t fsw_instance;
72 uuid_t fsw_device;
73 uuid_t fsw_host;
74 uuid_t fsw_agent;
75 } *utun_nx_t;
76
77 #endif // UTUN_NEXUS
78
79 /* Control block allocated for each kernel control connection */
80 struct utun_pcb {
81 TAILQ_ENTRY(utun_pcb) utun_chain;
82 kern_ctl_ref utun_ctlref;
83 ifnet_t utun_ifp;
84 u_int32_t utun_unit;
85 u_int32_t utun_unique_id;
86 u_int32_t utun_flags;
87 int utun_ext_ifdata_stats;
88 u_int32_t utun_max_pending_packets;
89 char utun_if_xname[IFXNAMSIZ];
90 char utun_unique_name[IFXNAMSIZ];
91 // PCB lock protects state fields and rings
92 decl_lck_rw_data(, utun_pcb_lock);
93 struct mbuf * utun_input_chain;
94 struct mbuf * utun_input_chain_last;
95 u_int32_t utun_input_chain_count;
96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
98 lck_mtx_t utun_input_chain_lock;
99
100 #if UTUN_NEXUS
101 struct utun_nx utun_nx;
102 int utun_kpipe_enabled;
103 uuid_t utun_kpipe_uuid;
104 void * utun_kpipe_rxring;
105 void * utun_kpipe_txring;
106 kern_pbufpool_t utun_kpipe_pp;
107 u_int32_t utun_kpipe_tx_ring_size;
108 u_int32_t utun_kpipe_rx_ring_size;
109
110 kern_nexus_t utun_netif_nexus;
111 kern_pbufpool_t utun_netif_pp;
112 void * utun_netif_rxring;
113 void * utun_netif_txring;
114 uint64_t utun_netif_txring_size;
115
116 u_int32_t utun_slot_size;
117 u_int32_t utun_netif_ring_size;
118 u_int32_t utun_tx_fsw_ring_size;
119 u_int32_t utun_rx_fsw_ring_size;
120 // Auto attach flowswitch when netif is enabled. When set to false,
121 // it allows userspace nexus controller to attach and own flowswitch.
122 bool utun_attach_fsw;
123 bool utun_netif_connected;
124 bool utun_use_netif;
125 bool utun_needs_netagent;
126 #endif // UTUN_NEXUS
127 };
128
129 /* Kernel Control functions */
130 static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
131 void **unitinfo);
132 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
133 void **unitinfo);
134 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
135 void *unitinfo);
136 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
137 void *unitinfo, mbuf_t m, int flags);
138 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
139 int opt, void *data, size_t *len);
140 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
141 int opt, void *data, size_t len);
142 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
143 int flags);
144
145 /* Network Interface functions */
146 static void utun_start(ifnet_t interface);
147 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
148 const struct sockaddr *dest, const char *desk_linkaddr,
149 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
150 static errno_t utun_output(ifnet_t interface, mbuf_t data);
151 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
152 protocol_family_t *protocol);
153 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
154 const struct ifnet_demux_desc *demux_array,
155 u_int32_t demux_count);
156 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
157 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
158 static void utun_detached(ifnet_t interface);
159
160 /* Protocol handlers */
161 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
162 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
163 mbuf_t m, char *frame_header);
164 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
165 mbuf_t *packet, const struct sockaddr *dest, void *route,
166 char *frame_type, char *link_layer_dest);
167 static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
168
169 #if UTUN_NEXUS
170
171 #define UTUN_IF_DEFAULT_SLOT_SIZE 2048
172 #define UTUN_IF_DEFAULT_RING_SIZE 64
173 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
174 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
175 #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
176 #define UTUN_IF_HEADROOM_SIZE 32
177
178 #define UTUN_IF_MIN_RING_SIZE 8
179 #define UTUN_IF_MAX_RING_SIZE 1024
180
181 #define UTUN_IF_MIN_SLOT_SIZE 1024
182 #define UTUN_IF_MAX_SLOT_SIZE 4096
183
184 #define UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT 512
185
186 static int if_utun_max_pending_input = UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT;
187
188 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
189 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
190 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
191
192 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
193 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
194 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
195
196 SYSCTL_DECL(_net_utun);
197 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
198
199 SYSCTL_INT(_net_utun, OID_AUTO, max_pending_input, CTLFLAG_LOCKED | CTLFLAG_RW, &if_utun_max_pending_input, 0, "");
200 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
201 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
202 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
203 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
204 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
205 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
206
207 static errno_t
208 utun_register_nexus(void);
209
210 static errno_t
211 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
212 static errno_t
213 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
214 proc_t p, kern_nexus_t nexus,
215 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
216 static errno_t
217 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
218 kern_channel_t channel);
219 static void
220 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
221 kern_channel_t channel);
222 static void
223 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
224 kern_channel_t channel);
225 static void
226 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
227 kern_channel_t channel);
228 static errno_t
229 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
230 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
231 void **ring_ctx);
232 static void
233 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
234 kern_channel_ring_t ring);
235 static errno_t
236 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
237 kern_channel_ring_t ring, uint32_t flags);
238 static errno_t
239 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
240 kern_channel_ring_t ring, uint32_t flags);
241 #endif // UTUN_NEXUS
242
243 #define UTUN_DEFAULT_MTU 1500
244 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
245
246 static kern_ctl_ref utun_kctlref;
247 static lck_attr_t *utun_lck_attr;
248 static lck_grp_attr_t *utun_lck_grp_attr;
249 static lck_grp_t *utun_lck_grp;
250 static lck_mtx_t utun_lock;
251
252 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
253
254 #define UTUN_PCB_ZONE_MAX 32
255 #define UTUN_PCB_ZONE_NAME "net.if_utun"
256
257 static unsigned int utun_pcb_size; /* size of zone element */
258 static struct zone *utun_pcb_zone; /* zone for utun_pcb */
259
260 #if UTUN_NEXUS
261
262 static int
263 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
264 {
265 #pragma unused(arg1, arg2)
266 int value = if_utun_ring_size;
267
268 int error = sysctl_handle_int(oidp, &value, 0, req);
269 if (error || !req->newptr) {
270 return error;
271 }
272
273 if (value < UTUN_IF_MIN_RING_SIZE ||
274 value > UTUN_IF_MAX_RING_SIZE) {
275 return EINVAL;
276 }
277
278 if_utun_ring_size = value;
279
280 return 0;
281 }
282
283 static int
284 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
285 {
286 #pragma unused(arg1, arg2)
287 int value = if_utun_tx_fsw_ring_size;
288
289 int error = sysctl_handle_int(oidp, &value, 0, req);
290 if (error || !req->newptr) {
291 return error;
292 }
293
294 if (value < UTUN_IF_MIN_RING_SIZE ||
295 value > UTUN_IF_MAX_RING_SIZE) {
296 return EINVAL;
297 }
298
299 if_utun_tx_fsw_ring_size = value;
300
301 return 0;
302 }
303
304 static int
305 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
306 {
307 #pragma unused(arg1, arg2)
308 int value = if_utun_rx_fsw_ring_size;
309
310 int error = sysctl_handle_int(oidp, &value, 0, req);
311 if (error || !req->newptr) {
312 return error;
313 }
314
315 if (value < UTUN_IF_MIN_RING_SIZE ||
316 value > UTUN_IF_MAX_RING_SIZE) {
317 return EINVAL;
318 }
319
320 if_utun_rx_fsw_ring_size = value;
321
322 return 0;
323 }
324
325 static errno_t
326 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
327 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
328 void **ring_ctx)
329 {
330 #pragma unused(nxprov)
331 #pragma unused(channel)
332 #pragma unused(ring_ctx)
333 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
334 if (!is_tx_ring) {
335 VERIFY(pcb->utun_netif_rxring == NULL);
336 pcb->utun_netif_rxring = ring;
337 } else {
338 VERIFY(pcb->utun_netif_txring == NULL);
339 pcb->utun_netif_txring = ring;
340 }
341 return 0;
342 }
343
344 static void
345 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
346 kern_channel_ring_t ring)
347 {
348 #pragma unused(nxprov)
349 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
350 if (pcb->utun_netif_rxring == ring) {
351 pcb->utun_netif_rxring = NULL;
352 } else if (pcb->utun_netif_txring == ring) {
353 pcb->utun_netif_txring = NULL;
354 }
355 }
356
357 static errno_t
358 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
359 kern_channel_ring_t tx_ring, uint32_t flags)
360 {
361 #pragma unused(nxprov)
362 #pragma unused(flags)
363 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
364
365 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
366
367 lck_rw_lock_shared(&pcb->utun_pcb_lock);
368
369 struct kern_channel_ring_stat_increment tx_ring_stats;
370 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
371 kern_channel_slot_t tx_pslot = NULL;
372 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
373
374 STATS_INC(nifs, NETIF_STATS_TX_SYNC);
375
376 if (tx_slot == NULL) {
377 // Nothing to write, don't bother signalling
378 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
379 return 0;
380 }
381
382 if (pcb->utun_kpipe_enabled) {
383 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
384 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
385
386 // Signal the kernel pipe ring to read
387 if (rx_ring != NULL) {
388 kern_channel_notify(rx_ring, 0);
389 }
390 return 0;
391 }
392
393 // If we're here, we're injecting into the utun kernel control socket
394 while (tx_slot != NULL) {
395 size_t length = 0;
396 mbuf_t data = NULL;
397
398 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
399
400 if (tx_ph == 0) {
401 // Advance TX ring
402 tx_pslot = tx_slot;
403 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
404 continue;
405 }
406 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
407
408 // Advance TX ring
409 tx_pslot = tx_slot;
410 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
411
412 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
413 VERIFY(tx_buf != NULL);
414
415 /* tx_baddr is the absolute buffer address */
416 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
417 VERIFY(tx_baddr != 0);
418
419 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
420
421 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
422 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
423
424 // The offset must be large enough for the headers
425 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
426
427 // Find family
428 uint32_t af = 0;
429 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
430 u_int ip_version = (vhl >> 4);
431 switch (ip_version) {
432 case 4: {
433 af = AF_INET;
434 break;
435 }
436 case 6: {
437 af = AF_INET6;
438 break;
439 }
440 default: {
441 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
442 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
443 UTUN_HEADER_SIZE(pcb));
444 break;
445 }
446 }
447
448 tx_offset -= UTUN_HEADER_SIZE(pcb);
449 tx_length += UTUN_HEADER_SIZE(pcb);
450 tx_baddr += tx_offset;
451
452 length = MIN(tx_length, pcb->utun_slot_size);
453
454 // Copy in family
455 memcpy(tx_baddr, &af, sizeof(af));
456 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
457 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
458 }
459
460 if (length > 0) {
461 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
462 if (error == 0) {
463 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
464 if (error == 0) {
465 error = utun_output(pcb->utun_ifp, data);
466 if (error != 0) {
467 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
468 }
469 } else {
470 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
471 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
472 STATS_INC(nifs, NETIF_STATS_DROP);
473 mbuf_freem(data);
474 data = NULL;
475 }
476 } else {
477 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
478 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
479 STATS_INC(nifs, NETIF_STATS_DROP);
480 }
481 } else {
482 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
483 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
484 STATS_INC(nifs, NETIF_STATS_DROP);
485 }
486
487 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
488
489 if (data == NULL) {
490 continue;
491 }
492
493 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
494 STATS_INC(nifs, NETIF_STATS_TX_COPY_MBUF);
495
496 tx_ring_stats.kcrsi_slots_transferred++;
497 tx_ring_stats.kcrsi_bytes_transferred += length;
498 }
499
500 if (tx_pslot) {
501 kern_channel_advance_slot(tx_ring, tx_pslot);
502 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
503 (void)kern_channel_reclaim(tx_ring);
504 }
505
506 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
507
508 return 0;
509 }
510
511 static errno_t
512 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
513 kern_channel_ring_t ring, __unused uint32_t flags)
514 {
515 #pragma unused(nxprov)
516 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
517 boolean_t more = false;
518 errno_t rc = 0;
519
520 /*
521 * Refill and sync the ring; we may be racing against another thread doing
522 * an RX sync that also wants to do kr_enter(), and so use the blocking
523 * variant here.
524 */
525 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
526 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
527 os_log_error(OS_LOG_DEFAULT, "%s, tx refill failed %d\n", __func__, rc);
528 }
529
530 (void) kr_enter(ring, TRUE);
531 lck_rw_lock_shared(&pcb->utun_pcb_lock);
532
533 if (pcb->utun_kpipe_enabled) {
534 uint32_t tx_available = kern_channel_available_slot_count(ring);
535 if (pcb->utun_netif_txring_size > 0 &&
536 tx_available >= pcb->utun_netif_txring_size - 1) {
537 // No room left in tx ring, disable output for now
538 errno_t error = ifnet_disable_output(pcb->utun_ifp);
539 if (error != 0) {
540 os_log_error(OS_LOG_DEFAULT, "utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
541 }
542 }
543 }
544
545 if (pcb->utun_kpipe_enabled) {
546 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
547
548 // Unlock while calling notify
549 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
550 // Signal the kernel pipe ring to read
551 if (rx_ring != NULL) {
552 kern_channel_notify(rx_ring, 0);
553 }
554 } else {
555 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
556 }
557
558 kr_exit(ring);
559
560 return 0;
561 }
562
563 static errno_t
564 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
565 kern_channel_ring_t rx_ring, uint32_t flags)
566 {
567 #pragma unused(nxprov)
568 #pragma unused(flags)
569 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
570 struct kern_channel_ring_stat_increment rx_ring_stats;
571
572 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
573
574 lck_rw_lock_shared(&pcb->utun_pcb_lock);
575
576 // Reclaim user-released slots
577 (void) kern_channel_reclaim(rx_ring);
578
579 STATS_INC(nifs, NETIF_STATS_RX_SYNC);
580
581 uint32_t avail = kern_channel_available_slot_count(rx_ring);
582 if (avail == 0) {
583 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
584 return 0;
585 }
586
587 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
588 VERIFY(rx_pp != NULL);
589 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
590 kern_channel_slot_t rx_pslot = NULL;
591 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
592
593 while (rx_slot != NULL) {
594 // Check for a waiting packet
595 lck_mtx_lock(&pcb->utun_input_chain_lock);
596 mbuf_t data = pcb->utun_input_chain;
597 if (data == NULL) {
598 lck_mtx_unlock(&pcb->utun_input_chain_lock);
599 break;
600 }
601
602 // Allocate rx packet
603 kern_packet_t rx_ph = 0;
604 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
605 if (__improbable(error != 0)) {
606 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
607 STATS_INC(nifs, NETIF_STATS_DROP);
608 lck_mtx_unlock(&pcb->utun_input_chain_lock);
609 break;
610 }
611
612 // Advance waiting packets
613 if (pcb->utun_input_chain_count > 0) {
614 pcb->utun_input_chain_count--;
615 }
616 pcb->utun_input_chain = data->m_nextpkt;
617 data->m_nextpkt = NULL;
618 if (pcb->utun_input_chain == NULL) {
619 pcb->utun_input_chain_last = NULL;
620 }
621 lck_mtx_unlock(&pcb->utun_input_chain_lock);
622
623 size_t header_offset = UTUN_HEADER_SIZE(pcb);
624 size_t length = mbuf_pkthdr_len(data);
625
626 if (length < header_offset) {
627 // mbuf is too small
628 mbuf_freem(data);
629 kern_pbufpool_free(rx_pp, rx_ph);
630 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
631 STATS_INC(nifs, NETIF_STATS_DROP);
632 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
633 pcb->utun_ifp->if_xname, length, header_offset);
634 continue;
635 }
636
637 length -= header_offset;
638 if (length > rx_pp->pp_buflet_size) {
639 // Flush data
640 mbuf_freem(data);
641 kern_pbufpool_free(rx_pp, rx_ph);
642 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
643 STATS_INC(nifs, NETIF_STATS_DROP);
644 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
645 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
646 continue;
647 }
648
649 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
650
651 // Fillout rx packet
652 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
653 VERIFY(rx_buf != NULL);
654 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
655 VERIFY(rx_baddr != NULL);
656
657 // Copy-in data from mbuf to buflet
658 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
659 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
660
661 // Finalize and attach the packet
662 error = kern_buflet_set_data_offset(rx_buf, 0);
663 VERIFY(error == 0);
664 error = kern_buflet_set_data_length(rx_buf, length);
665 VERIFY(error == 0);
666 error = kern_packet_set_headroom(rx_ph, 0);
667 VERIFY(error == 0);
668 error = kern_packet_finalize(rx_ph);
669 VERIFY(error == 0);
670 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
671 VERIFY(error == 0);
672
673 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
674 STATS_INC(nifs, NETIF_STATS_RX_COPY_MBUF);
675 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
676
677 rx_ring_stats.kcrsi_slots_transferred++;
678 rx_ring_stats.kcrsi_bytes_transferred += length;
679
680 mbuf_freem(data);
681
682 // Advance ring
683 rx_pslot = rx_slot;
684 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
685 }
686
687 struct kern_channel_ring_stat_increment tx_ring_stats;
688 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
689 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
690 kern_channel_slot_t tx_pslot = NULL;
691 kern_channel_slot_t tx_slot = NULL;
692 if (tx_ring == NULL) {
693 // Net-If TX ring not set up yet, nothing to read
694 goto done;
695 }
696
697 // Unlock utun before entering ring
698 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
699
700 (void)kr_enter(tx_ring, TRUE);
701
702 // Lock again after entering and validate
703 lck_rw_lock_shared(&pcb->utun_pcb_lock);
704 if (tx_ring != pcb->utun_kpipe_txring) {
705 goto done;
706 }
707
708 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
709 if (tx_slot == NULL) {
710 // Nothing to read, don't bother signalling
711 goto done;
712 }
713
714 while (rx_slot != NULL && tx_slot != NULL) {
715 // Allocate rx packet
716 kern_packet_t rx_ph = 0;
717 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
718
719 // Advance TX ring
720 tx_pslot = tx_slot;
721 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
722
723 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
724 if (tx_ph == 0) {
725 continue;
726 }
727
728 /* XXX We could try this alloc before advancing the slot to avoid
729 * dropping the packet on failure to allocate.
730 */
731 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
732 if (__improbable(error != 0)) {
733 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
734 STATS_INC(nifs, NETIF_STATS_DROP);
735 break;
736 }
737
738 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
739 VERIFY(tx_buf != NULL);
740 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
741 VERIFY(tx_baddr != 0);
742 tx_baddr += kern_buflet_get_data_offset(tx_buf);
743
744 // Check packet length
745 size_t header_offset = UTUN_HEADER_SIZE(pcb);
746 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
747 if (tx_length < header_offset) {
748 // Packet is too small
749 kern_pbufpool_free(rx_pp, rx_ph);
750 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
751 STATS_INC(nifs, NETIF_STATS_DROP);
752 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
753 pcb->utun_ifp->if_xname, tx_length, header_offset);
754 continue;
755 }
756
757 size_t length = MIN(tx_length - header_offset,
758 pcb->utun_slot_size);
759
760 tx_ring_stats.kcrsi_slots_transferred++;
761 tx_ring_stats.kcrsi_bytes_transferred += length;
762
763 // Fillout rx packet
764 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
765 VERIFY(rx_buf != NULL);
766 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
767 VERIFY(rx_baddr != NULL);
768
769 // Copy-in data from tx to rx
770 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
771 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
772
773 // Finalize and attach the packet
774 error = kern_buflet_set_data_offset(rx_buf, 0);
775 VERIFY(error == 0);
776 error = kern_buflet_set_data_length(rx_buf, length);
777 VERIFY(error == 0);
778 error = kern_packet_set_headroom(rx_ph, 0);
779 VERIFY(error == 0);
780 error = kern_packet_finalize(rx_ph);
781 VERIFY(error == 0);
782 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
783 VERIFY(error == 0);
784
785 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
786 STATS_INC(nifs, NETIF_STATS_RX_COPY_DIRECT);
787 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
788
789 rx_ring_stats.kcrsi_slots_transferred++;
790 rx_ring_stats.kcrsi_bytes_transferred += length;
791
792 rx_pslot = rx_slot;
793 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
794 }
795
796 done:
797 if (rx_pslot) {
798 kern_channel_advance_slot(rx_ring, rx_pslot);
799 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
800 }
801
802 if (tx_pslot) {
803 kern_channel_advance_slot(tx_ring, tx_pslot);
804 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
805 (void)kern_channel_reclaim(tx_ring);
806 }
807
808 // Unlock first, then exit ring
809 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
810 if (tx_ring != NULL) {
811 if (tx_pslot != NULL) {
812 kern_channel_notify(tx_ring, 0);
813 }
814 kr_exit(tx_ring);
815 }
816
817 return 0;
818 }
819
820 static errno_t
821 utun_nexus_ifattach(struct utun_pcb *pcb,
822 struct ifnet_init_eparams *init_params,
823 struct ifnet **ifp)
824 {
825 errno_t err;
826 nexus_controller_t controller = kern_nexus_shared_controller();
827 struct kern_nexus_net_init net_init;
828 struct kern_pbufpool_init pp_init;
829
830 nexus_name_t provider_name;
831 snprintf((char *)provider_name, sizeof(provider_name),
832 "com.apple.netif.%s", pcb->utun_if_xname);
833
834 struct kern_nexus_provider_init prov_init = {
835 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
836 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
837 .nxpi_pre_connect = utun_nexus_pre_connect,
838 .nxpi_connected = utun_nexus_connected,
839 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
840 .nxpi_disconnected = utun_nexus_disconnected,
841 .nxpi_ring_init = utun_netif_ring_init,
842 .nxpi_ring_fini = utun_netif_ring_fini,
843 .nxpi_slot_init = NULL,
844 .nxpi_slot_fini = NULL,
845 .nxpi_sync_tx = utun_netif_sync_tx,
846 .nxpi_sync_rx = utun_netif_sync_rx,
847 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
848 };
849
850 nexus_attr_t nxa = NULL;
851 err = kern_nexus_attr_create(&nxa);
852 if (err != 0) {
853 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
854 __func__, err);
855 goto failed;
856 }
857
858 uint64_t slot_buffer_size = pcb->utun_slot_size;
859 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
860 VERIFY(err == 0);
861
862 // Reset ring size for netif nexus to limit memory usage
863 uint64_t ring_size = pcb->utun_netif_ring_size;
864 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
865 VERIFY(err == 0);
866 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
867 VERIFY(err == 0);
868
869 pcb->utun_netif_txring_size = ring_size;
870
871 bzero(&pp_init, sizeof(pp_init));
872 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
873 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
874 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
875 pp_init.kbi_bufsize = pcb->utun_slot_size;
876 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
877 pp_init.kbi_max_frags = 1;
878 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
879 "%s", provider_name);
880 pp_init.kbi_ctx = NULL;
881 pp_init.kbi_ctx_retain = NULL;
882 pp_init.kbi_ctx_release = NULL;
883
884 err = kern_pbufpool_create(&pp_init, &pcb->utun_netif_pp, NULL);
885 if (err != 0) {
886 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, err);
887 goto failed;
888 }
889
890 err = kern_nexus_controller_register_provider(controller,
891 utun_nx_dom_prov,
892 provider_name,
893 &prov_init,
894 sizeof(prov_init),
895 nxa,
896 &pcb->utun_nx.if_provider);
897 if (err != 0) {
898 os_log_error(OS_LOG_DEFAULT, "%s register provider failed, error %d\n",
899 __func__, err);
900 goto failed;
901 }
902
903 bzero(&net_init, sizeof(net_init));
904 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
905 net_init.nxneti_flags = 0;
906 net_init.nxneti_eparams = init_params;
907 net_init.nxneti_lladdr = NULL;
908 net_init.nxneti_prepare = utun_netif_prepare;
909 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
910 err = kern_nexus_controller_alloc_net_provider_instance(controller,
911 pcb->utun_nx.if_provider,
912 pcb,
913 &pcb->utun_nx.if_instance,
914 &net_init,
915 ifp);
916 if (err != 0) {
917 os_log_error(OS_LOG_DEFAULT, "%s alloc_net_provider_instance failed, %d\n",
918 __func__, err);
919 kern_nexus_controller_deregister_provider(controller,
920 pcb->utun_nx.if_provider);
921 uuid_clear(pcb->utun_nx.if_provider);
922 goto failed;
923 }
924
925 failed:
926 if (nxa) {
927 kern_nexus_attr_destroy(nxa);
928 }
929 if (err && pcb->utun_netif_pp != NULL) {
930 kern_pbufpool_destroy(pcb->utun_netif_pp);
931 pcb->utun_netif_pp = NULL;
932 }
933 return err;
934 }
935
936 static void
937 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
938 {
939 nexus_controller_t controller = kern_nexus_shared_controller();
940 errno_t err;
941
942 if (!uuid_is_null(instance)) {
943 err = kern_nexus_controller_free_provider_instance(controller,
944 instance);
945 if (err != 0) {
946 os_log_error(OS_LOG_DEFAULT, "%s free_provider_instance failed %d\n",
947 __func__, err);
948 }
949 uuid_clear(instance);
950 }
951 if (!uuid_is_null(provider)) {
952 err = kern_nexus_controller_deregister_provider(controller,
953 provider);
954 if (err != 0) {
955 os_log_error(OS_LOG_DEFAULT, "%s deregister_provider %d\n", __func__, err);
956 }
957 uuid_clear(provider);
958 }
959 return;
960 }
961
962 static void
963 utun_nexus_detach(struct utun_pcb *pcb)
964 {
965 utun_nx_t nx = &pcb->utun_nx;
966 nexus_controller_t controller = kern_nexus_shared_controller();
967 errno_t err;
968
969 if (!uuid_is_null(nx->fsw_host)) {
970 err = kern_nexus_ifdetach(controller,
971 nx->fsw_instance,
972 nx->fsw_host);
973 if (err != 0) {
974 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms host failed %d\n",
975 __func__, err);
976 }
977 }
978
979 if (!uuid_is_null(nx->fsw_device)) {
980 err = kern_nexus_ifdetach(controller,
981 nx->fsw_instance,
982 nx->fsw_device);
983 if (err != 0) {
984 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms device failed %d\n",
985 __func__, err);
986 }
987 }
988
989 utun_detach_provider_and_instance(nx->if_provider,
990 nx->if_instance);
991 utun_detach_provider_and_instance(nx->fsw_provider,
992 nx->fsw_instance);
993
994 if (pcb->utun_netif_pp != NULL) {
995 kern_pbufpool_destroy(pcb->utun_netif_pp);
996 pcb->utun_netif_pp = NULL;
997 }
998 memset(nx, 0, sizeof(*nx));
999 }
1000
1001 static errno_t
1002 utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
1003 const char *type_name,
1004 const char *ifname,
1005 uuid_t *provider, uuid_t *instance)
1006 {
1007 nexus_attr_t attr = NULL;
1008 nexus_controller_t controller = kern_nexus_shared_controller();
1009 uuid_t dom_prov;
1010 errno_t err;
1011 struct kern_nexus_init init;
1012 nexus_name_t provider_name;
1013
1014 err = kern_nexus_get_default_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1015 &dom_prov);
1016 if (err != 0) {
1017 os_log_error(OS_LOG_DEFAULT, "%s can't get %s provider, error %d\n",
1018 __func__, type_name, err);
1019 goto failed;
1020 }
1021
1022 err = kern_nexus_attr_create(&attr);
1023 if (err != 0) {
1024 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1025 __func__, err);
1026 goto failed;
1027 }
1028
1029 uint64_t slot_buffer_size = pcb->utun_slot_size;
1030 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1031 VERIFY(err == 0);
1032
1033 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
1034 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
1035 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1036 VERIFY(err == 0);
1037 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
1038 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1039 VERIFY(err == 0);
1040
1041 snprintf((char *)provider_name, sizeof(provider_name),
1042 "com.apple.%s.%s", type_name, ifname);
1043 err = kern_nexus_controller_register_provider(controller,
1044 dom_prov,
1045 provider_name,
1046 NULL,
1047 0,
1048 attr,
1049 provider);
1050 kern_nexus_attr_destroy(attr);
1051 attr = NULL;
1052 if (err != 0) {
1053 os_log_error(OS_LOG_DEFAULT, "%s register %s provider failed, error %d\n",
1054 __func__, type_name, err);
1055 goto failed;
1056 }
1057 bzero(&init, sizeof(init));
1058 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1059 err = kern_nexus_controller_alloc_provider_instance(controller,
1060 *provider,
1061 NULL,
1062 instance, &init);
1063 if (err != 0) {
1064 os_log_error(OS_LOG_DEFAULT, "%s alloc_provider_instance %s failed, %d\n",
1065 __func__, type_name, err);
1066 kern_nexus_controller_deregister_provider(controller,
1067 *provider);
1068 uuid_clear(*provider);
1069 }
1070 failed:
1071 return err;
1072 }
1073
1074 static errno_t
1075 utun_flowswitch_attach(struct utun_pcb *pcb)
1076 {
1077 nexus_controller_t controller = kern_nexus_shared_controller();
1078 errno_t err = 0;
1079 utun_nx_t nx = &pcb->utun_nx;
1080
1081 // Allocate flowswitch
1082 err = utun_create_fs_provider_and_instance(pcb,
1083 "flowswitch",
1084 pcb->utun_ifp->if_xname,
1085 &nx->fsw_provider,
1086 &nx->fsw_instance);
1087 if (err != 0) {
1088 os_log_error(OS_LOG_DEFAULT, "%s: failed to create bridge provider and instance\n",
1089 __func__);
1090 goto failed;
1091 }
1092
1093 // Attach flowswitch to device port
1094 err = kern_nexus_ifattach(controller, nx->fsw_instance,
1095 NULL, nx->if_instance,
1096 FALSE, &nx->fsw_device);
1097 if (err != 0) {
1098 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms device %d\n", __func__, err);
1099 goto failed;
1100 }
1101
1102 // Attach flowswitch to host port
1103 err = kern_nexus_ifattach(controller, nx->fsw_instance,
1104 NULL, nx->if_instance,
1105 TRUE, &nx->fsw_host);
1106 if (err != 0) {
1107 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms host %d\n", __func__, err);
1108 goto failed;
1109 }
1110
1111 // Extract the agent UUID and save for later
1112 struct kern_nexus *flowswitch_nx = nx_find(nx->fsw_instance, false);
1113 if (flowswitch_nx != NULL) {
1114 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(flowswitch_nx);
1115 if (flowswitch != NULL) {
1116 FSW_RLOCK(flowswitch);
1117 uuid_copy(nx->fsw_agent, flowswitch->fsw_agent_uuid);
1118 FSW_UNLOCK(flowswitch);
1119 } else {
1120 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - flowswitch is NULL\n");
1121 }
1122 nx_release(flowswitch_nx);
1123 } else {
1124 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - unable to find flowswitch nexus\n");
1125 }
1126
1127 return 0;
1128
1129 failed:
1130 utun_nexus_detach(pcb);
1131
1132 errno_t detach_error = 0;
1133 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1134 panic("utun_flowswitch_attach - ifnet_detach failed: %d\n", detach_error);
1135 /* NOT REACHED */
1136 }
1137
1138 return err;
1139 }
1140
1141 static errno_t
1142 utun_register_kernel_pipe_nexus(struct utun_pcb *pcb)
1143 {
1144 nexus_attr_t nxa = NULL;
1145 errno_t result;
1146
1147 lck_mtx_lock(&utun_lock);
1148 if (utun_ncd_refcount++) {
1149 lck_mtx_unlock(&utun_lock);
1150 return 0;
1151 }
1152
1153 result = kern_nexus_controller_create(&utun_ncd);
1154 if (result) {
1155 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_create failed: %d\n",
1156 __FUNCTION__, result);
1157 goto done;
1158 }
1159
1160 uuid_t dom_prov;
1161 result = kern_nexus_get_default_domain_provider(
1162 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1163 if (result) {
1164 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_get_default_domain_provider failed: %d\n",
1165 __FUNCTION__, result);
1166 goto done;
1167 }
1168
1169 struct kern_nexus_provider_init prov_init = {
1170 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1171 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1172 .nxpi_pre_connect = utun_nexus_pre_connect,
1173 .nxpi_connected = utun_nexus_connected,
1174 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1175 .nxpi_disconnected = utun_nexus_disconnected,
1176 .nxpi_ring_init = utun_kpipe_ring_init,
1177 .nxpi_ring_fini = utun_kpipe_ring_fini,
1178 .nxpi_slot_init = NULL,
1179 .nxpi_slot_fini = NULL,
1180 .nxpi_sync_tx = utun_kpipe_sync_tx,
1181 .nxpi_sync_rx = utun_kpipe_sync_rx,
1182 .nxpi_tx_doorbell = NULL,
1183 };
1184
1185 result = kern_nexus_attr_create(&nxa);
1186 if (result) {
1187 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1188 __FUNCTION__, result);
1189 goto done;
1190 }
1191
1192 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1193 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1194 VERIFY(result == 0);
1195
1196 // Reset ring size for kernel pipe nexus to limit memory usage
1197 uint64_t ring_size =
1198 pcb->utun_kpipe_tx_ring_size != 0 ? pcb->utun_kpipe_tx_ring_size :
1199 if_utun_ring_size;
1200 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1201 VERIFY(result == 0);
1202
1203 ring_size =
1204 pcb->utun_kpipe_rx_ring_size != 0 ? pcb->utun_kpipe_rx_ring_size :
1205 if_utun_ring_size;
1206 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1207 VERIFY(result == 0);
1208
1209 result = kern_nexus_controller_register_provider(utun_ncd,
1210 dom_prov,
1211 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1212 &prov_init,
1213 sizeof(prov_init),
1214 nxa,
1215 &utun_kpipe_uuid);
1216 if (result) {
1217 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_register_provider failed: %d\n",
1218 __FUNCTION__, result);
1219 goto done;
1220 }
1221
1222 done:
1223 if (nxa) {
1224 kern_nexus_attr_destroy(nxa);
1225 }
1226
1227 if (result) {
1228 if (utun_ncd) {
1229 kern_nexus_controller_destroy(utun_ncd);
1230 utun_ncd = NULL;
1231 }
1232 utun_ncd_refcount = 0;
1233 }
1234
1235 lck_mtx_unlock(&utun_lock);
1236
1237 return result;
1238 }
1239
1240 static void
1241 utun_unregister_kernel_pipe_nexus(void)
1242 {
1243 lck_mtx_lock(&utun_lock);
1244
1245 VERIFY(utun_ncd_refcount > 0);
1246
1247 if (--utun_ncd_refcount == 0) {
1248 kern_nexus_controller_destroy(utun_ncd);
1249 utun_ncd = NULL;
1250 }
1251
1252 lck_mtx_unlock(&utun_lock);
1253 }
1254
1255 // For use by socket option, not internally
1256 static errno_t
1257 utun_disable_channel(struct utun_pcb *pcb)
1258 {
1259 errno_t result;
1260 int enabled;
1261 uuid_t uuid;
1262
1263 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1264
1265 enabled = pcb->utun_kpipe_enabled;
1266 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1267
1268 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1269
1270 pcb->utun_kpipe_enabled = 0;
1271 uuid_clear(pcb->utun_kpipe_uuid);
1272
1273 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1274
1275 if (enabled) {
1276 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1277 } else {
1278 result = ENXIO;
1279 }
1280
1281 if (!result) {
1282 if (pcb->utun_kpipe_pp != NULL) {
1283 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1284 pcb->utun_kpipe_pp = NULL;
1285 }
1286 utun_unregister_kernel_pipe_nexus();
1287 }
1288
1289 return result;
1290 }
1291
1292 static errno_t
1293 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1294 {
1295 struct kern_nexus_init init;
1296 struct kern_pbufpool_init pp_init;
1297 errno_t result;
1298
1299 kauth_cred_t cred = kauth_cred_get();
1300 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
1301 if (result) {
1302 return result;
1303 }
1304
1305 result = utun_register_kernel_pipe_nexus(pcb);
1306 if (result) {
1307 return result;
1308 }
1309
1310 VERIFY(utun_ncd);
1311
1312 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1313
1314 if (pcb->utun_kpipe_enabled) {
1315 result = EEXIST; // return success instead?
1316 goto done;
1317 }
1318
1319 /*
1320 * Make sure we can fit packets in the channel buffers and
1321 * Allow an extra 4 bytes for the protocol number header in the channel
1322 */
1323 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
1324 result = EOPNOTSUPP;
1325 goto done;
1326 }
1327
1328 bzero(&pp_init, sizeof(pp_init));
1329 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1330 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
1331 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1332 pp_init.kbi_bufsize = pcb->utun_slot_size;
1333 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1334 pp_init.kbi_max_frags = 1;
1335 pp_init.kbi_flags |= KBIF_QUANTUM;
1336 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
1337 "com.apple.kpipe.%s", pcb->utun_if_xname);
1338 pp_init.kbi_ctx = NULL;
1339 pp_init.kbi_ctx_retain = NULL;
1340 pp_init.kbi_ctx_release = NULL;
1341
1342 result = kern_pbufpool_create(&pp_init, &pcb->utun_kpipe_pp,
1343 NULL);
1344 if (result != 0) {
1345 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, result);
1346 goto done;
1347 }
1348
1349 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1350 bzero(&init, sizeof(init));
1351 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1352 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
1353 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1354 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1355 if (result) {
1356 goto done;
1357 }
1358
1359 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1360 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1361 pcb->utun_kpipe_uuid, &port,
1362 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1363 if (result) {
1364 kern_nexus_controller_free_provider_instance(utun_ncd,
1365 pcb->utun_kpipe_uuid);
1366 uuid_clear(pcb->utun_kpipe_uuid);
1367 goto done;
1368 }
1369
1370 pcb->utun_kpipe_enabled = 1;
1371
1372 done:
1373 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1374
1375 if (result) {
1376 if (pcb->utun_kpipe_pp != NULL) {
1377 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1378 pcb->utun_kpipe_pp = NULL;
1379 }
1380 utun_unregister_kernel_pipe_nexus();
1381 }
1382
1383 return result;
1384 }
1385
1386 #endif // UTUN_NEXUS
1387
1388 errno_t
1389 utun_register_control(void)
1390 {
1391 struct kern_ctl_reg kern_ctl;
1392 errno_t result = 0;
1393
1394 utun_pcb_size = sizeof(struct utun_pcb);
1395 utun_pcb_zone = zinit(utun_pcb_size,
1396 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1397 0, UTUN_PCB_ZONE_NAME);
1398 if (utun_pcb_zone == NULL) {
1399 os_log_error(OS_LOG_DEFAULT, "utun_register_control - zinit(utun_pcb) failed");
1400 return ENOMEM;
1401 }
1402
1403 #if UTUN_NEXUS
1404 utun_register_nexus();
1405 #endif // UTUN_NEXUS
1406
1407 TAILQ_INIT(&utun_head);
1408
1409 bzero(&kern_ctl, sizeof(kern_ctl));
1410 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1411 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1412 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
1413 kern_ctl.ctl_sendsize = 512 * 1024;
1414 kern_ctl.ctl_recvsize = 512 * 1024;
1415 kern_ctl.ctl_bind = utun_ctl_bind;
1416 kern_ctl.ctl_connect = utun_ctl_connect;
1417 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1418 kern_ctl.ctl_send = utun_ctl_send;
1419 kern_ctl.ctl_setopt = utun_ctl_setopt;
1420 kern_ctl.ctl_getopt = utun_ctl_getopt;
1421 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1422
1423 result = ctl_register(&kern_ctl, &utun_kctlref);
1424 if (result != 0) {
1425 os_log_error(OS_LOG_DEFAULT, "utun_register_control - ctl_register failed: %d\n", result);
1426 return result;
1427 }
1428
1429 /* Register the protocol plumbers */
1430 if ((result = proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN,
1431 utun_attach_proto, NULL)) != 0) {
1432 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN) failed: %d\n",
1433 result);
1434 ctl_deregister(utun_kctlref);
1435 return result;
1436 }
1437
1438 /* Register the protocol plumbers */
1439 if ((result = proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN,
1440 utun_attach_proto, NULL)) != 0) {
1441 proto_unregister_plumber(PF_INET, IFNET_FAMILY_UTUN);
1442 ctl_deregister(utun_kctlref);
1443 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN) failed: %d\n",
1444 result);
1445 return result;
1446 }
1447
1448 utun_lck_attr = lck_attr_alloc_init();
1449 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1450 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1451
1452 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1453
1454 return 0;
1455 }
1456
1457 /* Kernel control functions */
1458
1459 static inline void
1460 utun_free_pcb(struct utun_pcb *pcb, bool in_list)
1461 {
1462 #ifdef UTUN_NEXUS
1463 mbuf_freem_list(pcb->utun_input_chain);
1464 pcb->utun_input_chain_count = 0;
1465 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1466 #endif // UTUN_NEXUS
1467 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1468 if (in_list) {
1469 lck_mtx_lock(&utun_lock);
1470 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1471 lck_mtx_unlock(&utun_lock);
1472 }
1473 zfree(utun_pcb_zone, pcb);
1474 }
1475
1476 static errno_t
1477 utun_ctl_bind(kern_ctl_ref kctlref,
1478 struct sockaddr_ctl *sac,
1479 void **unitinfo)
1480 {
1481 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1482 memset(pcb, 0, sizeof(*pcb));
1483
1484 *unitinfo = pcb;
1485 pcb->utun_ctlref = kctlref;
1486 pcb->utun_unit = sac->sc_unit;
1487 pcb->utun_max_pending_packets = 1;
1488
1489 #if UTUN_NEXUS
1490 pcb->utun_use_netif = false;
1491 pcb->utun_attach_fsw = true;
1492 pcb->utun_netif_connected = false;
1493 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1494 pcb->utun_netif_ring_size = if_utun_ring_size;
1495 pcb->utun_tx_fsw_ring_size = if_utun_tx_fsw_ring_size;
1496 pcb->utun_rx_fsw_ring_size = if_utun_rx_fsw_ring_size;
1497 pcb->utun_input_chain_count = 0;
1498 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1499 #endif // UTUN_NEXUS
1500
1501 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1502
1503 return 0;
1504 }
1505
1506 static errno_t
1507 utun_ctl_connect(kern_ctl_ref kctlref,
1508 struct sockaddr_ctl *sac,
1509 void **unitinfo)
1510 {
1511 struct ifnet_init_eparams utun_init = {};
1512 errno_t result = 0;
1513
1514 if (*unitinfo == NULL) {
1515 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1516 }
1517
1518 struct utun_pcb *pcb = *unitinfo;
1519
1520 lck_mtx_lock(&utun_lock);
1521
1522 /* Find some open interface id */
1523 u_int32_t chosen_unique_id = 1;
1524 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1525 if (next_pcb != NULL) {
1526 /* List was not empty, add one to the last item */
1527 chosen_unique_id = next_pcb->utun_unique_id + 1;
1528 next_pcb = NULL;
1529
1530 /*
1531 * If this wrapped the id number, start looking at
1532 * the front of the list for an unused id.
1533 */
1534 if (chosen_unique_id == 0) {
1535 /* Find the next unused ID */
1536 chosen_unique_id = 1;
1537 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1538 if (next_pcb->utun_unique_id > chosen_unique_id) {
1539 /* We found a gap */
1540 break;
1541 }
1542
1543 chosen_unique_id = next_pcb->utun_unique_id + 1;
1544 }
1545 }
1546 }
1547
1548 pcb->utun_unique_id = chosen_unique_id;
1549
1550 if (next_pcb != NULL) {
1551 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1552 } else {
1553 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1554 }
1555 lck_mtx_unlock(&utun_lock);
1556
1557 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1558 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1559 os_log(OS_LOG_DEFAULT, "utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
1560
1561 /* Create the interface */
1562 bzero(&utun_init, sizeof(utun_init));
1563 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1564 utun_init.len = sizeof(utun_init);
1565
1566 #if UTUN_NEXUS
1567 if (pcb->utun_use_netif) {
1568 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1569 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1570 } else
1571 #endif // UTUN_NEXUS
1572 {
1573 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1574 utun_init.start = utun_start;
1575 utun_init.framer_extended = utun_framer;
1576 }
1577 utun_init.name = "utun";
1578 utun_init.unit = pcb->utun_unit - 1;
1579 utun_init.uniqueid = pcb->utun_unique_name;
1580 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1581 utun_init.family = IFNET_FAMILY_UTUN;
1582 utun_init.type = IFT_OTHER;
1583 utun_init.demux = utun_demux;
1584 utun_init.add_proto = utun_add_proto;
1585 utun_init.del_proto = utun_del_proto;
1586 utun_init.softc = pcb;
1587 utun_init.ioctl = utun_ioctl;
1588 utun_init.detach = utun_detached;
1589
1590 #if UTUN_NEXUS
1591 if (pcb->utun_use_netif) {
1592 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1593 if (result != 0) {
1594 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1595 utun_free_pcb(pcb, true);
1596 *unitinfo = NULL;
1597 return result;
1598 }
1599
1600 if (pcb->utun_attach_fsw) {
1601 result = utun_flowswitch_attach(pcb);
1602 if (result != 0) {
1603 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_flowswitch_attach failed: %d\n", result);
1604 *unitinfo = NULL;
1605 return result;
1606 }
1607 }
1608
1609 /* Attach to bpf */
1610 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1611 } else
1612 #endif // UTUN_NEXUS
1613 {
1614 /*
1615 * Upon success, this holds an ifnet reference which we will
1616 * release via ifnet_release() at final detach time.
1617 */
1618 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1619 if (result != 0) {
1620 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1621 utun_free_pcb(pcb, true);
1622 *unitinfo = NULL;
1623 return result;
1624 }
1625
1626 /* Set flags and additional information. */
1627 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1628 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1629
1630 /* The interface must generate its own IPv6 LinkLocal address,
1631 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1632 */
1633 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1634
1635 /* Reset the stats in case as the interface may have been recycled */
1636 struct ifnet_stats_param stats;
1637 bzero(&stats, sizeof(struct ifnet_stats_param));
1638 ifnet_set_stat(pcb->utun_ifp, &stats);
1639
1640 /* Attach the interface */
1641 result = ifnet_attach(pcb->utun_ifp, NULL);
1642 if (result != 0) {
1643 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_attach failed: %d\n", result);
1644 /* Release reference now since attach failed */
1645 ifnet_release(pcb->utun_ifp);
1646 utun_free_pcb(pcb, true);
1647 *unitinfo = NULL;
1648 return result;
1649 }
1650
1651 /* Attach to bpf */
1652 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1653 }
1654
1655 /* The interfaces resoures allocated, mark it as running */
1656 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1657
1658 return result;
1659 }
1660
1661 static errno_t
1662 utun_detach_ip(ifnet_t interface,
1663 protocol_family_t protocol,
1664 socket_t pf_socket)
1665 {
1666 errno_t result = EPROTONOSUPPORT;
1667
1668 /* Attempt a detach */
1669 if (protocol == PF_INET) {
1670 struct ifreq ifr;
1671
1672 bzero(&ifr, sizeof(ifr));
1673 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1674 ifnet_name(interface), ifnet_unit(interface));
1675
1676 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1677 } else if (protocol == PF_INET6) {
1678 struct in6_ifreq ifr6;
1679
1680 bzero(&ifr6, sizeof(ifr6));
1681 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1682 ifnet_name(interface), ifnet_unit(interface));
1683
1684 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1685 }
1686
1687 return result;
1688 }
1689
1690 static void
1691 utun_remove_address(ifnet_t interface,
1692 protocol_family_t protocol,
1693 ifaddr_t address,
1694 socket_t pf_socket)
1695 {
1696 errno_t result = 0;
1697
1698 /* Attempt a detach */
1699 if (protocol == PF_INET) {
1700 struct ifreq ifr;
1701
1702 bzero(&ifr, sizeof(ifr));
1703 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1704 ifnet_name(interface), ifnet_unit(interface));
1705 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1706 if (result != 0) {
1707 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed: %d", result);
1708 } else {
1709 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1710 if (result != 0) {
1711 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR failed: %d", result);
1712 }
1713 }
1714 } else if (protocol == PF_INET6) {
1715 struct in6_ifreq ifr6;
1716
1717 bzero(&ifr6, sizeof(ifr6));
1718 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1719 ifnet_name(interface), ifnet_unit(interface));
1720 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1721 sizeof(ifr6.ifr_addr));
1722 if (result != 0) {
1723 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed (v6): %d",
1724 result);
1725 } else {
1726 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1727 if (result != 0) {
1728 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1729 result);
1730 }
1731 }
1732 }
1733 }
1734
1735 static void
1736 utun_cleanup_family(ifnet_t interface,
1737 protocol_family_t protocol)
1738 {
1739 errno_t result = 0;
1740 socket_t pf_socket = NULL;
1741 ifaddr_t *addresses = NULL;
1742 int i;
1743
1744 if (protocol != PF_INET && protocol != PF_INET6) {
1745 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - invalid protocol family %d\n", protocol);
1746 return;
1747 }
1748
1749 /* Create a socket for removing addresses and detaching the protocol */
1750 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1751 if (result != 0) {
1752 if (result != EAFNOSUPPORT) {
1753 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - failed to create %s socket: %d\n",
1754 protocol == PF_INET ? "IP" : "IPv6", result);
1755 }
1756 goto cleanup;
1757 }
1758
1759 /* always set SS_PRIV, we want to close and detach regardless */
1760 sock_setpriv(pf_socket, 1);
1761
1762 result = utun_detach_ip(interface, protocol, pf_socket);
1763 if (result == 0 || result == ENXIO) {
1764 /* We are done! We either detached or weren't attached. */
1765 goto cleanup;
1766 } else if (result != EBUSY) {
1767 /* Uh, not really sure what happened here... */
1768 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1769 goto cleanup;
1770 }
1771
1772 /*
1773 * At this point, we received an EBUSY error. This means there are
1774 * addresses attached. We should detach them and then try again.
1775 */
1776 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1777 if (result != 0) {
1778 os_log_error(OS_LOG_DEFAULT, "fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1779 ifnet_name(interface), ifnet_unit(interface),
1780 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1781 goto cleanup;
1782 }
1783
1784 for (i = 0; addresses[i] != 0; i++) {
1785 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1786 }
1787 ifnet_free_address_list(addresses);
1788 addresses = NULL;
1789
1790 /*
1791 * The addresses should be gone, we should try the remove again.
1792 */
1793 result = utun_detach_ip(interface, protocol, pf_socket);
1794 if (result != 0 && result != ENXIO) {
1795 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1796 }
1797
1798 cleanup:
1799 if (pf_socket != NULL) {
1800 sock_close(pf_socket);
1801 }
1802
1803 if (addresses != NULL) {
1804 ifnet_free_address_list(addresses);
1805 }
1806 }
1807
1808 static errno_t
1809 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1810 __unused u_int32_t unit,
1811 void *unitinfo)
1812 {
1813 struct utun_pcb *pcb = unitinfo;
1814 ifnet_t ifp = NULL;
1815 errno_t result = 0;
1816
1817 if (pcb == NULL) {
1818 return EINVAL;
1819 }
1820
1821 #if UTUN_NEXUS
1822 // Tell the nexus to stop all rings
1823 if (pcb->utun_netif_nexus != NULL && pcb->utun_netif_connected) {
1824 kern_nexus_stop(pcb->utun_netif_nexus);
1825 }
1826 #endif // UTUN_NEXUS
1827
1828 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1829
1830 #if UTUN_NEXUS
1831 uuid_t kpipe_uuid;
1832 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1833 uuid_clear(pcb->utun_kpipe_uuid);
1834 pcb->utun_kpipe_enabled = FALSE;
1835 #endif // UTUN_NEXUS
1836
1837 pcb->utun_ctlref = NULL;
1838
1839 ifp = pcb->utun_ifp;
1840 if (ifp != NULL) {
1841 #if UTUN_NEXUS
1842 // Tell the nexus to stop all rings
1843 if (pcb->utun_netif_nexus != NULL) {
1844 /*
1845 * Quiesce the interface and flush any pending outbound packets.
1846 */
1847 if_down(ifp);
1848
1849 /* Increment refcnt, but detach interface */
1850 ifnet_incr_iorefcnt(ifp);
1851 if ((result = ifnet_detach(ifp)) != 0) {
1852 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1853 }
1854
1855 /*
1856 * We want to do everything in our power to ensure that the interface
1857 * really goes away when the socket is closed. We must remove IP/IPv6
1858 * addresses and detach the protocols. Finally, we can remove and
1859 * release the interface.
1860 */
1861 utun_cleanup_family(ifp, AF_INET);
1862 utun_cleanup_family(ifp, AF_INET6);
1863
1864 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1865
1866 if (!uuid_is_null(kpipe_uuid)) {
1867 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1868 if (pcb->utun_kpipe_pp != NULL) {
1869 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1870 pcb->utun_kpipe_pp = NULL;
1871 }
1872 utun_unregister_kernel_pipe_nexus();
1873 }
1874 }
1875 utun_nexus_detach(pcb);
1876
1877 /* Decrement refcnt to finish detaching and freeing */
1878 ifnet_decr_iorefcnt(ifp);
1879 } else
1880 #endif // UTUN_NEXUS
1881 {
1882 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1883
1884 #if UTUN_NEXUS
1885 if (!uuid_is_null(kpipe_uuid)) {
1886 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1887 if (pcb->utun_kpipe_pp != NULL) {
1888 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1889 pcb->utun_kpipe_pp = NULL;
1890 }
1891 utun_unregister_kernel_pipe_nexus();
1892 }
1893 }
1894 #endif // UTUN_NEXUS
1895
1896 /*
1897 * We want to do everything in our power to ensure that the interface
1898 * really goes away when the socket is closed. We must remove IP/IPv6
1899 * addresses and detach the protocols. Finally, we can remove and
1900 * release the interface.
1901 */
1902 utun_cleanup_family(ifp, AF_INET);
1903 utun_cleanup_family(ifp, AF_INET6);
1904
1905 /*
1906 * Detach now; utun_detach() will be called asynchronously once
1907 * the I/O reference count drops to 0. There we will invoke
1908 * ifnet_release().
1909 */
1910 if ((result = ifnet_detach(ifp)) != 0) {
1911 os_log_error(OS_LOG_DEFAULT, "utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1912 }
1913 }
1914 } else {
1915 // Bound, but not connected
1916 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1917 utun_free_pcb(pcb, false);
1918 }
1919
1920 return 0;
1921 }
1922
1923 static errno_t
1924 utun_ctl_send(__unused kern_ctl_ref kctlref,
1925 __unused u_int32_t unit,
1926 void *unitinfo,
1927 mbuf_t m,
1928 __unused int flags)
1929 {
1930 /*
1931 * The userland ABI requires the first four bytes have the protocol family
1932 * in network byte order: swap them
1933 */
1934 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
1935 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
1936 } else {
1937 os_log_error(OS_LOG_DEFAULT, "%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m));
1938 }
1939
1940 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
1941 }
1942
1943 static errno_t
1944 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1945 __unused u_int32_t unit,
1946 void *unitinfo,
1947 int opt,
1948 void *data,
1949 size_t len)
1950 {
1951 struct utun_pcb *pcb = unitinfo;
1952 errno_t result = 0;
1953 /* check for privileges for privileged options */
1954 switch (opt) {
1955 case UTUN_OPT_FLAGS:
1956 case UTUN_OPT_EXT_IFDATA_STATS:
1957 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1958 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1959 return EPERM;
1960 }
1961 break;
1962 }
1963
1964 switch (opt) {
1965 case UTUN_OPT_FLAGS:
1966 if (len != sizeof(u_int32_t)) {
1967 result = EMSGSIZE;
1968 } else {
1969 if (pcb->utun_ifp == NULL) {
1970 // Only can set after connecting
1971 result = EINVAL;
1972 break;
1973 }
1974 #if UTUN_NEXUS
1975 if (pcb->utun_use_netif) {
1976 pcb->utun_flags = *(u_int32_t *)data;
1977 } else
1978 #endif // UTUN_NEXUS
1979 {
1980 u_int32_t old_flags = pcb->utun_flags;
1981 pcb->utun_flags = *(u_int32_t *)data;
1982 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1983 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1984 bpfdetach(pcb->utun_ifp);
1985 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1986 }
1987 }
1988 }
1989 break;
1990
1991 case UTUN_OPT_EXT_IFDATA_STATS:
1992 if (len != sizeof(int)) {
1993 result = EMSGSIZE;
1994 break;
1995 }
1996 if (pcb->utun_ifp == NULL) {
1997 // Only can set after connecting
1998 result = EINVAL;
1999 break;
2000 }
2001 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
2002 break;
2003
2004 case UTUN_OPT_INC_IFDATA_STATS_IN:
2005 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
2006 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
2007
2008 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
2009 result = EINVAL;
2010 break;
2011 }
2012 if (pcb->utun_ifp == NULL) {
2013 // Only can set after connecting
2014 result = EINVAL;
2015 break;
2016 }
2017 if (!pcb->utun_ext_ifdata_stats) {
2018 result = EINVAL;
2019 break;
2020 }
2021 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) {
2022 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2023 utsp->utsp_bytes, utsp->utsp_errors);
2024 } else {
2025 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2026 utsp->utsp_bytes, utsp->utsp_errors);
2027 }
2028 break;
2029 }
2030 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2031 ifnet_t del_ifp = NULL;
2032 char name[IFNAMSIZ];
2033
2034 if (len > IFNAMSIZ - 1) {
2035 result = EMSGSIZE;
2036 break;
2037 }
2038 if (pcb->utun_ifp == NULL) {
2039 // Only can set after connecting
2040 result = EINVAL;
2041 break;
2042 }
2043 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2044 bcopy(data, name, len);
2045 name[len] = 0;
2046 result = ifnet_find_by_name(name, &del_ifp);
2047 }
2048 if (result == 0) {
2049 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2050 if (del_ifp) {
2051 ifnet_release(del_ifp);
2052 }
2053 }
2054 break;
2055 }
2056 case UTUN_OPT_MAX_PENDING_PACKETS: {
2057 u_int32_t max_pending_packets = 0;
2058 if (len != sizeof(u_int32_t)) {
2059 result = EMSGSIZE;
2060 break;
2061 }
2062 max_pending_packets = *(u_int32_t *)data;
2063 if (max_pending_packets == 0) {
2064 result = EINVAL;
2065 break;
2066 }
2067 pcb->utun_max_pending_packets = max_pending_packets;
2068 break;
2069 }
2070 #if UTUN_NEXUS
2071 case UTUN_OPT_ENABLE_CHANNEL: {
2072 if (len != sizeof(int)) {
2073 result = EMSGSIZE;
2074 break;
2075 }
2076 if (pcb->utun_ifp == NULL) {
2077 // Only can set after connecting
2078 result = EINVAL;
2079 break;
2080 }
2081 if (*(int *)data) {
2082 result = utun_enable_channel(pcb, current_proc());
2083 } else {
2084 result = utun_disable_channel(pcb);
2085 }
2086 break;
2087 }
2088 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2089 if (len != sizeof(int)) {
2090 result = EMSGSIZE;
2091 break;
2092 }
2093 if (pcb->utun_ifp == NULL) {
2094 // Only can set after connecting
2095 result = EINVAL;
2096 break;
2097 }
2098 if (!if_is_fsw_transport_netagent_enabled()) {
2099 result = ENOTSUP;
2100 break;
2101 }
2102 if (uuid_is_null(pcb->utun_nx.fsw_agent)) {
2103 result = ENOENT;
2104 break;
2105 }
2106
2107 uint32_t flags = netagent_get_flags(pcb->utun_nx.fsw_agent);
2108
2109 if (*(int *)data) {
2110 pcb->utun_needs_netagent = true;
2111 flags |= (NETAGENT_FLAG_NEXUS_PROVIDER |
2112 NETAGENT_FLAG_NEXUS_LISTENER);
2113 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2114 } else {
2115 flags &= ~(NETAGENT_FLAG_NEXUS_PROVIDER |
2116 NETAGENT_FLAG_NEXUS_LISTENER);
2117 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2118 pcb->utun_needs_netagent = false;
2119 }
2120 break;
2121 }
2122 case UTUN_OPT_ATTACH_FLOWSWITCH: {
2123 if (len != sizeof(int)) {
2124 result = EMSGSIZE;
2125 break;
2126 }
2127 if (pcb->utun_ifp != NULL) {
2128 // Only can set before connecting
2129 result = EINVAL;
2130 break;
2131 }
2132 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2133 pcb->utun_attach_fsw = !!(*(int *)data);
2134 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2135 break;
2136 }
2137 case UTUN_OPT_ENABLE_NETIF: {
2138 if (len != sizeof(int)) {
2139 result = EMSGSIZE;
2140 break;
2141 }
2142 if (pcb->utun_ifp != NULL) {
2143 // Only can set before connecting
2144 result = EINVAL;
2145 break;
2146 }
2147 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2148 pcb->utun_use_netif = !!(*(int *)data);
2149 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2150 break;
2151 }
2152 case UTUN_OPT_SLOT_SIZE: {
2153 if (len != sizeof(u_int32_t)) {
2154 result = EMSGSIZE;
2155 break;
2156 }
2157 if (pcb->utun_ifp != NULL) {
2158 // Only can set before connecting
2159 result = EINVAL;
2160 break;
2161 }
2162 u_int32_t slot_size = *(u_int32_t *)data;
2163 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2164 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2165 return EINVAL;
2166 }
2167 pcb->utun_slot_size = slot_size;
2168 break;
2169 }
2170 case UTUN_OPT_NETIF_RING_SIZE: {
2171 if (len != sizeof(u_int32_t)) {
2172 result = EMSGSIZE;
2173 break;
2174 }
2175 if (pcb->utun_ifp != NULL) {
2176 // Only can set before connecting
2177 result = EINVAL;
2178 break;
2179 }
2180 u_int32_t ring_size = *(u_int32_t *)data;
2181 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2182 ring_size > UTUN_IF_MAX_RING_SIZE) {
2183 return EINVAL;
2184 }
2185 pcb->utun_netif_ring_size = ring_size;
2186 break;
2187 }
2188 case UTUN_OPT_TX_FSW_RING_SIZE: {
2189 if (len != sizeof(u_int32_t)) {
2190 result = EMSGSIZE;
2191 break;
2192 }
2193 if (pcb->utun_ifp != NULL) {
2194 // Only can set before connecting
2195 result = EINVAL;
2196 break;
2197 }
2198 u_int32_t ring_size = *(u_int32_t *)data;
2199 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2200 ring_size > UTUN_IF_MAX_RING_SIZE) {
2201 return EINVAL;
2202 }
2203 pcb->utun_tx_fsw_ring_size = ring_size;
2204 break;
2205 }
2206 case UTUN_OPT_RX_FSW_RING_SIZE: {
2207 if (len != sizeof(u_int32_t)) {
2208 result = EMSGSIZE;
2209 break;
2210 }
2211 if (pcb->utun_ifp != NULL) {
2212 // Only can set before connecting
2213 result = EINVAL;
2214 break;
2215 }
2216 u_int32_t ring_size = *(u_int32_t *)data;
2217 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2218 ring_size > UTUN_IF_MAX_RING_SIZE) {
2219 return EINVAL;
2220 }
2221 pcb->utun_rx_fsw_ring_size = ring_size;
2222 break;
2223 }
2224 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2225 if (len != sizeof(u_int32_t)) {
2226 result = EMSGSIZE;
2227 break;
2228 }
2229 if (pcb->utun_ifp != NULL) {
2230 // Only can set before connecting
2231 result = EINVAL;
2232 break;
2233 }
2234 u_int32_t ring_size = *(u_int32_t *)data;
2235 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2236 ring_size > UTUN_IF_MAX_RING_SIZE) {
2237 return EINVAL;
2238 }
2239 pcb->utun_kpipe_tx_ring_size = ring_size;
2240 break;
2241 }
2242 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2243 if (len != sizeof(u_int32_t)) {
2244 result = EMSGSIZE;
2245 break;
2246 }
2247 if (pcb->utun_ifp != NULL) {
2248 // Only can set before connecting
2249 result = EINVAL;
2250 break;
2251 }
2252 u_int32_t ring_size = *(u_int32_t *)data;
2253 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2254 ring_size > UTUN_IF_MAX_RING_SIZE) {
2255 return EINVAL;
2256 }
2257 pcb->utun_kpipe_rx_ring_size = ring_size;
2258 break;
2259 }
2260 #endif // UTUN_NEXUS
2261 default: {
2262 result = ENOPROTOOPT;
2263 break;
2264 }
2265 }
2266
2267 return result;
2268 }
2269
2270 static errno_t
2271 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2272 __unused u_int32_t unit,
2273 void *unitinfo,
2274 int opt,
2275 void *data,
2276 size_t *len)
2277 {
2278 struct utun_pcb *pcb = unitinfo;
2279 errno_t result = 0;
2280
2281 switch (opt) {
2282 case UTUN_OPT_FLAGS:
2283 if (*len != sizeof(u_int32_t)) {
2284 result = EMSGSIZE;
2285 } else {
2286 *(u_int32_t *)data = pcb->utun_flags;
2287 }
2288 break;
2289
2290 case UTUN_OPT_EXT_IFDATA_STATS:
2291 if (*len != sizeof(int)) {
2292 result = EMSGSIZE;
2293 } else {
2294 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2295 }
2296 break;
2297
2298 case UTUN_OPT_IFNAME:
2299 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2300 result = EMSGSIZE;
2301 } else {
2302 if (pcb->utun_ifp == NULL) {
2303 // Only can get after connecting
2304 result = EINVAL;
2305 break;
2306 }
2307 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2308 }
2309 break;
2310
2311 case UTUN_OPT_MAX_PENDING_PACKETS: {
2312 if (*len != sizeof(u_int32_t)) {
2313 result = EMSGSIZE;
2314 } else {
2315 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2316 }
2317 break;
2318 }
2319
2320 #if UTUN_NEXUS
2321 case UTUN_OPT_ENABLE_CHANNEL: {
2322 if (*len != sizeof(int)) {
2323 result = EMSGSIZE;
2324 } else {
2325 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2326 *(int *)data = pcb->utun_kpipe_enabled;
2327 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2328 }
2329 break;
2330 }
2331
2332 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2333 if (*len != sizeof(int)) {
2334 result = EMSGSIZE;
2335 } else {
2336 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.fsw_agent);
2337 }
2338 break;
2339 }
2340
2341 case UTUN_OPT_ENABLE_NETIF: {
2342 if (*len != sizeof(int)) {
2343 result = EMSGSIZE;
2344 } else {
2345 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2346 *(int *)data = !!pcb->utun_use_netif;
2347 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2348 }
2349 break;
2350 }
2351
2352 case UTUN_OPT_GET_CHANNEL_UUID: {
2353 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2354 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2355 result = ENXIO;
2356 } else if (*len != sizeof(uuid_t)) {
2357 result = EMSGSIZE;
2358 } else {
2359 uuid_copy(data, pcb->utun_kpipe_uuid);
2360 }
2361 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2362 break;
2363 }
2364 case UTUN_OPT_SLOT_SIZE: {
2365 if (*len != sizeof(u_int32_t)) {
2366 result = EMSGSIZE;
2367 } else {
2368 *(u_int32_t *)data = pcb->utun_slot_size;
2369 }
2370 break;
2371 }
2372 case UTUN_OPT_NETIF_RING_SIZE: {
2373 if (*len != sizeof(u_int32_t)) {
2374 result = EMSGSIZE;
2375 } else {
2376 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2377 }
2378 break;
2379 }
2380 case UTUN_OPT_TX_FSW_RING_SIZE: {
2381 if (*len != sizeof(u_int32_t)) {
2382 result = EMSGSIZE;
2383 } else {
2384 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2385 }
2386 break;
2387 }
2388 case UTUN_OPT_RX_FSW_RING_SIZE: {
2389 if (*len != sizeof(u_int32_t)) {
2390 result = EMSGSIZE;
2391 } else {
2392 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2393 }
2394 break;
2395 }
2396 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2397 if (*len != sizeof(u_int32_t)) {
2398 result = EMSGSIZE;
2399 } else {
2400 *(u_int32_t *)data = pcb->utun_kpipe_tx_ring_size;
2401 }
2402 break;
2403 }
2404 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2405 if (*len != sizeof(u_int32_t)) {
2406 result = EMSGSIZE;
2407 } else {
2408 *(u_int32_t *)data = pcb->utun_kpipe_rx_ring_size;
2409 }
2410 break;
2411 }
2412 #endif // UTUN_NEXUS
2413
2414 default:
2415 result = ENOPROTOOPT;
2416 break;
2417 }
2418
2419 return result;
2420 }
2421
2422 static void
2423 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2424 {
2425 #pragma unused(flags)
2426 bool reenable_output = false;
2427 struct utun_pcb *pcb = unitinfo;
2428 if (pcb == NULL) {
2429 return;
2430 }
2431 ifnet_lock_exclusive(pcb->utun_ifp);
2432
2433 u_int32_t utun_packet_cnt;
2434 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2435 if (error_pc != 0) {
2436 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2437 utun_packet_cnt = 0;
2438 }
2439
2440 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2441 reenable_output = true;
2442 }
2443
2444 if (reenable_output) {
2445 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2446 if (error != 0) {
2447 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2448 }
2449 }
2450 ifnet_lock_done(pcb->utun_ifp);
2451 }
2452
2453 /* Network Interface functions */
2454 static void
2455 utun_start(ifnet_t interface)
2456 {
2457 mbuf_t data;
2458 struct utun_pcb *pcb = ifnet_softc(interface);
2459
2460 VERIFY(pcb != NULL);
2461
2462 #if UTUN_NEXUS
2463 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2464 if (pcb->utun_kpipe_enabled) {
2465 /* It's possible to have channels enabled, but not yet have the channel opened,
2466 * in which case the rxring will not be set
2467 */
2468 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2469 if (pcb->utun_kpipe_rxring != NULL) {
2470 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2471 }
2472 return;
2473 }
2474 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2475 #endif // UTUN_NEXUS
2476
2477 for (;;) {
2478 bool can_accept_packets = true;
2479 ifnet_lock_shared(pcb->utun_ifp);
2480
2481 u_int32_t utun_packet_cnt;
2482 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2483 if (error_pc != 0) {
2484 os_log_error(OS_LOG_DEFAULT, "utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2485 utun_packet_cnt = 0;
2486 }
2487
2488 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2489 if (!can_accept_packets && pcb->utun_ctlref) {
2490 u_int32_t difference = 0;
2491 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2492 if (difference > 0) {
2493 // If the low-water mark has not yet been reached, we still need to enqueue data
2494 // into the buffer
2495 can_accept_packets = true;
2496 }
2497 }
2498 }
2499 if (!can_accept_packets) {
2500 errno_t error = ifnet_disable_output(interface);
2501 if (error != 0) {
2502 os_log_error(OS_LOG_DEFAULT, "utun_start: ifnet_disable_output returned error %d\n", error);
2503 }
2504 ifnet_lock_done(pcb->utun_ifp);
2505 break;
2506 }
2507 ifnet_lock_done(pcb->utun_ifp);
2508 if (ifnet_dequeue(interface, &data) != 0) {
2509 break;
2510 }
2511 if (utun_output(interface, data) != 0) {
2512 break;
2513 }
2514 }
2515 }
2516
2517 static errno_t
2518 utun_output(ifnet_t interface,
2519 mbuf_t data)
2520 {
2521 struct utun_pcb *pcb = ifnet_softc(interface);
2522 errno_t result;
2523
2524 VERIFY(interface == pcb->utun_ifp);
2525
2526 #if UTUN_NEXUS
2527 if (!pcb->utun_use_netif)
2528 #endif // UTUN_NEXUS
2529 {
2530 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2531 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2532 }
2533 }
2534
2535 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2536 /* flush data */
2537 mbuf_freem(data);
2538 return 0;
2539 }
2540
2541 // otherwise, fall thru to ctl_enqueumbuf
2542 if (pcb->utun_ctlref) {
2543 int length;
2544
2545 /*
2546 * The ABI requires the protocol in network byte order
2547 */
2548 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2549 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2550 }
2551
2552 length = mbuf_pkthdr_len(data);
2553 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2554 if (result != 0) {
2555 mbuf_freem(data);
2556 os_log_error(OS_LOG_DEFAULT, "utun_output - ctl_enqueuembuf failed: %d\n", result);
2557 #if UTUN_NEXUS
2558 if (!pcb->utun_use_netif)
2559 #endif // UTUN_NEXUS
2560 {
2561 ifnet_stat_increment_out(interface, 0, 0, 1);
2562 }
2563 } else {
2564 #if UTUN_NEXUS
2565 if (!pcb->utun_use_netif)
2566 #endif // UTUN_NEXUS
2567 {
2568 if (!pcb->utun_ext_ifdata_stats) {
2569 ifnet_stat_increment_out(interface, 1, length, 0);
2570 }
2571 }
2572 }
2573 } else {
2574 mbuf_freem(data);
2575 }
2576
2577 return 0;
2578 }
2579
2580 static errno_t
2581 utun_demux(__unused ifnet_t interface,
2582 mbuf_t data,
2583 __unused char *frame_header,
2584 protocol_family_t *protocol)
2585 {
2586 #if UTUN_NEXUS
2587 struct utun_pcb *pcb = ifnet_softc(interface);
2588 struct ip *ip;
2589 u_int ip_version;
2590 #endif
2591
2592 while (data != NULL && mbuf_len(data) < 1) {
2593 data = mbuf_next(data);
2594 }
2595
2596 if (data == NULL) {
2597 return ENOENT;
2598 }
2599
2600 #if UTUN_NEXUS
2601 if (pcb->utun_use_netif) {
2602 ip = mtod(data, struct ip *);
2603 ip_version = ip->ip_v;
2604
2605 switch (ip_version) {
2606 case 4:
2607 *protocol = PF_INET;
2608 return 0;
2609 case 6:
2610 *protocol = PF_INET6;
2611 return 0;
2612 default:
2613 *protocol = 0;
2614 break;
2615 }
2616 } else
2617 #endif // UTUN_NEXUS
2618 {
2619 *protocol = *(u_int32_t *)mbuf_data(data);
2620 }
2621
2622 return 0;
2623 }
2624
2625 static errno_t
2626 utun_framer(ifnet_t interface,
2627 mbuf_t *packet,
2628 __unused const struct sockaddr *dest,
2629 __unused const char *desk_linkaddr,
2630 const char *frame_type,
2631 u_int32_t *prepend_len,
2632 u_int32_t *postpend_len)
2633 {
2634 struct utun_pcb *pcb = ifnet_softc(interface);
2635 VERIFY(interface == pcb->utun_ifp);
2636
2637 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2638 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2639 os_log_error(OS_LOG_DEFAULT, "utun_framer - ifnet_output prepend failed\n");
2640
2641 ifnet_stat_increment_out(interface, 0, 0, 1);
2642
2643 // just return, because the buffer was freed in mbuf_prepend
2644 return EJUSTRETURN;
2645 }
2646 if (prepend_len != NULL) {
2647 *prepend_len = header_length;
2648 }
2649 if (postpend_len != NULL) {
2650 *postpend_len = 0;
2651 }
2652
2653 // place protocol number at the beginning of the mbuf
2654 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2655
2656
2657 return 0;
2658 }
2659
2660 static errno_t
2661 utun_add_proto(__unused ifnet_t interface,
2662 protocol_family_t protocol,
2663 __unused const struct ifnet_demux_desc *demux_array,
2664 __unused u_int32_t demux_count)
2665 {
2666 switch (protocol) {
2667 case PF_INET:
2668 return 0;
2669 case PF_INET6:
2670 return 0;
2671 default:
2672 break;
2673 }
2674
2675 return ENOPROTOOPT;
2676 }
2677
2678 static errno_t
2679 utun_del_proto(__unused ifnet_t interface,
2680 __unused protocol_family_t protocol)
2681 {
2682 return 0;
2683 }
2684
2685 static errno_t
2686 utun_ioctl(ifnet_t interface,
2687 u_long command,
2688 void *data)
2689 {
2690 #if UTUN_NEXUS
2691 struct utun_pcb *pcb = ifnet_softc(interface);
2692 #endif
2693 errno_t result = 0;
2694
2695 switch (command) {
2696 case SIOCSIFMTU: {
2697 #if UTUN_NEXUS
2698 if (pcb->utun_use_netif) {
2699 // Make sure we can fit packets in the channel buffers
2700 // Allow for the headroom in the slot
2701 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2702 result = EINVAL;
2703 } else {
2704 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2705 }
2706 } else
2707 #endif // UTUN_NEXUS
2708 {
2709 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2710 }
2711 break;
2712 }
2713
2714 case SIOCSIFFLAGS:
2715 /* ifioctl() takes care of it */
2716 break;
2717
2718 default:
2719 result = EOPNOTSUPP;
2720 }
2721
2722 return result;
2723 }
2724
2725 static void
2726 utun_detached(ifnet_t interface)
2727 {
2728 struct utun_pcb *pcb = ifnet_softc(interface);
2729 (void)ifnet_release(interface);
2730 utun_free_pcb(pcb, true);
2731 }
2732
2733 /* Protocol Handlers */
2734
2735 static errno_t
2736 utun_proto_input(__unused ifnet_t interface,
2737 protocol_family_t protocol,
2738 mbuf_t m,
2739 __unused char *frame_header)
2740 {
2741 struct utun_pcb *pcb = ifnet_softc(interface);
2742 #if UTUN_NEXUS
2743 if (!pcb->utun_use_netif)
2744 #endif // UTUN_NEXUS
2745 {
2746 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2747 }
2748 int32_t pktlen = m->m_pkthdr.len;
2749 if (proto_input(protocol, m) != 0) {
2750 m_freem(m);
2751 #if UTUN_NEXUS
2752 if (!pcb->utun_use_netif)
2753 #endif // UTUN_NEXUS
2754 {
2755 ifnet_stat_increment_in(interface, 0, 0, 1);
2756 }
2757 } else {
2758 #if UTUN_NEXUS
2759 if (!pcb->utun_use_netif)
2760 #endif // UTUN_NEXUS
2761 {
2762 ifnet_stat_increment_in(interface, 1, pktlen, 0);
2763 }
2764 }
2765
2766 return 0;
2767 }
2768
2769 static errno_t
2770 utun_proto_pre_output(__unused ifnet_t interface,
2771 protocol_family_t protocol,
2772 __unused mbuf_t *packet,
2773 __unused const struct sockaddr *dest,
2774 __unused void *route,
2775 char *frame_type,
2776 __unused char *link_layer_dest)
2777 {
2778 *(protocol_family_t *)(void *)frame_type = protocol;
2779 return 0;
2780 }
2781
2782 static errno_t
2783 utun_attach_proto(ifnet_t interface,
2784 protocol_family_t protocol)
2785 {
2786 struct ifnet_attach_proto_param proto;
2787
2788 bzero(&proto, sizeof(proto));
2789 proto.input = utun_proto_input;
2790 proto.pre_output = utun_proto_pre_output;
2791
2792 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2793 if (result != 0 && result != EEXIST) {
2794 os_log_error(OS_LOG_DEFAULT, "utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2795 protocol, result);
2796 }
2797
2798 return result;
2799 }
2800
2801 static errno_t
2802 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2803 {
2804 #if UTUN_NEXUS
2805 if (pcb->utun_use_netif) {
2806 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2807
2808 lck_mtx_lock(&pcb->utun_input_chain_lock);
2809
2810 if (pcb->utun_input_chain_count > (u_int32_t)if_utun_max_pending_input) {
2811 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2812 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2813 return ENOSPC;
2814 }
2815
2816 if (pcb->utun_input_chain != NULL) {
2817 pcb->utun_input_chain_last->m_nextpkt = packet;
2818 } else {
2819 pcb->utun_input_chain = packet;
2820 }
2821 pcb->utun_input_chain_count++;
2822 while (packet->m_nextpkt) {
2823 VERIFY(packet != packet->m_nextpkt);
2824 packet = packet->m_nextpkt;
2825 pcb->utun_input_chain_count++;
2826 }
2827 pcb->utun_input_chain_last = packet;
2828 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2829
2830 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2831 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2832
2833 if (rx_ring != NULL) {
2834 kern_channel_notify(rx_ring, 0);
2835 }
2836
2837 return 0;
2838 } else
2839 #endif // UTUN_NEXUS
2840 {
2841 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
2842
2843 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2844 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2845 }
2846 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2847 /* flush data */
2848 mbuf_freem(packet);
2849 return 0;
2850 }
2851
2852 errno_t result = 0;
2853 if (!pcb->utun_ext_ifdata_stats) {
2854 struct ifnet_stat_increment_param incs = {};
2855 incs.packets_in = 1;
2856 incs.bytes_in = mbuf_pkthdr_len(packet);
2857 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2858 } else {
2859 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2860 }
2861 if (result != 0) {
2862 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2863
2864 os_log_error(OS_LOG_DEFAULT, "%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2865 mbuf_freem(packet);
2866 }
2867
2868 return 0;
2869 }
2870 }
2871
2872 #if UTUN_NEXUS
2873
2874 static errno_t
2875 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2876 {
2877 return 0;
2878 }
2879
2880 static void
2881 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2882 {
2883 // Ignore
2884 }
2885
2886 static errno_t
2887 utun_register_nexus(void)
2888 {
2889 const struct kern_nexus_domain_provider_init dp_init = {
2890 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2891 .nxdpi_flags = 0,
2892 .nxdpi_init = utun_nxdp_init,
2893 .nxdpi_fini = utun_nxdp_fini
2894 };
2895 errno_t err = 0;
2896
2897 /* utun_nxdp_init() is called before this function returns */
2898 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2899 (const uint8_t *) "com.apple.utun",
2900 &dp_init, sizeof(dp_init),
2901 &utun_nx_dom_prov);
2902 if (err != 0) {
2903 os_log_error(OS_LOG_DEFAULT, "%s: failed to register domain provider\n", __func__);
2904 return err;
2905 }
2906 return 0;
2907 }
2908 boolean_t
2909 utun_interface_needs_netagent(ifnet_t interface)
2910 {
2911 struct utun_pcb *pcb = NULL;
2912
2913 if (interface == NULL) {
2914 return FALSE;
2915 }
2916
2917 pcb = ifnet_softc(interface);
2918
2919 if (pcb == NULL) {
2920 return FALSE;
2921 }
2922
2923 return pcb->utun_needs_netagent == true;
2924 }
2925
2926 static errno_t
2927 utun_ifnet_set_attrs(ifnet_t ifp)
2928 {
2929 /* Set flags and additional information. */
2930 ifnet_set_mtu(ifp, 1500);
2931 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2932
2933 /* The interface must generate its own IPv6 LinkLocal address,
2934 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2935 */
2936 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2937
2938 return 0;
2939 }
2940
2941 static errno_t
2942 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2943 {
2944 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2945 pcb->utun_netif_nexus = nexus;
2946 return utun_ifnet_set_attrs(ifp);
2947 }
2948
2949 static errno_t
2950 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2951 proc_t p, kern_nexus_t nexus,
2952 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2953 {
2954 #pragma unused(nxprov, p)
2955 #pragma unused(nexus, nexus_port, channel, ch_ctx)
2956 return 0;
2957 }
2958
2959 static errno_t
2960 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2961 kern_channel_t channel)
2962 {
2963 #pragma unused(nxprov, channel)
2964 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2965 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2966 if (pcb->utun_netif_nexus == nexus) {
2967 pcb->utun_netif_connected = true;
2968 }
2969 return ok ? 0 : ENXIO;
2970 }
2971
2972 static void
2973 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2974 kern_channel_t channel)
2975 {
2976 #pragma unused(nxprov, nexus, channel)
2977 }
2978
2979 static void
2980 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2981 kern_channel_t channel)
2982 {
2983 #pragma unused(nxprov, nexus, channel)
2984 }
2985
2986 static void
2987 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2988 kern_channel_t channel)
2989 {
2990 #pragma unused(nxprov, channel)
2991 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2992 if (pcb->utun_netif_nexus == nexus) {
2993 pcb->utun_netif_connected = false;
2994 if (pcb->utun_attach_fsw) {
2995 // disconnected by flowswitch that was attached by us
2996 pcb->utun_netif_nexus = NULL;
2997 }
2998 }
2999 ifnet_decr_iorefcnt(pcb->utun_ifp);
3000 }
3001
3002 static errno_t
3003 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3004 kern_channel_t channel, kern_channel_ring_t ring,
3005 boolean_t is_tx_ring, void **ring_ctx)
3006 {
3007 #pragma unused(nxprov)
3008 #pragma unused(channel)
3009 #pragma unused(ring_ctx)
3010 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3011 if (!is_tx_ring) {
3012 VERIFY(pcb->utun_kpipe_rxring == NULL);
3013 pcb->utun_kpipe_rxring = ring;
3014 } else {
3015 VERIFY(pcb->utun_kpipe_txring == NULL);
3016 pcb->utun_kpipe_txring = ring;
3017 }
3018 return 0;
3019 }
3020
3021 static void
3022 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3023 kern_channel_ring_t ring)
3024 {
3025 #pragma unused(nxprov)
3026 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3027 if (pcb->utun_kpipe_rxring == ring) {
3028 pcb->utun_kpipe_rxring = NULL;
3029 } else if (pcb->utun_kpipe_txring == ring) {
3030 pcb->utun_kpipe_txring = NULL;
3031 }
3032 }
3033
3034 static errno_t
3035 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3036 kern_channel_ring_t tx_ring, uint32_t flags)
3037 {
3038 #pragma unused(nxprov)
3039 #pragma unused(flags)
3040 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3041
3042 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3043 int channel_enabled = pcb->utun_kpipe_enabled;
3044 if (!channel_enabled) {
3045 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3046 return 0;
3047 }
3048
3049 if (pcb->utun_use_netif) {
3050 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3051 if (tx_slot == NULL) {
3052 // Nothing to write, bail
3053 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3054 return 0;
3055 }
3056
3057 // Signal the netif ring to read
3058 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
3059 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3060 if (rx_ring != NULL) {
3061 kern_channel_notify(rx_ring, 0);
3062 }
3063 } else {
3064 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3065
3066 struct ifnet_stat_increment_param incs = {};
3067 struct kern_channel_ring_stat_increment tx_ring_stats = {};
3068 MBUFQ_HEAD(mbufq) mbq;
3069 MBUFQ_INIT(&mbq);
3070 kern_channel_slot_t tx_pslot = NULL;
3071 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3072 while (tx_slot != NULL) {
3073 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3074
3075 // Advance TX ring
3076 tx_pslot = tx_slot;
3077 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3078
3079 if (tx_ph == 0) {
3080 continue;
3081 }
3082
3083 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3084 VERIFY(tx_buf != NULL);
3085 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3086 VERIFY(tx_baddr != 0);
3087 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3088
3089 size_t length = MIN(kern_packet_get_data_length(tx_ph),
3090 pcb->utun_slot_size);
3091
3092 mbuf_t data = NULL;
3093 if (length >= UTUN_HEADER_SIZE(pcb) &&
3094 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
3095 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
3096 VERIFY(0 == error);
3097 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
3098 VERIFY(0 == error);
3099 /*
3100 * The userland ABI requires the first four bytes have
3101 * the protocol family in network byte order: swap them
3102 */
3103 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
3104 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
3105 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3106 incs.packets_in++;
3107 incs.bytes_in += length;
3108 MBUFQ_ENQUEUE(&mbq, data);
3109 }
3110 }
3111 if (tx_pslot) {
3112 kern_channel_advance_slot(tx_ring, tx_pslot);
3113 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
3114 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3115 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3116 (void) kern_channel_reclaim(tx_ring);
3117 }
3118 if (!MBUFQ_EMPTY(&mbq)) {
3119 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
3120 MBUFQ_LAST(&mbq), &incs);
3121 MBUFQ_INIT(&mbq);
3122 }
3123 }
3124
3125 return 0;
3126 }
3127
3128 static errno_t
3129 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3130 kern_channel_ring_t rx_ring, uint32_t flags)
3131 {
3132 #pragma unused(nxprov)
3133 #pragma unused(flags)
3134 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3135 struct kern_channel_ring_stat_increment rx_ring_stats = {};
3136
3137 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3138
3139 int channel_enabled = pcb->utun_kpipe_enabled;
3140 if (!channel_enabled) {
3141 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3142 return 0;
3143 }
3144
3145 /* reclaim user-released slots */
3146 (void) kern_channel_reclaim(rx_ring);
3147
3148 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3149 if (avail == 0) {
3150 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3151 return 0;
3152 }
3153
3154 if (pcb->utun_use_netif) {
3155 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3156 if (tx_ring == NULL ||
3157 pcb->utun_netif_nexus == NULL) {
3158 // Net-If TX ring not set up yet, nothing to read
3159 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3160 return 0;
3161 }
3162
3163 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
3164
3165 // Unlock utun before entering ring
3166 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3167
3168 (void)kr_enter(tx_ring, TRUE);
3169
3170 // Lock again after entering and validate
3171 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3172 if (tx_ring != pcb->utun_netif_txring) {
3173 // Ring no longer valid
3174 // Unlock first, then exit ring
3175 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3176 kr_exit(tx_ring);
3177 return 0;
3178 }
3179
3180 struct kern_channel_ring_stat_increment tx_ring_stats;
3181 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3182 kern_channel_slot_t tx_pslot = NULL;
3183 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3184 if (tx_slot == NULL) {
3185 // Nothing to read, don't bother signalling
3186 // Unlock first, then exit ring
3187 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3188 kr_exit(tx_ring);
3189 return 0;
3190 }
3191
3192 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3193 VERIFY(rx_pp != NULL);
3194 kern_channel_slot_t rx_pslot = NULL;
3195 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3196
3197 while (rx_slot != NULL && tx_slot != NULL) {
3198 size_t length;
3199 kern_buflet_t rx_buf;
3200 void *rx_baddr;
3201
3202 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3203
3204 // Advance TX ring
3205 tx_pslot = tx_slot;
3206 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3207
3208 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3209 if (tx_ph == 0) {
3210 continue;
3211 }
3212
3213 // Allocate rx packet
3214 kern_packet_t rx_ph = 0;
3215 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3216 if (__improbable(error != 0)) {
3217 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3218 pcb->utun_ifp->if_xname);
3219 break;
3220 }
3221
3222 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3223 VERIFY(tx_buf != NULL);
3224 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3225 VERIFY(tx_baddr != NULL);
3226 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3227
3228 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3229
3230 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
3231 pcb->utun_slot_size);
3232
3233 tx_ring_stats.kcrsi_slots_transferred++;
3234 tx_ring_stats.kcrsi_bytes_transferred += length;
3235
3236 if (length < UTUN_HEADER_SIZE(pcb) ||
3237 length > pcb->utun_slot_size ||
3238 length > rx_pp->pp_buflet_size ||
3239 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3240 /* flush data */
3241 kern_pbufpool_free(rx_pp, rx_ph);
3242 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3243 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3244 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
3245 STATS_INC(nifs, NETIF_STATS_DROP);
3246 continue;
3247 }
3248
3249 /* fillout packet */
3250 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3251 VERIFY(rx_buf != NULL);
3252 rx_baddr = kern_buflet_get_object_address(rx_buf);
3253 VERIFY(rx_baddr != NULL);
3254
3255 // Find family
3256 uint32_t af = 0;
3257 uint8_t vhl = *(uint8_t *)(tx_baddr);
3258 u_int ip_version = (vhl >> 4);
3259 switch (ip_version) {
3260 case 4: {
3261 af = AF_INET;
3262 break;
3263 }
3264 case 6: {
3265 af = AF_INET6;
3266 break;
3267 }
3268 default: {
3269 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3270 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3271 break;
3272 }
3273 }
3274
3275 // Copy header
3276 af = htonl(af);
3277 memcpy((void *)rx_baddr, &af, sizeof(af));
3278 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3279 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3280 }
3281
3282 // Copy data from tx to rx
3283 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3284 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
3285
3286 /* finalize and attach the packet */
3287 error = kern_buflet_set_data_offset(rx_buf, 0);
3288 VERIFY(error == 0);
3289 error = kern_buflet_set_data_length(rx_buf, length);
3290 VERIFY(error == 0);
3291 error = kern_packet_finalize(rx_ph);
3292 VERIFY(error == 0);
3293 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3294 VERIFY(error == 0);
3295
3296 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
3297 STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
3298
3299 rx_ring_stats.kcrsi_slots_transferred++;
3300 rx_ring_stats.kcrsi_bytes_transferred += length;
3301
3302 rx_pslot = rx_slot;
3303 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3304 }
3305
3306 if (rx_pslot) {
3307 kern_channel_advance_slot(rx_ring, rx_pslot);
3308 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3309 }
3310
3311 if (tx_pslot) {
3312 kern_channel_advance_slot(tx_ring, tx_pslot);
3313 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3314 (void)kern_channel_reclaim(tx_ring);
3315 }
3316
3317 /* just like utun_ctl_rcvd(), always reenable output */
3318 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3319 if (error != 0) {
3320 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
3321 }
3322
3323 // Unlock first, then exit ring
3324 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3325
3326 if (tx_pslot != NULL) {
3327 kern_channel_notify(tx_ring, 0);
3328 }
3329 kr_exit(tx_ring);
3330 } else {
3331 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3332
3333 uint32_t mb_cnt = 0;
3334 uint32_t mb_len = 0;
3335 struct mbuf *mb_head = NULL;
3336 struct mbuf *mb_tail = NULL;
3337
3338 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3339 &mb_tail, &mb_cnt, &mb_len) != 0) {
3340 return 0;
3341 }
3342 VERIFY(mb_cnt <= avail);
3343
3344 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3345 VERIFY(rx_pp != NULL);
3346 kern_channel_slot_t rx_pslot = NULL;
3347 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3348 while (rx_slot) {
3349 size_t length = 0;
3350 mbuf_t data = NULL;
3351 if ((data = mb_head) == NULL) {
3352 VERIFY(mb_cnt == 0);
3353 break;
3354 }
3355 mb_head = mbuf_nextpkt(mb_head);
3356 mbuf_setnextpkt(data, NULL);
3357 VERIFY(mb_cnt != 0);
3358 --mb_cnt;
3359 length = mbuf_pkthdr_len(data);
3360 if (length < UTUN_HEADER_SIZE(pcb) ||
3361 length > pcb->utun_slot_size ||
3362 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3363 /* flush data */
3364 mbuf_freem(data);
3365 continue;
3366 }
3367 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3368
3369 // Allocate rx packet
3370 kern_packet_t rx_ph = 0;
3371 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3372 if (__improbable(error != 0)) {
3373 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3374 pcb->utun_ifp->if_xname);
3375 break;
3376 }
3377
3378 /*
3379 * The ABI requires the protocol in network byte order
3380 */
3381 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3382
3383 // Fillout rx packet
3384 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3385 VERIFY(rx_buf != NULL);
3386 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3387 VERIFY(rx_baddr != NULL);
3388
3389 // Copy-in data from mbuf to buflet
3390 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3391 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3392
3393 // Finalize and attach the packet
3394 error = kern_buflet_set_data_offset(rx_buf, 0);
3395 VERIFY(error == 0);
3396 error = kern_buflet_set_data_length(rx_buf, length);
3397 VERIFY(error == 0);
3398 error = kern_packet_finalize(rx_ph);
3399 VERIFY(error == 0);
3400 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3401 VERIFY(error == 0);
3402
3403 rx_ring_stats.kcrsi_slots_transferred++;
3404 rx_ring_stats.kcrsi_bytes_transferred += length;
3405
3406 if (!pcb->utun_ext_ifdata_stats) {
3407 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3408 }
3409
3410 mbuf_freem(data);
3411
3412 rx_pslot = rx_slot;
3413 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3414 }
3415 if (rx_pslot) {
3416 kern_channel_advance_slot(rx_ring, rx_pslot);
3417 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3418 }
3419 if (mb_head != NULL) {
3420 VERIFY(mb_cnt != 0);
3421 mbuf_freem_list(mb_head);
3422 }
3423 }
3424
3425 return 0;
3426 }
3427
3428 #endif // UTUN_NEXUS
3429
3430
3431 /*
3432 * These are place holders until coreTLS kext stops calling them
3433 */
3434 errno_t utun_ctl_register_dtls(void *reg);
3435 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3436 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3437
3438 errno_t
3439 utun_ctl_register_dtls(void *reg)
3440 {
3441 #pragma unused(reg)
3442 return 0;
3443 }
3444
3445 int
3446 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3447 {
3448 #pragma unused(pcb)
3449 #pragma unused(pkt)
3450 #pragma unused(family)
3451 return 0;
3452 }
3453
3454 void
3455 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3456 {
3457 #pragma unused(pcb)
3458 }
mirror of XNU