]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/if_utun.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.51.1.tar.gz
[apple/xnu.git] / bsd / net / if_utun.c
1 /*
2 * Copyright (c) 2008-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 Application of kernel control for interface creation
33
34 Theory of operation:
35 utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 This kernel control will register an interface for every client that connects.
37 ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57
58 #define UTUN_NEXUS 0
59
60 #if UTUN_NEXUS
61 static nexus_controller_t utun_ncd;
62 static int utun_ncd_refcount;
63 static uuid_t utun_kpipe_uuid;
64 static uuid_t utun_nx_dom_prov;
65
66 typedef struct utun_nx {
67 uuid_t if_provider;
68 uuid_t if_instance;
69 uuid_t ms_provider;
70 uuid_t ms_instance;
71 uuid_t ms_device;
72 uuid_t ms_host;
73 uuid_t ms_agent;
74 } *utun_nx_t;
75
76 #endif // UTUN_NEXUS
77
78 /* Control block allocated for each kernel control connection */
79 struct utun_pcb {
80 TAILQ_ENTRY(utun_pcb) utun_chain;
81 kern_ctl_ref utun_ctlref;
82 ifnet_t utun_ifp;
83 u_int32_t utun_unit;
84 u_int32_t utun_unique_id;
85 u_int32_t utun_flags;
86 int utun_ext_ifdata_stats;
87 u_int32_t utun_max_pending_packets;
88 char utun_if_xname[IFXNAMSIZ];
89 char utun_unique_name[IFXNAMSIZ];
90 // PCB lock protects state fields and rings
91 decl_lck_rw_data(, utun_pcb_lock);
92 struct mbuf * utun_input_chain;
93 struct mbuf * utun_input_chain_last;
94 // Input chain lock protects the list of input mbufs
95 // The input chain lock must be taken AFTER the PCB lock if both are held
96 lck_mtx_t utun_input_chain_lock;
97
98 #if UTUN_NEXUS
99 struct utun_nx utun_nx;
100 int utun_kpipe_enabled;
101 uuid_t utun_kpipe_uuid;
102 void * utun_kpipe_rxring;
103 void * utun_kpipe_txring;
104 kern_pbufpool_t utun_kpipe_pp;
105
106 kern_nexus_t utun_netif_nexus;
107 kern_pbufpool_t utun_netif_pp;
108 void * utun_netif_rxring;
109 void * utun_netif_txring;
110 uint64_t utun_netif_txring_size;
111
112 u_int32_t utun_slot_size;
113 u_int32_t utun_netif_ring_size;
114 u_int32_t utun_tx_fsw_ring_size;
115 u_int32_t utun_rx_fsw_ring_size;
116 bool utun_use_netif;
117 #endif // UTUN_NEXUS
118 };
119
120 /* Kernel Control functions */
121 static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
122 void **unitinfo);
123 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
124 void **unitinfo);
125 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
126 void *unitinfo);
127 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
128 void *unitinfo, mbuf_t m, int flags);
129 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
130 int opt, void *data, size_t *len);
131 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
132 int opt, void *data, size_t len);
133 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
134 int flags);
135
136 /* Network Interface functions */
137 static void utun_start(ifnet_t interface);
138 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
139 const struct sockaddr *dest, const char *desk_linkaddr,
140 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
141 static errno_t utun_output(ifnet_t interface, mbuf_t data);
142 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
143 protocol_family_t *protocol);
144 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
145 const struct ifnet_demux_desc *demux_array,
146 u_int32_t demux_count);
147 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
148 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
149 static void utun_detached(ifnet_t interface);
150
151 /* Protocol handlers */
152 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
153 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
154 mbuf_t m, char *frame_header);
155 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
156 mbuf_t *packet, const struct sockaddr *dest, void *route,
157 char *frame_type, char *link_layer_dest);
158 static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
159
160 #if UTUN_NEXUS
161
162 #define UTUN_IF_DEFAULT_SLOT_SIZE 2048
163 #define UTUN_IF_DEFAULT_RING_SIZE 64
164 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
165 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
166 #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
167 #define UTUN_IF_HEADROOM_SIZE 32
168
169 #define UTUN_IF_MIN_RING_SIZE 16
170 #define UTUN_IF_MAX_RING_SIZE 1024
171
172 #define UTUN_IF_MIN_SLOT_SIZE 1024
173 #define UTUN_IF_MAX_SLOT_SIZE 4096
174
175 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
176 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
177 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
178
179 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
180 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
181 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
182
183 SYSCTL_DECL(_net_utun);
184 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
185
186 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
187 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
188 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
189 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
190 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
191 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
192
193 static errno_t
194 utun_register_nexus(void);
195
196 static errno_t
197 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
198 static errno_t
199 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
200 proc_t p, kern_nexus_t nexus,
201 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
202 static errno_t
203 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
204 kern_channel_t channel);
205 static void
206 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
207 kern_channel_t channel);
208 static void
209 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
210 kern_channel_t channel);
211 static void
212 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
213 kern_channel_t channel);
214 static errno_t
215 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
216 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
217 void **ring_ctx);
218 static void
219 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
220 kern_channel_ring_t ring);
221 static errno_t
222 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
223 kern_channel_ring_t ring, uint32_t flags);
224 static errno_t
225 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
226 kern_channel_ring_t ring, uint32_t flags);
227 #endif // UTUN_NEXUS
228
229 #define UTUN_DEFAULT_MTU 1500
230 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
231
232 static kern_ctl_ref utun_kctlref;
233 static u_int32_t utun_family;
234 static lck_attr_t *utun_lck_attr;
235 static lck_grp_attr_t *utun_lck_grp_attr;
236 static lck_grp_t *utun_lck_grp;
237 static lck_mtx_t utun_lock;
238
239 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
240
241 #define UTUN_PCB_ZONE_MAX 32
242 #define UTUN_PCB_ZONE_NAME "net.if_utun"
243
244 static unsigned int utun_pcb_size; /* size of zone element */
245 static struct zone *utun_pcb_zone; /* zone for utun_pcb */
246
247 #if UTUN_NEXUS
248
249 static int
250 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
251 {
252 #pragma unused(arg1, arg2)
253 int value = if_utun_ring_size;
254
255 int error = sysctl_handle_int(oidp, &value, 0, req);
256 if (error || !req->newptr) {
257 return (error);
258 }
259
260 if (value < UTUN_IF_MIN_RING_SIZE ||
261 value > UTUN_IF_MAX_RING_SIZE) {
262 return (EINVAL);
263 }
264
265 if_utun_ring_size = value;
266
267 return (0);
268 }
269
270 static int
271 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
272 {
273 #pragma unused(arg1, arg2)
274 int value = if_utun_tx_fsw_ring_size;
275
276 int error = sysctl_handle_int(oidp, &value, 0, req);
277 if (error || !req->newptr) {
278 return (error);
279 }
280
281 if (value < UTUN_IF_MIN_RING_SIZE ||
282 value > UTUN_IF_MAX_RING_SIZE) {
283 return (EINVAL);
284 }
285
286 if_utun_tx_fsw_ring_size = value;
287
288 return (0);
289 }
290
291 static int
292 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
293 {
294 #pragma unused(arg1, arg2)
295 int value = if_utun_rx_fsw_ring_size;
296
297 int error = sysctl_handle_int(oidp, &value, 0, req);
298 if (error || !req->newptr) {
299 return (error);
300 }
301
302 if (value < UTUN_IF_MIN_RING_SIZE ||
303 value > UTUN_IF_MAX_RING_SIZE) {
304 return (EINVAL);
305 }
306
307 if_utun_rx_fsw_ring_size = value;
308
309 return (0);
310 }
311
312 static errno_t
313 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
314 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
315 void **ring_ctx)
316 {
317 #pragma unused(nxprov)
318 #pragma unused(channel)
319 #pragma unused(ring_ctx)
320 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
321 if (!is_tx_ring) {
322 VERIFY(pcb->utun_netif_rxring == NULL);
323 pcb->utun_netif_rxring = ring;
324 } else {
325 VERIFY(pcb->utun_netif_txring == NULL);
326 pcb->utun_netif_txring = ring;
327 }
328 return 0;
329 }
330
331 static void
332 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
333 kern_channel_ring_t ring)
334 {
335 #pragma unused(nxprov)
336 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
337 if (pcb->utun_netif_rxring == ring) {
338 pcb->utun_netif_rxring = NULL;
339 } else if (pcb->utun_netif_txring == ring) {
340 pcb->utun_netif_txring = NULL;
341 }
342 }
343
344 static errno_t
345 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
346 kern_channel_ring_t tx_ring, uint32_t flags)
347 {
348 #pragma unused(nxprov)
349 #pragma unused(flags)
350 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
351
352 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
353
354 lck_rw_lock_shared(&pcb->utun_pcb_lock);
355
356 struct kern_channel_ring_stat_increment tx_ring_stats;
357 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
358 kern_channel_slot_t tx_pslot = NULL;
359 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
360
361 STATS_INC(nifs, NETIF_STATS_TXSYNC);
362
363 if (tx_slot == NULL) {
364 // Nothing to write, don't bother signalling
365 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
366 return 0;
367 }
368
369 if (pcb->utun_kpipe_enabled) {
370 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
371 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
372
373 // Signal the kernel pipe ring to read
374 if (rx_ring != NULL) {
375 kern_channel_notify(rx_ring, 0);
376 }
377 return 0;
378 }
379
380 // If we're here, we're injecting into the utun kernel control socket
381 while (tx_slot != NULL) {
382 size_t length = 0;
383 mbuf_t data = NULL;
384
385 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
386
387 if (tx_ph == 0) {
388 // Advance TX ring
389 tx_pslot = tx_slot;
390 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
391 continue;
392 }
393 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
394
395 // Advance TX ring
396 tx_pslot = tx_slot;
397 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
398
399 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
400 VERIFY(tx_buf != NULL);
401
402 /* tx_baddr is the absolute buffer address */
403 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
404 VERIFY(tx_baddr != 0);
405
406 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
407
408 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
409 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
410
411 // The offset must be large enough for the headers
412 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
413
414 // Find family
415 uint32_t af = 0;
416 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
417 u_int ip_version = (vhl >> 4);
418 switch (ip_version) {
419 case 4: {
420 af = AF_INET;
421 break;
422 }
423 case 6: {
424 af = AF_INET6;
425 break;
426 }
427 default: {
428 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
429 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
430 UTUN_HEADER_SIZE(pcb));
431 break;
432 }
433 }
434
435 tx_offset -= UTUN_HEADER_SIZE(pcb);
436 tx_length += UTUN_HEADER_SIZE(pcb);
437 tx_baddr += tx_offset;
438
439 length = MIN(tx_length, pcb->utun_slot_size);
440
441 // Copy in family
442 memcpy(tx_baddr, &af, sizeof(af));
443 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
444 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
445 }
446
447 if (length > 0) {
448 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
449 if (error == 0) {
450 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
451 if (error == 0) {
452 error = utun_output(pcb->utun_ifp, data);
453 if (error != 0) {
454 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
455 }
456 } else {
457 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
458 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
459 STATS_INC(nifs, NETIF_STATS_DROPPED);
460 mbuf_freem(data);
461 data = NULL;
462 }
463 } else {
464 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
465 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
466 STATS_INC(nifs, NETIF_STATS_DROPPED);
467 }
468 } else {
469 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
470 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
471 STATS_INC(nifs, NETIF_STATS_DROPPED);
472 }
473
474 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
475
476 if (data == NULL) {
477 continue;
478 }
479
480 STATS_INC(nifs, NETIF_STATS_TXPKTS);
481 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
482
483 tx_ring_stats.kcrsi_slots_transferred++;
484 tx_ring_stats.kcrsi_bytes_transferred += length;
485 }
486
487 if (tx_pslot) {
488 kern_channel_advance_slot(tx_ring, tx_pslot);
489 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
490 (void)kern_channel_reclaim(tx_ring);
491 }
492
493 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
494
495 return 0;
496 }
497
498 static errno_t
499 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
500 kern_channel_ring_t ring, __unused uint32_t flags)
501 {
502 #pragma unused(nxprov)
503 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
504 boolean_t more = false;
505 errno_t rc = 0;
506
507 /*
508 * Refill and sync the ring; we may be racing against another thread doing
509 * an RX sync that also wants to do kr_enter(), and so use the blocking
510 * variant here.
511 */
512 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
513 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
514 printf("%s, tx refill failed %d\n", __func__, rc);
515 }
516
517 (void) kr_enter(ring, TRUE);
518 lck_rw_lock_shared(&pcb->utun_pcb_lock);
519
520 if (pcb->utun_kpipe_enabled) {
521 uint32_t tx_available = kern_channel_available_slot_count(ring);
522 if (pcb->utun_netif_txring_size > 0 &&
523 tx_available >= pcb->utun_netif_txring_size - 1) {
524 // No room left in tx ring, disable output for now
525 errno_t error = ifnet_disable_output(pcb->utun_ifp);
526 if (error != 0) {
527 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
528 }
529 }
530 }
531
532 if (pcb->utun_kpipe_enabled) {
533 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
534
535 // Unlock while calling notify
536 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
537 // Signal the kernel pipe ring to read
538 if (rx_ring != NULL) {
539 kern_channel_notify(rx_ring, 0);
540 }
541 } else {
542 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
543 }
544
545 kr_exit(ring);
546
547 return (0);
548 }
549
550 static errno_t
551 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
552 kern_channel_ring_t rx_ring, uint32_t flags)
553 {
554 #pragma unused(nxprov)
555 #pragma unused(flags)
556 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
557 struct kern_channel_ring_stat_increment rx_ring_stats;
558
559 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
560
561 lck_rw_lock_shared(&pcb->utun_pcb_lock);
562
563 // Reclaim user-released slots
564 (void) kern_channel_reclaim(rx_ring);
565
566 STATS_INC(nifs, NETIF_STATS_RXSYNC);
567
568 uint32_t avail = kern_channel_available_slot_count(rx_ring);
569 if (avail == 0) {
570 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
571 return 0;
572 }
573
574 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
575 VERIFY(rx_pp != NULL);
576 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
577 kern_channel_slot_t rx_pslot = NULL;
578 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
579
580 while (rx_slot != NULL) {
581 // Check for a waiting packet
582 lck_mtx_lock(&pcb->utun_input_chain_lock);
583 mbuf_t data = pcb->utun_input_chain;
584 if (data == NULL) {
585 lck_mtx_unlock(&pcb->utun_input_chain_lock);
586 break;
587 }
588
589 // Allocate rx packet
590 kern_packet_t rx_ph = 0;
591 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
592 if (__improbable(error != 0)) {
593 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
594 STATS_INC(nifs, NETIF_STATS_DROPPED);
595 lck_mtx_unlock(&pcb->utun_input_chain_lock);
596 break;
597 }
598
599 // Advance waiting packets
600 pcb->utun_input_chain = data->m_nextpkt;
601 data->m_nextpkt = NULL;
602 if (pcb->utun_input_chain == NULL) {
603 pcb->utun_input_chain_last = NULL;
604 }
605 lck_mtx_unlock(&pcb->utun_input_chain_lock);
606
607 size_t header_offset = UTUN_HEADER_SIZE(pcb);
608 size_t length = mbuf_pkthdr_len(data);
609
610 if (length < header_offset) {
611 // mbuf is too small
612 mbuf_freem(data);
613 kern_pbufpool_free(rx_pp, rx_ph);
614 STATS_INC(nifs, NETIF_STATS_BADLEN);
615 STATS_INC(nifs, NETIF_STATS_DROPPED);
616 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
617 pcb->utun_ifp->if_xname, length, header_offset);
618 continue;
619 }
620
621 length -= header_offset;
622 if (length > rx_pp->pp_buflet_size) {
623 // Flush data
624 mbuf_freem(data);
625 kern_pbufpool_free(rx_pp, rx_ph);
626 STATS_INC(nifs, NETIF_STATS_BADLEN);
627 STATS_INC(nifs, NETIF_STATS_DROPPED);
628 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
629 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
630 continue;
631 }
632
633 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
634
635 // Fillout rx packet
636 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
637 VERIFY(rx_buf != NULL);
638 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
639 VERIFY(rx_baddr != NULL);
640
641 // Copy-in data from mbuf to buflet
642 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
643 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
644
645 // Finalize and attach the packet
646 error = kern_buflet_set_data_offset(rx_buf, 0);
647 VERIFY(error == 0);
648 error = kern_buflet_set_data_length(rx_buf, length);
649 VERIFY(error == 0);
650 error = kern_packet_set_link_header_offset(rx_ph, 0);
651 VERIFY(error == 0);
652 error = kern_packet_set_network_header_offset(rx_ph, 0);
653 VERIFY(error == 0);
654 error = kern_packet_finalize(rx_ph);
655 VERIFY(error == 0);
656 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
657 VERIFY(error == 0);
658
659 STATS_INC(nifs, NETIF_STATS_RXPKTS);
660 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
661 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
662
663 rx_ring_stats.kcrsi_slots_transferred++;
664 rx_ring_stats.kcrsi_bytes_transferred += length;
665
666 mbuf_freem(data);
667
668 // Advance ring
669 rx_pslot = rx_slot;
670 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
671 }
672
673 struct kern_channel_ring_stat_increment tx_ring_stats;
674 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
675 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
676 kern_channel_slot_t tx_pslot = NULL;
677 kern_channel_slot_t tx_slot = NULL;
678 if (tx_ring == NULL) {
679 // Net-If TX ring not set up yet, nothing to read
680 goto done;
681 }
682
683 // Unlock utun before entering ring
684 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
685
686 (void)kr_enter(tx_ring, TRUE);
687
688 // Lock again after entering and validate
689 lck_rw_lock_shared(&pcb->utun_pcb_lock);
690 if (tx_ring != pcb->utun_kpipe_txring) {
691 goto done;
692 }
693
694 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
695 if (tx_slot == NULL) {
696 // Nothing to read, don't bother signalling
697 goto done;
698 }
699
700 while (rx_slot != NULL && tx_slot != NULL) {
701 // Allocate rx packet
702 kern_packet_t rx_ph = 0;
703 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
704
705 // Advance TX ring
706 tx_pslot = tx_slot;
707 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
708
709 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
710 if (tx_ph == 0) {
711 continue;
712 }
713
714 /* XXX We could try this alloc before advancing the slot to avoid
715 * dropping the packet on failure to allocate.
716 */
717 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
718 if (__improbable(error != 0)) {
719 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
720 STATS_INC(nifs, NETIF_STATS_DROPPED);
721 break;
722 }
723
724 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
725 VERIFY(tx_buf != NULL);
726 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
727 VERIFY(tx_baddr != 0);
728 tx_baddr += kern_buflet_get_data_offset(tx_buf);
729
730 // Check packet length
731 size_t header_offset = UTUN_HEADER_SIZE(pcb);
732 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
733 if (tx_length < header_offset) {
734 // Packet is too small
735 kern_pbufpool_free(rx_pp, rx_ph);
736 STATS_INC(nifs, NETIF_STATS_BADLEN);
737 STATS_INC(nifs, NETIF_STATS_DROPPED);
738 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
739 pcb->utun_ifp->if_xname, tx_length, header_offset);
740 continue;
741 }
742
743 size_t length = MIN(tx_length - header_offset,
744 pcb->utun_slot_size);
745
746 tx_ring_stats.kcrsi_slots_transferred++;
747 tx_ring_stats.kcrsi_bytes_transferred += length;
748
749 // Fillout rx packet
750 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
751 VERIFY(rx_buf != NULL);
752 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
753 VERIFY(rx_baddr != NULL);
754
755 // Copy-in data from tx to rx
756 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
757 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
758
759 // Finalize and attach the packet
760 error = kern_buflet_set_data_offset(rx_buf, 0);
761 VERIFY(error == 0);
762 error = kern_buflet_set_data_length(rx_buf, length);
763 VERIFY(error == 0);
764 error = kern_packet_set_link_header_offset(rx_ph, 0);
765 VERIFY(error == 0);
766 error = kern_packet_set_network_header_offset(rx_ph, 0);
767 VERIFY(error == 0);
768 error = kern_packet_finalize(rx_ph);
769 VERIFY(error == 0);
770 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
771 VERIFY(error == 0);
772
773 STATS_INC(nifs, NETIF_STATS_RXPKTS);
774 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
775 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
776
777 rx_ring_stats.kcrsi_slots_transferred++;
778 rx_ring_stats.kcrsi_bytes_transferred += length;
779
780 rx_pslot = rx_slot;
781 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
782 }
783
784 done:
785 if (rx_pslot) {
786 kern_channel_advance_slot(rx_ring, rx_pslot);
787 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
788 }
789
790 if (tx_pslot) {
791 kern_channel_advance_slot(tx_ring, tx_pslot);
792 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
793 (void)kern_channel_reclaim(tx_ring);
794 }
795
796 // Unlock first, then exit ring
797 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
798 if (tx_ring != NULL) {
799 if (tx_pslot != NULL) {
800 kern_channel_notify(tx_ring, 0);
801 }
802 kr_exit(tx_ring);
803 }
804
805 return 0;
806 }
807
808 static errno_t
809 utun_nexus_ifattach(struct utun_pcb *pcb,
810 struct ifnet_init_eparams *init_params,
811 struct ifnet **ifp)
812 {
813 errno_t err;
814 nexus_controller_t controller = kern_nexus_shared_controller();
815 struct kern_nexus_net_init net_init;
816 struct kern_pbufpool_init pp_init;
817
818 nexus_name_t provider_name;
819 snprintf((char *)provider_name, sizeof(provider_name),
820 "com.apple.netif.%s", pcb->utun_if_xname);
821
822 struct kern_nexus_provider_init prov_init = {
823 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
824 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
825 .nxpi_pre_connect = utun_nexus_pre_connect,
826 .nxpi_connected = utun_nexus_connected,
827 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
828 .nxpi_disconnected = utun_nexus_disconnected,
829 .nxpi_ring_init = utun_netif_ring_init,
830 .nxpi_ring_fini = utun_netif_ring_fini,
831 .nxpi_slot_init = NULL,
832 .nxpi_slot_fini = NULL,
833 .nxpi_sync_tx = utun_netif_sync_tx,
834 .nxpi_sync_rx = utun_netif_sync_rx,
835 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
836 };
837
838 nexus_attr_t nxa = NULL;
839 err = kern_nexus_attr_create(&nxa);
840 if (err != 0) {
841 printf("%s: kern_nexus_attr_create failed: %d\n",
842 __func__, err);
843 goto failed;
844 }
845
846 uint64_t slot_buffer_size = pcb->utun_slot_size;
847 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
848 VERIFY(err == 0);
849
850 // Reset ring size for netif nexus to limit memory usage
851 uint64_t ring_size = pcb->utun_netif_ring_size;
852 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
853 VERIFY(err == 0);
854 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
855 VERIFY(err == 0);
856
857 pcb->utun_netif_txring_size = ring_size;
858
859 bzero(&pp_init, sizeof (pp_init));
860 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
861 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
862 pp_init.kbi_bufsize = pcb->utun_slot_size;
863 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
864 pp_init.kbi_max_frags = 1;
865 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
866 "%s", provider_name);
867
868 err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_netif_pp, NULL);
869 if (err != 0) {
870 printf("%s pbufbool create failed, error %d\n", __func__, err);
871 goto failed;
872 }
873
874 err = kern_nexus_controller_register_provider(controller,
875 utun_nx_dom_prov,
876 provider_name,
877 &prov_init,
878 sizeof(prov_init),
879 nxa,
880 &pcb->utun_nx.if_provider);
881 if (err != 0) {
882 printf("%s register provider failed, error %d\n",
883 __func__, err);
884 goto failed;
885 }
886
887 bzero(&net_init, sizeof(net_init));
888 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
889 net_init.nxneti_flags = 0;
890 net_init.nxneti_eparams = init_params;
891 net_init.nxneti_lladdr = NULL;
892 net_init.nxneti_prepare = utun_netif_prepare;
893 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
894 err = kern_nexus_controller_alloc_net_provider_instance(controller,
895 pcb->utun_nx.if_provider,
896 pcb,
897 &pcb->utun_nx.if_instance,
898 &net_init,
899 ifp);
900 if (err != 0) {
901 printf("%s alloc_net_provider_instance failed, %d\n",
902 __func__, err);
903 kern_nexus_controller_deregister_provider(controller,
904 pcb->utun_nx.if_provider);
905 uuid_clear(pcb->utun_nx.if_provider);
906 goto failed;
907 }
908
909 failed:
910 if (nxa) {
911 kern_nexus_attr_destroy(nxa);
912 }
913 if (err && pcb->utun_netif_pp != NULL) {
914 kern_pbufpool_destroy(pcb->utun_netif_pp);
915 pcb->utun_netif_pp = NULL;
916 }
917 return (err);
918 }
919
920 static void
921 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
922 {
923 nexus_controller_t controller = kern_nexus_shared_controller();
924 errno_t err;
925
926 if (!uuid_is_null(instance)) {
927 err = kern_nexus_controller_free_provider_instance(controller,
928 instance);
929 if (err != 0) {
930 printf("%s free_provider_instance failed %d\n",
931 __func__, err);
932 }
933 uuid_clear(instance);
934 }
935 if (!uuid_is_null(provider)) {
936 err = kern_nexus_controller_deregister_provider(controller,
937 provider);
938 if (err != 0) {
939 printf("%s deregister_provider %d\n", __func__, err);
940 }
941 uuid_clear(provider);
942 }
943 return;
944 }
945
946 static void
947 utun_nexus_detach(struct utun_pcb *pcb)
948 {
949 utun_nx_t nx = &pcb->utun_nx;
950 nexus_controller_t controller = kern_nexus_shared_controller();
951 errno_t err;
952
953 if (!uuid_is_null(nx->ms_host)) {
954 err = kern_nexus_ifdetach(controller,
955 nx->ms_instance,
956 nx->ms_host);
957 if (err != 0) {
958 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
959 __func__, err);
960 }
961 }
962
963 if (!uuid_is_null(nx->ms_device)) {
964 err = kern_nexus_ifdetach(controller,
965 nx->ms_instance,
966 nx->ms_device);
967 if (err != 0) {
968 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
969 __func__, err);
970 }
971 }
972
973 utun_detach_provider_and_instance(nx->if_provider,
974 nx->if_instance);
975 utun_detach_provider_and_instance(nx->ms_provider,
976 nx->ms_instance);
977
978 if (pcb->utun_netif_pp != NULL) {
979 kern_pbufpool_destroy(pcb->utun_netif_pp);
980 pcb->utun_netif_pp = NULL;
981
982 }
983 memset(nx, 0, sizeof(*nx));
984 }
985
986 static errno_t
987 utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
988 uint32_t subtype, const char *type_name,
989 const char *ifname,
990 uuid_t *provider, uuid_t *instance)
991 {
992 nexus_attr_t attr = NULL;
993 nexus_controller_t controller = kern_nexus_shared_controller();
994 uuid_t dom_prov;
995 errno_t err;
996 struct kern_nexus_init init;
997 nexus_name_t provider_name;
998
999 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1000 &dom_prov);
1001 if (err != 0) {
1002 printf("%s can't get %s provider, error %d\n",
1003 __func__, type_name, err);
1004 goto failed;
1005 }
1006
1007 err = kern_nexus_attr_create(&attr);
1008 if (err != 0) {
1009 printf("%s: kern_nexus_attr_create failed: %d\n",
1010 __func__, err);
1011 goto failed;
1012 }
1013
1014 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
1015 VERIFY(err == 0);
1016
1017 uint64_t slot_buffer_size = pcb->utun_slot_size;
1018 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1019 VERIFY(err == 0);
1020
1021 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
1022 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
1023 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1024 VERIFY(err == 0);
1025 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
1026 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1027 VERIFY(err == 0);
1028
1029 snprintf((char *)provider_name, sizeof(provider_name),
1030 "com.apple.%s.%s", type_name, ifname);
1031 err = kern_nexus_controller_register_provider(controller,
1032 dom_prov,
1033 provider_name,
1034 NULL,
1035 0,
1036 attr,
1037 provider);
1038 kern_nexus_attr_destroy(attr);
1039 attr = NULL;
1040 if (err != 0) {
1041 printf("%s register %s provider failed, error %d\n",
1042 __func__, type_name, err);
1043 goto failed;
1044 }
1045 bzero(&init, sizeof (init));
1046 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1047 err = kern_nexus_controller_alloc_provider_instance(controller,
1048 *provider,
1049 NULL,
1050 instance, &init);
1051 if (err != 0) {
1052 printf("%s alloc_provider_instance %s failed, %d\n",
1053 __func__, type_name, err);
1054 kern_nexus_controller_deregister_provider(controller,
1055 *provider);
1056 uuid_clear(*provider);
1057 }
1058 failed:
1059 return (err);
1060 }
1061
1062 static errno_t
1063 utun_multistack_attach(struct utun_pcb *pcb)
1064 {
1065 nexus_controller_t controller = kern_nexus_shared_controller();
1066 errno_t err = 0;
1067 utun_nx_t nx = &pcb->utun_nx;
1068
1069 // Allocate multistack flowswitch
1070 err = utun_create_fs_provider_and_instance(pcb,
1071 NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
1072 "multistack",
1073 pcb->utun_ifp->if_xname,
1074 &nx->ms_provider,
1075 &nx->ms_instance);
1076 if (err != 0) {
1077 printf("%s: failed to create bridge provider and instance\n",
1078 __func__);
1079 goto failed;
1080 }
1081
1082 // Attach multistack to device port
1083 err = kern_nexus_ifattach(controller, nx->ms_instance,
1084 NULL, nx->if_instance,
1085 FALSE, &nx->ms_device);
1086 if (err != 0) {
1087 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1088 goto failed;
1089 }
1090
1091 // Attach multistack to host port
1092 err = kern_nexus_ifattach(controller, nx->ms_instance,
1093 NULL, nx->if_instance,
1094 TRUE, &nx->ms_host);
1095 if (err != 0) {
1096 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1097 goto failed;
1098 }
1099
1100 // Extract the agent UUID and save for later
1101 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1102 if (multistack_nx != NULL) {
1103 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1104 if (flowswitch != NULL) {
1105 FSW_RLOCK(flowswitch);
1106 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1107 if (ms_context != NULL) {
1108 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1109 } else {
1110 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1111 }
1112 FSW_UNLOCK(flowswitch);
1113 } else {
1114 printf("utun_multistack_attach - flowswitch is NULL\n");
1115 }
1116 nx_release(multistack_nx);
1117 } else {
1118 printf("utun_multistack_attach - unable to find multistack nexus\n");
1119 }
1120
1121 return (0);
1122
1123 failed:
1124 utun_nexus_detach(pcb);
1125
1126 errno_t detach_error = 0;
1127 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1128 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1129 /* NOT REACHED */
1130 }
1131
1132 return (err);
1133 }
1134
1135 static errno_t
1136 utun_register_kernel_pipe_nexus(void)
1137 {
1138 nexus_attr_t nxa = NULL;
1139 errno_t result;
1140
1141 lck_mtx_lock(&utun_lock);
1142 if (utun_ncd_refcount++) {
1143 lck_mtx_unlock(&utun_lock);
1144 return 0;
1145 }
1146
1147 result = kern_nexus_controller_create(&utun_ncd);
1148 if (result) {
1149 printf("%s: kern_nexus_controller_create failed: %d\n",
1150 __FUNCTION__, result);
1151 goto done;
1152 }
1153
1154 uuid_t dom_prov;
1155 result = kern_nexus_get_builtin_domain_provider(
1156 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1157 if (result) {
1158 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1159 __FUNCTION__, result);
1160 goto done;
1161 }
1162
1163 struct kern_nexus_provider_init prov_init = {
1164 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1165 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1166 .nxpi_pre_connect = utun_nexus_pre_connect,
1167 .nxpi_connected = utun_nexus_connected,
1168 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1169 .nxpi_disconnected = utun_nexus_disconnected,
1170 .nxpi_ring_init = utun_kpipe_ring_init,
1171 .nxpi_ring_fini = utun_kpipe_ring_fini,
1172 .nxpi_slot_init = NULL,
1173 .nxpi_slot_fini = NULL,
1174 .nxpi_sync_tx = utun_kpipe_sync_tx,
1175 .nxpi_sync_rx = utun_kpipe_sync_rx,
1176 .nxpi_tx_doorbell = NULL,
1177 };
1178
1179 result = kern_nexus_attr_create(&nxa);
1180 if (result) {
1181 printf("%s: kern_nexus_attr_create failed: %d\n",
1182 __FUNCTION__, result);
1183 goto done;
1184 }
1185
1186 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1187 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1188 VERIFY(result == 0);
1189
1190 // Reset ring size for kernel pipe nexus to limit memory usage
1191 uint64_t ring_size = if_utun_ring_size;
1192 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1193 VERIFY(result == 0);
1194 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1195 VERIFY(result == 0);
1196
1197 result = kern_nexus_controller_register_provider(utun_ncd,
1198 dom_prov,
1199 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1200 &prov_init,
1201 sizeof(prov_init),
1202 nxa,
1203 &utun_kpipe_uuid);
1204 if (result) {
1205 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1206 __FUNCTION__, result);
1207 goto done;
1208 }
1209
1210 done:
1211 if (nxa) {
1212 kern_nexus_attr_destroy(nxa);
1213 }
1214
1215 if (result) {
1216 if (utun_ncd) {
1217 kern_nexus_controller_destroy(utun_ncd);
1218 utun_ncd = NULL;
1219 }
1220 utun_ncd_refcount = 0;
1221 }
1222
1223 lck_mtx_unlock(&utun_lock);
1224
1225 return result;
1226 }
1227
1228 static void
1229 utun_unregister_kernel_pipe_nexus(void)
1230 {
1231 lck_mtx_lock(&utun_lock);
1232
1233 VERIFY(utun_ncd_refcount > 0);
1234
1235 if (--utun_ncd_refcount == 0) {
1236 kern_nexus_controller_destroy(utun_ncd);
1237 utun_ncd = NULL;
1238 }
1239
1240 lck_mtx_unlock(&utun_lock);
1241 }
1242
1243 // For use by socket option, not internally
1244 static errno_t
1245 utun_disable_channel(struct utun_pcb *pcb)
1246 {
1247 errno_t result;
1248 int enabled;
1249 uuid_t uuid;
1250
1251 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1252
1253 enabled = pcb->utun_kpipe_enabled;
1254 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1255
1256 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1257
1258 pcb->utun_kpipe_enabled = 0;
1259 uuid_clear(pcb->utun_kpipe_uuid);
1260
1261 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1262
1263 if (enabled) {
1264 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1265 } else {
1266 result = ENXIO;
1267 }
1268
1269 if (!result) {
1270 if (pcb->utun_kpipe_pp != NULL) {
1271 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1272 pcb->utun_kpipe_pp = NULL;
1273 }
1274 utun_unregister_kernel_pipe_nexus();
1275 }
1276
1277 return result;
1278 }
1279
1280 static errno_t
1281 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1282 {
1283 struct kern_nexus_init init;
1284 struct kern_pbufpool_init pp_init;
1285 errno_t result;
1286
1287 result = utun_register_kernel_pipe_nexus();
1288 if (result) {
1289 return result;
1290 }
1291
1292 VERIFY(utun_ncd);
1293
1294 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1295
1296 if (pcb->utun_kpipe_enabled) {
1297 result = EEXIST; // return success instead?
1298 goto done;
1299 }
1300
1301 /*
1302 * Make sure we can fit packets in the channel buffers and
1303 * Allow an extra 4 bytes for the protocol number header in the channel
1304 */
1305 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
1306 result = EOPNOTSUPP;
1307 goto done;
1308 }
1309
1310 bzero(&pp_init, sizeof (pp_init));
1311 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1312 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1313 pp_init.kbi_bufsize = pcb->utun_slot_size;
1314 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1315 pp_init.kbi_max_frags = 1;
1316 pp_init.kbi_flags |= KBIF_QUANTUM;
1317 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
1318 "com.apple.kpipe.%s", pcb->utun_if_xname);
1319
1320 result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_kpipe_pp,
1321 NULL);
1322 if (result != 0) {
1323 printf("%s pbufbool create failed, error %d\n", __func__, result);
1324 goto done;
1325 }
1326
1327 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1328 bzero(&init, sizeof (init));
1329 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1330 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
1331 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1332 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1333 if (result) {
1334 goto done;
1335 }
1336
1337 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1338 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1339 pcb->utun_kpipe_uuid, &port,
1340 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1341 if (result) {
1342 kern_nexus_controller_free_provider_instance(utun_ncd,
1343 pcb->utun_kpipe_uuid);
1344 uuid_clear(pcb->utun_kpipe_uuid);
1345 goto done;
1346 }
1347
1348 pcb->utun_kpipe_enabled = 1;
1349
1350 done:
1351 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1352
1353 if (result) {
1354 if (pcb->utun_kpipe_pp != NULL) {
1355 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1356 pcb->utun_kpipe_pp = NULL;
1357 }
1358 utun_unregister_kernel_pipe_nexus();
1359 }
1360
1361 return result;
1362 }
1363
1364 #endif // UTUN_NEXUS
1365
1366 errno_t
1367 utun_register_control(void)
1368 {
1369 struct kern_ctl_reg kern_ctl;
1370 errno_t result = 0;
1371
1372 /* Find a unique value for our interface family */
1373 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1374 if (result != 0) {
1375 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1376 return result;
1377 }
1378
1379 utun_pcb_size = sizeof(struct utun_pcb);
1380 utun_pcb_zone = zinit(utun_pcb_size,
1381 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1382 0, UTUN_PCB_ZONE_NAME);
1383 if (utun_pcb_zone == NULL) {
1384 printf("utun_register_control - zinit(utun_pcb) failed");
1385 return ENOMEM;
1386 }
1387
1388 #if UTUN_NEXUS
1389 utun_register_nexus();
1390 #endif // UTUN_NEXUS
1391
1392 TAILQ_INIT(&utun_head);
1393
1394 bzero(&kern_ctl, sizeof(kern_ctl));
1395 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1396 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1397 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
1398 kern_ctl.ctl_sendsize = 512 * 1024;
1399 kern_ctl.ctl_recvsize = 512 * 1024;
1400 kern_ctl.ctl_bind = utun_ctl_bind;
1401 kern_ctl.ctl_connect = utun_ctl_connect;
1402 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1403 kern_ctl.ctl_send = utun_ctl_send;
1404 kern_ctl.ctl_setopt = utun_ctl_setopt;
1405 kern_ctl.ctl_getopt = utun_ctl_getopt;
1406 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1407
1408 result = ctl_register(&kern_ctl, &utun_kctlref);
1409 if (result != 0) {
1410 printf("utun_register_control - ctl_register failed: %d\n", result);
1411 return result;
1412 }
1413
1414 /* Register the protocol plumbers */
1415 if ((result = proto_register_plumber(PF_INET, utun_family,
1416 utun_attach_proto, NULL)) != 0) {
1417 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
1418 utun_family, result);
1419 ctl_deregister(utun_kctlref);
1420 return result;
1421 }
1422
1423 /* Register the protocol plumbers */
1424 if ((result = proto_register_plumber(PF_INET6, utun_family,
1425 utun_attach_proto, NULL)) != 0) {
1426 proto_unregister_plumber(PF_INET, utun_family);
1427 ctl_deregister(utun_kctlref);
1428 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
1429 utun_family, result);
1430 return result;
1431 }
1432
1433 utun_lck_attr = lck_attr_alloc_init();
1434 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1435 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1436
1437 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1438
1439 return 0;
1440 }
1441
1442 /* Kernel control functions */
1443
1444 static inline void
1445 utun_free_pcb(struct utun_pcb *pcb, bool in_list)
1446 {
1447 #ifdef UTUN_NEXUS
1448 mbuf_freem_list(pcb->utun_input_chain);
1449 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1450 #endif // UTUN_NEXUS
1451 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1452 if (in_list) {
1453 lck_mtx_lock(&utun_lock);
1454 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1455 lck_mtx_unlock(&utun_lock);
1456 }
1457 zfree(utun_pcb_zone, pcb);
1458 }
1459
1460 static errno_t
1461 utun_ctl_bind(kern_ctl_ref kctlref,
1462 struct sockaddr_ctl *sac,
1463 void **unitinfo)
1464 {
1465 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1466 memset(pcb, 0, sizeof(*pcb));
1467
1468 *unitinfo = pcb;
1469 pcb->utun_ctlref = kctlref;
1470 pcb->utun_unit = sac->sc_unit;
1471 pcb->utun_max_pending_packets = 1;
1472
1473 #if UTUN_NEXUS
1474 pcb->utun_use_netif = false;
1475 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1476 pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
1477 pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
1478 pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
1479 #endif // UTUN_NEXUS
1480
1481 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1482 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1483
1484 return (0);
1485 }
1486
1487 static errno_t
1488 utun_ctl_connect(kern_ctl_ref kctlref,
1489 struct sockaddr_ctl *sac,
1490 void **unitinfo)
1491 {
1492 struct ifnet_init_eparams utun_init = {};
1493 errno_t result = 0;
1494
1495 if (*unitinfo == NULL) {
1496 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1497 }
1498
1499 struct utun_pcb *pcb = *unitinfo;
1500
1501 lck_mtx_lock(&utun_lock);
1502
1503 /* Find some open interface id */
1504 u_int32_t chosen_unique_id = 1;
1505 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1506 if (next_pcb != NULL) {
1507 /* List was not empty, add one to the last item */
1508 chosen_unique_id = next_pcb->utun_unique_id + 1;
1509 next_pcb = NULL;
1510
1511 /*
1512 * If this wrapped the id number, start looking at
1513 * the front of the list for an unused id.
1514 */
1515 if (chosen_unique_id == 0) {
1516 /* Find the next unused ID */
1517 chosen_unique_id = 1;
1518 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1519 if (next_pcb->utun_unique_id > chosen_unique_id) {
1520 /* We found a gap */
1521 break;
1522 }
1523
1524 chosen_unique_id = next_pcb->utun_unique_id + 1;
1525 }
1526 }
1527 }
1528
1529 pcb->utun_unique_id = chosen_unique_id;
1530
1531 if (next_pcb != NULL) {
1532 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1533 } else {
1534 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1535 }
1536 lck_mtx_unlock(&utun_lock);
1537
1538 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1539 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1540 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
1541
1542 /* Create the interface */
1543 bzero(&utun_init, sizeof(utun_init));
1544 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1545 utun_init.len = sizeof (utun_init);
1546
1547 #if UTUN_NEXUS
1548 if (pcb->utun_use_netif) {
1549 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1550 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1551 } else
1552 #endif // UTUN_NEXUS
1553 {
1554 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1555 utun_init.start = utun_start;
1556 utun_init.framer_extended = utun_framer;
1557 }
1558 utun_init.name = "utun";
1559 utun_init.unit = pcb->utun_unit - 1;
1560 utun_init.uniqueid = pcb->utun_unique_name;
1561 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1562 utun_init.family = utun_family;
1563 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
1564 utun_init.type = IFT_OTHER;
1565 utun_init.demux = utun_demux;
1566 utun_init.add_proto = utun_add_proto;
1567 utun_init.del_proto = utun_del_proto;
1568 utun_init.softc = pcb;
1569 utun_init.ioctl = utun_ioctl;
1570 utun_init.detach = utun_detached;
1571
1572 #if UTUN_NEXUS
1573 if (pcb->utun_use_netif) {
1574 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1575 if (result != 0) {
1576 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1577 utun_free_pcb(pcb, true);
1578 *unitinfo = NULL;
1579 return result;
1580 }
1581
1582 result = utun_multistack_attach(pcb);
1583 if (result != 0) {
1584 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1585 *unitinfo = NULL;
1586 return result;
1587 }
1588
1589 /* Attach to bpf */
1590 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1591 } else
1592 #endif // UTUN_NEXUS
1593 {
1594 /*
1595 * Upon success, this holds an ifnet reference which we will
1596 * release via ifnet_release() at final detach time.
1597 */
1598 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1599 if (result != 0) {
1600 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1601 utun_free_pcb(pcb, true);
1602 *unitinfo = NULL;
1603 return result;
1604 }
1605
1606 /* Set flags and additional information. */
1607 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1608 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1609
1610 /* The interface must generate its own IPv6 LinkLocal address,
1611 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1612 */
1613 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1614
1615 /* Reset the stats in case as the interface may have been recycled */
1616 struct ifnet_stats_param stats;
1617 bzero(&stats, sizeof(struct ifnet_stats_param));
1618 ifnet_set_stat(pcb->utun_ifp, &stats);
1619
1620 /* Attach the interface */
1621 result = ifnet_attach(pcb->utun_ifp, NULL);
1622 if (result != 0) {
1623 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
1624 /* Release reference now since attach failed */
1625 ifnet_release(pcb->utun_ifp);
1626 utun_free_pcb(pcb, true);
1627 *unitinfo = NULL;
1628 return (result);
1629 }
1630
1631 /* Attach to bpf */
1632 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1633 }
1634
1635 /* The interfaces resoures allocated, mark it as running */
1636 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1637
1638 return result;
1639 }
1640
1641 static errno_t
1642 utun_detach_ip(ifnet_t interface,
1643 protocol_family_t protocol,
1644 socket_t pf_socket)
1645 {
1646 errno_t result = EPROTONOSUPPORT;
1647
1648 /* Attempt a detach */
1649 if (protocol == PF_INET) {
1650 struct ifreq ifr;
1651
1652 bzero(&ifr, sizeof(ifr));
1653 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1654 ifnet_name(interface), ifnet_unit(interface));
1655
1656 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1657 } else if (protocol == PF_INET6) {
1658 struct in6_ifreq ifr6;
1659
1660 bzero(&ifr6, sizeof(ifr6));
1661 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1662 ifnet_name(interface), ifnet_unit(interface));
1663
1664 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1665 }
1666
1667 return result;
1668 }
1669
1670 static void
1671 utun_remove_address(ifnet_t interface,
1672 protocol_family_t protocol,
1673 ifaddr_t address,
1674 socket_t pf_socket)
1675 {
1676 errno_t result = 0;
1677
1678 /* Attempt a detach */
1679 if (protocol == PF_INET) {
1680 struct ifreq ifr;
1681
1682 bzero(&ifr, sizeof(ifr));
1683 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1684 ifnet_name(interface), ifnet_unit(interface));
1685 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1686 if (result != 0) {
1687 printf("utun_remove_address - ifaddr_address failed: %d", result);
1688 } else {
1689 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1690 if (result != 0) {
1691 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1692 }
1693 }
1694 } else if (protocol == PF_INET6) {
1695 struct in6_ifreq ifr6;
1696
1697 bzero(&ifr6, sizeof(ifr6));
1698 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1699 ifnet_name(interface), ifnet_unit(interface));
1700 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1701 sizeof(ifr6.ifr_addr));
1702 if (result != 0) {
1703 printf("utun_remove_address - ifaddr_address failed (v6): %d",
1704 result);
1705 } else {
1706 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1707 if (result != 0) {
1708 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1709 result);
1710 }
1711 }
1712 }
1713 }
1714
1715 static void
1716 utun_cleanup_family(ifnet_t interface,
1717 protocol_family_t protocol)
1718 {
1719 errno_t result = 0;
1720 socket_t pf_socket = NULL;
1721 ifaddr_t *addresses = NULL;
1722 int i;
1723
1724 if (protocol != PF_INET && protocol != PF_INET6) {
1725 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1726 return;
1727 }
1728
1729 /* Create a socket for removing addresses and detaching the protocol */
1730 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1731 if (result != 0) {
1732 if (result != EAFNOSUPPORT)
1733 printf("utun_cleanup_family - failed to create %s socket: %d\n",
1734 protocol == PF_INET ? "IP" : "IPv6", result);
1735 goto cleanup;
1736 }
1737
1738 /* always set SS_PRIV, we want to close and detach regardless */
1739 sock_setpriv(pf_socket, 1);
1740
1741 result = utun_detach_ip(interface, protocol, pf_socket);
1742 if (result == 0 || result == ENXIO) {
1743 /* We are done! We either detached or weren't attached. */
1744 goto cleanup;
1745 } else if (result != EBUSY) {
1746 /* Uh, not really sure what happened here... */
1747 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1748 goto cleanup;
1749 }
1750
1751 /*
1752 * At this point, we received an EBUSY error. This means there are
1753 * addresses attached. We should detach them and then try again.
1754 */
1755 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1756 if (result != 0) {
1757 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1758 ifnet_name(interface), ifnet_unit(interface),
1759 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1760 goto cleanup;
1761 }
1762
1763 for (i = 0; addresses[i] != 0; i++) {
1764 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1765 }
1766 ifnet_free_address_list(addresses);
1767 addresses = NULL;
1768
1769 /*
1770 * The addresses should be gone, we should try the remove again.
1771 */
1772 result = utun_detach_ip(interface, protocol, pf_socket);
1773 if (result != 0 && result != ENXIO) {
1774 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1775 }
1776
1777 cleanup:
1778 if (pf_socket != NULL) {
1779 sock_close(pf_socket);
1780 }
1781
1782 if (addresses != NULL) {
1783 ifnet_free_address_list(addresses);
1784 }
1785 }
1786
1787 static errno_t
1788 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1789 __unused u_int32_t unit,
1790 void *unitinfo)
1791 {
1792 struct utun_pcb *pcb = unitinfo;
1793 ifnet_t ifp = NULL;
1794 errno_t result = 0;
1795
1796 if (pcb == NULL) {
1797 return EINVAL;
1798 }
1799
1800 #if UTUN_NEXUS
1801 // Tell the nexus to stop all rings
1802 if (pcb->utun_netif_nexus != NULL) {
1803 kern_nexus_stop(pcb->utun_netif_nexus);
1804 }
1805 #endif // UTUN_NEXUS
1806
1807 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1808
1809 #if UTUN_NEXUS
1810 uuid_t kpipe_uuid;
1811 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1812 uuid_clear(pcb->utun_kpipe_uuid);
1813 pcb->utun_kpipe_enabled = FALSE;
1814 #endif // UTUN_NEXUS
1815
1816 pcb->utun_ctlref = NULL;
1817
1818 ifp = pcb->utun_ifp;
1819 if (ifp != NULL) {
1820 #if UTUN_NEXUS
1821 // Tell the nexus to stop all rings
1822 if (pcb->utun_netif_nexus != NULL) {
1823 /*
1824 * Quiesce the interface and flush any pending outbound packets.
1825 */
1826 if_down(ifp);
1827
1828 /* Increment refcnt, but detach interface */
1829 ifnet_incr_iorefcnt(ifp);
1830 if ((result = ifnet_detach(ifp)) != 0) {
1831 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1832 }
1833
1834 /*
1835 * We want to do everything in our power to ensure that the interface
1836 * really goes away when the socket is closed. We must remove IP/IPv6
1837 * addresses and detach the protocols. Finally, we can remove and
1838 * release the interface.
1839 */
1840 utun_cleanup_family(ifp, AF_INET);
1841 utun_cleanup_family(ifp, AF_INET6);
1842
1843 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1844
1845 if (!uuid_is_null(kpipe_uuid)) {
1846 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1847 if (pcb->utun_kpipe_pp != NULL) {
1848 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1849 pcb->utun_kpipe_pp = NULL;
1850 }
1851 utun_unregister_kernel_pipe_nexus();
1852 }
1853 }
1854 utun_nexus_detach(pcb);
1855
1856 /* Decrement refcnt to finish detaching and freeing */
1857 ifnet_decr_iorefcnt(ifp);
1858 } else
1859 #endif // UTUN_NEXUS
1860 {
1861 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1862
1863 #if UTUN_NEXUS
1864 if (!uuid_is_null(kpipe_uuid)) {
1865 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1866 if (pcb->utun_kpipe_pp != NULL) {
1867 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1868 pcb->utun_kpipe_pp = NULL;
1869 }
1870 utun_unregister_kernel_pipe_nexus();
1871 }
1872 }
1873 #endif // UTUN_NEXUS
1874
1875 /*
1876 * We want to do everything in our power to ensure that the interface
1877 * really goes away when the socket is closed. We must remove IP/IPv6
1878 * addresses and detach the protocols. Finally, we can remove and
1879 * release the interface.
1880 */
1881 utun_cleanup_family(ifp, AF_INET);
1882 utun_cleanup_family(ifp, AF_INET6);
1883
1884 /*
1885 * Detach now; utun_detach() will be called asynchronously once
1886 * the I/O reference count drops to 0. There we will invoke
1887 * ifnet_release().
1888 */
1889 if ((result = ifnet_detach(ifp)) != 0) {
1890 printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1891 }
1892 }
1893 } else {
1894 // Bound, but not connected
1895 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1896 utun_free_pcb(pcb, false);
1897 }
1898
1899 return 0;
1900 }
1901
1902 static errno_t
1903 utun_ctl_send(__unused kern_ctl_ref kctlref,
1904 __unused u_int32_t unit,
1905 void *unitinfo,
1906 mbuf_t m,
1907 __unused int flags)
1908 {
1909 /*
1910 * The userland ABI requires the first four bytes have the protocol family
1911 * in network byte order: swap them
1912 */
1913 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
1914 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
1915 } else {
1916 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
1917 }
1918
1919 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
1920 }
1921
1922 static errno_t
1923 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1924 __unused u_int32_t unit,
1925 void *unitinfo,
1926 int opt,
1927 void *data,
1928 size_t len)
1929 {
1930 struct utun_pcb *pcb = unitinfo;
1931 errno_t result = 0;
1932 /* check for privileges for privileged options */
1933 switch (opt) {
1934 case UTUN_OPT_FLAGS:
1935 case UTUN_OPT_EXT_IFDATA_STATS:
1936 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1937 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1938 return EPERM;
1939 }
1940 break;
1941 }
1942
1943 switch (opt) {
1944 case UTUN_OPT_FLAGS:
1945 if (len != sizeof(u_int32_t)) {
1946 result = EMSGSIZE;
1947 } else {
1948 if (pcb->utun_ifp == NULL) {
1949 // Only can set after connecting
1950 result = EINVAL;
1951 break;
1952 }
1953 #if UTUN_NEXUS
1954 if (pcb->utun_use_netif) {
1955 pcb->utun_flags = *(u_int32_t *)data;
1956 } else
1957 #endif // UTUN_NEXUS
1958 {
1959 u_int32_t old_flags = pcb->utun_flags;
1960 pcb->utun_flags = *(u_int32_t *)data;
1961 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1962 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1963 bpfdetach(pcb->utun_ifp);
1964 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1965 }
1966 }
1967 }
1968 break;
1969
1970 case UTUN_OPT_EXT_IFDATA_STATS:
1971 if (len != sizeof(int)) {
1972 result = EMSGSIZE;
1973 break;
1974 }
1975 if (pcb->utun_ifp == NULL) {
1976 // Only can set after connecting
1977 result = EINVAL;
1978 break;
1979 }
1980 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1981 break;
1982
1983 case UTUN_OPT_INC_IFDATA_STATS_IN:
1984 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1985 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1986
1987 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1988 result = EINVAL;
1989 break;
1990 }
1991 if (pcb->utun_ifp == NULL) {
1992 // Only can set after connecting
1993 result = EINVAL;
1994 break;
1995 }
1996 if (!pcb->utun_ext_ifdata_stats) {
1997 result = EINVAL;
1998 break;
1999 }
2000 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN)
2001 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2002 utsp->utsp_bytes, utsp->utsp_errors);
2003 else
2004 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2005 utsp->utsp_bytes, utsp->utsp_errors);
2006 break;
2007 }
2008 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2009 ifnet_t del_ifp = NULL;
2010 char name[IFNAMSIZ];
2011
2012 if (len > IFNAMSIZ - 1) {
2013 result = EMSGSIZE;
2014 break;
2015 }
2016 if (pcb->utun_ifp == NULL) {
2017 // Only can set after connecting
2018 result = EINVAL;
2019 break;
2020 }
2021 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2022 bcopy(data, name, len);
2023 name[len] = 0;
2024 result = ifnet_find_by_name(name, &del_ifp);
2025 }
2026 if (result == 0) {
2027 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2028 if (del_ifp)
2029 ifnet_release(del_ifp);
2030 }
2031 break;
2032 }
2033 case UTUN_OPT_MAX_PENDING_PACKETS: {
2034 u_int32_t max_pending_packets = 0;
2035 if (len != sizeof(u_int32_t)) {
2036 result = EMSGSIZE;
2037 break;
2038 }
2039 max_pending_packets = *(u_int32_t *)data;
2040 if (max_pending_packets == 0) {
2041 result = EINVAL;
2042 break;
2043 }
2044 pcb->utun_max_pending_packets = max_pending_packets;
2045 break;
2046 }
2047 #if UTUN_NEXUS
2048 case UTUN_OPT_ENABLE_CHANNEL: {
2049 if (len != sizeof(int)) {
2050 result = EMSGSIZE;
2051 break;
2052 }
2053 if (pcb->utun_ifp == NULL) {
2054 // Only can set after connecting
2055 result = EINVAL;
2056 break;
2057 }
2058 if (*(int *)data) {
2059 result = utun_enable_channel(pcb, current_proc());
2060 } else {
2061 result = utun_disable_channel(pcb);
2062 }
2063 break;
2064 }
2065 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2066 if (len != sizeof(int)) {
2067 result = EMSGSIZE;
2068 break;
2069 }
2070 if (pcb->utun_ifp == NULL) {
2071 // Only can set after connecting
2072 result = EINVAL;
2073 break;
2074 }
2075 if (!if_is_netagent_enabled()) {
2076 result = ENOTSUP;
2077 break;
2078 }
2079 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
2080 result = ENOENT;
2081 break;
2082 }
2083
2084 if (*(int *)data) {
2085 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2086 } else {
2087 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2088 }
2089 break;
2090 }
2091 case UTUN_OPT_ENABLE_NETIF: {
2092 if (len != sizeof(int)) {
2093 result = EMSGSIZE;
2094 break;
2095 }
2096 if (pcb->utun_ifp != NULL) {
2097 // Only can set before connecting
2098 result = EINVAL;
2099 break;
2100 }
2101 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2102 pcb->utun_use_netif = !!(*(int *)data);
2103 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2104 break;
2105 }
2106 case UTUN_OPT_SLOT_SIZE: {
2107 if (len != sizeof(u_int32_t)) {
2108 result = EMSGSIZE;
2109 break;
2110 }
2111 if (pcb->utun_ifp != NULL) {
2112 // Only can set before connecting
2113 result = EINVAL;
2114 break;
2115 }
2116 u_int32_t slot_size = *(u_int32_t *)data;
2117 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2118 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2119 return (EINVAL);
2120 }
2121 pcb->utun_slot_size = slot_size;
2122 break;
2123 }
2124 case UTUN_OPT_NETIF_RING_SIZE: {
2125 if (len != sizeof(u_int32_t)) {
2126 result = EMSGSIZE;
2127 break;
2128 }
2129 if (pcb->utun_ifp != NULL) {
2130 // Only can set before connecting
2131 result = EINVAL;
2132 break;
2133 }
2134 u_int32_t ring_size = *(u_int32_t *)data;
2135 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2136 ring_size > UTUN_IF_MAX_RING_SIZE) {
2137 return (EINVAL);
2138 }
2139 pcb->utun_netif_ring_size = ring_size;
2140 break;
2141 }
2142 case UTUN_OPT_TX_FSW_RING_SIZE: {
2143 if (len != sizeof(u_int32_t)) {
2144 result = EMSGSIZE;
2145 break;
2146 }
2147 if (pcb->utun_ifp != NULL) {
2148 // Only can set before connecting
2149 result = EINVAL;
2150 break;
2151 }
2152 u_int32_t ring_size = *(u_int32_t *)data;
2153 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2154 ring_size > UTUN_IF_MAX_RING_SIZE) {
2155 return (EINVAL);
2156 }
2157 pcb->utun_tx_fsw_ring_size = ring_size;
2158 break;
2159 }
2160 case UTUN_OPT_RX_FSW_RING_SIZE: {
2161 if (len != sizeof(u_int32_t)) {
2162 result = EMSGSIZE;
2163 break;
2164 }
2165 if (pcb->utun_ifp != NULL) {
2166 // Only can set before connecting
2167 result = EINVAL;
2168 break;
2169 }
2170 u_int32_t ring_size = *(u_int32_t *)data;
2171 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2172 ring_size > UTUN_IF_MAX_RING_SIZE) {
2173 return (EINVAL);
2174 }
2175 pcb->utun_rx_fsw_ring_size = ring_size;
2176 break;
2177 }
2178 #endif // UTUN_NEXUS
2179 default: {
2180 result = ENOPROTOOPT;
2181 break;
2182 }
2183 }
2184
2185 return result;
2186 }
2187
2188 static errno_t
2189 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2190 __unused u_int32_t unit,
2191 void *unitinfo,
2192 int opt,
2193 void *data,
2194 size_t *len)
2195 {
2196 struct utun_pcb *pcb = unitinfo;
2197 errno_t result = 0;
2198
2199 switch (opt) {
2200 case UTUN_OPT_FLAGS:
2201 if (*len != sizeof(u_int32_t)) {
2202 result = EMSGSIZE;
2203 } else {
2204 *(u_int32_t *)data = pcb->utun_flags;
2205 }
2206 break;
2207
2208 case UTUN_OPT_EXT_IFDATA_STATS:
2209 if (*len != sizeof(int)) {
2210 result = EMSGSIZE;
2211 } else {
2212 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2213 }
2214 break;
2215
2216 case UTUN_OPT_IFNAME:
2217 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2218 result = EMSGSIZE;
2219 } else {
2220 if (pcb->utun_ifp == NULL) {
2221 // Only can get after connecting
2222 result = EINVAL;
2223 break;
2224 }
2225 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2226 }
2227 break;
2228
2229 case UTUN_OPT_MAX_PENDING_PACKETS: {
2230 if (*len != sizeof(u_int32_t)) {
2231 result = EMSGSIZE;
2232 } else {
2233 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2234 }
2235 break;
2236 }
2237
2238 #if UTUN_NEXUS
2239 case UTUN_OPT_ENABLE_CHANNEL: {
2240 if (*len != sizeof(int)) {
2241 result = EMSGSIZE;
2242 } else {
2243 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2244 *(int *)data = pcb->utun_kpipe_enabled;
2245 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2246 }
2247 break;
2248 }
2249
2250 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2251 if (*len != sizeof(int)) {
2252 result = EMSGSIZE;
2253 } else {
2254 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2255 }
2256 break;
2257 }
2258
2259 case UTUN_OPT_ENABLE_NETIF: {
2260 if (*len != sizeof(int)) {
2261 result = EMSGSIZE;
2262 } else {
2263 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2264 *(int *)data = !!pcb->utun_use_netif;
2265 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2266 }
2267 break;
2268 }
2269
2270 case UTUN_OPT_GET_CHANNEL_UUID: {
2271 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2272 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2273 result = ENXIO;
2274 } else if (*len != sizeof(uuid_t)) {
2275 result = EMSGSIZE;
2276 } else {
2277 uuid_copy(data, pcb->utun_kpipe_uuid);
2278 }
2279 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2280 break;
2281 }
2282 case UTUN_OPT_SLOT_SIZE: {
2283 if (*len != sizeof(u_int32_t)) {
2284 result = EMSGSIZE;
2285 } else {
2286 *(u_int32_t *)data = pcb->utun_slot_size;
2287 }
2288 break;
2289 }
2290 case UTUN_OPT_NETIF_RING_SIZE: {
2291 if (*len != sizeof(u_int32_t)) {
2292 result = EMSGSIZE;
2293 } else {
2294 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2295 }
2296 break;
2297 }
2298 case UTUN_OPT_TX_FSW_RING_SIZE: {
2299 if (*len != sizeof(u_int32_t)) {
2300 result = EMSGSIZE;
2301 } else {
2302 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2303 }
2304 break;
2305 }
2306 case UTUN_OPT_RX_FSW_RING_SIZE: {
2307 if (*len != sizeof(u_int32_t)) {
2308 result = EMSGSIZE;
2309 } else {
2310 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2311 }
2312 break;
2313 }
2314 #endif // UTUN_NEXUS
2315
2316 default:
2317 result = ENOPROTOOPT;
2318 break;
2319 }
2320
2321 return result;
2322 }
2323
2324 static void
2325 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2326 {
2327 #pragma unused(flags)
2328 bool reenable_output = false;
2329 struct utun_pcb *pcb = unitinfo;
2330 if (pcb == NULL) {
2331 return;
2332 }
2333 ifnet_lock_exclusive(pcb->utun_ifp);
2334
2335 u_int32_t utun_packet_cnt;
2336 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2337 if (error_pc != 0) {
2338 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2339 utun_packet_cnt = 0;
2340 }
2341
2342 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2343 reenable_output = true;
2344 }
2345
2346 if (reenable_output) {
2347 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2348 if (error != 0) {
2349 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2350 }
2351 }
2352 ifnet_lock_done(pcb->utun_ifp);
2353 }
2354
2355 /* Network Interface functions */
2356 static void
2357 utun_start(ifnet_t interface)
2358 {
2359 mbuf_t data;
2360 struct utun_pcb *pcb = ifnet_softc(interface);
2361
2362 VERIFY(pcb != NULL);
2363
2364 #if UTUN_NEXUS
2365 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2366 if (pcb->utun_kpipe_enabled) {
2367 /* It's possible to have channels enabled, but not yet have the channel opened,
2368 * in which case the rxring will not be set
2369 */
2370 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2371 if (pcb->utun_kpipe_rxring != NULL) {
2372 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2373 }
2374 return;
2375 }
2376 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2377 #endif // UTUN_NEXUS
2378
2379 for (;;) {
2380 bool can_accept_packets = true;
2381 ifnet_lock_shared(pcb->utun_ifp);
2382
2383 u_int32_t utun_packet_cnt;
2384 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2385 if (error_pc != 0) {
2386 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2387 utun_packet_cnt = 0;
2388 }
2389
2390 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2391 if (!can_accept_packets && pcb->utun_ctlref) {
2392 u_int32_t difference = 0;
2393 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2394 if (difference > 0) {
2395 // If the low-water mark has not yet been reached, we still need to enqueue data
2396 // into the buffer
2397 can_accept_packets = true;
2398 }
2399 }
2400 }
2401 if (!can_accept_packets) {
2402 errno_t error = ifnet_disable_output(interface);
2403 if (error != 0) {
2404 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2405 }
2406 ifnet_lock_done(pcb->utun_ifp);
2407 break;
2408 }
2409 ifnet_lock_done(pcb->utun_ifp);
2410 if (ifnet_dequeue(interface, &data) != 0) {
2411 break;
2412 }
2413 if (utun_output(interface, data) != 0) {
2414 break;
2415 }
2416 }
2417 }
2418
2419 static errno_t
2420 utun_output(ifnet_t interface,
2421 mbuf_t data)
2422 {
2423 struct utun_pcb *pcb = ifnet_softc(interface);
2424 errno_t result;
2425
2426 VERIFY(interface == pcb->utun_ifp);
2427
2428 #if UTUN_NEXUS
2429 if (!pcb->utun_use_netif)
2430 #endif // UTUN_NEXUS
2431 {
2432 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2433 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2434 }
2435 }
2436
2437 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2438 /* flush data */
2439 mbuf_freem(data);
2440 return 0;
2441 }
2442
2443 // otherwise, fall thru to ctl_enqueumbuf
2444 if (pcb->utun_ctlref) {
2445 int length;
2446
2447 /*
2448 * The ABI requires the protocol in network byte order
2449 */
2450 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2451 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2452 }
2453
2454 length = mbuf_pkthdr_len(data);
2455 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2456 if (result != 0) {
2457 mbuf_freem(data);
2458 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
2459 #if UTUN_NEXUS
2460 if (!pcb->utun_use_netif)
2461 #endif // UTUN_NEXUS
2462 {
2463 ifnet_stat_increment_out(interface, 0, 0, 1);
2464 }
2465 } else {
2466 #if UTUN_NEXUS
2467 if (!pcb->utun_use_netif)
2468 #endif // UTUN_NEXUS
2469 {
2470 if (!pcb->utun_ext_ifdata_stats) {
2471 ifnet_stat_increment_out(interface, 1, length, 0);
2472 }
2473 }
2474 }
2475 } else {
2476 mbuf_freem(data);
2477 }
2478
2479 return 0;
2480 }
2481
2482 static errno_t
2483 utun_demux(__unused ifnet_t interface,
2484 mbuf_t data,
2485 __unused char *frame_header,
2486 protocol_family_t *protocol)
2487 {
2488 while (data != NULL && mbuf_len(data) < 1) {
2489 data = mbuf_next(data);
2490 }
2491
2492 if (data == NULL) {
2493 return ENOENT;
2494 }
2495
2496 #if UTUN_NEXUS
2497 struct utun_pcb *pcb = ifnet_softc(interface);
2498 struct ip *ip;
2499 u_int ip_version;
2500
2501 if (pcb->utun_use_netif) {
2502 ip = mtod(data, struct ip *);
2503 ip_version = ip->ip_v;
2504
2505 switch(ip_version) {
2506 case 4:
2507 *protocol = PF_INET;
2508 return 0;
2509 case 6:
2510 *protocol = PF_INET6;
2511 return 0;
2512 default:
2513 *protocol = 0;
2514 break;
2515 }
2516 } else
2517 #endif // UTUN_NEXUS
2518 {
2519 *protocol = *(u_int32_t *)mbuf_data(data);
2520 }
2521
2522 return 0;
2523 }
2524
2525 static errno_t
2526 utun_framer(ifnet_t interface,
2527 mbuf_t *packet,
2528 __unused const struct sockaddr *dest,
2529 __unused const char *desk_linkaddr,
2530 const char *frame_type,
2531 u_int32_t *prepend_len,
2532 u_int32_t *postpend_len)
2533 {
2534 struct utun_pcb *pcb = ifnet_softc(interface);
2535 VERIFY(interface == pcb->utun_ifp);
2536
2537 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2538 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2539 printf("utun_framer - ifnet_output prepend failed\n");
2540
2541 ifnet_stat_increment_out(interface, 0, 0, 1);
2542
2543 // just return, because the buffer was freed in mbuf_prepend
2544 return EJUSTRETURN;
2545 }
2546 if (prepend_len != NULL) {
2547 *prepend_len = header_length;
2548 }
2549 if (postpend_len != NULL) {
2550 *postpend_len = 0;
2551 }
2552
2553 // place protocol number at the beginning of the mbuf
2554 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2555
2556
2557 return 0;
2558 }
2559
2560 static errno_t
2561 utun_add_proto(__unused ifnet_t interface,
2562 protocol_family_t protocol,
2563 __unused const struct ifnet_demux_desc *demux_array,
2564 __unused u_int32_t demux_count)
2565 {
2566 switch(protocol) {
2567 case PF_INET:
2568 return 0;
2569 case PF_INET6:
2570 return 0;
2571 default:
2572 break;
2573 }
2574
2575 return ENOPROTOOPT;
2576 }
2577
2578 static errno_t
2579 utun_del_proto(__unused ifnet_t interface,
2580 __unused protocol_family_t protocol)
2581 {
2582 return 0;
2583 }
2584
2585 static errno_t
2586 utun_ioctl(ifnet_t interface,
2587 u_long command,
2588 void *data)
2589 {
2590 errno_t result = 0;
2591
2592 switch(command) {
2593 case SIOCSIFMTU: {
2594 #if UTUN_NEXUS
2595 struct utun_pcb *pcb = ifnet_softc(interface);
2596 if (pcb->utun_use_netif) {
2597 // Make sure we can fit packets in the channel buffers
2598 // Allow for the headroom in the slot
2599 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2600 result = EINVAL;
2601 } else {
2602 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2603 }
2604 } else
2605 #endif // UTUN_NEXUS
2606 {
2607 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2608 }
2609 break;
2610 }
2611
2612 case SIOCSIFFLAGS:
2613 /* ifioctl() takes care of it */
2614 break;
2615
2616 default:
2617 result = EOPNOTSUPP;
2618 }
2619
2620 return result;
2621 }
2622
2623 static void
2624 utun_detached(ifnet_t interface)
2625 {
2626 struct utun_pcb *pcb = ifnet_softc(interface);
2627 (void)ifnet_release(interface);
2628 utun_free_pcb(pcb, true);
2629 }
2630
2631 /* Protocol Handlers */
2632
2633 static errno_t
2634 utun_proto_input(__unused ifnet_t interface,
2635 protocol_family_t protocol,
2636 mbuf_t m,
2637 __unused char *frame_header)
2638 {
2639 struct utun_pcb *pcb = ifnet_softc(interface);
2640 #if UTUN_NEXUS
2641 if (!pcb->utun_use_netif)
2642 #endif // UTUN_NEXUS
2643 {
2644 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2645 }
2646 int32_t pktlen = m->m_pkthdr.len;
2647 if (proto_input(protocol, m) != 0) {
2648 m_freem(m);
2649 #if UTUN_NEXUS
2650 if (!pcb->utun_use_netif)
2651 #endif // UTUN_NEXUS
2652 {
2653 ifnet_stat_increment_in(interface, 0, 0, 1);
2654 }
2655 } else {
2656 #if UTUN_NEXUS
2657 if (!pcb->utun_use_netif)
2658 #endif // UTUN_NEXUS
2659 {
2660 ifnet_stat_increment_in(interface, 1, pktlen, 0);
2661 }
2662 }
2663
2664 return 0;
2665 }
2666
2667 static errno_t
2668 utun_proto_pre_output(__unused ifnet_t interface,
2669 protocol_family_t protocol,
2670 __unused mbuf_t *packet,
2671 __unused const struct sockaddr *dest,
2672 __unused void *route,
2673 char *frame_type,
2674 __unused char *link_layer_dest)
2675 {
2676 *(protocol_family_t *)(void *)frame_type = protocol;
2677 return 0;
2678 }
2679
2680 static errno_t
2681 utun_attach_proto(ifnet_t interface,
2682 protocol_family_t protocol)
2683 {
2684 struct ifnet_attach_proto_param proto;
2685
2686 bzero(&proto, sizeof(proto));
2687 proto.input = utun_proto_input;
2688 proto.pre_output = utun_proto_pre_output;
2689
2690 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2691 if (result != 0 && result != EEXIST) {
2692 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2693 protocol, result);
2694 }
2695
2696 return result;
2697 }
2698
2699 static errno_t
2700 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2701 {
2702 #if UTUN_NEXUS
2703 if (pcb->utun_use_netif) {
2704 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2705
2706 lck_mtx_lock(&pcb->utun_input_chain_lock);
2707 if (pcb->utun_input_chain != NULL) {
2708 pcb->utun_input_chain_last->m_nextpkt = packet;
2709 } else {
2710 pcb->utun_input_chain = packet;
2711 }
2712 while (packet->m_nextpkt) {
2713 VERIFY(packet != packet->m_nextpkt);
2714 packet = packet->m_nextpkt;
2715 }
2716 pcb->utun_input_chain_last = packet;
2717 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2718
2719 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2720 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2721
2722 if (rx_ring != NULL) {
2723 kern_channel_notify(rx_ring, 0);
2724 }
2725
2726 return (0);
2727 } else
2728 #endif // IPSEC_NEXUS
2729 {
2730 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
2731
2732 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2733 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2734 }
2735 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2736 /* flush data */
2737 mbuf_freem(packet);
2738 return 0;
2739 }
2740
2741 errno_t result = 0;
2742 if (!pcb->utun_ext_ifdata_stats) {
2743 struct ifnet_stat_increment_param incs = {};
2744 incs.packets_in = 1;
2745 incs.bytes_in = mbuf_pkthdr_len(packet);
2746 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2747 } else {
2748 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2749 }
2750 if (result != 0) {
2751 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2752
2753 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2754 mbuf_freem(packet);
2755 }
2756
2757 return (0);
2758 }
2759 }
2760
2761 #if UTUN_NEXUS
2762
2763 static errno_t
2764 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2765 {
2766 return 0;
2767 }
2768
2769 static void
2770 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2771 {
2772 // Ignore
2773 }
2774
2775 static errno_t
2776 utun_register_nexus(void)
2777 {
2778 const struct kern_nexus_domain_provider_init dp_init = {
2779 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2780 .nxdpi_flags = 0,
2781 .nxdpi_init = utun_nxdp_init,
2782 .nxdpi_fini = utun_nxdp_fini
2783 };
2784 errno_t err = 0;
2785
2786 /* utun_nxdp_init() is called before this function returns */
2787 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2788 (const uint8_t *) "com.apple.utun",
2789 &dp_init, sizeof(dp_init),
2790 &utun_nx_dom_prov);
2791 if (err != 0) {
2792 printf("%s: failed to register domain provider\n", __func__);
2793 return (err);
2794 }
2795 return (0);
2796 }
2797
2798 static errno_t
2799 utun_ifnet_set_attrs(ifnet_t ifp)
2800 {
2801 /* Set flags and additional information. */
2802 ifnet_set_mtu(ifp, 1500);
2803 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2804
2805 /* The interface must generate its own IPv6 LinkLocal address,
2806 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2807 */
2808 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2809
2810 return (0);
2811 }
2812
2813 static errno_t
2814 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2815 {
2816 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2817 pcb->utun_netif_nexus = nexus;
2818 return (utun_ifnet_set_attrs(ifp));
2819 }
2820
2821 static errno_t
2822 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2823 proc_t p, kern_nexus_t nexus,
2824 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2825 {
2826 #pragma unused(nxprov, p)
2827 #pragma unused(nexus, nexus_port, channel, ch_ctx)
2828 return (0);
2829 }
2830
2831 static errno_t
2832 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2833 kern_channel_t channel)
2834 {
2835 #pragma unused(nxprov, channel)
2836 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2837 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2838 return (ok ? 0 : ENXIO);
2839 }
2840
2841 static void
2842 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2843 kern_channel_t channel)
2844 {
2845 #pragma unused(nxprov, nexus, channel)
2846 }
2847
2848 static void
2849 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2850 kern_channel_t channel)
2851 {
2852 #pragma unused(nxprov, nexus, channel)
2853 }
2854
2855 static void
2856 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2857 kern_channel_t channel)
2858 {
2859 #pragma unused(nxprov, channel)
2860 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2861 if (pcb->utun_netif_nexus == nexus) {
2862 pcb->utun_netif_nexus = NULL;
2863 }
2864 ifnet_decr_iorefcnt(pcb->utun_ifp);
2865 }
2866
2867 static errno_t
2868 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2869 kern_channel_t channel, kern_channel_ring_t ring,
2870 boolean_t is_tx_ring, void **ring_ctx)
2871 {
2872 #pragma unused(nxprov)
2873 #pragma unused(channel)
2874 #pragma unused(ring_ctx)
2875 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2876 if (!is_tx_ring) {
2877 VERIFY(pcb->utun_kpipe_rxring == NULL);
2878 pcb->utun_kpipe_rxring = ring;
2879 } else {
2880 VERIFY(pcb->utun_kpipe_txring == NULL);
2881 pcb->utun_kpipe_txring = ring;
2882 }
2883 return 0;
2884 }
2885
2886 static void
2887 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2888 kern_channel_ring_t ring)
2889 {
2890 #pragma unused(nxprov)
2891 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2892 if (pcb->utun_kpipe_rxring == ring) {
2893 pcb->utun_kpipe_rxring = NULL;
2894 } else if (pcb->utun_kpipe_txring == ring) {
2895 pcb->utun_kpipe_txring = NULL;
2896 }
2897 }
2898
2899 static errno_t
2900 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2901 kern_channel_ring_t tx_ring, uint32_t flags)
2902 {
2903 #pragma unused(nxprov)
2904 #pragma unused(flags)
2905 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2906
2907 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2908 int channel_enabled = pcb->utun_kpipe_enabled;
2909 if (!channel_enabled) {
2910 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2911 return 0;
2912 }
2913
2914 if (pcb->utun_use_netif) {
2915 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2916 if (tx_slot == NULL) {
2917 // Nothing to write, bail
2918 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2919 return 0;
2920 }
2921
2922 // Signal the netif ring to read
2923 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2924 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2925 if (rx_ring != NULL) {
2926 kern_channel_notify(rx_ring, 0);
2927 }
2928 } else {
2929 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2930
2931 struct ifnet_stat_increment_param incs = {};
2932 struct kern_channel_ring_stat_increment tx_ring_stats = {};
2933 MBUFQ_HEAD(mbufq) mbq;
2934 MBUFQ_INIT(&mbq);
2935 kern_channel_slot_t tx_pslot = NULL;
2936 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2937 while (tx_slot != NULL) {
2938 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2939
2940 // Advance TX ring
2941 tx_pslot = tx_slot;
2942 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2943
2944 if (tx_ph == 0) {
2945 continue;
2946 }
2947
2948 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2949 VERIFY(tx_buf != NULL);
2950 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2951 VERIFY(tx_baddr != 0);
2952 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2953
2954 size_t length = MIN(kern_packet_get_data_length(tx_ph),
2955 pcb->utun_slot_size);
2956
2957 mbuf_t data = NULL;
2958 if (length >= UTUN_HEADER_SIZE(pcb) &&
2959 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
2960 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
2961 VERIFY(0 == error);
2962 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
2963 VERIFY(0 == error);
2964 /*
2965 * The userland ABI requires the first four bytes have
2966 * the protocol family in network byte order: swap them
2967 */
2968 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
2969 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
2970 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2971 incs.packets_in++;
2972 incs.bytes_in += length;
2973 MBUFQ_ENQUEUE(&mbq, data);
2974 }
2975 }
2976 if (tx_pslot) {
2977 kern_channel_advance_slot(tx_ring, tx_pslot);
2978 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
2979 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
2980 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
2981 (void) kern_channel_reclaim(tx_ring);
2982 }
2983 if (!MBUFQ_EMPTY(&mbq)) {
2984 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
2985 MBUFQ_LAST(&mbq), &incs);
2986 MBUFQ_INIT(&mbq);
2987 }
2988 }
2989
2990 return 0;
2991 }
2992
2993 static errno_t
2994 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2995 kern_channel_ring_t rx_ring, uint32_t flags)
2996 {
2997 #pragma unused(nxprov)
2998 #pragma unused(flags)
2999 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3000 struct kern_channel_ring_stat_increment rx_ring_stats = {};
3001
3002 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3003
3004 int channel_enabled = pcb->utun_kpipe_enabled;
3005 if (!channel_enabled) {
3006 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3007 return 0;
3008 }
3009
3010 /* reclaim user-released slots */
3011 (void) kern_channel_reclaim(rx_ring);
3012
3013 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3014 if (avail == 0) {
3015 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3016 return 0;
3017 }
3018
3019 if (pcb->utun_use_netif) {
3020 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3021 if (tx_ring == NULL ||
3022 pcb->utun_netif_nexus == NULL) {
3023 // Net-If TX ring not set up yet, nothing to read
3024 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3025 return 0;
3026 }
3027
3028 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
3029
3030 // Unlock utun before entering ring
3031 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3032
3033 (void)kr_enter(tx_ring, TRUE);
3034
3035 // Lock again after entering and validate
3036 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3037 if (tx_ring != pcb->utun_netif_txring) {
3038 // Ring no longer valid
3039 // Unlock first, then exit ring
3040 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3041 kr_exit(tx_ring);
3042 return 0;
3043 }
3044
3045 struct kern_channel_ring_stat_increment tx_ring_stats;
3046 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3047 kern_channel_slot_t tx_pslot = NULL;
3048 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3049 if (tx_slot == NULL) {
3050 // Nothing to read, don't bother signalling
3051 // Unlock first, then exit ring
3052 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3053 kr_exit(tx_ring);
3054 return 0;
3055 }
3056
3057 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3058 VERIFY(rx_pp != NULL);
3059 kern_channel_slot_t rx_pslot = NULL;
3060 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3061
3062 while (rx_slot != NULL && tx_slot != NULL) {
3063 size_t length;
3064 kern_buflet_t rx_buf;
3065 void *rx_baddr;
3066
3067 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3068
3069 // Advance TX ring
3070 tx_pslot = tx_slot;
3071 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3072
3073 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3074 if (tx_ph == 0) {
3075 continue;
3076 }
3077
3078 // Allocate rx packet
3079 kern_packet_t rx_ph = 0;
3080 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3081 if (__improbable(error != 0)) {
3082 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
3083 pcb->utun_ifp->if_xname);
3084 break;
3085 }
3086
3087 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3088 VERIFY(tx_buf != NULL);
3089 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3090 VERIFY(tx_baddr != NULL);
3091 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3092
3093 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3094
3095 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
3096 pcb->utun_slot_size);
3097
3098 tx_ring_stats.kcrsi_slots_transferred++;
3099 tx_ring_stats.kcrsi_bytes_transferred += length;
3100
3101 if (length < UTUN_HEADER_SIZE(pcb) ||
3102 length > pcb->utun_slot_size ||
3103 length > rx_pp->pp_buflet_size ||
3104 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3105 /* flush data */
3106 kern_pbufpool_free(rx_pp, rx_ph);
3107 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3108 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3109 STATS_INC(nifs, NETIF_STATS_BADLEN);
3110 STATS_INC(nifs, NETIF_STATS_DROPPED);
3111 continue;
3112 }
3113
3114 /* fillout packet */
3115 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3116 VERIFY(rx_buf != NULL);
3117 rx_baddr = kern_buflet_get_object_address(rx_buf);
3118 VERIFY(rx_baddr != NULL);
3119
3120 // Find family
3121 uint32_t af = 0;
3122 uint8_t vhl = *(uint8_t *)(tx_baddr);
3123 u_int ip_version = (vhl >> 4);
3124 switch (ip_version) {
3125 case 4: {
3126 af = AF_INET;
3127 break;
3128 }
3129 case 6: {
3130 af = AF_INET6;
3131 break;
3132 }
3133 default: {
3134 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3135 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3136 break;
3137 }
3138 }
3139
3140 // Copy header
3141 af = htonl(af);
3142 memcpy((void *)rx_baddr, &af, sizeof(af));
3143 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3144 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3145 }
3146
3147 // Copy data from tx to rx
3148 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3149 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
3150
3151 /* finalize and attach the packet */
3152 error = kern_buflet_set_data_offset(rx_buf, 0);
3153 VERIFY(error == 0);
3154 error = kern_buflet_set_data_length(rx_buf, length);
3155 VERIFY(error == 0);
3156 error = kern_packet_finalize(rx_ph);
3157 VERIFY(error == 0);
3158 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3159 VERIFY(error == 0);
3160
3161 STATS_INC(nifs, NETIF_STATS_TXPKTS);
3162 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
3163
3164 rx_ring_stats.kcrsi_slots_transferred++;
3165 rx_ring_stats.kcrsi_bytes_transferred += length;
3166
3167 rx_pslot = rx_slot;
3168 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3169 }
3170
3171 if (rx_pslot) {
3172 kern_channel_advance_slot(rx_ring, rx_pslot);
3173 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3174 }
3175
3176 if (tx_pslot) {
3177 kern_channel_advance_slot(tx_ring, tx_pslot);
3178 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3179 (void)kern_channel_reclaim(tx_ring);
3180 }
3181
3182 /* just like utun_ctl_rcvd(), always reenable output */
3183 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3184 if (error != 0) {
3185 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
3186 }
3187
3188 // Unlock first, then exit ring
3189 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3190
3191 if (tx_pslot != NULL) {
3192 kern_channel_notify(tx_ring, 0);
3193 }
3194 kr_exit(tx_ring);
3195 } else {
3196 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3197
3198 uint32_t mb_cnt = 0;
3199 uint32_t mb_len = 0;
3200 struct mbuf *mb_head = NULL;
3201 struct mbuf *mb_tail = NULL;
3202
3203 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3204 &mb_tail, &mb_cnt, &mb_len) != 0) {
3205 return 0;
3206 }
3207 VERIFY(mb_cnt <= avail);
3208
3209 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3210 VERIFY(rx_pp != NULL);
3211 kern_channel_slot_t rx_pslot = NULL;
3212 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3213 while (rx_slot) {
3214 size_t length = 0;
3215 mbuf_t data = NULL;
3216 if ((data = mb_head) == NULL) {
3217 VERIFY(mb_cnt == 0);
3218 break;
3219 }
3220 mb_head = mbuf_nextpkt(mb_head);
3221 mbuf_setnextpkt(data, NULL);
3222 VERIFY(mb_cnt != 0);
3223 --mb_cnt;
3224 length = mbuf_pkthdr_len(data);
3225 if (length < UTUN_HEADER_SIZE(pcb) ||
3226 length > pcb->utun_slot_size ||
3227 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3228 /* flush data */
3229 mbuf_freem(data);
3230 continue;
3231 }
3232 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3233
3234 // Allocate rx packet
3235 kern_packet_t rx_ph = 0;
3236 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3237 if (__improbable(error != 0)) {
3238 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
3239 pcb->utun_ifp->if_xname);
3240 break;
3241 }
3242
3243 /*
3244 * The ABI requires the protocol in network byte order
3245 */
3246 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3247
3248 // Fillout rx packet
3249 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3250 VERIFY(rx_buf != NULL);
3251 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3252 VERIFY(rx_baddr != NULL);
3253
3254 // Copy-in data from mbuf to buflet
3255 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3256 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3257
3258 // Finalize and attach the packet
3259 error = kern_buflet_set_data_offset(rx_buf, 0);
3260 VERIFY(error == 0);
3261 error = kern_buflet_set_data_length(rx_buf, length);
3262 VERIFY(error == 0);
3263 error = kern_packet_finalize(rx_ph);
3264 VERIFY(error == 0);
3265 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3266 VERIFY(error == 0);
3267
3268 rx_ring_stats.kcrsi_slots_transferred++;
3269 rx_ring_stats.kcrsi_bytes_transferred += length;
3270
3271 if (!pcb->utun_ext_ifdata_stats) {
3272 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3273 }
3274
3275 mbuf_freem(data);
3276
3277 rx_pslot = rx_slot;
3278 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3279 }
3280 if (rx_pslot) {
3281 kern_channel_advance_slot(rx_ring, rx_pslot);
3282 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3283 }
3284 if (mb_head != NULL) {
3285 VERIFY(mb_cnt != 0);
3286 mbuf_freem_list(mb_head);
3287 }
3288 }
3289
3290 return 0;
3291 }
3292
3293 #endif // UTUN_NEXUS
3294
3295
3296 /*
3297 * These are place holders until coreTLS kext stops calling them
3298 */
3299 errno_t utun_ctl_register_dtls (void *reg);
3300 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3301 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3302
3303 errno_t
3304 utun_ctl_register_dtls (void *reg)
3305 {
3306 #pragma unused(reg)
3307 return 0;
3308 }
3309
3310 int
3311 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3312 {
3313 #pragma unused(pcb)
3314 #pragma unused(pkt)
3315 #pragma unused(family)
3316 return 0;
3317 }
3318
3319 void
3320 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3321 {
3322 #pragma unused(pcb)
3323 }
mirror of XNU