]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/if_utun.c
projects / apple / xnu.git / blob
? search:


225d9c21cc4481a2e2d9c4de8f7005a0a44246c6
[apple/xnu.git] / bsd / net / if_utun.c
1 /*
2 * Copyright (c) 2008-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 Application of kernel control for interface creation
33
34 Theory of operation:
35 utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 This kernel control will register an interface for every client that connects.
37 ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57
58 #define UTUN_NEXUS 0
59
60 #if UTUN_NEXUS
61 static nexus_controller_t utun_ncd;
62 static int utun_ncd_refcount;
63 static uuid_t utun_kpipe_uuid;
64 static uuid_t utun_nx_dom_prov;
65
66 typedef struct utun_nx {
67 uuid_t if_provider;
68 uuid_t if_instance;
69 uuid_t ms_provider;
70 uuid_t ms_instance;
71 uuid_t ms_device;
72 uuid_t ms_host;
73 uuid_t ms_agent;
74 } *utun_nx_t;
75
76 #endif // UTUN_NEXUS
77
78 /* Control block allocated for each kernel control connection */
79 struct utun_pcb {
80 TAILQ_ENTRY(utun_pcb) utun_chain;
81 kern_ctl_ref utun_ctlref;
82 ifnet_t utun_ifp;
83 u_int32_t utun_unit;
84 u_int32_t utun_unique_id;
85 u_int32_t utun_flags;
86 int utun_ext_ifdata_stats;
87 u_int32_t utun_max_pending_packets;
88 char utun_if_xname[IFXNAMSIZ];
89 char utun_unique_name[IFXNAMSIZ];
90 // PCB lock protects state fields and rings
91 decl_lck_rw_data(, utun_pcb_lock);
92 struct mbuf * utun_input_chain;
93 struct mbuf * utun_input_chain_last;
94 // Input chain lock protects the list of input mbufs
95 // The input chain lock must be taken AFTER the PCB lock if both are held
96 lck_mtx_t utun_input_chain_lock;
97
98 #if UTUN_NEXUS
99 struct utun_nx utun_nx;
100 int utun_kpipe_enabled;
101 uuid_t utun_kpipe_uuid;
102 void * utun_kpipe_rxring;
103 void * utun_kpipe_txring;
104 kern_pbufpool_t utun_kpipe_pp;
105
106 kern_nexus_t utun_netif_nexus;
107 kern_pbufpool_t utun_netif_pp;
108 void * utun_netif_rxring;
109 void * utun_netif_txring;
110 uint64_t utun_netif_txring_size;
111
112 u_int32_t utun_slot_size;
113 u_int32_t utun_netif_ring_size;
114 u_int32_t utun_tx_fsw_ring_size;
115 u_int32_t utun_rx_fsw_ring_size;
116 bool utun_use_netif;
117 bool utun_needs_netagent;
118 #endif // UTUN_NEXUS
119 };
120
121 /* Kernel Control functions */
122 static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
123 void **unitinfo);
124 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
125 void **unitinfo);
126 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
127 void *unitinfo);
128 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
129 void *unitinfo, mbuf_t m, int flags);
130 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
131 int opt, void *data, size_t *len);
132 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
133 int opt, void *data, size_t len);
134 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
135 int flags);
136
137 /* Network Interface functions */
138 static void utun_start(ifnet_t interface);
139 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
140 const struct sockaddr *dest, const char *desk_linkaddr,
141 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
142 static errno_t utun_output(ifnet_t interface, mbuf_t data);
143 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
144 protocol_family_t *protocol);
145 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
146 const struct ifnet_demux_desc *demux_array,
147 u_int32_t demux_count);
148 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
149 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
150 static void utun_detached(ifnet_t interface);
151
152 /* Protocol handlers */
153 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
154 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
155 mbuf_t m, char *frame_header);
156 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
157 mbuf_t *packet, const struct sockaddr *dest, void *route,
158 char *frame_type, char *link_layer_dest);
159 static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
160
161 #if UTUN_NEXUS
162
163 #define UTUN_IF_DEFAULT_SLOT_SIZE 2048
164 #define UTUN_IF_DEFAULT_RING_SIZE 64
165 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
166 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
167 #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
168 #define UTUN_IF_HEADROOM_SIZE 32
169
170 #define UTUN_IF_MIN_RING_SIZE 16
171 #define UTUN_IF_MAX_RING_SIZE 1024
172
173 #define UTUN_IF_MIN_SLOT_SIZE 1024
174 #define UTUN_IF_MAX_SLOT_SIZE 4096
175
176 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
177 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
178 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
179
180 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
181 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
182 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
183
184 SYSCTL_DECL(_net_utun);
185 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
186
187 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
188 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
189 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
190 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
191 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
192 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
193
194 static errno_t
195 utun_register_nexus(void);
196
197 static errno_t
198 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
199 static errno_t
200 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
201 proc_t p, kern_nexus_t nexus,
202 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
203 static errno_t
204 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
205 kern_channel_t channel);
206 static void
207 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
208 kern_channel_t channel);
209 static void
210 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
211 kern_channel_t channel);
212 static void
213 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
214 kern_channel_t channel);
215 static errno_t
216 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
217 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
218 void **ring_ctx);
219 static void
220 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
221 kern_channel_ring_t ring);
222 static errno_t
223 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
224 kern_channel_ring_t ring, uint32_t flags);
225 static errno_t
226 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
227 kern_channel_ring_t ring, uint32_t flags);
228 #endif // UTUN_NEXUS
229
230 #define UTUN_DEFAULT_MTU 1500
231 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
232
233 static kern_ctl_ref utun_kctlref;
234 static u_int32_t utun_family;
235 static lck_attr_t *utun_lck_attr;
236 static lck_grp_attr_t *utun_lck_grp_attr;
237 static lck_grp_t *utun_lck_grp;
238 static lck_mtx_t utun_lock;
239
240 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
241
242 #define UTUN_PCB_ZONE_MAX 32
243 #define UTUN_PCB_ZONE_NAME "net.if_utun"
244
245 static unsigned int utun_pcb_size; /* size of zone element */
246 static struct zone *utun_pcb_zone; /* zone for utun_pcb */
247
248 #if UTUN_NEXUS
249
250 static int
251 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
252 {
253 #pragma unused(arg1, arg2)
254 int value = if_utun_ring_size;
255
256 int error = sysctl_handle_int(oidp, &value, 0, req);
257 if (error || !req->newptr) {
258 return (error);
259 }
260
261 if (value < UTUN_IF_MIN_RING_SIZE ||
262 value > UTUN_IF_MAX_RING_SIZE) {
263 return (EINVAL);
264 }
265
266 if_utun_ring_size = value;
267
268 return (0);
269 }
270
271 static int
272 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
273 {
274 #pragma unused(arg1, arg2)
275 int value = if_utun_tx_fsw_ring_size;
276
277 int error = sysctl_handle_int(oidp, &value, 0, req);
278 if (error || !req->newptr) {
279 return (error);
280 }
281
282 if (value < UTUN_IF_MIN_RING_SIZE ||
283 value > UTUN_IF_MAX_RING_SIZE) {
284 return (EINVAL);
285 }
286
287 if_utun_tx_fsw_ring_size = value;
288
289 return (0);
290 }
291
292 static int
293 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
294 {
295 #pragma unused(arg1, arg2)
296 int value = if_utun_rx_fsw_ring_size;
297
298 int error = sysctl_handle_int(oidp, &value, 0, req);
299 if (error || !req->newptr) {
300 return (error);
301 }
302
303 if (value < UTUN_IF_MIN_RING_SIZE ||
304 value > UTUN_IF_MAX_RING_SIZE) {
305 return (EINVAL);
306 }
307
308 if_utun_rx_fsw_ring_size = value;
309
310 return (0);
311 }
312
313 static errno_t
314 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
315 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
316 void **ring_ctx)
317 {
318 #pragma unused(nxprov)
319 #pragma unused(channel)
320 #pragma unused(ring_ctx)
321 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
322 if (!is_tx_ring) {
323 VERIFY(pcb->utun_netif_rxring == NULL);
324 pcb->utun_netif_rxring = ring;
325 } else {
326 VERIFY(pcb->utun_netif_txring == NULL);
327 pcb->utun_netif_txring = ring;
328 }
329 return 0;
330 }
331
332 static void
333 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
334 kern_channel_ring_t ring)
335 {
336 #pragma unused(nxprov)
337 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
338 if (pcb->utun_netif_rxring == ring) {
339 pcb->utun_netif_rxring = NULL;
340 } else if (pcb->utun_netif_txring == ring) {
341 pcb->utun_netif_txring = NULL;
342 }
343 }
344
345 static errno_t
346 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
347 kern_channel_ring_t tx_ring, uint32_t flags)
348 {
349 #pragma unused(nxprov)
350 #pragma unused(flags)
351 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
352
353 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
354
355 lck_rw_lock_shared(&pcb->utun_pcb_lock);
356
357 struct kern_channel_ring_stat_increment tx_ring_stats;
358 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
359 kern_channel_slot_t tx_pslot = NULL;
360 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
361
362 STATS_INC(nifs, NETIF_STATS_TXSYNC);
363
364 if (tx_slot == NULL) {
365 // Nothing to write, don't bother signalling
366 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
367 return 0;
368 }
369
370 if (pcb->utun_kpipe_enabled) {
371 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
372 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
373
374 // Signal the kernel pipe ring to read
375 if (rx_ring != NULL) {
376 kern_channel_notify(rx_ring, 0);
377 }
378 return 0;
379 }
380
381 // If we're here, we're injecting into the utun kernel control socket
382 while (tx_slot != NULL) {
383 size_t length = 0;
384 mbuf_t data = NULL;
385
386 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
387
388 if (tx_ph == 0) {
389 // Advance TX ring
390 tx_pslot = tx_slot;
391 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
392 continue;
393 }
394 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
395
396 // Advance TX ring
397 tx_pslot = tx_slot;
398 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
399
400 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
401 VERIFY(tx_buf != NULL);
402
403 /* tx_baddr is the absolute buffer address */
404 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
405 VERIFY(tx_baddr != 0);
406
407 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
408
409 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
410 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
411
412 // The offset must be large enough for the headers
413 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
414
415 // Find family
416 uint32_t af = 0;
417 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
418 u_int ip_version = (vhl >> 4);
419 switch (ip_version) {
420 case 4: {
421 af = AF_INET;
422 break;
423 }
424 case 6: {
425 af = AF_INET6;
426 break;
427 }
428 default: {
429 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
430 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
431 UTUN_HEADER_SIZE(pcb));
432 break;
433 }
434 }
435
436 tx_offset -= UTUN_HEADER_SIZE(pcb);
437 tx_length += UTUN_HEADER_SIZE(pcb);
438 tx_baddr += tx_offset;
439
440 length = MIN(tx_length, pcb->utun_slot_size);
441
442 // Copy in family
443 memcpy(tx_baddr, &af, sizeof(af));
444 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
445 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
446 }
447
448 if (length > 0) {
449 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
450 if (error == 0) {
451 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
452 if (error == 0) {
453 error = utun_output(pcb->utun_ifp, data);
454 if (error != 0) {
455 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
456 }
457 } else {
458 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
459 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
460 STATS_INC(nifs, NETIF_STATS_DROPPED);
461 mbuf_freem(data);
462 data = NULL;
463 }
464 } else {
465 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
466 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
467 STATS_INC(nifs, NETIF_STATS_DROPPED);
468 }
469 } else {
470 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
471 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
472 STATS_INC(nifs, NETIF_STATS_DROPPED);
473 }
474
475 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
476
477 if (data == NULL) {
478 continue;
479 }
480
481 STATS_INC(nifs, NETIF_STATS_TXPKTS);
482 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
483
484 tx_ring_stats.kcrsi_slots_transferred++;
485 tx_ring_stats.kcrsi_bytes_transferred += length;
486 }
487
488 if (tx_pslot) {
489 kern_channel_advance_slot(tx_ring, tx_pslot);
490 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
491 (void)kern_channel_reclaim(tx_ring);
492 }
493
494 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
495
496 return 0;
497 }
498
499 static errno_t
500 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
501 kern_channel_ring_t ring, __unused uint32_t flags)
502 {
503 #pragma unused(nxprov)
504 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
505 boolean_t more = false;
506 errno_t rc = 0;
507
508 /*
509 * Refill and sync the ring; we may be racing against another thread doing
510 * an RX sync that also wants to do kr_enter(), and so use the blocking
511 * variant here.
512 */
513 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
514 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
515 printf("%s, tx refill failed %d\n", __func__, rc);
516 }
517
518 (void) kr_enter(ring, TRUE);
519 lck_rw_lock_shared(&pcb->utun_pcb_lock);
520
521 if (pcb->utun_kpipe_enabled) {
522 uint32_t tx_available = kern_channel_available_slot_count(ring);
523 if (pcb->utun_netif_txring_size > 0 &&
524 tx_available >= pcb->utun_netif_txring_size - 1) {
525 // No room left in tx ring, disable output for now
526 errno_t error = ifnet_disable_output(pcb->utun_ifp);
527 if (error != 0) {
528 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
529 }
530 }
531 }
532
533 if (pcb->utun_kpipe_enabled) {
534 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
535
536 // Unlock while calling notify
537 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
538 // Signal the kernel pipe ring to read
539 if (rx_ring != NULL) {
540 kern_channel_notify(rx_ring, 0);
541 }
542 } else {
543 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
544 }
545
546 kr_exit(ring);
547
548 return (0);
549 }
550
551 static errno_t
552 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
553 kern_channel_ring_t rx_ring, uint32_t flags)
554 {
555 #pragma unused(nxprov)
556 #pragma unused(flags)
557 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
558 struct kern_channel_ring_stat_increment rx_ring_stats;
559
560 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
561
562 lck_rw_lock_shared(&pcb->utun_pcb_lock);
563
564 // Reclaim user-released slots
565 (void) kern_channel_reclaim(rx_ring);
566
567 STATS_INC(nifs, NETIF_STATS_RXSYNC);
568
569 uint32_t avail = kern_channel_available_slot_count(rx_ring);
570 if (avail == 0) {
571 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
572 return 0;
573 }
574
575 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
576 VERIFY(rx_pp != NULL);
577 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
578 kern_channel_slot_t rx_pslot = NULL;
579 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
580
581 while (rx_slot != NULL) {
582 // Check for a waiting packet
583 lck_mtx_lock(&pcb->utun_input_chain_lock);
584 mbuf_t data = pcb->utun_input_chain;
585 if (data == NULL) {
586 lck_mtx_unlock(&pcb->utun_input_chain_lock);
587 break;
588 }
589
590 // Allocate rx packet
591 kern_packet_t rx_ph = 0;
592 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
593 if (__improbable(error != 0)) {
594 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
595 STATS_INC(nifs, NETIF_STATS_DROPPED);
596 lck_mtx_unlock(&pcb->utun_input_chain_lock);
597 break;
598 }
599
600 // Advance waiting packets
601 pcb->utun_input_chain = data->m_nextpkt;
602 data->m_nextpkt = NULL;
603 if (pcb->utun_input_chain == NULL) {
604 pcb->utun_input_chain_last = NULL;
605 }
606 lck_mtx_unlock(&pcb->utun_input_chain_lock);
607
608 size_t header_offset = UTUN_HEADER_SIZE(pcb);
609 size_t length = mbuf_pkthdr_len(data);
610
611 if (length < header_offset) {
612 // mbuf is too small
613 mbuf_freem(data);
614 kern_pbufpool_free(rx_pp, rx_ph);
615 STATS_INC(nifs, NETIF_STATS_BADLEN);
616 STATS_INC(nifs, NETIF_STATS_DROPPED);
617 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
618 pcb->utun_ifp->if_xname, length, header_offset);
619 continue;
620 }
621
622 length -= header_offset;
623 if (length > rx_pp->pp_buflet_size) {
624 // Flush data
625 mbuf_freem(data);
626 kern_pbufpool_free(rx_pp, rx_ph);
627 STATS_INC(nifs, NETIF_STATS_BADLEN);
628 STATS_INC(nifs, NETIF_STATS_DROPPED);
629 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
630 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
631 continue;
632 }
633
634 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
635
636 // Fillout rx packet
637 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
638 VERIFY(rx_buf != NULL);
639 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
640 VERIFY(rx_baddr != NULL);
641
642 // Copy-in data from mbuf to buflet
643 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
644 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
645
646 // Finalize and attach the packet
647 error = kern_buflet_set_data_offset(rx_buf, 0);
648 VERIFY(error == 0);
649 error = kern_buflet_set_data_length(rx_buf, length);
650 VERIFY(error == 0);
651 error = kern_packet_set_link_header_offset(rx_ph, 0);
652 VERIFY(error == 0);
653 error = kern_packet_set_network_header_offset(rx_ph, 0);
654 VERIFY(error == 0);
655 error = kern_packet_finalize(rx_ph);
656 VERIFY(error == 0);
657 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
658 VERIFY(error == 0);
659
660 STATS_INC(nifs, NETIF_STATS_RXPKTS);
661 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
662 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
663
664 rx_ring_stats.kcrsi_slots_transferred++;
665 rx_ring_stats.kcrsi_bytes_transferred += length;
666
667 mbuf_freem(data);
668
669 // Advance ring
670 rx_pslot = rx_slot;
671 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
672 }
673
674 struct kern_channel_ring_stat_increment tx_ring_stats;
675 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
676 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
677 kern_channel_slot_t tx_pslot = NULL;
678 kern_channel_slot_t tx_slot = NULL;
679 if (tx_ring == NULL) {
680 // Net-If TX ring not set up yet, nothing to read
681 goto done;
682 }
683
684 // Unlock utun before entering ring
685 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
686
687 (void)kr_enter(tx_ring, TRUE);
688
689 // Lock again after entering and validate
690 lck_rw_lock_shared(&pcb->utun_pcb_lock);
691 if (tx_ring != pcb->utun_kpipe_txring) {
692 goto done;
693 }
694
695 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
696 if (tx_slot == NULL) {
697 // Nothing to read, don't bother signalling
698 goto done;
699 }
700
701 while (rx_slot != NULL && tx_slot != NULL) {
702 // Allocate rx packet
703 kern_packet_t rx_ph = 0;
704 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
705
706 // Advance TX ring
707 tx_pslot = tx_slot;
708 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
709
710 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
711 if (tx_ph == 0) {
712 continue;
713 }
714
715 /* XXX We could try this alloc before advancing the slot to avoid
716 * dropping the packet on failure to allocate.
717 */
718 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
719 if (__improbable(error != 0)) {
720 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
721 STATS_INC(nifs, NETIF_STATS_DROPPED);
722 break;
723 }
724
725 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
726 VERIFY(tx_buf != NULL);
727 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
728 VERIFY(tx_baddr != 0);
729 tx_baddr += kern_buflet_get_data_offset(tx_buf);
730
731 // Check packet length
732 size_t header_offset = UTUN_HEADER_SIZE(pcb);
733 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
734 if (tx_length < header_offset) {
735 // Packet is too small
736 kern_pbufpool_free(rx_pp, rx_ph);
737 STATS_INC(nifs, NETIF_STATS_BADLEN);
738 STATS_INC(nifs, NETIF_STATS_DROPPED);
739 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
740 pcb->utun_ifp->if_xname, tx_length, header_offset);
741 continue;
742 }
743
744 size_t length = MIN(tx_length - header_offset,
745 pcb->utun_slot_size);
746
747 tx_ring_stats.kcrsi_slots_transferred++;
748 tx_ring_stats.kcrsi_bytes_transferred += length;
749
750 // Fillout rx packet
751 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
752 VERIFY(rx_buf != NULL);
753 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
754 VERIFY(rx_baddr != NULL);
755
756 // Copy-in data from tx to rx
757 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
758 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
759
760 // Finalize and attach the packet
761 error = kern_buflet_set_data_offset(rx_buf, 0);
762 VERIFY(error == 0);
763 error = kern_buflet_set_data_length(rx_buf, length);
764 VERIFY(error == 0);
765 error = kern_packet_set_link_header_offset(rx_ph, 0);
766 VERIFY(error == 0);
767 error = kern_packet_set_network_header_offset(rx_ph, 0);
768 VERIFY(error == 0);
769 error = kern_packet_finalize(rx_ph);
770 VERIFY(error == 0);
771 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
772 VERIFY(error == 0);
773
774 STATS_INC(nifs, NETIF_STATS_RXPKTS);
775 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
776 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
777
778 rx_ring_stats.kcrsi_slots_transferred++;
779 rx_ring_stats.kcrsi_bytes_transferred += length;
780
781 rx_pslot = rx_slot;
782 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
783 }
784
785 done:
786 if (rx_pslot) {
787 kern_channel_advance_slot(rx_ring, rx_pslot);
788 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
789 }
790
791 if (tx_pslot) {
792 kern_channel_advance_slot(tx_ring, tx_pslot);
793 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
794 (void)kern_channel_reclaim(tx_ring);
795 }
796
797 // Unlock first, then exit ring
798 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
799 if (tx_ring != NULL) {
800 if (tx_pslot != NULL) {
801 kern_channel_notify(tx_ring, 0);
802 }
803 kr_exit(tx_ring);
804 }
805
806 return 0;
807 }
808
809 static errno_t
810 utun_nexus_ifattach(struct utun_pcb *pcb,
811 struct ifnet_init_eparams *init_params,
812 struct ifnet **ifp)
813 {
814 errno_t err;
815 nexus_controller_t controller = kern_nexus_shared_controller();
816 struct kern_nexus_net_init net_init;
817 struct kern_pbufpool_init pp_init;
818
819 nexus_name_t provider_name;
820 snprintf((char *)provider_name, sizeof(provider_name),
821 "com.apple.netif.%s", pcb->utun_if_xname);
822
823 struct kern_nexus_provider_init prov_init = {
824 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
825 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
826 .nxpi_pre_connect = utun_nexus_pre_connect,
827 .nxpi_connected = utun_nexus_connected,
828 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
829 .nxpi_disconnected = utun_nexus_disconnected,
830 .nxpi_ring_init = utun_netif_ring_init,
831 .nxpi_ring_fini = utun_netif_ring_fini,
832 .nxpi_slot_init = NULL,
833 .nxpi_slot_fini = NULL,
834 .nxpi_sync_tx = utun_netif_sync_tx,
835 .nxpi_sync_rx = utun_netif_sync_rx,
836 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
837 };
838
839 nexus_attr_t nxa = NULL;
840 err = kern_nexus_attr_create(&nxa);
841 if (err != 0) {
842 printf("%s: kern_nexus_attr_create failed: %d\n",
843 __func__, err);
844 goto failed;
845 }
846
847 uint64_t slot_buffer_size = pcb->utun_slot_size;
848 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
849 VERIFY(err == 0);
850
851 // Reset ring size for netif nexus to limit memory usage
852 uint64_t ring_size = pcb->utun_netif_ring_size;
853 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
854 VERIFY(err == 0);
855 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
856 VERIFY(err == 0);
857
858 pcb->utun_netif_txring_size = ring_size;
859
860 bzero(&pp_init, sizeof (pp_init));
861 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
862 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
863 pp_init.kbi_bufsize = pcb->utun_slot_size;
864 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
865 pp_init.kbi_max_frags = 1;
866 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
867 "%s", provider_name);
868
869 err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_netif_pp, NULL);
870 if (err != 0) {
871 printf("%s pbufbool create failed, error %d\n", __func__, err);
872 goto failed;
873 }
874
875 err = kern_nexus_controller_register_provider(controller,
876 utun_nx_dom_prov,
877 provider_name,
878 &prov_init,
879 sizeof(prov_init),
880 nxa,
881 &pcb->utun_nx.if_provider);
882 if (err != 0) {
883 printf("%s register provider failed, error %d\n",
884 __func__, err);
885 goto failed;
886 }
887
888 bzero(&net_init, sizeof(net_init));
889 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
890 net_init.nxneti_flags = 0;
891 net_init.nxneti_eparams = init_params;
892 net_init.nxneti_lladdr = NULL;
893 net_init.nxneti_prepare = utun_netif_prepare;
894 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
895 err = kern_nexus_controller_alloc_net_provider_instance(controller,
896 pcb->utun_nx.if_provider,
897 pcb,
898 &pcb->utun_nx.if_instance,
899 &net_init,
900 ifp);
901 if (err != 0) {
902 printf("%s alloc_net_provider_instance failed, %d\n",
903 __func__, err);
904 kern_nexus_controller_deregister_provider(controller,
905 pcb->utun_nx.if_provider);
906 uuid_clear(pcb->utun_nx.if_provider);
907 goto failed;
908 }
909
910 failed:
911 if (nxa) {
912 kern_nexus_attr_destroy(nxa);
913 }
914 if (err && pcb->utun_netif_pp != NULL) {
915 kern_pbufpool_destroy(pcb->utun_netif_pp);
916 pcb->utun_netif_pp = NULL;
917 }
918 return (err);
919 }
920
921 static void
922 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
923 {
924 nexus_controller_t controller = kern_nexus_shared_controller();
925 errno_t err;
926
927 if (!uuid_is_null(instance)) {
928 err = kern_nexus_controller_free_provider_instance(controller,
929 instance);
930 if (err != 0) {
931 printf("%s free_provider_instance failed %d\n",
932 __func__, err);
933 }
934 uuid_clear(instance);
935 }
936 if (!uuid_is_null(provider)) {
937 err = kern_nexus_controller_deregister_provider(controller,
938 provider);
939 if (err != 0) {
940 printf("%s deregister_provider %d\n", __func__, err);
941 }
942 uuid_clear(provider);
943 }
944 return;
945 }
946
947 static void
948 utun_nexus_detach(struct utun_pcb *pcb)
949 {
950 utun_nx_t nx = &pcb->utun_nx;
951 nexus_controller_t controller = kern_nexus_shared_controller();
952 errno_t err;
953
954 if (!uuid_is_null(nx->ms_host)) {
955 err = kern_nexus_ifdetach(controller,
956 nx->ms_instance,
957 nx->ms_host);
958 if (err != 0) {
959 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
960 __func__, err);
961 }
962 }
963
964 if (!uuid_is_null(nx->ms_device)) {
965 err = kern_nexus_ifdetach(controller,
966 nx->ms_instance,
967 nx->ms_device);
968 if (err != 0) {
969 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
970 __func__, err);
971 }
972 }
973
974 utun_detach_provider_and_instance(nx->if_provider,
975 nx->if_instance);
976 utun_detach_provider_and_instance(nx->ms_provider,
977 nx->ms_instance);
978
979 if (pcb->utun_netif_pp != NULL) {
980 kern_pbufpool_destroy(pcb->utun_netif_pp);
981 pcb->utun_netif_pp = NULL;
982
983 }
984 memset(nx, 0, sizeof(*nx));
985 }
986
987 static errno_t
988 utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
989 uint32_t subtype, const char *type_name,
990 const char *ifname,
991 uuid_t *provider, uuid_t *instance)
992 {
993 nexus_attr_t attr = NULL;
994 nexus_controller_t controller = kern_nexus_shared_controller();
995 uuid_t dom_prov;
996 errno_t err;
997 struct kern_nexus_init init;
998 nexus_name_t provider_name;
999
1000 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1001 &dom_prov);
1002 if (err != 0) {
1003 printf("%s can't get %s provider, error %d\n",
1004 __func__, type_name, err);
1005 goto failed;
1006 }
1007
1008 err = kern_nexus_attr_create(&attr);
1009 if (err != 0) {
1010 printf("%s: kern_nexus_attr_create failed: %d\n",
1011 __func__, err);
1012 goto failed;
1013 }
1014
1015 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
1016 VERIFY(err == 0);
1017
1018 uint64_t slot_buffer_size = pcb->utun_slot_size;
1019 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1020 VERIFY(err == 0);
1021
1022 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
1023 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
1024 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1025 VERIFY(err == 0);
1026 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
1027 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1028 VERIFY(err == 0);
1029
1030 snprintf((char *)provider_name, sizeof(provider_name),
1031 "com.apple.%s.%s", type_name, ifname);
1032 err = kern_nexus_controller_register_provider(controller,
1033 dom_prov,
1034 provider_name,
1035 NULL,
1036 0,
1037 attr,
1038 provider);
1039 kern_nexus_attr_destroy(attr);
1040 attr = NULL;
1041 if (err != 0) {
1042 printf("%s register %s provider failed, error %d\n",
1043 __func__, type_name, err);
1044 goto failed;
1045 }
1046 bzero(&init, sizeof (init));
1047 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1048 err = kern_nexus_controller_alloc_provider_instance(controller,
1049 *provider,
1050 NULL,
1051 instance, &init);
1052 if (err != 0) {
1053 printf("%s alloc_provider_instance %s failed, %d\n",
1054 __func__, type_name, err);
1055 kern_nexus_controller_deregister_provider(controller,
1056 *provider);
1057 uuid_clear(*provider);
1058 }
1059 failed:
1060 return (err);
1061 }
1062
1063 static errno_t
1064 utun_multistack_attach(struct utun_pcb *pcb)
1065 {
1066 nexus_controller_t controller = kern_nexus_shared_controller();
1067 errno_t err = 0;
1068 utun_nx_t nx = &pcb->utun_nx;
1069
1070 // Allocate multistack flowswitch
1071 err = utun_create_fs_provider_and_instance(pcb,
1072 NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
1073 "multistack",
1074 pcb->utun_ifp->if_xname,
1075 &nx->ms_provider,
1076 &nx->ms_instance);
1077 if (err != 0) {
1078 printf("%s: failed to create bridge provider and instance\n",
1079 __func__);
1080 goto failed;
1081 }
1082
1083 // Attach multistack to device port
1084 err = kern_nexus_ifattach(controller, nx->ms_instance,
1085 NULL, nx->if_instance,
1086 FALSE, &nx->ms_device);
1087 if (err != 0) {
1088 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1089 goto failed;
1090 }
1091
1092 // Attach multistack to host port
1093 err = kern_nexus_ifattach(controller, nx->ms_instance,
1094 NULL, nx->if_instance,
1095 TRUE, &nx->ms_host);
1096 if (err != 0) {
1097 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1098 goto failed;
1099 }
1100
1101 // Extract the agent UUID and save for later
1102 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1103 if (multistack_nx != NULL) {
1104 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1105 if (flowswitch != NULL) {
1106 FSW_RLOCK(flowswitch);
1107 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1108 if (ms_context != NULL) {
1109 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1110 } else {
1111 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1112 }
1113 FSW_UNLOCK(flowswitch);
1114 } else {
1115 printf("utun_multistack_attach - flowswitch is NULL\n");
1116 }
1117 nx_release(multistack_nx);
1118 } else {
1119 printf("utun_multistack_attach - unable to find multistack nexus\n");
1120 }
1121
1122 return (0);
1123
1124 failed:
1125 utun_nexus_detach(pcb);
1126
1127 errno_t detach_error = 0;
1128 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1129 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1130 /* NOT REACHED */
1131 }
1132
1133 return (err);
1134 }
1135
1136 static errno_t
1137 utun_register_kernel_pipe_nexus(void)
1138 {
1139 nexus_attr_t nxa = NULL;
1140 errno_t result;
1141
1142 lck_mtx_lock(&utun_lock);
1143 if (utun_ncd_refcount++) {
1144 lck_mtx_unlock(&utun_lock);
1145 return 0;
1146 }
1147
1148 result = kern_nexus_controller_create(&utun_ncd);
1149 if (result) {
1150 printf("%s: kern_nexus_controller_create failed: %d\n",
1151 __FUNCTION__, result);
1152 goto done;
1153 }
1154
1155 uuid_t dom_prov;
1156 result = kern_nexus_get_builtin_domain_provider(
1157 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1158 if (result) {
1159 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1160 __FUNCTION__, result);
1161 goto done;
1162 }
1163
1164 struct kern_nexus_provider_init prov_init = {
1165 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1166 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1167 .nxpi_pre_connect = utun_nexus_pre_connect,
1168 .nxpi_connected = utun_nexus_connected,
1169 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1170 .nxpi_disconnected = utun_nexus_disconnected,
1171 .nxpi_ring_init = utun_kpipe_ring_init,
1172 .nxpi_ring_fini = utun_kpipe_ring_fini,
1173 .nxpi_slot_init = NULL,
1174 .nxpi_slot_fini = NULL,
1175 .nxpi_sync_tx = utun_kpipe_sync_tx,
1176 .nxpi_sync_rx = utun_kpipe_sync_rx,
1177 .nxpi_tx_doorbell = NULL,
1178 };
1179
1180 result = kern_nexus_attr_create(&nxa);
1181 if (result) {
1182 printf("%s: kern_nexus_attr_create failed: %d\n",
1183 __FUNCTION__, result);
1184 goto done;
1185 }
1186
1187 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1188 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1189 VERIFY(result == 0);
1190
1191 // Reset ring size for kernel pipe nexus to limit memory usage
1192 uint64_t ring_size = if_utun_ring_size;
1193 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1194 VERIFY(result == 0);
1195 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1196 VERIFY(result == 0);
1197
1198 result = kern_nexus_controller_register_provider(utun_ncd,
1199 dom_prov,
1200 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1201 &prov_init,
1202 sizeof(prov_init),
1203 nxa,
1204 &utun_kpipe_uuid);
1205 if (result) {
1206 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1207 __FUNCTION__, result);
1208 goto done;
1209 }
1210
1211 done:
1212 if (nxa) {
1213 kern_nexus_attr_destroy(nxa);
1214 }
1215
1216 if (result) {
1217 if (utun_ncd) {
1218 kern_nexus_controller_destroy(utun_ncd);
1219 utun_ncd = NULL;
1220 }
1221 utun_ncd_refcount = 0;
1222 }
1223
1224 lck_mtx_unlock(&utun_lock);
1225
1226 return result;
1227 }
1228
1229 static void
1230 utun_unregister_kernel_pipe_nexus(void)
1231 {
1232 lck_mtx_lock(&utun_lock);
1233
1234 VERIFY(utun_ncd_refcount > 0);
1235
1236 if (--utun_ncd_refcount == 0) {
1237 kern_nexus_controller_destroy(utun_ncd);
1238 utun_ncd = NULL;
1239 }
1240
1241 lck_mtx_unlock(&utun_lock);
1242 }
1243
1244 // For use by socket option, not internally
1245 static errno_t
1246 utun_disable_channel(struct utun_pcb *pcb)
1247 {
1248 errno_t result;
1249 int enabled;
1250 uuid_t uuid;
1251
1252 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1253
1254 enabled = pcb->utun_kpipe_enabled;
1255 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1256
1257 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1258
1259 pcb->utun_kpipe_enabled = 0;
1260 uuid_clear(pcb->utun_kpipe_uuid);
1261
1262 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1263
1264 if (enabled) {
1265 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1266 } else {
1267 result = ENXIO;
1268 }
1269
1270 if (!result) {
1271 if (pcb->utun_kpipe_pp != NULL) {
1272 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1273 pcb->utun_kpipe_pp = NULL;
1274 }
1275 utun_unregister_kernel_pipe_nexus();
1276 }
1277
1278 return result;
1279 }
1280
1281 static errno_t
1282 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1283 {
1284 struct kern_nexus_init init;
1285 struct kern_pbufpool_init pp_init;
1286 errno_t result;
1287
1288 result = utun_register_kernel_pipe_nexus();
1289 if (result) {
1290 return result;
1291 }
1292
1293 VERIFY(utun_ncd);
1294
1295 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1296
1297 if (pcb->utun_kpipe_enabled) {
1298 result = EEXIST; // return success instead?
1299 goto done;
1300 }
1301
1302 /*
1303 * Make sure we can fit packets in the channel buffers and
1304 * Allow an extra 4 bytes for the protocol number header in the channel
1305 */
1306 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
1307 result = EOPNOTSUPP;
1308 goto done;
1309 }
1310
1311 bzero(&pp_init, sizeof (pp_init));
1312 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1313 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1314 pp_init.kbi_bufsize = pcb->utun_slot_size;
1315 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1316 pp_init.kbi_max_frags = 1;
1317 pp_init.kbi_flags |= KBIF_QUANTUM;
1318 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
1319 "com.apple.kpipe.%s", pcb->utun_if_xname);
1320
1321 result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->utun_kpipe_pp,
1322 NULL);
1323 if (result != 0) {
1324 printf("%s pbufbool create failed, error %d\n", __func__, result);
1325 goto done;
1326 }
1327
1328 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1329 bzero(&init, sizeof (init));
1330 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1331 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
1332 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1333 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1334 if (result) {
1335 goto done;
1336 }
1337
1338 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1339 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1340 pcb->utun_kpipe_uuid, &port,
1341 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1342 if (result) {
1343 kern_nexus_controller_free_provider_instance(utun_ncd,
1344 pcb->utun_kpipe_uuid);
1345 uuid_clear(pcb->utun_kpipe_uuid);
1346 goto done;
1347 }
1348
1349 pcb->utun_kpipe_enabled = 1;
1350
1351 done:
1352 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1353
1354 if (result) {
1355 if (pcb->utun_kpipe_pp != NULL) {
1356 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1357 pcb->utun_kpipe_pp = NULL;
1358 }
1359 utun_unregister_kernel_pipe_nexus();
1360 }
1361
1362 return result;
1363 }
1364
1365 #endif // UTUN_NEXUS
1366
1367 errno_t
1368 utun_register_control(void)
1369 {
1370 struct kern_ctl_reg kern_ctl;
1371 errno_t result = 0;
1372
1373 /* Find a unique value for our interface family */
1374 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1375 if (result != 0) {
1376 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1377 return result;
1378 }
1379
1380 utun_pcb_size = sizeof(struct utun_pcb);
1381 utun_pcb_zone = zinit(utun_pcb_size,
1382 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1383 0, UTUN_PCB_ZONE_NAME);
1384 if (utun_pcb_zone == NULL) {
1385 printf("utun_register_control - zinit(utun_pcb) failed");
1386 return ENOMEM;
1387 }
1388
1389 #if UTUN_NEXUS
1390 utun_register_nexus();
1391 #endif // UTUN_NEXUS
1392
1393 TAILQ_INIT(&utun_head);
1394
1395 bzero(&kern_ctl, sizeof(kern_ctl));
1396 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1397 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1398 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
1399 kern_ctl.ctl_sendsize = 512 * 1024;
1400 kern_ctl.ctl_recvsize = 512 * 1024;
1401 kern_ctl.ctl_bind = utun_ctl_bind;
1402 kern_ctl.ctl_connect = utun_ctl_connect;
1403 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1404 kern_ctl.ctl_send = utun_ctl_send;
1405 kern_ctl.ctl_setopt = utun_ctl_setopt;
1406 kern_ctl.ctl_getopt = utun_ctl_getopt;
1407 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1408
1409 result = ctl_register(&kern_ctl, &utun_kctlref);
1410 if (result != 0) {
1411 printf("utun_register_control - ctl_register failed: %d\n", result);
1412 return result;
1413 }
1414
1415 /* Register the protocol plumbers */
1416 if ((result = proto_register_plumber(PF_INET, utun_family,
1417 utun_attach_proto, NULL)) != 0) {
1418 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
1419 utun_family, result);
1420 ctl_deregister(utun_kctlref);
1421 return result;
1422 }
1423
1424 /* Register the protocol plumbers */
1425 if ((result = proto_register_plumber(PF_INET6, utun_family,
1426 utun_attach_proto, NULL)) != 0) {
1427 proto_unregister_plumber(PF_INET, utun_family);
1428 ctl_deregister(utun_kctlref);
1429 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
1430 utun_family, result);
1431 return result;
1432 }
1433
1434 utun_lck_attr = lck_attr_alloc_init();
1435 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1436 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1437
1438 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1439
1440 return 0;
1441 }
1442
1443 /* Kernel control functions */
1444
1445 static inline void
1446 utun_free_pcb(struct utun_pcb *pcb, bool in_list)
1447 {
1448 #ifdef UTUN_NEXUS
1449 mbuf_freem_list(pcb->utun_input_chain);
1450 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1451 #endif // UTUN_NEXUS
1452 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1453 if (in_list) {
1454 lck_mtx_lock(&utun_lock);
1455 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1456 lck_mtx_unlock(&utun_lock);
1457 }
1458 zfree(utun_pcb_zone, pcb);
1459 }
1460
1461 static errno_t
1462 utun_ctl_bind(kern_ctl_ref kctlref,
1463 struct sockaddr_ctl *sac,
1464 void **unitinfo)
1465 {
1466 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1467 memset(pcb, 0, sizeof(*pcb));
1468
1469 *unitinfo = pcb;
1470 pcb->utun_ctlref = kctlref;
1471 pcb->utun_unit = sac->sc_unit;
1472 pcb->utun_max_pending_packets = 1;
1473
1474 #if UTUN_NEXUS
1475 pcb->utun_use_netif = false;
1476 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1477 pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
1478 pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
1479 pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
1480 #endif // UTUN_NEXUS
1481
1482 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1483 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1484
1485 return (0);
1486 }
1487
1488 static errno_t
1489 utun_ctl_connect(kern_ctl_ref kctlref,
1490 struct sockaddr_ctl *sac,
1491 void **unitinfo)
1492 {
1493 struct ifnet_init_eparams utun_init = {};
1494 errno_t result = 0;
1495
1496 if (*unitinfo == NULL) {
1497 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1498 }
1499
1500 struct utun_pcb *pcb = *unitinfo;
1501
1502 lck_mtx_lock(&utun_lock);
1503
1504 /* Find some open interface id */
1505 u_int32_t chosen_unique_id = 1;
1506 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1507 if (next_pcb != NULL) {
1508 /* List was not empty, add one to the last item */
1509 chosen_unique_id = next_pcb->utun_unique_id + 1;
1510 next_pcb = NULL;
1511
1512 /*
1513 * If this wrapped the id number, start looking at
1514 * the front of the list for an unused id.
1515 */
1516 if (chosen_unique_id == 0) {
1517 /* Find the next unused ID */
1518 chosen_unique_id = 1;
1519 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1520 if (next_pcb->utun_unique_id > chosen_unique_id) {
1521 /* We found a gap */
1522 break;
1523 }
1524
1525 chosen_unique_id = next_pcb->utun_unique_id + 1;
1526 }
1527 }
1528 }
1529
1530 pcb->utun_unique_id = chosen_unique_id;
1531
1532 if (next_pcb != NULL) {
1533 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1534 } else {
1535 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1536 }
1537 lck_mtx_unlock(&utun_lock);
1538
1539 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1540 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1541 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
1542
1543 /* Create the interface */
1544 bzero(&utun_init, sizeof(utun_init));
1545 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1546 utun_init.len = sizeof (utun_init);
1547
1548 #if UTUN_NEXUS
1549 if (pcb->utun_use_netif) {
1550 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1551 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1552 } else
1553 #endif // UTUN_NEXUS
1554 {
1555 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1556 utun_init.start = utun_start;
1557 utun_init.framer_extended = utun_framer;
1558 }
1559 utun_init.name = "utun";
1560 utun_init.unit = pcb->utun_unit - 1;
1561 utun_init.uniqueid = pcb->utun_unique_name;
1562 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1563 utun_init.family = utun_family;
1564 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
1565 utun_init.type = IFT_OTHER;
1566 utun_init.demux = utun_demux;
1567 utun_init.add_proto = utun_add_proto;
1568 utun_init.del_proto = utun_del_proto;
1569 utun_init.softc = pcb;
1570 utun_init.ioctl = utun_ioctl;
1571 utun_init.detach = utun_detached;
1572
1573 #if UTUN_NEXUS
1574 if (pcb->utun_use_netif) {
1575 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1576 if (result != 0) {
1577 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1578 utun_free_pcb(pcb, true);
1579 *unitinfo = NULL;
1580 return result;
1581 }
1582
1583 result = utun_multistack_attach(pcb);
1584 if (result != 0) {
1585 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1586 *unitinfo = NULL;
1587 return result;
1588 }
1589
1590 /* Attach to bpf */
1591 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1592 } else
1593 #endif // UTUN_NEXUS
1594 {
1595 /*
1596 * Upon success, this holds an ifnet reference which we will
1597 * release via ifnet_release() at final detach time.
1598 */
1599 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1600 if (result != 0) {
1601 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1602 utun_free_pcb(pcb, true);
1603 *unitinfo = NULL;
1604 return result;
1605 }
1606
1607 /* Set flags and additional information. */
1608 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1609 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1610
1611 /* The interface must generate its own IPv6 LinkLocal address,
1612 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1613 */
1614 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1615
1616 /* Reset the stats in case as the interface may have been recycled */
1617 struct ifnet_stats_param stats;
1618 bzero(&stats, sizeof(struct ifnet_stats_param));
1619 ifnet_set_stat(pcb->utun_ifp, &stats);
1620
1621 /* Attach the interface */
1622 result = ifnet_attach(pcb->utun_ifp, NULL);
1623 if (result != 0) {
1624 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
1625 /* Release reference now since attach failed */
1626 ifnet_release(pcb->utun_ifp);
1627 utun_free_pcb(pcb, true);
1628 *unitinfo = NULL;
1629 return (result);
1630 }
1631
1632 /* Attach to bpf */
1633 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1634 }
1635
1636 /* The interfaces resoures allocated, mark it as running */
1637 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1638
1639 return result;
1640 }
1641
1642 static errno_t
1643 utun_detach_ip(ifnet_t interface,
1644 protocol_family_t protocol,
1645 socket_t pf_socket)
1646 {
1647 errno_t result = EPROTONOSUPPORT;
1648
1649 /* Attempt a detach */
1650 if (protocol == PF_INET) {
1651 struct ifreq ifr;
1652
1653 bzero(&ifr, sizeof(ifr));
1654 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1655 ifnet_name(interface), ifnet_unit(interface));
1656
1657 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1658 } else if (protocol == PF_INET6) {
1659 struct in6_ifreq ifr6;
1660
1661 bzero(&ifr6, sizeof(ifr6));
1662 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1663 ifnet_name(interface), ifnet_unit(interface));
1664
1665 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1666 }
1667
1668 return result;
1669 }
1670
1671 static void
1672 utun_remove_address(ifnet_t interface,
1673 protocol_family_t protocol,
1674 ifaddr_t address,
1675 socket_t pf_socket)
1676 {
1677 errno_t result = 0;
1678
1679 /* Attempt a detach */
1680 if (protocol == PF_INET) {
1681 struct ifreq ifr;
1682
1683 bzero(&ifr, sizeof(ifr));
1684 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1685 ifnet_name(interface), ifnet_unit(interface));
1686 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1687 if (result != 0) {
1688 printf("utun_remove_address - ifaddr_address failed: %d", result);
1689 } else {
1690 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1691 if (result != 0) {
1692 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1693 }
1694 }
1695 } else if (protocol == PF_INET6) {
1696 struct in6_ifreq ifr6;
1697
1698 bzero(&ifr6, sizeof(ifr6));
1699 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1700 ifnet_name(interface), ifnet_unit(interface));
1701 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1702 sizeof(ifr6.ifr_addr));
1703 if (result != 0) {
1704 printf("utun_remove_address - ifaddr_address failed (v6): %d",
1705 result);
1706 } else {
1707 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1708 if (result != 0) {
1709 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1710 result);
1711 }
1712 }
1713 }
1714 }
1715
1716 static void
1717 utun_cleanup_family(ifnet_t interface,
1718 protocol_family_t protocol)
1719 {
1720 errno_t result = 0;
1721 socket_t pf_socket = NULL;
1722 ifaddr_t *addresses = NULL;
1723 int i;
1724
1725 if (protocol != PF_INET && protocol != PF_INET6) {
1726 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1727 return;
1728 }
1729
1730 /* Create a socket for removing addresses and detaching the protocol */
1731 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1732 if (result != 0) {
1733 if (result != EAFNOSUPPORT)
1734 printf("utun_cleanup_family - failed to create %s socket: %d\n",
1735 protocol == PF_INET ? "IP" : "IPv6", result);
1736 goto cleanup;
1737 }
1738
1739 /* always set SS_PRIV, we want to close and detach regardless */
1740 sock_setpriv(pf_socket, 1);
1741
1742 result = utun_detach_ip(interface, protocol, pf_socket);
1743 if (result == 0 || result == ENXIO) {
1744 /* We are done! We either detached or weren't attached. */
1745 goto cleanup;
1746 } else if (result != EBUSY) {
1747 /* Uh, not really sure what happened here... */
1748 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1749 goto cleanup;
1750 }
1751
1752 /*
1753 * At this point, we received an EBUSY error. This means there are
1754 * addresses attached. We should detach them and then try again.
1755 */
1756 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1757 if (result != 0) {
1758 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1759 ifnet_name(interface), ifnet_unit(interface),
1760 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1761 goto cleanup;
1762 }
1763
1764 for (i = 0; addresses[i] != 0; i++) {
1765 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1766 }
1767 ifnet_free_address_list(addresses);
1768 addresses = NULL;
1769
1770 /*
1771 * The addresses should be gone, we should try the remove again.
1772 */
1773 result = utun_detach_ip(interface, protocol, pf_socket);
1774 if (result != 0 && result != ENXIO) {
1775 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1776 }
1777
1778 cleanup:
1779 if (pf_socket != NULL) {
1780 sock_close(pf_socket);
1781 }
1782
1783 if (addresses != NULL) {
1784 ifnet_free_address_list(addresses);
1785 }
1786 }
1787
1788 static errno_t
1789 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1790 __unused u_int32_t unit,
1791 void *unitinfo)
1792 {
1793 struct utun_pcb *pcb = unitinfo;
1794 ifnet_t ifp = NULL;
1795 errno_t result = 0;
1796
1797 if (pcb == NULL) {
1798 return EINVAL;
1799 }
1800
1801 #if UTUN_NEXUS
1802 // Tell the nexus to stop all rings
1803 if (pcb->utun_netif_nexus != NULL) {
1804 kern_nexus_stop(pcb->utun_netif_nexus);
1805 }
1806 #endif // UTUN_NEXUS
1807
1808 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1809
1810 #if UTUN_NEXUS
1811 uuid_t kpipe_uuid;
1812 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1813 uuid_clear(pcb->utun_kpipe_uuid);
1814 pcb->utun_kpipe_enabled = FALSE;
1815 #endif // UTUN_NEXUS
1816
1817 pcb->utun_ctlref = NULL;
1818
1819 ifp = pcb->utun_ifp;
1820 if (ifp != NULL) {
1821 #if UTUN_NEXUS
1822 // Tell the nexus to stop all rings
1823 if (pcb->utun_netif_nexus != NULL) {
1824 /*
1825 * Quiesce the interface and flush any pending outbound packets.
1826 */
1827 if_down(ifp);
1828
1829 /* Increment refcnt, but detach interface */
1830 ifnet_incr_iorefcnt(ifp);
1831 if ((result = ifnet_detach(ifp)) != 0) {
1832 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1833 }
1834
1835 /*
1836 * We want to do everything in our power to ensure that the interface
1837 * really goes away when the socket is closed. We must remove IP/IPv6
1838 * addresses and detach the protocols. Finally, we can remove and
1839 * release the interface.
1840 */
1841 utun_cleanup_family(ifp, AF_INET);
1842 utun_cleanup_family(ifp, AF_INET6);
1843
1844 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1845
1846 if (!uuid_is_null(kpipe_uuid)) {
1847 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1848 if (pcb->utun_kpipe_pp != NULL) {
1849 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1850 pcb->utun_kpipe_pp = NULL;
1851 }
1852 utun_unregister_kernel_pipe_nexus();
1853 }
1854 }
1855 utun_nexus_detach(pcb);
1856
1857 /* Decrement refcnt to finish detaching and freeing */
1858 ifnet_decr_iorefcnt(ifp);
1859 } else
1860 #endif // UTUN_NEXUS
1861 {
1862 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1863
1864 #if UTUN_NEXUS
1865 if (!uuid_is_null(kpipe_uuid)) {
1866 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1867 if (pcb->utun_kpipe_pp != NULL) {
1868 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1869 pcb->utun_kpipe_pp = NULL;
1870 }
1871 utun_unregister_kernel_pipe_nexus();
1872 }
1873 }
1874 #endif // UTUN_NEXUS
1875
1876 /*
1877 * We want to do everything in our power to ensure that the interface
1878 * really goes away when the socket is closed. We must remove IP/IPv6
1879 * addresses and detach the protocols. Finally, we can remove and
1880 * release the interface.
1881 */
1882 utun_cleanup_family(ifp, AF_INET);
1883 utun_cleanup_family(ifp, AF_INET6);
1884
1885 /*
1886 * Detach now; utun_detach() will be called asynchronously once
1887 * the I/O reference count drops to 0. There we will invoke
1888 * ifnet_release().
1889 */
1890 if ((result = ifnet_detach(ifp)) != 0) {
1891 printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1892 }
1893 }
1894 } else {
1895 // Bound, but not connected
1896 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1897 utun_free_pcb(pcb, false);
1898 }
1899
1900 return 0;
1901 }
1902
1903 static errno_t
1904 utun_ctl_send(__unused kern_ctl_ref kctlref,
1905 __unused u_int32_t unit,
1906 void *unitinfo,
1907 mbuf_t m,
1908 __unused int flags)
1909 {
1910 /*
1911 * The userland ABI requires the first four bytes have the protocol family
1912 * in network byte order: swap them
1913 */
1914 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
1915 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
1916 } else {
1917 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
1918 }
1919
1920 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
1921 }
1922
1923 static errno_t
1924 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1925 __unused u_int32_t unit,
1926 void *unitinfo,
1927 int opt,
1928 void *data,
1929 size_t len)
1930 {
1931 struct utun_pcb *pcb = unitinfo;
1932 errno_t result = 0;
1933 /* check for privileges for privileged options */
1934 switch (opt) {
1935 case UTUN_OPT_FLAGS:
1936 case UTUN_OPT_EXT_IFDATA_STATS:
1937 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1938 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1939 return EPERM;
1940 }
1941 break;
1942 }
1943
1944 switch (opt) {
1945 case UTUN_OPT_FLAGS:
1946 if (len != sizeof(u_int32_t)) {
1947 result = EMSGSIZE;
1948 } else {
1949 if (pcb->utun_ifp == NULL) {
1950 // Only can set after connecting
1951 result = EINVAL;
1952 break;
1953 }
1954 #if UTUN_NEXUS
1955 if (pcb->utun_use_netif) {
1956 pcb->utun_flags = *(u_int32_t *)data;
1957 } else
1958 #endif // UTUN_NEXUS
1959 {
1960 u_int32_t old_flags = pcb->utun_flags;
1961 pcb->utun_flags = *(u_int32_t *)data;
1962 if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
1963 // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
1964 bpfdetach(pcb->utun_ifp);
1965 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1966 }
1967 }
1968 }
1969 break;
1970
1971 case UTUN_OPT_EXT_IFDATA_STATS:
1972 if (len != sizeof(int)) {
1973 result = EMSGSIZE;
1974 break;
1975 }
1976 if (pcb->utun_ifp == NULL) {
1977 // Only can set after connecting
1978 result = EINVAL;
1979 break;
1980 }
1981 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1982 break;
1983
1984 case UTUN_OPT_INC_IFDATA_STATS_IN:
1985 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1986 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1987
1988 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1989 result = EINVAL;
1990 break;
1991 }
1992 if (pcb->utun_ifp == NULL) {
1993 // Only can set after connecting
1994 result = EINVAL;
1995 break;
1996 }
1997 if (!pcb->utun_ext_ifdata_stats) {
1998 result = EINVAL;
1999 break;
2000 }
2001 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN)
2002 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2003 utsp->utsp_bytes, utsp->utsp_errors);
2004 else
2005 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2006 utsp->utsp_bytes, utsp->utsp_errors);
2007 break;
2008 }
2009 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2010 ifnet_t del_ifp = NULL;
2011 char name[IFNAMSIZ];
2012
2013 if (len > IFNAMSIZ - 1) {
2014 result = EMSGSIZE;
2015 break;
2016 }
2017 if (pcb->utun_ifp == NULL) {
2018 // Only can set after connecting
2019 result = EINVAL;
2020 break;
2021 }
2022 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2023 bcopy(data, name, len);
2024 name[len] = 0;
2025 result = ifnet_find_by_name(name, &del_ifp);
2026 }
2027 if (result == 0) {
2028 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2029 if (del_ifp)
2030 ifnet_release(del_ifp);
2031 }
2032 break;
2033 }
2034 case UTUN_OPT_MAX_PENDING_PACKETS: {
2035 u_int32_t max_pending_packets = 0;
2036 if (len != sizeof(u_int32_t)) {
2037 result = EMSGSIZE;
2038 break;
2039 }
2040 max_pending_packets = *(u_int32_t *)data;
2041 if (max_pending_packets == 0) {
2042 result = EINVAL;
2043 break;
2044 }
2045 pcb->utun_max_pending_packets = max_pending_packets;
2046 break;
2047 }
2048 #if UTUN_NEXUS
2049 case UTUN_OPT_ENABLE_CHANNEL: {
2050 if (len != sizeof(int)) {
2051 result = EMSGSIZE;
2052 break;
2053 }
2054 if (pcb->utun_ifp == NULL) {
2055 // Only can set after connecting
2056 result = EINVAL;
2057 break;
2058 }
2059 if (*(int *)data) {
2060 result = utun_enable_channel(pcb, current_proc());
2061 } else {
2062 result = utun_disable_channel(pcb);
2063 }
2064 break;
2065 }
2066 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2067 if (len != sizeof(int)) {
2068 result = EMSGSIZE;
2069 break;
2070 }
2071 if (pcb->utun_ifp == NULL) {
2072 // Only can set after connecting
2073 result = EINVAL;
2074 break;
2075 }
2076 if (!if_is_netagent_enabled()) {
2077 result = ENOTSUP;
2078 break;
2079 }
2080 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
2081 result = ENOENT;
2082 break;
2083 }
2084
2085 if (*(int *)data) {
2086 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2087 pcb->utun_needs_netagent = true;
2088 } else {
2089 pcb->utun_needs_netagent = false;
2090 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2091 }
2092 break;
2093 }
2094 case UTUN_OPT_ENABLE_NETIF: {
2095 if (len != sizeof(int)) {
2096 result = EMSGSIZE;
2097 break;
2098 }
2099 if (pcb->utun_ifp != NULL) {
2100 // Only can set before connecting
2101 result = EINVAL;
2102 break;
2103 }
2104 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2105 pcb->utun_use_netif = !!(*(int *)data);
2106 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2107 break;
2108 }
2109 case UTUN_OPT_SLOT_SIZE: {
2110 if (len != sizeof(u_int32_t)) {
2111 result = EMSGSIZE;
2112 break;
2113 }
2114 if (pcb->utun_ifp != NULL) {
2115 // Only can set before connecting
2116 result = EINVAL;
2117 break;
2118 }
2119 u_int32_t slot_size = *(u_int32_t *)data;
2120 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2121 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2122 return (EINVAL);
2123 }
2124 pcb->utun_slot_size = slot_size;
2125 break;
2126 }
2127 case UTUN_OPT_NETIF_RING_SIZE: {
2128 if (len != sizeof(u_int32_t)) {
2129 result = EMSGSIZE;
2130 break;
2131 }
2132 if (pcb->utun_ifp != NULL) {
2133 // Only can set before connecting
2134 result = EINVAL;
2135 break;
2136 }
2137 u_int32_t ring_size = *(u_int32_t *)data;
2138 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2139 ring_size > UTUN_IF_MAX_RING_SIZE) {
2140 return (EINVAL);
2141 }
2142 pcb->utun_netif_ring_size = ring_size;
2143 break;
2144 }
2145 case UTUN_OPT_TX_FSW_RING_SIZE: {
2146 if (len != sizeof(u_int32_t)) {
2147 result = EMSGSIZE;
2148 break;
2149 }
2150 if (pcb->utun_ifp != NULL) {
2151 // Only can set before connecting
2152 result = EINVAL;
2153 break;
2154 }
2155 u_int32_t ring_size = *(u_int32_t *)data;
2156 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2157 ring_size > UTUN_IF_MAX_RING_SIZE) {
2158 return (EINVAL);
2159 }
2160 pcb->utun_tx_fsw_ring_size = ring_size;
2161 break;
2162 }
2163 case UTUN_OPT_RX_FSW_RING_SIZE: {
2164 if (len != sizeof(u_int32_t)) {
2165 result = EMSGSIZE;
2166 break;
2167 }
2168 if (pcb->utun_ifp != NULL) {
2169 // Only can set before connecting
2170 result = EINVAL;
2171 break;
2172 }
2173 u_int32_t ring_size = *(u_int32_t *)data;
2174 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2175 ring_size > UTUN_IF_MAX_RING_SIZE) {
2176 return (EINVAL);
2177 }
2178 pcb->utun_rx_fsw_ring_size = ring_size;
2179 break;
2180 }
2181 #endif // UTUN_NEXUS
2182 default: {
2183 result = ENOPROTOOPT;
2184 break;
2185 }
2186 }
2187
2188 return result;
2189 }
2190
2191 static errno_t
2192 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2193 __unused u_int32_t unit,
2194 void *unitinfo,
2195 int opt,
2196 void *data,
2197 size_t *len)
2198 {
2199 struct utun_pcb *pcb = unitinfo;
2200 errno_t result = 0;
2201
2202 switch (opt) {
2203 case UTUN_OPT_FLAGS:
2204 if (*len != sizeof(u_int32_t)) {
2205 result = EMSGSIZE;
2206 } else {
2207 *(u_int32_t *)data = pcb->utun_flags;
2208 }
2209 break;
2210
2211 case UTUN_OPT_EXT_IFDATA_STATS:
2212 if (*len != sizeof(int)) {
2213 result = EMSGSIZE;
2214 } else {
2215 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2216 }
2217 break;
2218
2219 case UTUN_OPT_IFNAME:
2220 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2221 result = EMSGSIZE;
2222 } else {
2223 if (pcb->utun_ifp == NULL) {
2224 // Only can get after connecting
2225 result = EINVAL;
2226 break;
2227 }
2228 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2229 }
2230 break;
2231
2232 case UTUN_OPT_MAX_PENDING_PACKETS: {
2233 if (*len != sizeof(u_int32_t)) {
2234 result = EMSGSIZE;
2235 } else {
2236 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2237 }
2238 break;
2239 }
2240
2241 #if UTUN_NEXUS
2242 case UTUN_OPT_ENABLE_CHANNEL: {
2243 if (*len != sizeof(int)) {
2244 result = EMSGSIZE;
2245 } else {
2246 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2247 *(int *)data = pcb->utun_kpipe_enabled;
2248 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2249 }
2250 break;
2251 }
2252
2253 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2254 if (*len != sizeof(int)) {
2255 result = EMSGSIZE;
2256 } else {
2257 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
2258 }
2259 break;
2260 }
2261
2262 case UTUN_OPT_ENABLE_NETIF: {
2263 if (*len != sizeof(int)) {
2264 result = EMSGSIZE;
2265 } else {
2266 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2267 *(int *)data = !!pcb->utun_use_netif;
2268 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2269 }
2270 break;
2271 }
2272
2273 case UTUN_OPT_GET_CHANNEL_UUID: {
2274 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2275 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2276 result = ENXIO;
2277 } else if (*len != sizeof(uuid_t)) {
2278 result = EMSGSIZE;
2279 } else {
2280 uuid_copy(data, pcb->utun_kpipe_uuid);
2281 }
2282 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2283 break;
2284 }
2285 case UTUN_OPT_SLOT_SIZE: {
2286 if (*len != sizeof(u_int32_t)) {
2287 result = EMSGSIZE;
2288 } else {
2289 *(u_int32_t *)data = pcb->utun_slot_size;
2290 }
2291 break;
2292 }
2293 case UTUN_OPT_NETIF_RING_SIZE: {
2294 if (*len != sizeof(u_int32_t)) {
2295 result = EMSGSIZE;
2296 } else {
2297 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2298 }
2299 break;
2300 }
2301 case UTUN_OPT_TX_FSW_RING_SIZE: {
2302 if (*len != sizeof(u_int32_t)) {
2303 result = EMSGSIZE;
2304 } else {
2305 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2306 }
2307 break;
2308 }
2309 case UTUN_OPT_RX_FSW_RING_SIZE: {
2310 if (*len != sizeof(u_int32_t)) {
2311 result = EMSGSIZE;
2312 } else {
2313 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2314 }
2315 break;
2316 }
2317 #endif // UTUN_NEXUS
2318
2319 default:
2320 result = ENOPROTOOPT;
2321 break;
2322 }
2323
2324 return result;
2325 }
2326
2327 static void
2328 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2329 {
2330 #pragma unused(flags)
2331 bool reenable_output = false;
2332 struct utun_pcb *pcb = unitinfo;
2333 if (pcb == NULL) {
2334 return;
2335 }
2336 ifnet_lock_exclusive(pcb->utun_ifp);
2337
2338 u_int32_t utun_packet_cnt;
2339 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2340 if (error_pc != 0) {
2341 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2342 utun_packet_cnt = 0;
2343 }
2344
2345 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2346 reenable_output = true;
2347 }
2348
2349 if (reenable_output) {
2350 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2351 if (error != 0) {
2352 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2353 }
2354 }
2355 ifnet_lock_done(pcb->utun_ifp);
2356 }
2357
2358 /* Network Interface functions */
2359 static void
2360 utun_start(ifnet_t interface)
2361 {
2362 mbuf_t data;
2363 struct utun_pcb *pcb = ifnet_softc(interface);
2364
2365 VERIFY(pcb != NULL);
2366
2367 #if UTUN_NEXUS
2368 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2369 if (pcb->utun_kpipe_enabled) {
2370 /* It's possible to have channels enabled, but not yet have the channel opened,
2371 * in which case the rxring will not be set
2372 */
2373 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2374 if (pcb->utun_kpipe_rxring != NULL) {
2375 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2376 }
2377 return;
2378 }
2379 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2380 #endif // UTUN_NEXUS
2381
2382 for (;;) {
2383 bool can_accept_packets = true;
2384 ifnet_lock_shared(pcb->utun_ifp);
2385
2386 u_int32_t utun_packet_cnt;
2387 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2388 if (error_pc != 0) {
2389 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2390 utun_packet_cnt = 0;
2391 }
2392
2393 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2394 if (!can_accept_packets && pcb->utun_ctlref) {
2395 u_int32_t difference = 0;
2396 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2397 if (difference > 0) {
2398 // If the low-water mark has not yet been reached, we still need to enqueue data
2399 // into the buffer
2400 can_accept_packets = true;
2401 }
2402 }
2403 }
2404 if (!can_accept_packets) {
2405 errno_t error = ifnet_disable_output(interface);
2406 if (error != 0) {
2407 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2408 }
2409 ifnet_lock_done(pcb->utun_ifp);
2410 break;
2411 }
2412 ifnet_lock_done(pcb->utun_ifp);
2413 if (ifnet_dequeue(interface, &data) != 0) {
2414 break;
2415 }
2416 if (utun_output(interface, data) != 0) {
2417 break;
2418 }
2419 }
2420 }
2421
2422 static errno_t
2423 utun_output(ifnet_t interface,
2424 mbuf_t data)
2425 {
2426 struct utun_pcb *pcb = ifnet_softc(interface);
2427 errno_t result;
2428
2429 VERIFY(interface == pcb->utun_ifp);
2430
2431 #if UTUN_NEXUS
2432 if (!pcb->utun_use_netif)
2433 #endif // UTUN_NEXUS
2434 {
2435 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2436 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2437 }
2438 }
2439
2440 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2441 /* flush data */
2442 mbuf_freem(data);
2443 return 0;
2444 }
2445
2446 // otherwise, fall thru to ctl_enqueumbuf
2447 if (pcb->utun_ctlref) {
2448 int length;
2449
2450 /*
2451 * The ABI requires the protocol in network byte order
2452 */
2453 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2454 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2455 }
2456
2457 length = mbuf_pkthdr_len(data);
2458 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2459 if (result != 0) {
2460 mbuf_freem(data);
2461 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
2462 #if UTUN_NEXUS
2463 if (!pcb->utun_use_netif)
2464 #endif // UTUN_NEXUS
2465 {
2466 ifnet_stat_increment_out(interface, 0, 0, 1);
2467 }
2468 } else {
2469 #if UTUN_NEXUS
2470 if (!pcb->utun_use_netif)
2471 #endif // UTUN_NEXUS
2472 {
2473 if (!pcb->utun_ext_ifdata_stats) {
2474 ifnet_stat_increment_out(interface, 1, length, 0);
2475 }
2476 }
2477 }
2478 } else {
2479 mbuf_freem(data);
2480 }
2481
2482 return 0;
2483 }
2484
2485 static errno_t
2486 utun_demux(__unused ifnet_t interface,
2487 mbuf_t data,
2488 __unused char *frame_header,
2489 protocol_family_t *protocol)
2490 {
2491 while (data != NULL && mbuf_len(data) < 1) {
2492 data = mbuf_next(data);
2493 }
2494
2495 if (data == NULL) {
2496 return ENOENT;
2497 }
2498
2499 #if UTUN_NEXUS
2500 struct utun_pcb *pcb = ifnet_softc(interface);
2501 struct ip *ip;
2502 u_int ip_version;
2503
2504 if (pcb->utun_use_netif) {
2505 ip = mtod(data, struct ip *);
2506 ip_version = ip->ip_v;
2507
2508 switch(ip_version) {
2509 case 4:
2510 *protocol = PF_INET;
2511 return 0;
2512 case 6:
2513 *protocol = PF_INET6;
2514 return 0;
2515 default:
2516 *protocol = 0;
2517 break;
2518 }
2519 } else
2520 #endif // UTUN_NEXUS
2521 {
2522 *protocol = *(u_int32_t *)mbuf_data(data);
2523 }
2524
2525 return 0;
2526 }
2527
2528 static errno_t
2529 utun_framer(ifnet_t interface,
2530 mbuf_t *packet,
2531 __unused const struct sockaddr *dest,
2532 __unused const char *desk_linkaddr,
2533 const char *frame_type,
2534 u_int32_t *prepend_len,
2535 u_int32_t *postpend_len)
2536 {
2537 struct utun_pcb *pcb = ifnet_softc(interface);
2538 VERIFY(interface == pcb->utun_ifp);
2539
2540 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2541 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2542 printf("utun_framer - ifnet_output prepend failed\n");
2543
2544 ifnet_stat_increment_out(interface, 0, 0, 1);
2545
2546 // just return, because the buffer was freed in mbuf_prepend
2547 return EJUSTRETURN;
2548 }
2549 if (prepend_len != NULL) {
2550 *prepend_len = header_length;
2551 }
2552 if (postpend_len != NULL) {
2553 *postpend_len = 0;
2554 }
2555
2556 // place protocol number at the beginning of the mbuf
2557 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2558
2559
2560 return 0;
2561 }
2562
2563 static errno_t
2564 utun_add_proto(__unused ifnet_t interface,
2565 protocol_family_t protocol,
2566 __unused const struct ifnet_demux_desc *demux_array,
2567 __unused u_int32_t demux_count)
2568 {
2569 switch(protocol) {
2570 case PF_INET:
2571 return 0;
2572 case PF_INET6:
2573 return 0;
2574 default:
2575 break;
2576 }
2577
2578 return ENOPROTOOPT;
2579 }
2580
2581 static errno_t
2582 utun_del_proto(__unused ifnet_t interface,
2583 __unused protocol_family_t protocol)
2584 {
2585 return 0;
2586 }
2587
2588 static errno_t
2589 utun_ioctl(ifnet_t interface,
2590 u_long command,
2591 void *data)
2592 {
2593 errno_t result = 0;
2594
2595 switch(command) {
2596 case SIOCSIFMTU: {
2597 #if UTUN_NEXUS
2598 struct utun_pcb *pcb = ifnet_softc(interface);
2599 if (pcb->utun_use_netif) {
2600 // Make sure we can fit packets in the channel buffers
2601 // Allow for the headroom in the slot
2602 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2603 result = EINVAL;
2604 } else {
2605 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2606 }
2607 } else
2608 #endif // UTUN_NEXUS
2609 {
2610 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2611 }
2612 break;
2613 }
2614
2615 case SIOCSIFFLAGS:
2616 /* ifioctl() takes care of it */
2617 break;
2618
2619 default:
2620 result = EOPNOTSUPP;
2621 }
2622
2623 return result;
2624 }
2625
2626 static void
2627 utun_detached(ifnet_t interface)
2628 {
2629 struct utun_pcb *pcb = ifnet_softc(interface);
2630 (void)ifnet_release(interface);
2631 utun_free_pcb(pcb, true);
2632 }
2633
2634 /* Protocol Handlers */
2635
2636 static errno_t
2637 utun_proto_input(__unused ifnet_t interface,
2638 protocol_family_t protocol,
2639 mbuf_t m,
2640 __unused char *frame_header)
2641 {
2642 struct utun_pcb *pcb = ifnet_softc(interface);
2643 #if UTUN_NEXUS
2644 if (!pcb->utun_use_netif)
2645 #endif // UTUN_NEXUS
2646 {
2647 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2648 }
2649 int32_t pktlen = m->m_pkthdr.len;
2650 if (proto_input(protocol, m) != 0) {
2651 m_freem(m);
2652 #if UTUN_NEXUS
2653 if (!pcb->utun_use_netif)
2654 #endif // UTUN_NEXUS
2655 {
2656 ifnet_stat_increment_in(interface, 0, 0, 1);
2657 }
2658 } else {
2659 #if UTUN_NEXUS
2660 if (!pcb->utun_use_netif)
2661 #endif // UTUN_NEXUS
2662 {
2663 ifnet_stat_increment_in(interface, 1, pktlen, 0);
2664 }
2665 }
2666
2667 return 0;
2668 }
2669
2670 static errno_t
2671 utun_proto_pre_output(__unused ifnet_t interface,
2672 protocol_family_t protocol,
2673 __unused mbuf_t *packet,
2674 __unused const struct sockaddr *dest,
2675 __unused void *route,
2676 char *frame_type,
2677 __unused char *link_layer_dest)
2678 {
2679 *(protocol_family_t *)(void *)frame_type = protocol;
2680 return 0;
2681 }
2682
2683 static errno_t
2684 utun_attach_proto(ifnet_t interface,
2685 protocol_family_t protocol)
2686 {
2687 struct ifnet_attach_proto_param proto;
2688
2689 bzero(&proto, sizeof(proto));
2690 proto.input = utun_proto_input;
2691 proto.pre_output = utun_proto_pre_output;
2692
2693 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2694 if (result != 0 && result != EEXIST) {
2695 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2696 protocol, result);
2697 }
2698
2699 return result;
2700 }
2701
2702 static errno_t
2703 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2704 {
2705 #if UTUN_NEXUS
2706 if (pcb->utun_use_netif) {
2707 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2708
2709 lck_mtx_lock(&pcb->utun_input_chain_lock);
2710 if (pcb->utun_input_chain != NULL) {
2711 pcb->utun_input_chain_last->m_nextpkt = packet;
2712 } else {
2713 pcb->utun_input_chain = packet;
2714 }
2715 while (packet->m_nextpkt) {
2716 VERIFY(packet != packet->m_nextpkt);
2717 packet = packet->m_nextpkt;
2718 }
2719 pcb->utun_input_chain_last = packet;
2720 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2721
2722 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2723 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2724
2725 if (rx_ring != NULL) {
2726 kern_channel_notify(rx_ring, 0);
2727 }
2728
2729 return (0);
2730 } else
2731 #endif // IPSEC_NEXUS
2732 {
2733 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
2734
2735 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2736 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2737 }
2738 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2739 /* flush data */
2740 mbuf_freem(packet);
2741 return 0;
2742 }
2743
2744 errno_t result = 0;
2745 if (!pcb->utun_ext_ifdata_stats) {
2746 struct ifnet_stat_increment_param incs = {};
2747 incs.packets_in = 1;
2748 incs.bytes_in = mbuf_pkthdr_len(packet);
2749 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2750 } else {
2751 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2752 }
2753 if (result != 0) {
2754 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2755
2756 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2757 mbuf_freem(packet);
2758 }
2759
2760 return (0);
2761 }
2762 }
2763
2764 #if UTUN_NEXUS
2765
2766 static errno_t
2767 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2768 {
2769 return 0;
2770 }
2771
2772 static void
2773 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2774 {
2775 // Ignore
2776 }
2777
2778 static errno_t
2779 utun_register_nexus(void)
2780 {
2781 const struct kern_nexus_domain_provider_init dp_init = {
2782 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2783 .nxdpi_flags = 0,
2784 .nxdpi_init = utun_nxdp_init,
2785 .nxdpi_fini = utun_nxdp_fini
2786 };
2787 errno_t err = 0;
2788
2789 /* utun_nxdp_init() is called before this function returns */
2790 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2791 (const uint8_t *) "com.apple.utun",
2792 &dp_init, sizeof(dp_init),
2793 &utun_nx_dom_prov);
2794 if (err != 0) {
2795 printf("%s: failed to register domain provider\n", __func__);
2796 return (err);
2797 }
2798 return (0);
2799 }
2800 boolean_t
2801 utun_interface_needs_netagent(ifnet_t interface)
2802 {
2803 struct utun_pcb *pcb = NULL;
2804
2805 if (interface == NULL) {
2806 return (FALSE);
2807 }
2808
2809 pcb = ifnet_softc(interface);
2810
2811 if (pcb == NULL) {
2812 return (FALSE);
2813 }
2814
2815 return (pcb->utun_needs_netagent == true);
2816 }
2817
2818 static errno_t
2819 utun_ifnet_set_attrs(ifnet_t ifp)
2820 {
2821 /* Set flags and additional information. */
2822 ifnet_set_mtu(ifp, 1500);
2823 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2824
2825 /* The interface must generate its own IPv6 LinkLocal address,
2826 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2827 */
2828 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2829
2830 return (0);
2831 }
2832
2833 static errno_t
2834 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2835 {
2836 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2837 pcb->utun_netif_nexus = nexus;
2838 return (utun_ifnet_set_attrs(ifp));
2839 }
2840
2841 static errno_t
2842 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2843 proc_t p, kern_nexus_t nexus,
2844 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2845 {
2846 #pragma unused(nxprov, p)
2847 #pragma unused(nexus, nexus_port, channel, ch_ctx)
2848 return (0);
2849 }
2850
2851 static errno_t
2852 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2853 kern_channel_t channel)
2854 {
2855 #pragma unused(nxprov, channel)
2856 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2857 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2858 return (ok ? 0 : ENXIO);
2859 }
2860
2861 static void
2862 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2863 kern_channel_t channel)
2864 {
2865 #pragma unused(nxprov, nexus, channel)
2866 }
2867
2868 static void
2869 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2870 kern_channel_t channel)
2871 {
2872 #pragma unused(nxprov, nexus, channel)
2873 }
2874
2875 static void
2876 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2877 kern_channel_t channel)
2878 {
2879 #pragma unused(nxprov, channel)
2880 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2881 if (pcb->utun_netif_nexus == nexus) {
2882 pcb->utun_netif_nexus = NULL;
2883 }
2884 ifnet_decr_iorefcnt(pcb->utun_ifp);
2885 }
2886
2887 static errno_t
2888 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2889 kern_channel_t channel, kern_channel_ring_t ring,
2890 boolean_t is_tx_ring, void **ring_ctx)
2891 {
2892 #pragma unused(nxprov)
2893 #pragma unused(channel)
2894 #pragma unused(ring_ctx)
2895 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2896 if (!is_tx_ring) {
2897 VERIFY(pcb->utun_kpipe_rxring == NULL);
2898 pcb->utun_kpipe_rxring = ring;
2899 } else {
2900 VERIFY(pcb->utun_kpipe_txring == NULL);
2901 pcb->utun_kpipe_txring = ring;
2902 }
2903 return 0;
2904 }
2905
2906 static void
2907 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2908 kern_channel_ring_t ring)
2909 {
2910 #pragma unused(nxprov)
2911 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2912 if (pcb->utun_kpipe_rxring == ring) {
2913 pcb->utun_kpipe_rxring = NULL;
2914 } else if (pcb->utun_kpipe_txring == ring) {
2915 pcb->utun_kpipe_txring = NULL;
2916 }
2917 }
2918
2919 static errno_t
2920 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2921 kern_channel_ring_t tx_ring, uint32_t flags)
2922 {
2923 #pragma unused(nxprov)
2924 #pragma unused(flags)
2925 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2926
2927 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2928 int channel_enabled = pcb->utun_kpipe_enabled;
2929 if (!channel_enabled) {
2930 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2931 return 0;
2932 }
2933
2934 if (pcb->utun_use_netif) {
2935 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2936 if (tx_slot == NULL) {
2937 // Nothing to write, bail
2938 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2939 return 0;
2940 }
2941
2942 // Signal the netif ring to read
2943 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2944 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2945 if (rx_ring != NULL) {
2946 kern_channel_notify(rx_ring, 0);
2947 }
2948 } else {
2949 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2950
2951 struct ifnet_stat_increment_param incs = {};
2952 struct kern_channel_ring_stat_increment tx_ring_stats = {};
2953 MBUFQ_HEAD(mbufq) mbq;
2954 MBUFQ_INIT(&mbq);
2955 kern_channel_slot_t tx_pslot = NULL;
2956 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2957 while (tx_slot != NULL) {
2958 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2959
2960 // Advance TX ring
2961 tx_pslot = tx_slot;
2962 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2963
2964 if (tx_ph == 0) {
2965 continue;
2966 }
2967
2968 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2969 VERIFY(tx_buf != NULL);
2970 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2971 VERIFY(tx_baddr != 0);
2972 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2973
2974 size_t length = MIN(kern_packet_get_data_length(tx_ph),
2975 pcb->utun_slot_size);
2976
2977 mbuf_t data = NULL;
2978 if (length >= UTUN_HEADER_SIZE(pcb) &&
2979 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
2980 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
2981 VERIFY(0 == error);
2982 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
2983 VERIFY(0 == error);
2984 /*
2985 * The userland ABI requires the first four bytes have
2986 * the protocol family in network byte order: swap them
2987 */
2988 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
2989 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
2990 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2991 incs.packets_in++;
2992 incs.bytes_in += length;
2993 MBUFQ_ENQUEUE(&mbq, data);
2994 }
2995 }
2996 if (tx_pslot) {
2997 kern_channel_advance_slot(tx_ring, tx_pslot);
2998 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
2999 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3000 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3001 (void) kern_channel_reclaim(tx_ring);
3002 }
3003 if (!MBUFQ_EMPTY(&mbq)) {
3004 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
3005 MBUFQ_LAST(&mbq), &incs);
3006 MBUFQ_INIT(&mbq);
3007 }
3008 }
3009
3010 return 0;
3011 }
3012
3013 static errno_t
3014 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3015 kern_channel_ring_t rx_ring, uint32_t flags)
3016 {
3017 #pragma unused(nxprov)
3018 #pragma unused(flags)
3019 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3020 struct kern_channel_ring_stat_increment rx_ring_stats = {};
3021
3022 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3023
3024 int channel_enabled = pcb->utun_kpipe_enabled;
3025 if (!channel_enabled) {
3026 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3027 return 0;
3028 }
3029
3030 /* reclaim user-released slots */
3031 (void) kern_channel_reclaim(rx_ring);
3032
3033 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3034 if (avail == 0) {
3035 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3036 return 0;
3037 }
3038
3039 if (pcb->utun_use_netif) {
3040 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3041 if (tx_ring == NULL ||
3042 pcb->utun_netif_nexus == NULL) {
3043 // Net-If TX ring not set up yet, nothing to read
3044 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3045 return 0;
3046 }
3047
3048 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
3049
3050 // Unlock utun before entering ring
3051 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3052
3053 (void)kr_enter(tx_ring, TRUE);
3054
3055 // Lock again after entering and validate
3056 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3057 if (tx_ring != pcb->utun_netif_txring) {
3058 // Ring no longer valid
3059 // Unlock first, then exit ring
3060 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3061 kr_exit(tx_ring);
3062 return 0;
3063 }
3064
3065 struct kern_channel_ring_stat_increment tx_ring_stats;
3066 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3067 kern_channel_slot_t tx_pslot = NULL;
3068 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3069 if (tx_slot == NULL) {
3070 // Nothing to read, don't bother signalling
3071 // Unlock first, then exit ring
3072 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3073 kr_exit(tx_ring);
3074 return 0;
3075 }
3076
3077 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3078 VERIFY(rx_pp != NULL);
3079 kern_channel_slot_t rx_pslot = NULL;
3080 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3081
3082 while (rx_slot != NULL && tx_slot != NULL) {
3083 size_t length;
3084 kern_buflet_t rx_buf;
3085 void *rx_baddr;
3086
3087 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3088
3089 // Advance TX ring
3090 tx_pslot = tx_slot;
3091 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3092
3093 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3094 if (tx_ph == 0) {
3095 continue;
3096 }
3097
3098 // Allocate rx packet
3099 kern_packet_t rx_ph = 0;
3100 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3101 if (__improbable(error != 0)) {
3102 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
3103 pcb->utun_ifp->if_xname);
3104 break;
3105 }
3106
3107 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3108 VERIFY(tx_buf != NULL);
3109 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
3110 VERIFY(tx_baddr != NULL);
3111 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3112
3113 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3114
3115 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
3116 pcb->utun_slot_size);
3117
3118 tx_ring_stats.kcrsi_slots_transferred++;
3119 tx_ring_stats.kcrsi_bytes_transferred += length;
3120
3121 if (length < UTUN_HEADER_SIZE(pcb) ||
3122 length > pcb->utun_slot_size ||
3123 length > rx_pp->pp_buflet_size ||
3124 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3125 /* flush data */
3126 kern_pbufpool_free(rx_pp, rx_ph);
3127 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3128 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3129 STATS_INC(nifs, NETIF_STATS_BADLEN);
3130 STATS_INC(nifs, NETIF_STATS_DROPPED);
3131 continue;
3132 }
3133
3134 /* fillout packet */
3135 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3136 VERIFY(rx_buf != NULL);
3137 rx_baddr = kern_buflet_get_object_address(rx_buf);
3138 VERIFY(rx_baddr != NULL);
3139
3140 // Find family
3141 uint32_t af = 0;
3142 uint8_t vhl = *(uint8_t *)(tx_baddr);
3143 u_int ip_version = (vhl >> 4);
3144 switch (ip_version) {
3145 case 4: {
3146 af = AF_INET;
3147 break;
3148 }
3149 case 6: {
3150 af = AF_INET6;
3151 break;
3152 }
3153 default: {
3154 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3155 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3156 break;
3157 }
3158 }
3159
3160 // Copy header
3161 af = htonl(af);
3162 memcpy((void *)rx_baddr, &af, sizeof(af));
3163 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3164 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3165 }
3166
3167 // Copy data from tx to rx
3168 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3169 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
3170
3171 /* finalize and attach the packet */
3172 error = kern_buflet_set_data_offset(rx_buf, 0);
3173 VERIFY(error == 0);
3174 error = kern_buflet_set_data_length(rx_buf, length);
3175 VERIFY(error == 0);
3176 error = kern_packet_finalize(rx_ph);
3177 VERIFY(error == 0);
3178 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3179 VERIFY(error == 0);
3180
3181 STATS_INC(nifs, NETIF_STATS_TXPKTS);
3182 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
3183
3184 rx_ring_stats.kcrsi_slots_transferred++;
3185 rx_ring_stats.kcrsi_bytes_transferred += length;
3186
3187 rx_pslot = rx_slot;
3188 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3189 }
3190
3191 if (rx_pslot) {
3192 kern_channel_advance_slot(rx_ring, rx_pslot);
3193 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3194 }
3195
3196 if (tx_pslot) {
3197 kern_channel_advance_slot(tx_ring, tx_pslot);
3198 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3199 (void)kern_channel_reclaim(tx_ring);
3200 }
3201
3202 /* just like utun_ctl_rcvd(), always reenable output */
3203 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3204 if (error != 0) {
3205 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
3206 }
3207
3208 // Unlock first, then exit ring
3209 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3210
3211 if (tx_pslot != NULL) {
3212 kern_channel_notify(tx_ring, 0);
3213 }
3214 kr_exit(tx_ring);
3215 } else {
3216 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3217
3218 uint32_t mb_cnt = 0;
3219 uint32_t mb_len = 0;
3220 struct mbuf *mb_head = NULL;
3221 struct mbuf *mb_tail = NULL;
3222
3223 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3224 &mb_tail, &mb_cnt, &mb_len) != 0) {
3225 return 0;
3226 }
3227 VERIFY(mb_cnt <= avail);
3228
3229 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3230 VERIFY(rx_pp != NULL);
3231 kern_channel_slot_t rx_pslot = NULL;
3232 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3233 while (rx_slot) {
3234 size_t length = 0;
3235 mbuf_t data = NULL;
3236 if ((data = mb_head) == NULL) {
3237 VERIFY(mb_cnt == 0);
3238 break;
3239 }
3240 mb_head = mbuf_nextpkt(mb_head);
3241 mbuf_setnextpkt(data, NULL);
3242 VERIFY(mb_cnt != 0);
3243 --mb_cnt;
3244 length = mbuf_pkthdr_len(data);
3245 if (length < UTUN_HEADER_SIZE(pcb) ||
3246 length > pcb->utun_slot_size ||
3247 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3248 /* flush data */
3249 mbuf_freem(data);
3250 continue;
3251 }
3252 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3253
3254 // Allocate rx packet
3255 kern_packet_t rx_ph = 0;
3256 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3257 if (__improbable(error != 0)) {
3258 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
3259 pcb->utun_ifp->if_xname);
3260 break;
3261 }
3262
3263 /*
3264 * The ABI requires the protocol in network byte order
3265 */
3266 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3267
3268 // Fillout rx packet
3269 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3270 VERIFY(rx_buf != NULL);
3271 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
3272 VERIFY(rx_baddr != NULL);
3273
3274 // Copy-in data from mbuf to buflet
3275 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3276 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3277
3278 // Finalize and attach the packet
3279 error = kern_buflet_set_data_offset(rx_buf, 0);
3280 VERIFY(error == 0);
3281 error = kern_buflet_set_data_length(rx_buf, length);
3282 VERIFY(error == 0);
3283 error = kern_packet_finalize(rx_ph);
3284 VERIFY(error == 0);
3285 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3286 VERIFY(error == 0);
3287
3288 rx_ring_stats.kcrsi_slots_transferred++;
3289 rx_ring_stats.kcrsi_bytes_transferred += length;
3290
3291 if (!pcb->utun_ext_ifdata_stats) {
3292 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3293 }
3294
3295 mbuf_freem(data);
3296
3297 rx_pslot = rx_slot;
3298 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3299 }
3300 if (rx_pslot) {
3301 kern_channel_advance_slot(rx_ring, rx_pslot);
3302 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3303 }
3304 if (mb_head != NULL) {
3305 VERIFY(mb_cnt != 0);
3306 mbuf_freem_list(mb_head);
3307 }
3308 }
3309
3310 return 0;
3311 }
3312
3313 #endif // UTUN_NEXUS
3314
3315
3316 /*
3317 * These are place holders until coreTLS kext stops calling them
3318 */
3319 errno_t utun_ctl_register_dtls (void *reg);
3320 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3321 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3322
3323 errno_t
3324 utun_ctl_register_dtls (void *reg)
3325 {
3326 #pragma unused(reg)
3327 return 0;
3328 }
3329
3330 int
3331 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3332 {
3333 #pragma unused(pcb)
3334 #pragma unused(pkt)
3335 #pragma unused(family)
3336 return 0;
3337 }
3338
3339 void
3340 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3341 {
3342 #pragma unused(pcb)
3343 }
mirror of XNU