]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/if_utun.c
projects / apple / xnu.git / blob
? search:


e98d9ec870f2f8da0ab247acc7418f7126990235
[apple/xnu.git] / bsd / net / if_utun.c
1 /*
2 * Copyright (c) 2008-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 * Application of kernel control for interface creation
33 *
34 * Theory of operation:
35 * utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 * This kernel control will register an interface for every client that connects.
37 * ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57 #include <os/log.h>
58
59 #define UTUN_NEXUS 0
60
61 #if UTUN_NEXUS
62 static nexus_controller_t utun_ncd;
63 static int utun_ncd_refcount;
64 static uuid_t utun_kpipe_uuid;
65 static uuid_t utun_nx_dom_prov;
66
67 typedef struct utun_nx {
68 uuid_t if_provider;
69 uuid_t if_instance;
70 uuid_t fsw_provider;
71 uuid_t fsw_instance;
72 uuid_t fsw_device;
73 uuid_t fsw_host;
74 uuid_t fsw_agent;
75 } *utun_nx_t;
76
77 #endif // UTUN_NEXUS
78
79 /* Control block allocated for each kernel control connection */
80 struct utun_pcb {
81 TAILQ_ENTRY(utun_pcb) utun_chain;
82 kern_ctl_ref utun_ctlref;
83 ifnet_t utun_ifp;
84 u_int32_t utun_unit;
85 u_int32_t utun_unique_id;
86 u_int32_t utun_flags;
87 int utun_ext_ifdata_stats;
88 u_int32_t utun_max_pending_packets;
89 char utun_if_xname[IFXNAMSIZ];
90 char utun_unique_name[IFXNAMSIZ];
91 // PCB lock protects state fields and rings
92 decl_lck_rw_data(, utun_pcb_lock);
93 struct mbuf * utun_input_chain;
94 struct mbuf * utun_input_chain_last;
95 u_int32_t utun_input_chain_count;
96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
98 lck_mtx_t utun_input_chain_lock;
99
100 #if UTUN_NEXUS
101 struct utun_nx utun_nx;
102 int utun_kpipe_enabled;
103 uuid_t utun_kpipe_uuid;
104 void * utun_kpipe_rxring;
105 void * utun_kpipe_txring;
106 kern_pbufpool_t utun_kpipe_pp;
107 u_int32_t utun_kpipe_tx_ring_size;
108 u_int32_t utun_kpipe_rx_ring_size;
109
110 kern_nexus_t utun_netif_nexus;
111 kern_pbufpool_t utun_netif_pp;
112 void * utun_netif_rxring;
113 void * utun_netif_txring;
114 uint64_t utun_netif_txring_size;
115
116 u_int32_t utun_slot_size;
117 u_int32_t utun_netif_ring_size;
118 u_int32_t utun_tx_fsw_ring_size;
119 u_int32_t utun_rx_fsw_ring_size;
120 // Auto attach flowswitch when netif is enabled. When set to false,
121 // it allows userspace nexus controller to attach and own flowswitch.
122 bool utun_attach_fsw;
123 bool utun_netif_connected;
124 bool utun_use_netif;
125 bool utun_needs_netagent;
126 #endif // UTUN_NEXUS
127 };
128
129 /* Kernel Control functions */
130 static errno_t utun_ctl_setup(u_int32_t *unit, void **unitinfo);
131 static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
132 void **unitinfo);
133 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
134 void **unitinfo);
135 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
136 void *unitinfo);
137 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
138 void *unitinfo, mbuf_t m, int flags);
139 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
140 int opt, void *data, size_t *len);
141 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
142 int opt, void *data, size_t len);
143 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
144 int flags);
145
146 /* Network Interface functions */
147 static void utun_start(ifnet_t interface);
148 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
149 const struct sockaddr *dest, const char *desk_linkaddr,
150 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
151 static errno_t utun_output(ifnet_t interface, mbuf_t data);
152 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
153 protocol_family_t *protocol);
154 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
155 const struct ifnet_demux_desc *demux_array,
156 u_int32_t demux_count);
157 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
158 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
159 static void utun_detached(ifnet_t interface);
160
161 /* Protocol handlers */
162 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
163 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
164 mbuf_t m, char *frame_header);
165 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
166 mbuf_t *packet, const struct sockaddr *dest, void *route,
167 char *frame_type, char *link_layer_dest);
168 static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);
169
170 #if UTUN_NEXUS
171
172 #define UTUN_IF_DEFAULT_SLOT_SIZE 2048
173 #define UTUN_IF_DEFAULT_RING_SIZE 64
174 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
175 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
176 #define UTUN_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
177 #define UTUN_IF_HEADROOM_SIZE 32
178
179 #define UTUN_IF_MIN_RING_SIZE 8
180 #define UTUN_IF_MAX_RING_SIZE 1024
181
182 #define UTUN_IF_MIN_SLOT_SIZE 1024
183 #define UTUN_IF_MAX_SLOT_SIZE 4096
184
185 #define UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT 512
186
187 static int if_utun_max_pending_input = UTUN_DEFAULT_MAX_PENDING_INPUT_COUNT;
188
189 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
190 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
191 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
192
193 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
194 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
195 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
196
197 SYSCTL_DECL(_net_utun);
198 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
199
200 SYSCTL_INT(_net_utun, OID_AUTO, max_pending_input, CTLFLAG_LOCKED | CTLFLAG_RW, &if_utun_max_pending_input, 0, "");
201 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
202 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
203 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
204 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
205 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
206 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
207
208 static errno_t
209 utun_register_nexus(void);
210
211 static errno_t
212 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
213 static errno_t
214 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
215 proc_t p, kern_nexus_t nexus,
216 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
217 static errno_t
218 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
219 kern_channel_t channel);
220 static void
221 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
222 kern_channel_t channel);
223 static void
224 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
225 kern_channel_t channel);
226 static void
227 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
228 kern_channel_t channel);
229 static errno_t
230 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
231 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
232 void **ring_ctx);
233 static void
234 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
235 kern_channel_ring_t ring);
236 static errno_t
237 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
238 kern_channel_ring_t ring, uint32_t flags);
239 static errno_t
240 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
241 kern_channel_ring_t ring, uint32_t flags);
242 #endif // UTUN_NEXUS
243
244 #define UTUN_DEFAULT_MTU 1500
245 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
246
247 static kern_ctl_ref utun_kctlref;
248 static lck_attr_t *utun_lck_attr;
249 static lck_grp_attr_t *utun_lck_grp_attr;
250 static lck_grp_t *utun_lck_grp;
251 static lck_mtx_t utun_lock;
252
253 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
254
255 static ZONE_DECLARE(utun_pcb_zone, "net.if_utun",
256 sizeof(struct utun_pcb), ZC_ZFREE_CLEARMEM);
257
258 #if UTUN_NEXUS
259
260 static int
261 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
262 {
263 #pragma unused(arg1, arg2)
264 int value = if_utun_ring_size;
265
266 int error = sysctl_handle_int(oidp, &value, 0, req);
267 if (error || !req->newptr) {
268 return error;
269 }
270
271 if (value < UTUN_IF_MIN_RING_SIZE ||
272 value > UTUN_IF_MAX_RING_SIZE) {
273 return EINVAL;
274 }
275
276 if_utun_ring_size = value;
277
278 return 0;
279 }
280
281 static int
282 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
283 {
284 #pragma unused(arg1, arg2)
285 int value = if_utun_tx_fsw_ring_size;
286
287 int error = sysctl_handle_int(oidp, &value, 0, req);
288 if (error || !req->newptr) {
289 return error;
290 }
291
292 if (value < UTUN_IF_MIN_RING_SIZE ||
293 value > UTUN_IF_MAX_RING_SIZE) {
294 return EINVAL;
295 }
296
297 if_utun_tx_fsw_ring_size = value;
298
299 return 0;
300 }
301
302 static int
303 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
304 {
305 #pragma unused(arg1, arg2)
306 int value = if_utun_rx_fsw_ring_size;
307
308 int error = sysctl_handle_int(oidp, &value, 0, req);
309 if (error || !req->newptr) {
310 return error;
311 }
312
313 if (value < UTUN_IF_MIN_RING_SIZE ||
314 value > UTUN_IF_MAX_RING_SIZE) {
315 return EINVAL;
316 }
317
318 if_utun_rx_fsw_ring_size = value;
319
320 return 0;
321 }
322
323 static errno_t
324 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
325 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
326 void **ring_ctx)
327 {
328 #pragma unused(nxprov)
329 #pragma unused(channel)
330 #pragma unused(ring_ctx)
331 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
332 if (!is_tx_ring) {
333 VERIFY(pcb->utun_netif_rxring == NULL);
334 pcb->utun_netif_rxring = ring;
335 } else {
336 VERIFY(pcb->utun_netif_txring == NULL);
337 pcb->utun_netif_txring = ring;
338 }
339 return 0;
340 }
341
342 static void
343 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
344 kern_channel_ring_t ring)
345 {
346 #pragma unused(nxprov)
347 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
348 if (pcb->utun_netif_rxring == ring) {
349 pcb->utun_netif_rxring = NULL;
350 } else if (pcb->utun_netif_txring == ring) {
351 pcb->utun_netif_txring = NULL;
352 }
353 }
354
355 static errno_t
356 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
357 kern_channel_ring_t tx_ring, uint32_t flags)
358 {
359 #pragma unused(nxprov)
360 #pragma unused(flags)
361 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
362
363 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
364
365 lck_rw_lock_shared(&pcb->utun_pcb_lock);
366
367 struct kern_channel_ring_stat_increment tx_ring_stats;
368 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
369 kern_channel_slot_t tx_pslot = NULL;
370 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
371
372 STATS_INC(nifs, NETIF_STATS_TX_SYNC);
373
374 if (tx_slot == NULL) {
375 // Nothing to write, don't bother signalling
376 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
377 return 0;
378 }
379
380 if (pcb->utun_kpipe_enabled) {
381 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
382 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
383
384 // Signal the kernel pipe ring to read
385 if (rx_ring != NULL) {
386 kern_channel_notify(rx_ring, 0);
387 }
388 return 0;
389 }
390
391 // If we're here, we're injecting into the utun kernel control socket
392 while (tx_slot != NULL) {
393 size_t length = 0;
394 mbuf_t data = NULL;
395
396 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
397
398 if (tx_ph == 0) {
399 // Advance TX ring
400 tx_pslot = tx_slot;
401 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
402 continue;
403 }
404 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
405
406 // Advance TX ring
407 tx_pslot = tx_slot;
408 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
409
410 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
411 VERIFY(tx_buf != NULL);
412
413 /* tx_baddr is the absolute buffer address */
414 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
415 VERIFY(tx_baddr != 0);
416
417 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
418
419 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
420 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
421
422 // The offset must be large enough for the headers
423 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
424
425 // Find family
426 uint32_t af = 0;
427 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
428 u_int ip_version = (vhl >> 4);
429 switch (ip_version) {
430 case 4: {
431 af = AF_INET;
432 break;
433 }
434 case 6: {
435 af = AF_INET6;
436 break;
437 }
438 default: {
439 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
440 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
441 UTUN_HEADER_SIZE(pcb));
442 break;
443 }
444 }
445
446 tx_offset -= UTUN_HEADER_SIZE(pcb);
447 tx_length += UTUN_HEADER_SIZE(pcb);
448 tx_baddr += tx_offset;
449
450 length = MIN(tx_length, pcb->utun_slot_size);
451
452 // Copy in family
453 memcpy(tx_baddr, &af, sizeof(af));
454 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
455 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
456 }
457
458 if (length > 0) {
459 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
460 if (error == 0) {
461 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
462 if (error == 0) {
463 error = utun_output(pcb->utun_ifp, data);
464 if (error != 0) {
465 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
466 }
467 } else {
468 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
469 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
470 STATS_INC(nifs, NETIF_STATS_DROP);
471 mbuf_freem(data);
472 data = NULL;
473 }
474 } else {
475 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
476 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
477 STATS_INC(nifs, NETIF_STATS_DROP);
478 }
479 } else {
480 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
481 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
482 STATS_INC(nifs, NETIF_STATS_DROP);
483 }
484
485 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
486
487 if (data == NULL) {
488 continue;
489 }
490
491 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
492 STATS_INC(nifs, NETIF_STATS_TX_COPY_MBUF);
493
494 tx_ring_stats.kcrsi_slots_transferred++;
495 tx_ring_stats.kcrsi_bytes_transferred += length;
496 }
497
498 if (tx_pslot) {
499 kern_channel_advance_slot(tx_ring, tx_pslot);
500 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
501 (void)kern_channel_reclaim(tx_ring);
502 }
503
504 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
505
506 return 0;
507 }
508
509 static errno_t
510 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
511 kern_channel_ring_t ring, __unused uint32_t flags)
512 {
513 #pragma unused(nxprov)
514 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
515 boolean_t more = false;
516 errno_t rc = 0;
517
518 /*
519 * Refill and sync the ring; we may be racing against another thread doing
520 * an RX sync that also wants to do kr_enter(), and so use the blocking
521 * variant here.
522 */
523 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
524 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
525 os_log_error(OS_LOG_DEFAULT, "%s, tx refill failed %d\n", __func__, rc);
526 }
527
528 (void) kr_enter(ring, TRUE);
529 lck_rw_lock_shared(&pcb->utun_pcb_lock);
530
531 if (pcb->utun_kpipe_enabled) {
532 uint32_t tx_available = kern_channel_available_slot_count(ring);
533 if (pcb->utun_netif_txring_size > 0 &&
534 tx_available >= pcb->utun_netif_txring_size - 1) {
535 // No room left in tx ring, disable output for now
536 errno_t error = ifnet_disable_output(pcb->utun_ifp);
537 if (error != 0) {
538 os_log_error(OS_LOG_DEFAULT, "utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
539 }
540 }
541 }
542
543 if (pcb->utun_kpipe_enabled) {
544 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
545
546 // Unlock while calling notify
547 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
548 // Signal the kernel pipe ring to read
549 if (rx_ring != NULL) {
550 kern_channel_notify(rx_ring, 0);
551 }
552 } else {
553 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
554 }
555
556 kr_exit(ring);
557
558 return 0;
559 }
560
561 static errno_t
562 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
563 kern_channel_ring_t rx_ring, uint32_t flags)
564 {
565 #pragma unused(nxprov)
566 #pragma unused(flags)
567 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
568 struct kern_channel_ring_stat_increment rx_ring_stats;
569
570 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
571
572 lck_rw_lock_shared(&pcb->utun_pcb_lock);
573
574 // Reclaim user-released slots
575 (void) kern_channel_reclaim(rx_ring);
576
577 STATS_INC(nifs, NETIF_STATS_RX_SYNC);
578
579 uint32_t avail = kern_channel_available_slot_count(rx_ring);
580 if (avail == 0) {
581 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
582 return 0;
583 }
584
585 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
586 VERIFY(rx_pp != NULL);
587 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
588 kern_channel_slot_t rx_pslot = NULL;
589 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
590
591 while (rx_slot != NULL) {
592 // Check for a waiting packet
593 lck_mtx_lock(&pcb->utun_input_chain_lock);
594 mbuf_t data = pcb->utun_input_chain;
595 if (data == NULL) {
596 lck_mtx_unlock(&pcb->utun_input_chain_lock);
597 break;
598 }
599
600 // Allocate rx packet
601 kern_packet_t rx_ph = 0;
602 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
603 if (__improbable(error != 0)) {
604 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
605 STATS_INC(nifs, NETIF_STATS_DROP);
606 lck_mtx_unlock(&pcb->utun_input_chain_lock);
607 break;
608 }
609
610 // Advance waiting packets
611 if (pcb->utun_input_chain_count > 0) {
612 pcb->utun_input_chain_count--;
613 }
614 pcb->utun_input_chain = data->m_nextpkt;
615 data->m_nextpkt = NULL;
616 if (pcb->utun_input_chain == NULL) {
617 pcb->utun_input_chain_last = NULL;
618 }
619 lck_mtx_unlock(&pcb->utun_input_chain_lock);
620
621 size_t header_offset = UTUN_HEADER_SIZE(pcb);
622 size_t length = mbuf_pkthdr_len(data);
623
624 if (length < header_offset) {
625 // mbuf is too small
626 mbuf_freem(data);
627 kern_pbufpool_free(rx_pp, rx_ph);
628 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
629 STATS_INC(nifs, NETIF_STATS_DROP);
630 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
631 pcb->utun_ifp->if_xname, length, header_offset);
632 continue;
633 }
634
635 length -= header_offset;
636 if (length > rx_pp->pp_buflet_size) {
637 // Flush data
638 mbuf_freem(data);
639 kern_pbufpool_free(rx_pp, rx_ph);
640 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
641 STATS_INC(nifs, NETIF_STATS_DROP);
642 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
643 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
644 continue;
645 }
646
647 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
648
649 // Fillout rx packet
650 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
651 VERIFY(rx_buf != NULL);
652 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
653 VERIFY(rx_baddr != NULL);
654
655 // Copy-in data from mbuf to buflet
656 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
657 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
658
659 // Finalize and attach the packet
660 error = kern_buflet_set_data_offset(rx_buf, 0);
661 VERIFY(error == 0);
662 error = kern_buflet_set_data_length(rx_buf, length);
663 VERIFY(error == 0);
664 error = kern_packet_set_headroom(rx_ph, 0);
665 VERIFY(error == 0);
666 error = kern_packet_finalize(rx_ph);
667 VERIFY(error == 0);
668 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
669 VERIFY(error == 0);
670
671 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
672 STATS_INC(nifs, NETIF_STATS_RX_COPY_MBUF);
673 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
674
675 rx_ring_stats.kcrsi_slots_transferred++;
676 rx_ring_stats.kcrsi_bytes_transferred += length;
677
678 mbuf_freem(data);
679
680 // Advance ring
681 rx_pslot = rx_slot;
682 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
683 }
684
685 struct kern_channel_ring_stat_increment tx_ring_stats;
686 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
687 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
688 kern_channel_slot_t tx_pslot = NULL;
689 kern_channel_slot_t tx_slot = NULL;
690 if (tx_ring == NULL) {
691 // Net-If TX ring not set up yet, nothing to read
692 goto done;
693 }
694
695 // Unlock utun before entering ring
696 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
697
698 (void)kr_enter(tx_ring, TRUE);
699
700 // Lock again after entering and validate
701 lck_rw_lock_shared(&pcb->utun_pcb_lock);
702 if (tx_ring != pcb->utun_kpipe_txring) {
703 goto done;
704 }
705
706 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
707 if (tx_slot == NULL) {
708 // Nothing to read, don't bother signalling
709 goto done;
710 }
711
712 while (rx_slot != NULL && tx_slot != NULL) {
713 // Allocate rx packet
714 kern_packet_t rx_ph = 0;
715 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
716
717 // Advance TX ring
718 tx_pslot = tx_slot;
719 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
720
721 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
722 if (tx_ph == 0) {
723 continue;
724 }
725
726 /* XXX We could try this alloc before advancing the slot to avoid
727 * dropping the packet on failure to allocate.
728 */
729 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
730 if (__improbable(error != 0)) {
731 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
732 STATS_INC(nifs, NETIF_STATS_DROP);
733 break;
734 }
735
736 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
737 VERIFY(tx_buf != NULL);
738 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
739 VERIFY(tx_baddr != 0);
740 tx_baddr += kern_buflet_get_data_offset(tx_buf);
741
742 // Check packet length
743 size_t header_offset = UTUN_HEADER_SIZE(pcb);
744 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
745 if (tx_length < header_offset) {
746 // Packet is too small
747 kern_pbufpool_free(rx_pp, rx_ph);
748 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
749 STATS_INC(nifs, NETIF_STATS_DROP);
750 os_log_error(OS_LOG_DEFAULT, "utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
751 pcb->utun_ifp->if_xname, tx_length, header_offset);
752 continue;
753 }
754
755 size_t length = MIN(tx_length - header_offset,
756 pcb->utun_slot_size);
757
758 tx_ring_stats.kcrsi_slots_transferred++;
759 tx_ring_stats.kcrsi_bytes_transferred += length;
760
761 // Fillout rx packet
762 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
763 VERIFY(rx_buf != NULL);
764 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
765 VERIFY(rx_baddr != NULL);
766
767 // Copy-in data from tx to rx
768 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
769 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
770
771 // Finalize and attach the packet
772 error = kern_buflet_set_data_offset(rx_buf, 0);
773 VERIFY(error == 0);
774 error = kern_buflet_set_data_length(rx_buf, length);
775 VERIFY(error == 0);
776 error = kern_packet_set_headroom(rx_ph, 0);
777 VERIFY(error == 0);
778 error = kern_packet_finalize(rx_ph);
779 VERIFY(error == 0);
780 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
781 VERIFY(error == 0);
782
783 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
784 STATS_INC(nifs, NETIF_STATS_RX_COPY_DIRECT);
785 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
786
787 rx_ring_stats.kcrsi_slots_transferred++;
788 rx_ring_stats.kcrsi_bytes_transferred += length;
789
790 rx_pslot = rx_slot;
791 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
792 }
793
794 done:
795 if (rx_pslot) {
796 kern_channel_advance_slot(rx_ring, rx_pslot);
797 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
798 }
799
800 if (tx_pslot) {
801 kern_channel_advance_slot(tx_ring, tx_pslot);
802 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
803 (void)kern_channel_reclaim(tx_ring);
804 }
805
806 // Unlock first, then exit ring
807 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
808 if (tx_ring != NULL) {
809 if (tx_pslot != NULL) {
810 kern_channel_notify(tx_ring, 0);
811 }
812 kr_exit(tx_ring);
813 }
814
815 return 0;
816 }
817
818 static errno_t
819 utun_nexus_ifattach(struct utun_pcb *pcb,
820 struct ifnet_init_eparams *init_params,
821 struct ifnet **ifp)
822 {
823 errno_t err;
824 nexus_controller_t controller = kern_nexus_shared_controller();
825 struct kern_nexus_net_init net_init;
826 struct kern_pbufpool_init pp_init;
827
828 nexus_name_t provider_name;
829 snprintf((char *)provider_name, sizeof(provider_name),
830 "com.apple.netif.%s", pcb->utun_if_xname);
831
832 struct kern_nexus_provider_init prov_init = {
833 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
834 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
835 .nxpi_pre_connect = utun_nexus_pre_connect,
836 .nxpi_connected = utun_nexus_connected,
837 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
838 .nxpi_disconnected = utun_nexus_disconnected,
839 .nxpi_ring_init = utun_netif_ring_init,
840 .nxpi_ring_fini = utun_netif_ring_fini,
841 .nxpi_slot_init = NULL,
842 .nxpi_slot_fini = NULL,
843 .nxpi_sync_tx = utun_netif_sync_tx,
844 .nxpi_sync_rx = utun_netif_sync_rx,
845 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
846 };
847
848 nexus_attr_t nxa = NULL;
849 err = kern_nexus_attr_create(&nxa);
850 if (err != 0) {
851 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
852 __func__, err);
853 goto failed;
854 }
855
856 uint64_t slot_buffer_size = pcb->utun_slot_size;
857 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
858 VERIFY(err == 0);
859
860 // Reset ring size for netif nexus to limit memory usage
861 uint64_t ring_size = pcb->utun_netif_ring_size;
862 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
863 VERIFY(err == 0);
864 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
865 VERIFY(err == 0);
866
867 pcb->utun_netif_txring_size = ring_size;
868
869 bzero(&pp_init, sizeof(pp_init));
870 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
871 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
872 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
873 pp_init.kbi_bufsize = pcb->utun_slot_size;
874 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
875 pp_init.kbi_max_frags = 1;
876 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
877 "%s", provider_name);
878 pp_init.kbi_ctx = NULL;
879 pp_init.kbi_ctx_retain = NULL;
880 pp_init.kbi_ctx_release = NULL;
881
882 err = kern_pbufpool_create(&pp_init, &pcb->utun_netif_pp, NULL);
883 if (err != 0) {
884 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, err);
885 goto failed;
886 }
887
888 err = kern_nexus_controller_register_provider(controller,
889 utun_nx_dom_prov,
890 provider_name,
891 &prov_init,
892 sizeof(prov_init),
893 nxa,
894 &pcb->utun_nx.if_provider);
895 if (err != 0) {
896 os_log_error(OS_LOG_DEFAULT, "%s register provider failed, error %d\n",
897 __func__, err);
898 goto failed;
899 }
900
901 bzero(&net_init, sizeof(net_init));
902 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
903 net_init.nxneti_flags = 0;
904 net_init.nxneti_eparams = init_params;
905 net_init.nxneti_lladdr = NULL;
906 net_init.nxneti_prepare = utun_netif_prepare;
907 net_init.nxneti_rx_pbufpool = pcb->utun_netif_pp;
908 net_init.nxneti_tx_pbufpool = pcb->utun_netif_pp;
909 err = kern_nexus_controller_alloc_net_provider_instance(controller,
910 pcb->utun_nx.if_provider,
911 pcb,
912 &pcb->utun_nx.if_instance,
913 &net_init,
914 ifp);
915 if (err != 0) {
916 os_log_error(OS_LOG_DEFAULT, "%s alloc_net_provider_instance failed, %d\n",
917 __func__, err);
918 kern_nexus_controller_deregister_provider(controller,
919 pcb->utun_nx.if_provider);
920 uuid_clear(pcb->utun_nx.if_provider);
921 goto failed;
922 }
923
924 failed:
925 if (nxa) {
926 kern_nexus_attr_destroy(nxa);
927 }
928 if (err && pcb->utun_netif_pp != NULL) {
929 kern_pbufpool_destroy(pcb->utun_netif_pp);
930 pcb->utun_netif_pp = NULL;
931 }
932 return err;
933 }
934
935 static void
936 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
937 {
938 nexus_controller_t controller = kern_nexus_shared_controller();
939 errno_t err;
940
941 if (!uuid_is_null(instance)) {
942 err = kern_nexus_controller_free_provider_instance(controller,
943 instance);
944 if (err != 0) {
945 os_log_error(OS_LOG_DEFAULT, "%s free_provider_instance failed %d\n",
946 __func__, err);
947 }
948 uuid_clear(instance);
949 }
950 if (!uuid_is_null(provider)) {
951 err = kern_nexus_controller_deregister_provider(controller,
952 provider);
953 if (err != 0) {
954 os_log_error(OS_LOG_DEFAULT, "%s deregister_provider %d\n", __func__, err);
955 }
956 uuid_clear(provider);
957 }
958 return;
959 }
960
961 static void
962 utun_nexus_detach(struct utun_pcb *pcb)
963 {
964 utun_nx_t nx = &pcb->utun_nx;
965 nexus_controller_t controller = kern_nexus_shared_controller();
966 errno_t err;
967
968 if (!uuid_is_null(nx->fsw_host)) {
969 err = kern_nexus_ifdetach(controller,
970 nx->fsw_instance,
971 nx->fsw_host);
972 if (err != 0) {
973 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms host failed %d\n",
974 __func__, err);
975 }
976 }
977
978 if (!uuid_is_null(nx->fsw_device)) {
979 err = kern_nexus_ifdetach(controller,
980 nx->fsw_instance,
981 nx->fsw_device);
982 if (err != 0) {
983 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms device failed %d\n",
984 __func__, err);
985 }
986 }
987
988 utun_detach_provider_and_instance(nx->if_provider,
989 nx->if_instance);
990 utun_detach_provider_and_instance(nx->fsw_provider,
991 nx->fsw_instance);
992
993 if (pcb->utun_netif_pp != NULL) {
994 kern_pbufpool_destroy(pcb->utun_netif_pp);
995 pcb->utun_netif_pp = NULL;
996 }
997 memset(nx, 0, sizeof(*nx));
998 }
999
1000 static errno_t
1001 utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
1002 const char *type_name,
1003 const char *ifname,
1004 uuid_t *provider, uuid_t *instance)
1005 {
1006 nexus_attr_t attr = NULL;
1007 nexus_controller_t controller = kern_nexus_shared_controller();
1008 uuid_t dom_prov;
1009 errno_t err;
1010 struct kern_nexus_init init;
1011 nexus_name_t provider_name;
1012
1013 err = kern_nexus_get_default_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1014 &dom_prov);
1015 if (err != 0) {
1016 os_log_error(OS_LOG_DEFAULT, "%s can't get %s provider, error %d\n",
1017 __func__, type_name, err);
1018 goto failed;
1019 }
1020
1021 err = kern_nexus_attr_create(&attr);
1022 if (err != 0) {
1023 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1024 __func__, err);
1025 goto failed;
1026 }
1027
1028 uint64_t slot_buffer_size = pcb->utun_slot_size;
1029 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1030 VERIFY(err == 0);
1031
1032 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
1033 uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;
1034 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1035 VERIFY(err == 0);
1036 uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;
1037 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1038 VERIFY(err == 0);
1039 /*
1040 * Configure flowswitch to use super-packet (multi-buflet).
1041 * This allows flowswitch to perform intra-stack packet aggregation.
1042 */
1043 err = kern_nexus_attr_set(attr, NEXUS_ATTR_MAX_FRAGS,
1044 sk_fsw_rx_agg_tcp ? NX_PBUF_FRAGS_MAX : 1);
1045 VERIFY(err == 0);
1046
1047 snprintf((char *)provider_name, sizeof(provider_name),
1048 "com.apple.%s.%s", type_name, ifname);
1049 err = kern_nexus_controller_register_provider(controller,
1050 dom_prov,
1051 provider_name,
1052 NULL,
1053 0,
1054 attr,
1055 provider);
1056 kern_nexus_attr_destroy(attr);
1057 attr = NULL;
1058 if (err != 0) {
1059 os_log_error(OS_LOG_DEFAULT, "%s register %s provider failed, error %d\n",
1060 __func__, type_name, err);
1061 goto failed;
1062 }
1063 bzero(&init, sizeof(init));
1064 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1065 err = kern_nexus_controller_alloc_provider_instance(controller,
1066 *provider,
1067 NULL,
1068 instance, &init);
1069 if (err != 0) {
1070 os_log_error(OS_LOG_DEFAULT, "%s alloc_provider_instance %s failed, %d\n",
1071 __func__, type_name, err);
1072 kern_nexus_controller_deregister_provider(controller,
1073 *provider);
1074 uuid_clear(*provider);
1075 }
1076 failed:
1077 return err;
1078 }
1079
1080 static errno_t
1081 utun_flowswitch_attach(struct utun_pcb *pcb)
1082 {
1083 nexus_controller_t controller = kern_nexus_shared_controller();
1084 errno_t err = 0;
1085 utun_nx_t nx = &pcb->utun_nx;
1086
1087 // Allocate flowswitch
1088 err = utun_create_fs_provider_and_instance(pcb,
1089 "flowswitch",
1090 pcb->utun_ifp->if_xname,
1091 &nx->fsw_provider,
1092 &nx->fsw_instance);
1093 if (err != 0) {
1094 os_log_error(OS_LOG_DEFAULT, "%s: failed to create bridge provider and instance\n",
1095 __func__);
1096 goto failed;
1097 }
1098
1099 // Attach flowswitch to device port
1100 err = kern_nexus_ifattach(controller, nx->fsw_instance,
1101 NULL, nx->if_instance,
1102 FALSE, &nx->fsw_device);
1103 if (err != 0) {
1104 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms device %d\n", __func__, err);
1105 goto failed;
1106 }
1107
1108 // Attach flowswitch to host port
1109 err = kern_nexus_ifattach(controller, nx->fsw_instance,
1110 NULL, nx->if_instance,
1111 TRUE, &nx->fsw_host);
1112 if (err != 0) {
1113 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms host %d\n", __func__, err);
1114 goto failed;
1115 }
1116
1117 // Extract the agent UUID and save for later
1118 struct kern_nexus *flowswitch_nx = nx_find(nx->fsw_instance, false);
1119 if (flowswitch_nx != NULL) {
1120 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(flowswitch_nx);
1121 if (flowswitch != NULL) {
1122 FSW_RLOCK(flowswitch);
1123 uuid_copy(nx->fsw_agent, flowswitch->fsw_agent_uuid);
1124 FSW_UNLOCK(flowswitch);
1125 } else {
1126 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - flowswitch is NULL\n");
1127 }
1128 nx_release(flowswitch_nx);
1129 } else {
1130 os_log_error(OS_LOG_DEFAULT, "utun_flowswitch_attach - unable to find flowswitch nexus\n");
1131 }
1132
1133 return 0;
1134
1135 failed:
1136 utun_nexus_detach(pcb);
1137
1138 errno_t detach_error = 0;
1139 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1140 panic("utun_flowswitch_attach - ifnet_detach failed: %d\n", detach_error);
1141 /* NOT REACHED */
1142 }
1143
1144 return err;
1145 }
1146
1147 static errno_t
1148 utun_register_kernel_pipe_nexus(struct utun_pcb *pcb)
1149 {
1150 nexus_attr_t nxa = NULL;
1151 errno_t result;
1152
1153 lck_mtx_lock(&utun_lock);
1154 if (utun_ncd_refcount++) {
1155 lck_mtx_unlock(&utun_lock);
1156 return 0;
1157 }
1158
1159 result = kern_nexus_controller_create(&utun_ncd);
1160 if (result) {
1161 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_create failed: %d\n",
1162 __FUNCTION__, result);
1163 goto done;
1164 }
1165
1166 uuid_t dom_prov;
1167 result = kern_nexus_get_default_domain_provider(
1168 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1169 if (result) {
1170 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_get_default_domain_provider failed: %d\n",
1171 __FUNCTION__, result);
1172 goto done;
1173 }
1174
1175 struct kern_nexus_provider_init prov_init = {
1176 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1177 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1178 .nxpi_pre_connect = utun_nexus_pre_connect,
1179 .nxpi_connected = utun_nexus_connected,
1180 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1181 .nxpi_disconnected = utun_nexus_disconnected,
1182 .nxpi_ring_init = utun_kpipe_ring_init,
1183 .nxpi_ring_fini = utun_kpipe_ring_fini,
1184 .nxpi_slot_init = NULL,
1185 .nxpi_slot_fini = NULL,
1186 .nxpi_sync_tx = utun_kpipe_sync_tx,
1187 .nxpi_sync_rx = utun_kpipe_sync_rx,
1188 .nxpi_tx_doorbell = NULL,
1189 };
1190
1191 result = kern_nexus_attr_create(&nxa);
1192 if (result) {
1193 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
1194 __FUNCTION__, result);
1195 goto done;
1196 }
1197
1198 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1199 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1200 VERIFY(result == 0);
1201
1202 // Reset ring size for kernel pipe nexus to limit memory usage
1203 uint64_t ring_size =
1204 pcb->utun_kpipe_tx_ring_size != 0 ? pcb->utun_kpipe_tx_ring_size :
1205 if_utun_ring_size;
1206 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1207 VERIFY(result == 0);
1208
1209 ring_size =
1210 pcb->utun_kpipe_rx_ring_size != 0 ? pcb->utun_kpipe_rx_ring_size :
1211 if_utun_ring_size;
1212 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1213 VERIFY(result == 0);
1214
1215 result = kern_nexus_controller_register_provider(utun_ncd,
1216 dom_prov,
1217 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1218 &prov_init,
1219 sizeof(prov_init),
1220 nxa,
1221 &utun_kpipe_uuid);
1222 if (result) {
1223 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_register_provider failed: %d\n",
1224 __FUNCTION__, result);
1225 goto done;
1226 }
1227
1228 done:
1229 if (nxa) {
1230 kern_nexus_attr_destroy(nxa);
1231 }
1232
1233 if (result) {
1234 if (utun_ncd) {
1235 kern_nexus_controller_destroy(utun_ncd);
1236 utun_ncd = NULL;
1237 }
1238 utun_ncd_refcount = 0;
1239 }
1240
1241 lck_mtx_unlock(&utun_lock);
1242
1243 return result;
1244 }
1245
1246 static void
1247 utun_unregister_kernel_pipe_nexus(void)
1248 {
1249 lck_mtx_lock(&utun_lock);
1250
1251 VERIFY(utun_ncd_refcount > 0);
1252
1253 if (--utun_ncd_refcount == 0) {
1254 kern_nexus_controller_destroy(utun_ncd);
1255 utun_ncd = NULL;
1256 }
1257
1258 lck_mtx_unlock(&utun_lock);
1259 }
1260
1261 // For use by socket option, not internally
1262 static errno_t
1263 utun_disable_channel(struct utun_pcb *pcb)
1264 {
1265 errno_t result;
1266 int enabled;
1267 uuid_t uuid;
1268
1269 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1270
1271 enabled = pcb->utun_kpipe_enabled;
1272 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1273
1274 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1275
1276 pcb->utun_kpipe_enabled = 0;
1277 uuid_clear(pcb->utun_kpipe_uuid);
1278
1279 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1280
1281 if (enabled) {
1282 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1283 } else {
1284 result = ENXIO;
1285 }
1286
1287 if (!result) {
1288 if (pcb->utun_kpipe_pp != NULL) {
1289 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1290 pcb->utun_kpipe_pp = NULL;
1291 }
1292 utun_unregister_kernel_pipe_nexus();
1293 }
1294
1295 return result;
1296 }
1297
1298 static errno_t
1299 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1300 {
1301 struct kern_nexus_init init;
1302 struct kern_pbufpool_init pp_init;
1303 errno_t result;
1304
1305 kauth_cred_t cred = kauth_cred_get();
1306 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
1307 if (result) {
1308 return result;
1309 }
1310
1311 result = utun_register_kernel_pipe_nexus(pcb);
1312 if (result) {
1313 return result;
1314 }
1315
1316 VERIFY(utun_ncd);
1317
1318 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1319
1320 if (pcb->utun_kpipe_enabled) {
1321 result = EEXIST; // return success instead?
1322 goto done;
1323 }
1324
1325 /*
1326 * Make sure we can fit packets in the channel buffers and
1327 * Allow an extra 4 bytes for the protocol number header in the channel
1328 */
1329 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {
1330 result = EOPNOTSUPP;
1331 goto done;
1332 }
1333
1334 bzero(&pp_init, sizeof(pp_init));
1335 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1336 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
1337 pp_init.kbi_packets = pcb->utun_netif_ring_size * 2;
1338 pp_init.kbi_bufsize = pcb->utun_slot_size;
1339 pp_init.kbi_buf_seg_size = UTUN_IF_DEFAULT_BUF_SEG_SIZE;
1340 pp_init.kbi_max_frags = 1;
1341 pp_init.kbi_flags |= KBIF_QUANTUM;
1342 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
1343 "com.apple.kpipe.%s", pcb->utun_if_xname);
1344 pp_init.kbi_ctx = NULL;
1345 pp_init.kbi_ctx_retain = NULL;
1346 pp_init.kbi_ctx_release = NULL;
1347
1348 result = kern_pbufpool_create(&pp_init, &pcb->utun_kpipe_pp,
1349 NULL);
1350 if (result != 0) {
1351 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, result);
1352 goto done;
1353 }
1354
1355 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1356 bzero(&init, sizeof(init));
1357 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1358 init.nxi_tx_pbufpool = pcb->utun_kpipe_pp;
1359 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1360 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1361 if (result) {
1362 goto done;
1363 }
1364
1365 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1366 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1367 pcb->utun_kpipe_uuid, &port,
1368 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1369 if (result) {
1370 kern_nexus_controller_free_provider_instance(utun_ncd,
1371 pcb->utun_kpipe_uuid);
1372 uuid_clear(pcb->utun_kpipe_uuid);
1373 goto done;
1374 }
1375
1376 pcb->utun_kpipe_enabled = 1;
1377
1378 done:
1379 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1380
1381 if (result) {
1382 if (pcb->utun_kpipe_pp != NULL) {
1383 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1384 pcb->utun_kpipe_pp = NULL;
1385 }
1386 utun_unregister_kernel_pipe_nexus();
1387 }
1388
1389 return result;
1390 }
1391
1392 #endif // UTUN_NEXUS
1393
1394 errno_t
1395 utun_register_control(void)
1396 {
1397 struct kern_ctl_reg kern_ctl;
1398 errno_t result = 0;
1399
1400 #if UTUN_NEXUS
1401 utun_register_nexus();
1402 #endif // UTUN_NEXUS
1403
1404 TAILQ_INIT(&utun_head);
1405
1406 bzero(&kern_ctl, sizeof(kern_ctl));
1407 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1408 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1409 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_SETUP | CTL_FLAG_REG_EXTENDED; /* Require root */
1410 kern_ctl.ctl_sendsize = 512 * 1024;
1411 kern_ctl.ctl_recvsize = 512 * 1024;
1412 kern_ctl.ctl_setup = utun_ctl_setup;
1413 kern_ctl.ctl_bind = utun_ctl_bind;
1414 kern_ctl.ctl_connect = utun_ctl_connect;
1415 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1416 kern_ctl.ctl_send = utun_ctl_send;
1417 kern_ctl.ctl_setopt = utun_ctl_setopt;
1418 kern_ctl.ctl_getopt = utun_ctl_getopt;
1419 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1420
1421 result = ctl_register(&kern_ctl, &utun_kctlref);
1422 if (result != 0) {
1423 os_log_error(OS_LOG_DEFAULT, "utun_register_control - ctl_register failed: %d\n", result);
1424 return result;
1425 }
1426
1427 /* Register the protocol plumbers */
1428 if ((result = proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN,
1429 utun_attach_proto, NULL)) != 0) {
1430 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET, IFNET_FAMILY_UTUN) failed: %d\n",
1431 result);
1432 ctl_deregister(utun_kctlref);
1433 return result;
1434 }
1435
1436 /* Register the protocol plumbers */
1437 if ((result = proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN,
1438 utun_attach_proto, NULL)) != 0) {
1439 proto_unregister_plumber(PF_INET, IFNET_FAMILY_UTUN);
1440 ctl_deregister(utun_kctlref);
1441 os_log_error(OS_LOG_DEFAULT, "utun_register_control - proto_register_plumber(PF_INET6, IFNET_FAMILY_UTUN) failed: %d\n",
1442 result);
1443 return result;
1444 }
1445
1446 utun_lck_attr = lck_attr_alloc_init();
1447 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1448 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1449
1450 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1451
1452 return 0;
1453 }
1454
1455 /* Kernel control functions */
1456
1457 static inline int
1458 utun_find_by_unit(u_int32_t unit)
1459 {
1460 struct utun_pcb *next_pcb = NULL;
1461 int found = 0;
1462
1463 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1464 if (next_pcb->utun_unit == unit) {
1465 found = 1;
1466 break;
1467 }
1468 }
1469
1470 return found;
1471 }
1472
1473 static inline void
1474 utun_free_pcb(struct utun_pcb *pcb, bool locked)
1475 {
1476 #if UTUN_NEXUS
1477 mbuf_freem_list(pcb->utun_input_chain);
1478 pcb->utun_input_chain_count = 0;
1479 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1480 #endif // UTUN_NEXUS
1481 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1482 if (!locked) {
1483 lck_mtx_lock(&utun_lock);
1484 }
1485 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1486 if (!locked) {
1487 lck_mtx_unlock(&utun_lock);
1488 }
1489 zfree(utun_pcb_zone, pcb);
1490 }
1491
1492 static errno_t
1493 utun_ctl_setup(u_int32_t *unit, void **unitinfo)
1494 {
1495 if (unit == NULL || unitinfo == NULL) {
1496 return EINVAL;
1497 }
1498
1499 lck_mtx_lock(&utun_lock);
1500
1501 /* Find next available unit */
1502 if (*unit == 0) {
1503 *unit = 1;
1504 while (*unit != ctl_maxunit) {
1505 if (utun_find_by_unit(*unit)) {
1506 (*unit)++;
1507 } else {
1508 break;
1509 }
1510 }
1511 if (*unit == ctl_maxunit) {
1512 lck_mtx_unlock(&utun_lock);
1513 return EBUSY;
1514 }
1515 } else if (utun_find_by_unit(*unit)) {
1516 lck_mtx_unlock(&utun_lock);
1517 return EBUSY;
1518 }
1519
1520 /* Find some open interface id */
1521 u_int32_t chosen_unique_id = 1;
1522 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1523 if (next_pcb != NULL) {
1524 /* List was not empty, add one to the last item */
1525 chosen_unique_id = next_pcb->utun_unique_id + 1;
1526 next_pcb = NULL;
1527
1528 /*
1529 * If this wrapped the id number, start looking at
1530 * the front of the list for an unused id.
1531 */
1532 if (chosen_unique_id == 0) {
1533 /* Find the next unused ID */
1534 chosen_unique_id = 1;
1535 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1536 if (next_pcb->utun_unique_id > chosen_unique_id) {
1537 /* We found a gap */
1538 break;
1539 }
1540
1541 chosen_unique_id = next_pcb->utun_unique_id + 1;
1542 }
1543 }
1544 }
1545
1546 struct utun_pcb *pcb = zalloc_flags(utun_pcb_zone, Z_WAITOK | Z_ZERO);
1547
1548 *unitinfo = pcb;
1549 pcb->utun_unit = *unit;
1550 pcb->utun_unique_id = chosen_unique_id;
1551
1552 if (next_pcb != NULL) {
1553 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1554 } else {
1555 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1556 }
1557
1558 lck_mtx_unlock(&utun_lock);
1559
1560 return 0;
1561 }
1562
1563 static errno_t
1564 utun_ctl_bind(kern_ctl_ref kctlref,
1565 struct sockaddr_ctl *sac,
1566 void **unitinfo)
1567 {
1568 if (*unitinfo == NULL) {
1569 u_int32_t unit = 0;
1570 (void)utun_ctl_setup(&unit, unitinfo);
1571 }
1572
1573 struct utun_pcb *pcb = (struct utun_pcb *)*unitinfo;
1574 if (pcb == NULL) {
1575 return EINVAL;
1576 }
1577
1578 pcb->utun_ctlref = kctlref;
1579 pcb->utun_unit = sac->sc_unit;
1580 pcb->utun_max_pending_packets = 1;
1581
1582 #if UTUN_NEXUS
1583 pcb->utun_use_netif = false;
1584 pcb->utun_attach_fsw = true;
1585 pcb->utun_netif_connected = false;
1586 pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1587 pcb->utun_netif_ring_size = if_utun_ring_size;
1588 pcb->utun_tx_fsw_ring_size = if_utun_tx_fsw_ring_size;
1589 pcb->utun_rx_fsw_ring_size = if_utun_rx_fsw_ring_size;
1590 pcb->utun_input_chain_count = 0;
1591 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1592 #endif // UTUN_NEXUS
1593
1594 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1595
1596 return 0;
1597 }
1598
1599 static errno_t
1600 utun_ctl_connect(kern_ctl_ref kctlref,
1601 struct sockaddr_ctl *sac,
1602 void **unitinfo)
1603 {
1604 struct ifnet_init_eparams utun_init = {};
1605 errno_t result = 0;
1606
1607 if (*unitinfo == NULL) {
1608 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1609 }
1610
1611 struct utun_pcb *pcb = *unitinfo;
1612 if (pcb == NULL) {
1613 return EINVAL;
1614 }
1615
1616 /* Handle case where utun_ctl_setup() was called, but ipsec_ctl_bind() was not */
1617 if (pcb->utun_ctlref == NULL) {
1618 (void)utun_ctl_bind(kctlref, sac, unitinfo);
1619 }
1620
1621 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1622 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1623
1624 /* Create the interface */
1625 bzero(&utun_init, sizeof(utun_init));
1626 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1627 utun_init.len = sizeof(utun_init);
1628
1629 #if UTUN_NEXUS
1630 if (pcb->utun_use_netif) {
1631 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1632 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1633 } else
1634 #endif // UTUN_NEXUS
1635 {
1636 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1637 utun_init.start = utun_start;
1638 utun_init.framer_extended = utun_framer;
1639 }
1640 utun_init.name = "utun";
1641 utun_init.unit = pcb->utun_unit - 1;
1642 utun_init.uniqueid = pcb->utun_unique_name;
1643 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1644 utun_init.family = IFNET_FAMILY_UTUN;
1645 utun_init.type = IFT_OTHER;
1646 utun_init.demux = utun_demux;
1647 utun_init.add_proto = utun_add_proto;
1648 utun_init.del_proto = utun_del_proto;
1649 utun_init.softc = pcb;
1650 utun_init.ioctl = utun_ioctl;
1651 utun_init.free = utun_detached;
1652
1653 #if UTUN_NEXUS
1654 if (pcb->utun_use_netif) {
1655 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1656 if (result != 0) {
1657 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1658 utun_free_pcb(pcb, false);
1659 *unitinfo = NULL;
1660 return result;
1661 }
1662
1663 if (pcb->utun_attach_fsw) {
1664 result = utun_flowswitch_attach(pcb);
1665 if (result != 0) {
1666 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - utun_flowswitch_attach failed: %d\n", result);
1667 // Do not call utun_free_pcb(). We will be attached already, and will be freed later
1668 // in utun_detached().
1669 *unitinfo = NULL;
1670 return result;
1671 }
1672 }
1673
1674 /* Attach to bpf */
1675 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1676 } else
1677 #endif // UTUN_NEXUS
1678 {
1679 /*
1680 * Upon success, this holds an ifnet reference which we will
1681 * release via ifnet_release() at final detach time.
1682 */
1683 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1684 if (result != 0) {
1685 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1686 utun_free_pcb(pcb, false);
1687 *unitinfo = NULL;
1688 return result;
1689 }
1690
1691 /* Set flags and additional information. */
1692 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1693 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1694
1695 /* The interface must generate its own IPv6 LinkLocal address,
1696 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1697 */
1698 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1699
1700 /* Reset the stats in case as the interface may have been recycled */
1701 struct ifnet_stats_param stats;
1702 bzero(&stats, sizeof(struct ifnet_stats_param));
1703 ifnet_set_stat(pcb->utun_ifp, &stats);
1704
1705 /* Attach the interface */
1706 result = ifnet_attach(pcb->utun_ifp, NULL);
1707 if (result != 0) {
1708 os_log_error(OS_LOG_DEFAULT, "utun_ctl_connect - ifnet_attach failed: %d\n", result);
1709 /* Release reference now since attach failed */
1710 ifnet_release(pcb->utun_ifp);
1711 utun_free_pcb(pcb, false);
1712 *unitinfo = NULL;
1713 return result;
1714 }
1715
1716 /* Attach to bpf */
1717 bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
1718 }
1719
1720 /* The interfaces resoures allocated, mark it as running */
1721 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1722
1723 return result;
1724 }
1725
1726 static errno_t
1727 utun_detach_ip(ifnet_t interface,
1728 protocol_family_t protocol,
1729 socket_t pf_socket)
1730 {
1731 errno_t result = EPROTONOSUPPORT;
1732
1733 /* Attempt a detach */
1734 if (protocol == PF_INET) {
1735 struct ifreq ifr;
1736
1737 bzero(&ifr, sizeof(ifr));
1738 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1739 ifnet_name(interface), ifnet_unit(interface));
1740
1741 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1742 } else if (protocol == PF_INET6) {
1743 struct in6_ifreq ifr6;
1744
1745 bzero(&ifr6, sizeof(ifr6));
1746 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1747 ifnet_name(interface), ifnet_unit(interface));
1748
1749 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1750 }
1751
1752 return result;
1753 }
1754
1755 static void
1756 utun_remove_address(ifnet_t interface,
1757 protocol_family_t protocol,
1758 ifaddr_t address,
1759 socket_t pf_socket)
1760 {
1761 errno_t result = 0;
1762
1763 /* Attempt a detach */
1764 if (protocol == PF_INET) {
1765 struct ifreq ifr;
1766
1767 bzero(&ifr, sizeof(ifr));
1768 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1769 ifnet_name(interface), ifnet_unit(interface));
1770 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1771 if (result != 0) {
1772 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed: %d", result);
1773 } else {
1774 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1775 if (result != 0) {
1776 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR failed: %d", result);
1777 }
1778 }
1779 } else if (protocol == PF_INET6) {
1780 struct in6_ifreq ifr6;
1781
1782 bzero(&ifr6, sizeof(ifr6));
1783 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1784 ifnet_name(interface), ifnet_unit(interface));
1785 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1786 sizeof(ifr6.ifr_addr));
1787 if (result != 0) {
1788 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - ifaddr_address failed (v6): %d",
1789 result);
1790 } else {
1791 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1792 if (result != 0) {
1793 os_log_error(OS_LOG_DEFAULT, "utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1794 result);
1795 }
1796 }
1797 }
1798 }
1799
1800 static void
1801 utun_cleanup_family(ifnet_t interface,
1802 protocol_family_t protocol)
1803 {
1804 errno_t result = 0;
1805 socket_t pf_socket = NULL;
1806 ifaddr_t *addresses = NULL;
1807 int i;
1808
1809 if (protocol != PF_INET && protocol != PF_INET6) {
1810 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - invalid protocol family %d\n", protocol);
1811 return;
1812 }
1813
1814 /* Create a socket for removing addresses and detaching the protocol */
1815 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1816 if (result != 0) {
1817 if (result != EAFNOSUPPORT) {
1818 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - failed to create %s socket: %d\n",
1819 protocol == PF_INET ? "IP" : "IPv6", result);
1820 }
1821 goto cleanup;
1822 }
1823
1824 /* always set SS_PRIV, we want to close and detach regardless */
1825 sock_setpriv(pf_socket, 1);
1826
1827 result = utun_detach_ip(interface, protocol, pf_socket);
1828 if (result == 0 || result == ENXIO) {
1829 /* We are done! We either detached or weren't attached. */
1830 goto cleanup;
1831 } else if (result != EBUSY) {
1832 /* Uh, not really sure what happened here... */
1833 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1834 goto cleanup;
1835 }
1836
1837 /*
1838 * At this point, we received an EBUSY error. This means there are
1839 * addresses attached. We should detach them and then try again.
1840 */
1841 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1842 if (result != 0) {
1843 os_log_error(OS_LOG_DEFAULT, "fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1844 ifnet_name(interface), ifnet_unit(interface),
1845 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1846 goto cleanup;
1847 }
1848
1849 for (i = 0; addresses[i] != 0; i++) {
1850 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1851 }
1852 ifnet_free_address_list(addresses);
1853 addresses = NULL;
1854
1855 /*
1856 * The addresses should be gone, we should try the remove again.
1857 */
1858 result = utun_detach_ip(interface, protocol, pf_socket);
1859 if (result != 0 && result != ENXIO) {
1860 os_log_error(OS_LOG_DEFAULT, "utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1861 }
1862
1863 cleanup:
1864 if (pf_socket != NULL) {
1865 sock_close(pf_socket);
1866 }
1867
1868 if (addresses != NULL) {
1869 ifnet_free_address_list(addresses);
1870 }
1871 }
1872
1873 static errno_t
1874 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1875 __unused u_int32_t unit,
1876 void *unitinfo)
1877 {
1878 struct utun_pcb *pcb = unitinfo;
1879 ifnet_t ifp = NULL;
1880 errno_t result = 0;
1881
1882 if (pcb == NULL) {
1883 return EINVAL;
1884 }
1885
1886 #if UTUN_NEXUS
1887 // Tell the nexus to stop all rings
1888 if (pcb->utun_netif_nexus != NULL && pcb->utun_netif_connected) {
1889 kern_nexus_stop(pcb->utun_netif_nexus);
1890 }
1891 #endif // UTUN_NEXUS
1892
1893 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1894
1895 #if UTUN_NEXUS
1896 uuid_t kpipe_uuid;
1897 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1898 uuid_clear(pcb->utun_kpipe_uuid);
1899 pcb->utun_kpipe_enabled = FALSE;
1900 #endif // UTUN_NEXUS
1901
1902 pcb->utun_ctlref = NULL;
1903
1904 ifp = pcb->utun_ifp;
1905 if (ifp != NULL) {
1906 #if UTUN_NEXUS
1907 // Tell the nexus to stop all rings
1908 if (pcb->utun_netif_nexus != NULL) {
1909 /*
1910 * Quiesce the interface and flush any pending outbound packets.
1911 */
1912 if_down(ifp);
1913
1914 /* Increment refcnt, but detach interface */
1915 ifnet_incr_iorefcnt(ifp);
1916 if ((result = ifnet_detach(ifp)) != 0) {
1917 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1918 }
1919
1920 /*
1921 * We want to do everything in our power to ensure that the interface
1922 * really goes away when the socket is closed. We must remove IP/IPv6
1923 * addresses and detach the protocols. Finally, we can remove and
1924 * release the interface.
1925 */
1926 utun_cleanup_family(ifp, AF_INET);
1927 utun_cleanup_family(ifp, AF_INET6);
1928
1929 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1930
1931 if (!uuid_is_null(kpipe_uuid)) {
1932 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1933 if (pcb->utun_kpipe_pp != NULL) {
1934 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1935 pcb->utun_kpipe_pp = NULL;
1936 }
1937 utun_unregister_kernel_pipe_nexus();
1938 }
1939 }
1940 utun_nexus_detach(pcb);
1941
1942 /* Decrement refcnt to finish detaching and freeing */
1943 ifnet_decr_iorefcnt(ifp);
1944 } else
1945 #endif // UTUN_NEXUS
1946 {
1947 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1948
1949 #if UTUN_NEXUS
1950 if (!uuid_is_null(kpipe_uuid)) {
1951 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1952 if (pcb->utun_kpipe_pp != NULL) {
1953 kern_pbufpool_destroy(pcb->utun_kpipe_pp);
1954 pcb->utun_kpipe_pp = NULL;
1955 }
1956 utun_unregister_kernel_pipe_nexus();
1957 }
1958 }
1959 #endif // UTUN_NEXUS
1960
1961 /*
1962 * We want to do everything in our power to ensure that the interface
1963 * really goes away when the socket is closed. We must remove IP/IPv6
1964 * addresses and detach the protocols. Finally, we can remove and
1965 * release the interface.
1966 */
1967 utun_cleanup_family(ifp, AF_INET);
1968 utun_cleanup_family(ifp, AF_INET6);
1969
1970 /*
1971 * Detach now; utun_detach() will be called asynchronously once
1972 * the I/O reference count drops to 0. There we will invoke
1973 * ifnet_release().
1974 */
1975 if ((result = ifnet_detach(ifp)) != 0) {
1976 os_log_error(OS_LOG_DEFAULT, "utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1977 }
1978 }
1979 } else {
1980 // Bound, but not connected
1981 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1982 utun_free_pcb(pcb, false);
1983 }
1984
1985 return 0;
1986 }
1987
1988 static errno_t
1989 utun_ctl_send(__unused kern_ctl_ref kctlref,
1990 __unused u_int32_t unit,
1991 void *unitinfo,
1992 mbuf_t m,
1993 __unused int flags)
1994 {
1995 /*
1996 * The userland ABI requires the first four bytes have the protocol family
1997 * in network byte order: swap them
1998 */
1999 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
2000 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
2001 } else {
2002 os_log_error(OS_LOG_DEFAULT, "%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m));
2003 }
2004
2005 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
2006 }
2007
2008 static errno_t
2009 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
2010 __unused u_int32_t unit,
2011 void *unitinfo,
2012 int opt,
2013 void *data,
2014 size_t len)
2015 {
2016 struct utun_pcb *pcb = unitinfo;
2017 errno_t result = 0;
2018 /* check for privileges for privileged options */
2019 switch (opt) {
2020 case UTUN_OPT_FLAGS:
2021 case UTUN_OPT_EXT_IFDATA_STATS:
2022 case UTUN_OPT_SET_DELEGATE_INTERFACE:
2023 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
2024 return EPERM;
2025 }
2026 break;
2027 }
2028
2029 switch (opt) {
2030 case UTUN_OPT_FLAGS:
2031 if (len != sizeof(u_int32_t)) {
2032 result = EMSGSIZE;
2033 break;
2034 }
2035 if (pcb->utun_ifp != NULL) {
2036 // Only can set before connecting
2037 result = EINVAL;
2038 break;
2039 }
2040 pcb->utun_flags = *(u_int32_t *)data;
2041 break;
2042
2043 case UTUN_OPT_EXT_IFDATA_STATS:
2044 if (len != sizeof(int)) {
2045 result = EMSGSIZE;
2046 break;
2047 }
2048 if (pcb->utun_ifp == NULL) {
2049 // Only can set after connecting
2050 result = EINVAL;
2051 break;
2052 }
2053 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
2054 break;
2055
2056 case UTUN_OPT_INC_IFDATA_STATS_IN:
2057 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
2058 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
2059
2060 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
2061 result = EINVAL;
2062 break;
2063 }
2064 if (pcb->utun_ifp == NULL) {
2065 // Only can set after connecting
2066 result = EINVAL;
2067 break;
2068 }
2069 if (!pcb->utun_ext_ifdata_stats) {
2070 result = EINVAL;
2071 break;
2072 }
2073 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN) {
2074 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
2075 utsp->utsp_bytes, utsp->utsp_errors);
2076 } else {
2077 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
2078 utsp->utsp_bytes, utsp->utsp_errors);
2079 }
2080 break;
2081 }
2082 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
2083 ifnet_t del_ifp = NULL;
2084 char name[IFNAMSIZ];
2085
2086 if (len > IFNAMSIZ - 1) {
2087 result = EMSGSIZE;
2088 break;
2089 }
2090 if (pcb->utun_ifp == NULL) {
2091 // Only can set after connecting
2092 result = EINVAL;
2093 break;
2094 }
2095 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2096 bcopy(data, name, len);
2097 name[len] = 0;
2098 result = ifnet_find_by_name(name, &del_ifp);
2099 }
2100 if (result == 0) {
2101 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
2102 if (del_ifp) {
2103 ifnet_release(del_ifp);
2104 }
2105 }
2106 break;
2107 }
2108 case UTUN_OPT_MAX_PENDING_PACKETS: {
2109 u_int32_t max_pending_packets = 0;
2110 if (len != sizeof(u_int32_t)) {
2111 result = EMSGSIZE;
2112 break;
2113 }
2114 max_pending_packets = *(u_int32_t *)data;
2115 if (max_pending_packets == 0) {
2116 result = EINVAL;
2117 break;
2118 }
2119 pcb->utun_max_pending_packets = max_pending_packets;
2120 break;
2121 }
2122 #if UTUN_NEXUS
2123 case UTUN_OPT_ENABLE_CHANNEL: {
2124 if (len != sizeof(int)) {
2125 result = EMSGSIZE;
2126 break;
2127 }
2128 if (pcb->utun_ifp == NULL) {
2129 // Only can set after connecting
2130 result = EINVAL;
2131 break;
2132 }
2133 if (*(int *)data) {
2134 result = utun_enable_channel(pcb, current_proc());
2135 } else {
2136 result = utun_disable_channel(pcb);
2137 }
2138 break;
2139 }
2140 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2141 if (len != sizeof(int)) {
2142 result = EMSGSIZE;
2143 break;
2144 }
2145 if (pcb->utun_ifp == NULL) {
2146 // Only can set after connecting
2147 result = EINVAL;
2148 break;
2149 }
2150 if (!if_is_fsw_transport_netagent_enabled()) {
2151 result = ENOTSUP;
2152 break;
2153 }
2154 if (uuid_is_null(pcb->utun_nx.fsw_agent)) {
2155 result = ENOENT;
2156 break;
2157 }
2158
2159 uint32_t flags = netagent_get_flags(pcb->utun_nx.fsw_agent);
2160
2161 if (*(int *)data) {
2162 pcb->utun_needs_netagent = true;
2163 flags |= (NETAGENT_FLAG_NEXUS_PROVIDER |
2164 NETAGENT_FLAG_NEXUS_LISTENER);
2165 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2166 } else {
2167 flags &= ~(NETAGENT_FLAG_NEXUS_PROVIDER |
2168 NETAGENT_FLAG_NEXUS_LISTENER);
2169 result = netagent_set_flags(pcb->utun_nx.fsw_agent, flags);
2170 pcb->utun_needs_netagent = false;
2171 }
2172 break;
2173 }
2174 case UTUN_OPT_ATTACH_FLOWSWITCH: {
2175 if (len != sizeof(int)) {
2176 result = EMSGSIZE;
2177 break;
2178 }
2179 if (pcb->utun_ifp != NULL) {
2180 // Only can set before connecting
2181 result = EINVAL;
2182 break;
2183 }
2184 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2185 pcb->utun_attach_fsw = !!(*(int *)data);
2186 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2187 break;
2188 }
2189 case UTUN_OPT_ENABLE_NETIF: {
2190 if (len != sizeof(int)) {
2191 result = EMSGSIZE;
2192 break;
2193 }
2194 if (pcb->utun_ifp != NULL) {
2195 // Only can set before connecting
2196 result = EINVAL;
2197 break;
2198 }
2199 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
2200 pcb->utun_use_netif = !!(*(int *)data);
2201 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
2202 break;
2203 }
2204 case UTUN_OPT_SLOT_SIZE: {
2205 if (len != sizeof(u_int32_t)) {
2206 result = EMSGSIZE;
2207 break;
2208 }
2209 if (pcb->utun_ifp != NULL) {
2210 // Only can set before connecting
2211 result = EINVAL;
2212 break;
2213 }
2214 u_int32_t slot_size = *(u_int32_t *)data;
2215 if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
2216 slot_size > UTUN_IF_MAX_SLOT_SIZE) {
2217 return EINVAL;
2218 }
2219 pcb->utun_slot_size = slot_size;
2220 break;
2221 }
2222 case UTUN_OPT_NETIF_RING_SIZE: {
2223 if (len != sizeof(u_int32_t)) {
2224 result = EMSGSIZE;
2225 break;
2226 }
2227 if (pcb->utun_ifp != NULL) {
2228 // Only can set before connecting
2229 result = EINVAL;
2230 break;
2231 }
2232 u_int32_t ring_size = *(u_int32_t *)data;
2233 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2234 ring_size > UTUN_IF_MAX_RING_SIZE) {
2235 return EINVAL;
2236 }
2237 pcb->utun_netif_ring_size = ring_size;
2238 break;
2239 }
2240 case UTUN_OPT_TX_FSW_RING_SIZE: {
2241 if (len != sizeof(u_int32_t)) {
2242 result = EMSGSIZE;
2243 break;
2244 }
2245 if (pcb->utun_ifp != NULL) {
2246 // Only can set before connecting
2247 result = EINVAL;
2248 break;
2249 }
2250 u_int32_t ring_size = *(u_int32_t *)data;
2251 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2252 ring_size > UTUN_IF_MAX_RING_SIZE) {
2253 return EINVAL;
2254 }
2255 pcb->utun_tx_fsw_ring_size = ring_size;
2256 break;
2257 }
2258 case UTUN_OPT_RX_FSW_RING_SIZE: {
2259 if (len != sizeof(u_int32_t)) {
2260 result = EMSGSIZE;
2261 break;
2262 }
2263 if (pcb->utun_ifp != NULL) {
2264 // Only can set before connecting
2265 result = EINVAL;
2266 break;
2267 }
2268 u_int32_t ring_size = *(u_int32_t *)data;
2269 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2270 ring_size > UTUN_IF_MAX_RING_SIZE) {
2271 return EINVAL;
2272 }
2273 pcb->utun_rx_fsw_ring_size = ring_size;
2274 break;
2275 }
2276 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2277 if (len != sizeof(u_int32_t)) {
2278 result = EMSGSIZE;
2279 break;
2280 }
2281 if (pcb->utun_ifp != NULL) {
2282 // Only can set before connecting
2283 result = EINVAL;
2284 break;
2285 }
2286 u_int32_t ring_size = *(u_int32_t *)data;
2287 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2288 ring_size > UTUN_IF_MAX_RING_SIZE) {
2289 return EINVAL;
2290 }
2291 pcb->utun_kpipe_tx_ring_size = ring_size;
2292 break;
2293 }
2294 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2295 if (len != sizeof(u_int32_t)) {
2296 result = EMSGSIZE;
2297 break;
2298 }
2299 if (pcb->utun_ifp != NULL) {
2300 // Only can set before connecting
2301 result = EINVAL;
2302 break;
2303 }
2304 u_int32_t ring_size = *(u_int32_t *)data;
2305 if (ring_size < UTUN_IF_MIN_RING_SIZE ||
2306 ring_size > UTUN_IF_MAX_RING_SIZE) {
2307 return EINVAL;
2308 }
2309 pcb->utun_kpipe_rx_ring_size = ring_size;
2310 break;
2311 }
2312 #endif // UTUN_NEXUS
2313 default: {
2314 result = ENOPROTOOPT;
2315 break;
2316 }
2317 }
2318
2319 return result;
2320 }
2321
2322 static errno_t
2323 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
2324 __unused u_int32_t unit,
2325 void *unitinfo,
2326 int opt,
2327 void *data,
2328 size_t *len)
2329 {
2330 struct utun_pcb *pcb = unitinfo;
2331 errno_t result = 0;
2332
2333 switch (opt) {
2334 case UTUN_OPT_FLAGS:
2335 if (*len != sizeof(u_int32_t)) {
2336 result = EMSGSIZE;
2337 } else {
2338 *(u_int32_t *)data = pcb->utun_flags;
2339 }
2340 break;
2341
2342 case UTUN_OPT_EXT_IFDATA_STATS:
2343 if (*len != sizeof(int)) {
2344 result = EMSGSIZE;
2345 } else {
2346 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
2347 }
2348 break;
2349
2350 case UTUN_OPT_IFNAME:
2351 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
2352 result = EMSGSIZE;
2353 } else {
2354 if (pcb->utun_ifp == NULL) {
2355 // Only can get after connecting
2356 result = EINVAL;
2357 break;
2358 }
2359 *len = scnprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
2360 }
2361 break;
2362
2363 case UTUN_OPT_MAX_PENDING_PACKETS: {
2364 if (*len != sizeof(u_int32_t)) {
2365 result = EMSGSIZE;
2366 } else {
2367 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
2368 }
2369 break;
2370 }
2371
2372 #if UTUN_NEXUS
2373 case UTUN_OPT_ENABLE_CHANNEL: {
2374 if (*len != sizeof(int)) {
2375 result = EMSGSIZE;
2376 } else {
2377 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2378 *(int *)data = pcb->utun_kpipe_enabled;
2379 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2380 }
2381 break;
2382 }
2383
2384 case UTUN_OPT_ENABLE_FLOWSWITCH: {
2385 if (*len != sizeof(int)) {
2386 result = EMSGSIZE;
2387 } else {
2388 *(int *)data = if_check_netagent(pcb->utun_ifp, pcb->utun_nx.fsw_agent);
2389 }
2390 break;
2391 }
2392
2393 case UTUN_OPT_ENABLE_NETIF: {
2394 if (*len != sizeof(int)) {
2395 result = EMSGSIZE;
2396 } else {
2397 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2398 *(int *)data = !!pcb->utun_use_netif;
2399 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2400 }
2401 break;
2402 }
2403
2404 case UTUN_OPT_GET_CHANNEL_UUID: {
2405 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2406 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
2407 result = ENXIO;
2408 } else if (*len != sizeof(uuid_t)) {
2409 result = EMSGSIZE;
2410 } else {
2411 uuid_copy(data, pcb->utun_kpipe_uuid);
2412 }
2413 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2414 break;
2415 }
2416 case UTUN_OPT_SLOT_SIZE: {
2417 if (*len != sizeof(u_int32_t)) {
2418 result = EMSGSIZE;
2419 } else {
2420 *(u_int32_t *)data = pcb->utun_slot_size;
2421 }
2422 break;
2423 }
2424 case UTUN_OPT_NETIF_RING_SIZE: {
2425 if (*len != sizeof(u_int32_t)) {
2426 result = EMSGSIZE;
2427 } else {
2428 *(u_int32_t *)data = pcb->utun_netif_ring_size;
2429 }
2430 break;
2431 }
2432 case UTUN_OPT_TX_FSW_RING_SIZE: {
2433 if (*len != sizeof(u_int32_t)) {
2434 result = EMSGSIZE;
2435 } else {
2436 *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
2437 }
2438 break;
2439 }
2440 case UTUN_OPT_RX_FSW_RING_SIZE: {
2441 if (*len != sizeof(u_int32_t)) {
2442 result = EMSGSIZE;
2443 } else {
2444 *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
2445 }
2446 break;
2447 }
2448 case UTUN_OPT_KPIPE_TX_RING_SIZE: {
2449 if (*len != sizeof(u_int32_t)) {
2450 result = EMSGSIZE;
2451 } else {
2452 *(u_int32_t *)data = pcb->utun_kpipe_tx_ring_size;
2453 }
2454 break;
2455 }
2456 case UTUN_OPT_KPIPE_RX_RING_SIZE: {
2457 if (*len != sizeof(u_int32_t)) {
2458 result = EMSGSIZE;
2459 } else {
2460 *(u_int32_t *)data = pcb->utun_kpipe_rx_ring_size;
2461 }
2462 break;
2463 }
2464 #endif // UTUN_NEXUS
2465
2466 default:
2467 result = ENOPROTOOPT;
2468 break;
2469 }
2470
2471 return result;
2472 }
2473
2474 static void
2475 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
2476 {
2477 #pragma unused(flags)
2478 bool reenable_output = false;
2479 struct utun_pcb *pcb = unitinfo;
2480 if (pcb == NULL) {
2481 return;
2482 }
2483 ifnet_lock_exclusive(pcb->utun_ifp);
2484
2485 u_int32_t utun_packet_cnt;
2486 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
2487 if (error_pc != 0) {
2488 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2489 utun_packet_cnt = 0;
2490 }
2491
2492 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2493 reenable_output = true;
2494 }
2495
2496 if (reenable_output) {
2497 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2498 if (error != 0) {
2499 os_log_error(OS_LOG_DEFAULT, "utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2500 }
2501 }
2502 ifnet_lock_done(pcb->utun_ifp);
2503 }
2504
2505 /* Network Interface functions */
2506 static void
2507 utun_start(ifnet_t interface)
2508 {
2509 mbuf_t data;
2510 struct utun_pcb *pcb = ifnet_softc(interface);
2511
2512 VERIFY(pcb != NULL);
2513
2514 #if UTUN_NEXUS
2515 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2516 if (pcb->utun_kpipe_enabled) {
2517 /* It's possible to have channels enabled, but not yet have the channel opened,
2518 * in which case the rxring will not be set
2519 */
2520 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2521 if (pcb->utun_kpipe_rxring != NULL) {
2522 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2523 }
2524 return;
2525 }
2526 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2527 #endif // UTUN_NEXUS
2528
2529 for (;;) {
2530 bool can_accept_packets = true;
2531 ifnet_lock_shared(pcb->utun_ifp);
2532
2533 u_int32_t utun_packet_cnt;
2534 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2535 if (error_pc != 0) {
2536 os_log_error(OS_LOG_DEFAULT, "utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2537 utun_packet_cnt = 0;
2538 }
2539
2540 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2541 if (!can_accept_packets && pcb->utun_ctlref) {
2542 u_int32_t difference = 0;
2543 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2544 if (difference > 0) {
2545 // If the low-water mark has not yet been reached, we still need to enqueue data
2546 // into the buffer
2547 can_accept_packets = true;
2548 }
2549 }
2550 }
2551 if (!can_accept_packets) {
2552 errno_t error = ifnet_disable_output(interface);
2553 if (error != 0) {
2554 os_log_error(OS_LOG_DEFAULT, "utun_start: ifnet_disable_output returned error %d\n", error);
2555 }
2556 ifnet_lock_done(pcb->utun_ifp);
2557 break;
2558 }
2559 ifnet_lock_done(pcb->utun_ifp);
2560 if (ifnet_dequeue(interface, &data) != 0) {
2561 break;
2562 }
2563 if (utun_output(interface, data) != 0) {
2564 break;
2565 }
2566 }
2567 }
2568
2569 static errno_t
2570 utun_output(ifnet_t interface,
2571 mbuf_t data)
2572 {
2573 struct utun_pcb *pcb = ifnet_softc(interface);
2574 errno_t result;
2575
2576 VERIFY(interface == pcb->utun_ifp);
2577
2578 #if UTUN_NEXUS
2579 if (!pcb->utun_use_netif)
2580 #endif // UTUN_NEXUS
2581 {
2582 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2583 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
2584 }
2585 }
2586
2587 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2588 /* flush data */
2589 mbuf_freem(data);
2590 return 0;
2591 }
2592
2593 // otherwise, fall thru to ctl_enqueumbuf
2594 if (pcb->utun_ctlref) {
2595 int length;
2596
2597 /*
2598 * The ABI requires the protocol in network byte order
2599 */
2600 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2601 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2602 }
2603
2604 length = mbuf_pkthdr_len(data);
2605 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2606 if (result != 0) {
2607 mbuf_freem(data);
2608 os_log_error(OS_LOG_DEFAULT, "utun_output - ctl_enqueuembuf failed: %d\n", result);
2609 #if UTUN_NEXUS
2610 if (!pcb->utun_use_netif)
2611 #endif // UTUN_NEXUS
2612 {
2613 ifnet_stat_increment_out(interface, 0, 0, 1);
2614 }
2615 } else {
2616 #if UTUN_NEXUS
2617 if (!pcb->utun_use_netif)
2618 #endif // UTUN_NEXUS
2619 {
2620 if (!pcb->utun_ext_ifdata_stats) {
2621 ifnet_stat_increment_out(interface, 1, length, 0);
2622 }
2623 }
2624 }
2625 } else {
2626 mbuf_freem(data);
2627 }
2628
2629 return 0;
2630 }
2631
2632 static errno_t
2633 utun_demux(__unused ifnet_t interface,
2634 mbuf_t data,
2635 __unused char *frame_header,
2636 protocol_family_t *protocol)
2637 {
2638 #if UTUN_NEXUS
2639 struct utun_pcb *pcb = ifnet_softc(interface);
2640 struct ip *ip;
2641 u_int ip_version;
2642 #endif
2643
2644 while (data != NULL && mbuf_len(data) < 1) {
2645 data = mbuf_next(data);
2646 }
2647
2648 if (data == NULL) {
2649 return ENOENT;
2650 }
2651
2652 #if UTUN_NEXUS
2653 if (pcb->utun_use_netif) {
2654 ip = mtod(data, struct ip *);
2655 ip_version = ip->ip_v;
2656
2657 switch (ip_version) {
2658 case 4:
2659 *protocol = PF_INET;
2660 return 0;
2661 case 6:
2662 *protocol = PF_INET6;
2663 return 0;
2664 default:
2665 *protocol = 0;
2666 break;
2667 }
2668 } else
2669 #endif // UTUN_NEXUS
2670 {
2671 *protocol = *(u_int32_t *)mbuf_data(data);
2672 }
2673
2674 return 0;
2675 }
2676
2677 static errno_t
2678 utun_framer(ifnet_t interface,
2679 mbuf_t *packet,
2680 __unused const struct sockaddr *dest,
2681 __unused const char *desk_linkaddr,
2682 const char *frame_type,
2683 u_int32_t *prepend_len,
2684 u_int32_t *postpend_len)
2685 {
2686 struct utun_pcb *pcb = ifnet_softc(interface);
2687 VERIFY(interface == pcb->utun_ifp);
2688
2689 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2690 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2691 os_log_error(OS_LOG_DEFAULT, "utun_framer - ifnet_output prepend failed\n");
2692
2693 ifnet_stat_increment_out(interface, 0, 0, 1);
2694
2695 // just return, because the buffer was freed in mbuf_prepend
2696 return EJUSTRETURN;
2697 }
2698 if (prepend_len != NULL) {
2699 *prepend_len = header_length;
2700 }
2701 if (postpend_len != NULL) {
2702 *postpend_len = 0;
2703 }
2704
2705 // place protocol number at the beginning of the mbuf
2706 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2707
2708
2709 return 0;
2710 }
2711
2712 static errno_t
2713 utun_add_proto(__unused ifnet_t interface,
2714 protocol_family_t protocol,
2715 __unused const struct ifnet_demux_desc *demux_array,
2716 __unused u_int32_t demux_count)
2717 {
2718 switch (protocol) {
2719 case PF_INET:
2720 return 0;
2721 case PF_INET6:
2722 return 0;
2723 default:
2724 break;
2725 }
2726
2727 return ENOPROTOOPT;
2728 }
2729
2730 static errno_t
2731 utun_del_proto(__unused ifnet_t interface,
2732 __unused protocol_family_t protocol)
2733 {
2734 return 0;
2735 }
2736
2737 static errno_t
2738 utun_ioctl(ifnet_t interface,
2739 u_long command,
2740 void *data)
2741 {
2742 #if UTUN_NEXUS
2743 struct utun_pcb *pcb = ifnet_softc(interface);
2744 #endif
2745 errno_t result = 0;
2746
2747 switch (command) {
2748 case SIOCSIFMTU: {
2749 #if UTUN_NEXUS
2750 if (pcb->utun_use_netif) {
2751 // Make sure we can fit packets in the channel buffers
2752 // Allow for the headroom in the slot
2753 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
2754 result = EINVAL;
2755 } else {
2756 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
2757 }
2758 } else
2759 #endif // UTUN_NEXUS
2760 {
2761 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2762 }
2763 break;
2764 }
2765
2766 case SIOCSIFFLAGS:
2767 /* ifioctl() takes care of it */
2768 break;
2769
2770 default:
2771 result = EOPNOTSUPP;
2772 }
2773
2774 return result;
2775 }
2776
2777 static void
2778 utun_detached(ifnet_t interface)
2779 {
2780 struct utun_pcb *pcb = ifnet_softc(interface);
2781 (void)ifnet_release(interface);
2782 lck_mtx_lock(&utun_lock);
2783 utun_free_pcb(pcb, true);
2784 (void)ifnet_dispose(interface);
2785 lck_mtx_unlock(&utun_lock);
2786 }
2787
2788 /* Protocol Handlers */
2789
2790 static errno_t
2791 utun_proto_input(__unused ifnet_t interface,
2792 protocol_family_t protocol,
2793 mbuf_t m,
2794 __unused char *frame_header)
2795 {
2796 struct utun_pcb *pcb = ifnet_softc(interface);
2797 #if UTUN_NEXUS
2798 if (!pcb->utun_use_netif)
2799 #endif // UTUN_NEXUS
2800 {
2801 mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
2802 }
2803 int32_t pktlen = m->m_pkthdr.len;
2804 if (proto_input(protocol, m) != 0) {
2805 m_freem(m);
2806 #if UTUN_NEXUS
2807 if (!pcb->utun_use_netif)
2808 #endif // UTUN_NEXUS
2809 {
2810 ifnet_stat_increment_in(interface, 0, 0, 1);
2811 }
2812 } else {
2813 #if UTUN_NEXUS
2814 if (!pcb->utun_use_netif)
2815 #endif // UTUN_NEXUS
2816 {
2817 ifnet_stat_increment_in(interface, 1, pktlen, 0);
2818 }
2819 }
2820
2821 return 0;
2822 }
2823
2824 static errno_t
2825 utun_proto_pre_output(__unused ifnet_t interface,
2826 protocol_family_t protocol,
2827 __unused mbuf_t *packet,
2828 __unused const struct sockaddr *dest,
2829 __unused void *route,
2830 char *frame_type,
2831 __unused char *link_layer_dest)
2832 {
2833 *(protocol_family_t *)(void *)frame_type = protocol;
2834 return 0;
2835 }
2836
2837 static errno_t
2838 utun_attach_proto(ifnet_t interface,
2839 protocol_family_t protocol)
2840 {
2841 struct ifnet_attach_proto_param proto;
2842
2843 bzero(&proto, sizeof(proto));
2844 proto.input = utun_proto_input;
2845 proto.pre_output = utun_proto_pre_output;
2846
2847 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2848 if (result != 0 && result != EEXIST) {
2849 os_log_error(OS_LOG_DEFAULT, "utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2850 protocol, result);
2851 }
2852
2853 return result;
2854 }
2855
2856 static errno_t
2857 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2858 {
2859 #if UTUN_NEXUS
2860 if (pcb->utun_use_netif) {
2861 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2862
2863 lck_mtx_lock(&pcb->utun_input_chain_lock);
2864
2865 if (pcb->utun_input_chain_count > (u_int32_t)if_utun_max_pending_input) {
2866 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2867 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2868 return ENOSPC;
2869 }
2870
2871 if (pcb->utun_input_chain != NULL) {
2872 pcb->utun_input_chain_last->m_nextpkt = packet;
2873 } else {
2874 pcb->utun_input_chain = packet;
2875 }
2876 pcb->utun_input_chain_count++;
2877 while (packet->m_nextpkt) {
2878 VERIFY(packet != packet->m_nextpkt);
2879 packet = packet->m_nextpkt;
2880 pcb->utun_input_chain_count++;
2881 }
2882 pcb->utun_input_chain_last = packet;
2883 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2884
2885 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2886 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2887
2888 if (rx_ring != NULL) {
2889 kern_channel_notify(rx_ring, 0);
2890 }
2891
2892 return 0;
2893 } else
2894 #endif // UTUN_NEXUS
2895 {
2896 mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);
2897
2898 if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2899 bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
2900 }
2901 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2902 /* flush data */
2903 mbuf_freem(packet);
2904 return 0;
2905 }
2906
2907 errno_t result = 0;
2908 if (!pcb->utun_ext_ifdata_stats) {
2909 struct ifnet_stat_increment_param incs = {};
2910 incs.packets_in = 1;
2911 incs.bytes_in = mbuf_pkthdr_len(packet);
2912 result = ifnet_input(pcb->utun_ifp, packet, &incs);
2913 } else {
2914 result = ifnet_input(pcb->utun_ifp, packet, NULL);
2915 }
2916 if (result != 0) {
2917 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2918
2919 os_log_error(OS_LOG_DEFAULT, "%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2920 mbuf_freem(packet);
2921 }
2922
2923 return 0;
2924 }
2925 }
2926
2927 #if UTUN_NEXUS
2928
2929 static errno_t
2930 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2931 {
2932 return 0;
2933 }
2934
2935 static void
2936 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2937 {
2938 // Ignore
2939 }
2940
2941 static errno_t
2942 utun_register_nexus(void)
2943 {
2944 const struct kern_nexus_domain_provider_init dp_init = {
2945 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2946 .nxdpi_flags = 0,
2947 .nxdpi_init = utun_nxdp_init,
2948 .nxdpi_fini = utun_nxdp_fini
2949 };
2950 errno_t err = 0;
2951
2952 /* utun_nxdp_init() is called before this function returns */
2953 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2954 (const uint8_t *) "com.apple.utun",
2955 &dp_init, sizeof(dp_init),
2956 &utun_nx_dom_prov);
2957 if (err != 0) {
2958 os_log_error(OS_LOG_DEFAULT, "%s: failed to register domain provider\n", __func__);
2959 return err;
2960 }
2961 return 0;
2962 }
2963 boolean_t
2964 utun_interface_needs_netagent(ifnet_t interface)
2965 {
2966 struct utun_pcb *pcb = NULL;
2967
2968 if (interface == NULL) {
2969 return FALSE;
2970 }
2971
2972 pcb = ifnet_softc(interface);
2973
2974 if (pcb == NULL) {
2975 return FALSE;
2976 }
2977
2978 return pcb->utun_needs_netagent == true;
2979 }
2980
2981 static errno_t
2982 utun_ifnet_set_attrs(ifnet_t ifp)
2983 {
2984 /* Set flags and additional information. */
2985 ifnet_set_mtu(ifp, 1500);
2986 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2987
2988 /* The interface must generate its own IPv6 LinkLocal address,
2989 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2990 */
2991 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2992
2993 return 0;
2994 }
2995
2996 static errno_t
2997 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2998 {
2999 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3000 pcb->utun_netif_nexus = nexus;
3001 return utun_ifnet_set_attrs(ifp);
3002 }
3003
3004 static errno_t
3005 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
3006 proc_t p, kern_nexus_t nexus,
3007 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
3008 {
3009 #pragma unused(nxprov, p)
3010 #pragma unused(nexus, nexus_port, channel, ch_ctx)
3011 return 0;
3012 }
3013
3014 static errno_t
3015 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3016 kern_channel_t channel)
3017 {
3018 #pragma unused(nxprov, channel)
3019 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3020 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
3021 if (pcb->utun_netif_nexus == nexus) {
3022 pcb->utun_netif_connected = true;
3023 }
3024 return ok ? 0 : ENXIO;
3025 }
3026
3027 static void
3028 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3029 kern_channel_t channel)
3030 {
3031 #pragma unused(nxprov, nexus, channel)
3032 }
3033
3034 static void
3035 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3036 kern_channel_t channel)
3037 {
3038 #pragma unused(nxprov, nexus, channel)
3039 }
3040
3041 static void
3042 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3043 kern_channel_t channel)
3044 {
3045 #pragma unused(nxprov, channel)
3046 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3047 if (pcb->utun_netif_nexus == nexus) {
3048 pcb->utun_netif_connected = false;
3049 if (pcb->utun_attach_fsw) {
3050 // disconnected by flowswitch that was attached by us
3051 pcb->utun_netif_nexus = NULL;
3052 }
3053 }
3054 ifnet_decr_iorefcnt(pcb->utun_ifp);
3055 }
3056
3057 static errno_t
3058 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3059 kern_channel_t channel, kern_channel_ring_t ring,
3060 boolean_t is_tx_ring, void **ring_ctx)
3061 {
3062 #pragma unused(nxprov)
3063 #pragma unused(channel)
3064 #pragma unused(ring_ctx)
3065 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3066 if (!is_tx_ring) {
3067 VERIFY(pcb->utun_kpipe_rxring == NULL);
3068 pcb->utun_kpipe_rxring = ring;
3069 } else {
3070 VERIFY(pcb->utun_kpipe_txring == NULL);
3071 pcb->utun_kpipe_txring = ring;
3072 }
3073 return 0;
3074 }
3075
3076 static void
3077 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3078 kern_channel_ring_t ring)
3079 {
3080 #pragma unused(nxprov)
3081 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3082 if (pcb->utun_kpipe_rxring == ring) {
3083 pcb->utun_kpipe_rxring = NULL;
3084 } else if (pcb->utun_kpipe_txring == ring) {
3085 pcb->utun_kpipe_txring = NULL;
3086 }
3087 }
3088
3089 static errno_t
3090 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3091 kern_channel_ring_t tx_ring, uint32_t flags)
3092 {
3093 #pragma unused(nxprov)
3094 #pragma unused(flags)
3095 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3096
3097 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3098 int channel_enabled = pcb->utun_kpipe_enabled;
3099 if (!channel_enabled) {
3100 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3101 return 0;
3102 }
3103
3104 if (pcb->utun_use_netif) {
3105 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3106 if (tx_slot == NULL) {
3107 // Nothing to write, bail
3108 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3109 return 0;
3110 }
3111
3112 // Signal the netif ring to read
3113 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
3114 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3115 if (rx_ring != NULL) {
3116 kern_channel_notify(rx_ring, 0);
3117 }
3118 } else {
3119 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3120
3121 struct ifnet_stat_increment_param incs = {};
3122 struct kern_channel_ring_stat_increment tx_ring_stats = {};
3123 MBUFQ_HEAD(mbufq) mbq;
3124 MBUFQ_INIT(&mbq);
3125 kern_channel_slot_t tx_pslot = NULL;
3126 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3127 while (tx_slot != NULL) {
3128 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3129
3130 // Advance TX ring
3131 tx_pslot = tx_slot;
3132 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3133
3134 if (tx_ph == 0) {
3135 continue;
3136 }
3137
3138 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3139 VERIFY(tx_buf != NULL);
3140 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
3141 VERIFY(tx_baddr != 0);
3142 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3143
3144 size_t length = MIN(kern_packet_get_data_length(tx_ph),
3145 pcb->utun_slot_size);
3146
3147 mbuf_t data = NULL;
3148 if (length >= UTUN_HEADER_SIZE(pcb) &&
3149 !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
3150 errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
3151 VERIFY(0 == error);
3152 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
3153 VERIFY(0 == error);
3154 /*
3155 * The userland ABI requires the first four bytes have
3156 * the protocol family in network byte order: swap them
3157 */
3158 *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
3159 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
3160 bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3161 incs.packets_in++;
3162 incs.bytes_in += length;
3163 MBUFQ_ENQUEUE(&mbq, data);
3164 }
3165 }
3166 if (tx_pslot) {
3167 kern_channel_advance_slot(tx_ring, tx_pslot);
3168 tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
3169 tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
3170 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3171 (void) kern_channel_reclaim(tx_ring);
3172 }
3173 if (!MBUFQ_EMPTY(&mbq)) {
3174 (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
3175 MBUFQ_LAST(&mbq), &incs);
3176 MBUFQ_INIT(&mbq);
3177 }
3178 }
3179
3180 return 0;
3181 }
3182
3183 static errno_t
3184 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
3185 kern_channel_ring_t rx_ring, uint32_t flags)
3186 {
3187 #pragma unused(nxprov)
3188 #pragma unused(flags)
3189 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
3190 struct kern_channel_ring_stat_increment rx_ring_stats = {};
3191
3192 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3193
3194 int channel_enabled = pcb->utun_kpipe_enabled;
3195 if (!channel_enabled) {
3196 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3197 return 0;
3198 }
3199
3200 /* reclaim user-released slots */
3201 (void) kern_channel_reclaim(rx_ring);
3202
3203 uint32_t avail = kern_channel_available_slot_count(rx_ring);
3204 if (avail == 0) {
3205 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3206 return 0;
3207 }
3208
3209 if (pcb->utun_use_netif) {
3210 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
3211 if (tx_ring == NULL ||
3212 pcb->utun_netif_nexus == NULL) {
3213 // Net-If TX ring not set up yet, nothing to read
3214 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3215 return 0;
3216 }
3217
3218 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
3219
3220 // Unlock utun before entering ring
3221 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3222
3223 (void)kr_enter(tx_ring, TRUE);
3224
3225 // Lock again after entering and validate
3226 lck_rw_lock_shared(&pcb->utun_pcb_lock);
3227 if (tx_ring != pcb->utun_netif_txring) {
3228 // Ring no longer valid
3229 // Unlock first, then exit ring
3230 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3231 kr_exit(tx_ring);
3232 return 0;
3233 }
3234
3235 struct kern_channel_ring_stat_increment tx_ring_stats;
3236 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
3237 kern_channel_slot_t tx_pslot = NULL;
3238 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
3239 if (tx_slot == NULL) {
3240 // Nothing to read, don't bother signalling
3241 // Unlock first, then exit ring
3242 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3243 kr_exit(tx_ring);
3244 return 0;
3245 }
3246
3247 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3248 VERIFY(rx_pp != NULL);
3249 kern_channel_slot_t rx_pslot = NULL;
3250 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3251
3252 while (rx_slot != NULL && tx_slot != NULL) {
3253 size_t length;
3254 kern_buflet_t rx_buf;
3255 void *rx_baddr;
3256
3257 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
3258
3259 // Advance TX ring
3260 tx_pslot = tx_slot;
3261 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
3262
3263 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
3264 if (tx_ph == 0) {
3265 continue;
3266 }
3267
3268 // Allocate rx packet
3269 kern_packet_t rx_ph = 0;
3270 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3271 if (__improbable(error != 0)) {
3272 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3273 pcb->utun_ifp->if_xname);
3274 break;
3275 }
3276
3277 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
3278 VERIFY(tx_buf != NULL);
3279 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
3280 VERIFY(tx_baddr != NULL);
3281 tx_baddr += kern_buflet_get_data_offset(tx_buf);
3282
3283 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
3284
3285 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
3286 pcb->utun_slot_size);
3287
3288 tx_ring_stats.kcrsi_slots_transferred++;
3289 tx_ring_stats.kcrsi_bytes_transferred += length;
3290
3291 if (length < UTUN_HEADER_SIZE(pcb) ||
3292 length > pcb->utun_slot_size ||
3293 length > rx_pp->pp_buflet_size ||
3294 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3295 /* flush data */
3296 kern_pbufpool_free(rx_pp, rx_ph);
3297 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
3298 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
3299 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
3300 STATS_INC(nifs, NETIF_STATS_DROP);
3301 continue;
3302 }
3303
3304 /* fillout packet */
3305 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3306 VERIFY(rx_buf != NULL);
3307 rx_baddr = kern_buflet_get_data_address(rx_buf);
3308 VERIFY(rx_baddr != NULL);
3309
3310 // Find family
3311 uint32_t af = 0;
3312 uint8_t vhl = *(uint8_t *)(tx_baddr);
3313 u_int ip_version = (vhl >> 4);
3314 switch (ip_version) {
3315 case 4: {
3316 af = AF_INET;
3317 break;
3318 }
3319 case 6: {
3320 af = AF_INET6;
3321 break;
3322 }
3323 default: {
3324 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
3325 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
3326 break;
3327 }
3328 }
3329
3330 // Copy header
3331 af = htonl(af);
3332 memcpy((void *)rx_baddr, &af, sizeof(af));
3333 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
3334 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
3335 }
3336
3337 // Copy data from tx to rx
3338 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
3339 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
3340
3341 /* finalize and attach the packet */
3342 error = kern_buflet_set_data_offset(rx_buf, 0);
3343 VERIFY(error == 0);
3344 error = kern_buflet_set_data_length(rx_buf, length);
3345 VERIFY(error == 0);
3346 error = kern_packet_finalize(rx_ph);
3347 VERIFY(error == 0);
3348 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3349 VERIFY(error == 0);
3350
3351 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
3352 STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
3353
3354 rx_ring_stats.kcrsi_slots_transferred++;
3355 rx_ring_stats.kcrsi_bytes_transferred += length;
3356
3357 rx_pslot = rx_slot;
3358 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3359 }
3360
3361 if (rx_pslot) {
3362 kern_channel_advance_slot(rx_ring, rx_pslot);
3363 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
3364 }
3365
3366 if (tx_pslot) {
3367 kern_channel_advance_slot(tx_ring, tx_pslot);
3368 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
3369 (void)kern_channel_reclaim(tx_ring);
3370 }
3371
3372 /* just like utun_ctl_rcvd(), always reenable output */
3373 errno_t error = ifnet_enable_output(pcb->utun_ifp);
3374 if (error != 0) {
3375 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
3376 }
3377
3378 // Unlock first, then exit ring
3379 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3380
3381 if (tx_pslot != NULL) {
3382 kern_channel_notify(tx_ring, 0);
3383 }
3384 kr_exit(tx_ring);
3385 } else {
3386 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
3387
3388 uint32_t mb_cnt = 0;
3389 uint32_t mb_len = 0;
3390 struct mbuf *mb_head = NULL;
3391 struct mbuf *mb_tail = NULL;
3392
3393 if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
3394 &mb_tail, &mb_cnt, &mb_len) != 0) {
3395 return 0;
3396 }
3397 VERIFY(mb_cnt <= avail);
3398
3399 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
3400 VERIFY(rx_pp != NULL);
3401 kern_channel_slot_t rx_pslot = NULL;
3402 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
3403 while (rx_slot) {
3404 size_t length = 0;
3405 mbuf_t data = NULL;
3406 if ((data = mb_head) == NULL) {
3407 VERIFY(mb_cnt == 0);
3408 break;
3409 }
3410 mb_head = mbuf_nextpkt(mb_head);
3411 mbuf_setnextpkt(data, NULL);
3412 VERIFY(mb_cnt != 0);
3413 --mb_cnt;
3414 length = mbuf_pkthdr_len(data);
3415 if (length < UTUN_HEADER_SIZE(pcb) ||
3416 length > pcb->utun_slot_size ||
3417 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
3418 /* flush data */
3419 mbuf_freem(data);
3420 continue;
3421 }
3422 bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
3423
3424 // Allocate rx packet
3425 kern_packet_t rx_ph = 0;
3426 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
3427 if (__improbable(error != 0)) {
3428 os_log_error(OS_LOG_DEFAULT, "utun_kpipe_sync_rx %s: failed to allocate packet\n",
3429 pcb->utun_ifp->if_xname);
3430 break;
3431 }
3432
3433 /*
3434 * The ABI requires the protocol in network byte order
3435 */
3436 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
3437
3438 // Fillout rx packet
3439 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
3440 VERIFY(rx_buf != NULL);
3441 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
3442 VERIFY(rx_baddr != NULL);
3443
3444 // Copy-in data from mbuf to buflet
3445 mbuf_copydata(data, 0, length, (void *)rx_baddr);
3446 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
3447
3448 // Finalize and attach the packet
3449 error = kern_buflet_set_data_offset(rx_buf, 0);
3450 VERIFY(error == 0);
3451 error = kern_buflet_set_data_length(rx_buf, length);
3452 VERIFY(error == 0);
3453 error = kern_packet_finalize(rx_ph);
3454 VERIFY(error == 0);
3455 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
3456 VERIFY(error == 0);
3457
3458 rx_ring_stats.kcrsi_slots_transferred++;
3459 rx_ring_stats.kcrsi_bytes_transferred += length;
3460
3461 if (!pcb->utun_ext_ifdata_stats) {
3462 ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
3463 }
3464
3465 mbuf_freem(data);
3466
3467 rx_pslot = rx_slot;
3468 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
3469 }
3470 if (rx_pslot) {
3471 kern_channel_advance_slot(rx_ring, rx_pslot);
3472 kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
3473 }
3474 if (mb_head != NULL) {
3475 VERIFY(mb_cnt != 0);
3476 mbuf_freem_list(mb_head);
3477 }
3478 }
3479
3480 return 0;
3481 }
3482
3483 #endif // UTUN_NEXUS
3484
3485
3486 /*
3487 * These are place holders until coreTLS kext stops calling them
3488 */
3489 errno_t utun_ctl_register_dtls(void *reg);
3490 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
3491 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
3492
3493 errno_t
3494 utun_ctl_register_dtls(void *reg)
3495 {
3496 #pragma unused(reg)
3497 return 0;
3498 }
3499
3500 int
3501 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
3502 {
3503 #pragma unused(pcb)
3504 #pragma unused(pkt)
3505 #pragma unused(family)
3506 return 0;
3507 }
3508
3509 void
3510 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
3511 {
3512 #pragma unused(pcb)
3513 }
mirror of XNU