]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/if_utun.c
projects / apple / xnu.git / blob
? search:


5e01509dfad8c961497fbe20acc8b97b29b1ac0e
[apple/xnu.git] / bsd / net / if_utun.c
1 /*
2 * Copyright (c) 2008-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29
30
31 /* ----------------------------------------------------------------------------------
32 Application of kernel control for interface creation
33
34 Theory of operation:
35 utun (user tunnel) acts as glue between kernel control sockets and network interfaces.
36 This kernel control will register an interface for every client that connects.
37 ---------------------------------------------------------------------------------- */
38
39 #include <sys/systm.h>
40 #include <sys/kern_control.h>
41 #include <net/kpi_protocol.h>
42 #include <net/kpi_interface.h>
43 #include <sys/socket.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/bpf.h>
47 #include <net/if_utun.h>
48 #include <sys/mbuf.h>
49 #include <sys/sockio.h>
50 #include <netinet/in.h>
51 #include <netinet/ip.h>
52 #include <netinet6/in6_var.h>
53 #include <netinet6/in6_var.h>
54 #include <sys/kauth.h>
55 #include <net/necp.h>
56 #include <kern/zalloc.h>
57
58 #define UTUN_NEXUS 0
59
60 extern unsigned int if_enable_netagent;
61
62 #if UTUN_NEXUS
63 static nexus_controller_t utun_ncd;
64 static int utun_ncd_refcount;
65 static uuid_t utun_kpipe_uuid;
66 static uuid_t utun_nx_dom_prov;
67
68 typedef struct utun_nx {
69 uuid_t if_provider;
70 uuid_t if_instance;
71 uuid_t ms_provider;
72 uuid_t ms_instance;
73 uuid_t ms_device;
74 uuid_t ms_host;
75 uuid_t ms_agent;
76 } *utun_nx_t;
77
78 #endif // UTUN_NEXUS
79
80 /* Control block allocated for each kernel control connection */
81 struct utun_pcb {
82 TAILQ_ENTRY(utun_pcb) utun_chain;
83 kern_ctl_ref utun_ctlref;
84 ifnet_t utun_ifp;
85 u_int32_t utun_unit;
86 u_int32_t utun_unique_id;
87 u_int32_t utun_flags;
88 int utun_ext_ifdata_stats;
89 u_int32_t utun_max_pending_packets;
90 char utun_if_xname[IFXNAMSIZ];
91 char utun_unique_name[IFXNAMSIZ];
92 // PCB lock protects state fields and rings
93 decl_lck_rw_data(, utun_pcb_lock);
94 struct mbuf * utun_input_chain;
95 struct mbuf * utun_input_chain_last;
96 // Input chain lock protects the list of input mbufs
97 // The input chain lock must be taken AFTER the PCB lock if both are held
98 lck_mtx_t utun_input_chain_lock;
99 bool utun_output_disabled;
100
101 #if UTUN_NEXUS
102 struct utun_nx utun_nx;
103 int utun_kpipe_enabled;
104 uuid_t utun_kpipe_uuid;
105 void * utun_kpipe_rxring;
106 void * utun_kpipe_txring;
107
108 kern_nexus_t utun_netif_nexus;
109 void * utun_netif_rxring;
110 void * utun_netif_txring;
111 uint64_t utun_netif_txring_size;
112 #endif // UTUN_NEXUS
113 };
114
115 /* Kernel Control functions */
116 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
117 void **unitinfo);
118 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
119 void *unitinfo);
120 static errno_t utun_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
121 void *unitinfo, mbuf_t m, int flags);
122 static errno_t utun_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
123 int opt, void *data, size_t *len);
124 static errno_t utun_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
125 int opt, void *data, size_t len);
126 static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
127 int flags);
128
129 /* Network Interface functions */
130 #if !UTUN_NEXUS
131 static void utun_start(ifnet_t interface);
132 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
133 const struct sockaddr *dest, const char *desk_linkaddr,
134 const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
135 #endif // !UTUN_NEXUS
136 static errno_t utun_output(ifnet_t interface, mbuf_t data);
137 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
138 protocol_family_t *protocol);
139 static errno_t utun_add_proto(ifnet_t interface, protocol_family_t protocol,
140 const struct ifnet_demux_desc *demux_array,
141 u_int32_t demux_count);
142 static errno_t utun_del_proto(ifnet_t interface, protocol_family_t protocol);
143 static errno_t utun_ioctl(ifnet_t interface, u_long cmd, void *data);
144 static void utun_detached(ifnet_t interface);
145
146 /* Protocol handlers */
147 static errno_t utun_attach_proto(ifnet_t interface, protocol_family_t proto);
148 static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol,
149 mbuf_t m, char *frame_header);
150 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
151 mbuf_t *packet, const struct sockaddr *dest, void *route,
152 char *frame_type, char *link_layer_dest);
153 static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m);
154
155 #if UTUN_NEXUS
156
157 #define UTUN_IF_DEFAULT_SLOT_SIZE 4096
158 #define UTUN_IF_DEFAULT_RING_SIZE 64
159 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
160 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
161 #define UTUN_IF_HEADROOM_SIZE 32
162
163 #define UTUN_IF_MIN_RING_SIZE 16
164 #define UTUN_IF_MAX_RING_SIZE 1024
165
166 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
167 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
168 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
169
170 static int if_utun_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
171 static int if_utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
172 static int if_utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
173
174 SYSCTL_DECL(_net_utun);
175 SYSCTL_NODE(_net, OID_AUTO, utun, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "UTun");
176
177 SYSCTL_PROC(_net_utun, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
178 &if_utun_ring_size, UTUN_IF_DEFAULT_RING_SIZE, &sysctl_if_utun_ring_size, "I", "");
179 SYSCTL_PROC(_net_utun, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
180 &if_utun_tx_fsw_ring_size, UTUN_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_utun_tx_fsw_ring_size, "I", "");
181 SYSCTL_PROC(_net_utun, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
182 &if_utun_rx_fsw_ring_size, UTUN_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_utun_rx_fsw_ring_size, "I", "");
183
184 static errno_t
185 utun_register_nexus(void);
186
187 static errno_t
188 utun_netif_prepare(__unused kern_nexus_t nexus, ifnet_t ifp);
189 static errno_t
190 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
191 proc_t p, kern_nexus_t nexus,
192 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx);
193 static errno_t
194 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
195 kern_channel_t channel);
196 static void
197 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
198 kern_channel_t channel);
199 static void
200 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
201 kern_channel_t channel);
202 static void
203 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
204 kern_channel_t channel);
205 static errno_t
206 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
207 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
208 void **ring_ctx);
209 static void
210 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
211 kern_channel_ring_t ring);
212 static errno_t
213 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
214 kern_channel_ring_t ring, uint32_t flags);
215 static errno_t
216 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
217 kern_channel_ring_t ring, uint32_t flags);
218 #endif // UTUN_NEXUS
219
220 #define UTUN_DEFAULT_MTU 1500
221 #define UTUN_HEADER_SIZE(_pcb) (sizeof(u_int32_t) + (((_pcb)->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) ? sizeof(uuid_t) : 0))
222
223 static kern_ctl_ref utun_kctlref;
224 static u_int32_t utun_family;
225 static lck_attr_t *utun_lck_attr;
226 static lck_grp_attr_t *utun_lck_grp_attr;
227 static lck_grp_t *utun_lck_grp;
228 static lck_mtx_t utun_lock;
229
230 TAILQ_HEAD(utun_list, utun_pcb) utun_head;
231
232 #define UTUN_PCB_ZONE_MAX 32
233 #define UTUN_PCB_ZONE_NAME "net.if_utun"
234
235 static unsigned int utun_pcb_size; /* size of zone element */
236 static struct zone *utun_pcb_zone; /* zone for utun_pcb */
237
238 #if UTUN_NEXUS
239
240 static int
241 sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS
242 {
243 #pragma unused(arg1, arg2)
244 int value = if_utun_ring_size;
245
246 int error = sysctl_handle_int(oidp, &value, 0, req);
247 if (error || !req->newptr) {
248 return (error);
249 }
250
251 if (value < UTUN_IF_MIN_RING_SIZE ||
252 value > UTUN_IF_MAX_RING_SIZE) {
253 return (EINVAL);
254 }
255
256 if_utun_ring_size = value;
257
258 return (0);
259 }
260
261 static int
262 sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
263 {
264 #pragma unused(arg1, arg2)
265 int value = if_utun_tx_fsw_ring_size;
266
267 int error = sysctl_handle_int(oidp, &value, 0, req);
268 if (error || !req->newptr) {
269 return (error);
270 }
271
272 if (value < UTUN_IF_MIN_RING_SIZE ||
273 value > UTUN_IF_MAX_RING_SIZE) {
274 return (EINVAL);
275 }
276
277 if_utun_tx_fsw_ring_size = value;
278
279 return (0);
280 }
281
282 static int
283 sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
284 {
285 #pragma unused(arg1, arg2)
286 int value = if_utun_rx_fsw_ring_size;
287
288 int error = sysctl_handle_int(oidp, &value, 0, req);
289 if (error || !req->newptr) {
290 return (error);
291 }
292
293 if (value < UTUN_IF_MIN_RING_SIZE ||
294 value > UTUN_IF_MAX_RING_SIZE) {
295 return (EINVAL);
296 }
297
298 if_utun_rx_fsw_ring_size = value;
299
300 return (0);
301 }
302
303 static errno_t
304 utun_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
305 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
306 void **ring_ctx)
307 {
308 #pragma unused(nxprov)
309 #pragma unused(channel)
310 #pragma unused(ring_ctx)
311 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
312 if (!is_tx_ring) {
313 VERIFY(pcb->utun_netif_rxring == NULL);
314 pcb->utun_netif_rxring = ring;
315 } else {
316 VERIFY(pcb->utun_netif_txring == NULL);
317 pcb->utun_netif_txring = ring;
318 }
319 return 0;
320 }
321
322 static void
323 utun_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
324 kern_channel_ring_t ring)
325 {
326 #pragma unused(nxprov)
327 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
328 if (pcb->utun_netif_rxring == ring) {
329 pcb->utun_netif_rxring = NULL;
330 } else if (pcb->utun_netif_txring == ring) {
331 pcb->utun_netif_txring = NULL;
332 }
333 }
334
335 static errno_t
336 utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
337 kern_channel_ring_t tx_ring, uint32_t flags)
338 {
339 #pragma unused(nxprov)
340 #pragma unused(flags)
341 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
342
343 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
344
345 lck_rw_lock_shared(&pcb->utun_pcb_lock);
346
347 struct kern_channel_ring_stat_increment tx_ring_stats;
348 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
349 kern_channel_slot_t tx_pslot = NULL;
350 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
351
352 STATS_INC(nifs, NETIF_STATS_TXSYNC);
353
354 if (tx_slot == NULL) {
355 // Nothing to write, don't bother signalling
356 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
357 return 0;
358 }
359
360 if (pcb->utun_kpipe_enabled) {
361 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
362 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
363
364 // Signal the kernel pipe ring to read
365 if (rx_ring != NULL) {
366 kern_channel_notify(rx_ring, 0);
367 }
368 return 0;
369 }
370
371 // If we're here, we're injecting into the utun kernel control socket
372 while (tx_slot != NULL) {
373 size_t length = 0;
374 mbuf_t data = NULL;
375
376 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
377
378 if (tx_ph == 0) {
379 // Advance TX ring
380 tx_pslot = tx_slot;
381 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
382 continue;
383 }
384 (void) kern_channel_slot_detach_packet(tx_ring, tx_slot, tx_ph);
385
386 // Advance TX ring
387 tx_pslot = tx_slot;
388 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
389
390 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
391 VERIFY(tx_buf != NULL);
392
393 /* tx_baddr is the absolute buffer address */
394 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
395 VERIFY(tx_baddr != 0);
396
397 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
398
399 uint16_t tx_offset = kern_buflet_get_data_offset(tx_buf);
400 uint32_t tx_length = kern_buflet_get_data_length(tx_buf);
401
402 // The offset must be large enough for the headers
403 VERIFY(tx_offset >= UTUN_HEADER_SIZE(pcb));
404
405 // Find family
406 uint32_t af = 0;
407 uint8_t vhl = *(uint8_t *)(tx_baddr + tx_offset);
408 u_int ip_version = (vhl >> 4);
409 switch (ip_version) {
410 case 4: {
411 af = AF_INET;
412 break;
413 }
414 case 6: {
415 af = AF_INET6;
416 break;
417 }
418 default: {
419 printf("utun_netif_sync_tx %s: unknown ip version %u vhl %u tx_offset %u len %u header_size %zu\n",
420 pcb->utun_ifp->if_xname, ip_version, vhl, tx_offset, tx_length,
421 UTUN_HEADER_SIZE(pcb));
422 break;
423 }
424 }
425
426 tx_offset -= UTUN_HEADER_SIZE(pcb);
427 tx_length += UTUN_HEADER_SIZE(pcb);
428 tx_baddr += tx_offset;
429
430 length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE);
431
432 // Copy in family
433 memcpy(tx_baddr, &af, sizeof(af));
434 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
435 kern_packet_get_euuid(tx_ph, (void *)(tx_baddr + sizeof(af)));
436 }
437
438 if (length > 0) {
439 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
440 if (error == 0) {
441 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
442 if (error == 0) {
443 error = utun_output(pcb->utun_ifp, data);
444 if (error != 0) {
445 printf("utun_netif_sync_tx %s - utun_output error %d\n", pcb->utun_ifp->if_xname, error);
446 }
447 } else {
448 printf("utun_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->utun_ifp->if_xname, length, error);
449 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
450 STATS_INC(nifs, NETIF_STATS_DROPPED);
451 mbuf_freem(data);
452 data = NULL;
453 }
454 } else {
455 printf("utun_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->utun_ifp->if_xname, error);
456 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
457 STATS_INC(nifs, NETIF_STATS_DROPPED);
458 }
459 } else {
460 printf("utun_netif_sync_tx %s - 0 length packet\n", pcb->utun_ifp->if_xname);
461 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
462 STATS_INC(nifs, NETIF_STATS_DROPPED);
463 }
464
465 kern_pbufpool_free(tx_ring->ckr_pp, tx_ph);
466
467 if (data == NULL) {
468 continue;
469 }
470
471 STATS_INC(nifs, NETIF_STATS_TXPKTS);
472 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
473
474 tx_ring_stats.kcrsi_slots_transferred++;
475 tx_ring_stats.kcrsi_bytes_transferred += length;
476 }
477
478 if (tx_pslot) {
479 kern_channel_advance_slot(tx_ring, tx_pslot);
480 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
481 (void)kern_channel_reclaim(tx_ring);
482 }
483
484 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
485
486 return 0;
487 }
488
489 static errno_t
490 utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
491 kern_channel_ring_t ring, __unused uint32_t flags)
492 {
493 #pragma unused(nxprov)
494 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
495
496 lck_rw_lock_shared(&pcb->utun_pcb_lock);
497
498 boolean_t more = false;
499 errno_t rc = 0;
500 do {
501 /* Refill and sync the ring */
502 rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more);
503 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
504 printf("%s, tx refill failed %d\n", __func__, rc);
505 }
506 } while ((rc == 0) && more);
507
508 if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) {
509 uint32_t tx_available = kern_channel_available_slot_count(ring);
510 if (pcb->utun_netif_txring_size > 0 &&
511 tx_available >= pcb->utun_netif_txring_size - 1) {
512 // No room left in tx ring, disable output for now
513 errno_t error = ifnet_disable_output(pcb->utun_ifp);
514 if (error != 0) {
515 printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
516 } else {
517 pcb->utun_output_disabled = true;
518 }
519 }
520 }
521
522 if (pcb->utun_kpipe_enabled &&
523 (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) {
524 kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
525
526 // Unlock while calling notify
527 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
528 // Signal the kernel pipe ring to read
529 if (rx_ring != NULL) {
530 kern_channel_notify(rx_ring, 0);
531 }
532 } else {
533 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
534 }
535
536 return (0);
537 }
538
539 static errno_t
540 utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
541 kern_channel_ring_t rx_ring, uint32_t flags)
542 {
543 #pragma unused(nxprov)
544 #pragma unused(flags)
545 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
546 struct kern_channel_ring_stat_increment rx_ring_stats;
547
548 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
549
550 lck_rw_lock_shared(&pcb->utun_pcb_lock);
551
552 // Reclaim user-released slots
553 (void) kern_channel_reclaim(rx_ring);
554
555 STATS_INC(nifs, NETIF_STATS_RXSYNC);
556
557 uint32_t avail = kern_channel_available_slot_count(rx_ring);
558 if (avail == 0) {
559 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
560 return 0;
561 }
562
563 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
564 VERIFY(rx_pp != NULL);
565 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
566 kern_channel_slot_t rx_pslot = NULL;
567 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
568
569 while (rx_slot != NULL) {
570 // Check for a waiting packet
571 lck_mtx_lock(&pcb->utun_input_chain_lock);
572 mbuf_t data = pcb->utun_input_chain;
573 if (data == NULL) {
574 lck_mtx_unlock(&pcb->utun_input_chain_lock);
575 break;
576 }
577
578 // Allocate rx packet
579 kern_packet_t rx_ph = 0;
580 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
581 if (unlikely(error != 0)) {
582 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
583 STATS_INC(nifs, NETIF_STATS_DROPPED);
584 printf("utun_netif_sync_rx %s: failed to allocate packet\n",
585 pcb->utun_ifp->if_xname);
586 lck_mtx_unlock(&pcb->utun_input_chain_lock);
587 break;
588 }
589
590 // Advance waiting packets
591 pcb->utun_input_chain = data->m_nextpkt;
592 data->m_nextpkt = NULL;
593 if (pcb->utun_input_chain == NULL) {
594 pcb->utun_input_chain_last = NULL;
595 }
596 lck_mtx_unlock(&pcb->utun_input_chain_lock);
597
598 size_t header_offset = UTUN_HEADER_SIZE(pcb);
599 size_t length = mbuf_pkthdr_len(data);
600
601 if (length < header_offset) {
602 // mbuf is too small
603 mbuf_freem(data);
604 kern_pbufpool_free(rx_pp, rx_ph);
605 STATS_INC(nifs, NETIF_STATS_BADLEN);
606 STATS_INC(nifs, NETIF_STATS_DROPPED);
607 printf("utun_netif_sync_rx %s: legacy packet length too short for header %zu < %zu\n",
608 pcb->utun_ifp->if_xname, length, header_offset);
609 continue;
610 }
611
612 length -= header_offset;
613 if (length > rx_pp->pp_buflet_size) {
614 // Flush data
615 mbuf_freem(data);
616 kern_pbufpool_free(rx_pp, rx_ph);
617 STATS_INC(nifs, NETIF_STATS_BADLEN);
618 STATS_INC(nifs, NETIF_STATS_DROPPED);
619 printf("utun_netif_sync_rx %s: legacy packet length %zu > %u\n",
620 pcb->utun_ifp->if_xname, length, rx_pp->pp_buflet_size);
621 continue;
622 }
623
624 mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
625
626 // Fillout rx packet
627 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
628 VERIFY(rx_buf != NULL);
629 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
630 VERIFY(rx_baddr != NULL);
631
632 // Copy-in data from mbuf to buflet
633 mbuf_copydata(data, header_offset, length, (void *)rx_baddr);
634 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
635
636 // Finalize and attach the packet
637 error = kern_buflet_set_data_offset(rx_buf, 0);
638 VERIFY(error == 0);
639 error = kern_buflet_set_data_length(rx_buf, length);
640 VERIFY(error == 0);
641 error = kern_packet_set_link_header_offset(rx_ph, 0);
642 VERIFY(error == 0);
643 error = kern_packet_set_network_header_offset(rx_ph, 0);
644 VERIFY(error == 0);
645 error = kern_packet_finalize(rx_ph);
646 VERIFY(error == 0);
647 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
648 VERIFY(error == 0);
649
650 STATS_INC(nifs, NETIF_STATS_RXPKTS);
651 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
652 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
653
654 rx_ring_stats.kcrsi_slots_transferred++;
655 rx_ring_stats.kcrsi_bytes_transferred += length;
656
657 mbuf_freem(data);
658
659 // Advance ring
660 rx_pslot = rx_slot;
661 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
662 }
663
664 struct kern_channel_ring_stat_increment tx_ring_stats;
665 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
666 kern_channel_ring_t tx_ring = pcb->utun_kpipe_txring;
667 kern_channel_slot_t tx_pslot = NULL;
668 kern_channel_slot_t tx_slot = NULL;
669 if (tx_ring == NULL) {
670 // Net-If TX ring not set up yet, nothing to read
671 goto done;
672 }
673
674 // Unlock utun before entering ring
675 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
676
677 (void)kr_enter(tx_ring, TRUE);
678
679 // Lock again after entering and validate
680 lck_rw_lock_shared(&pcb->utun_pcb_lock);
681 if (tx_ring != pcb->utun_kpipe_txring) {
682 goto done;
683 }
684
685 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
686 if (tx_slot == NULL) {
687 // Nothing to read, don't bother signalling
688 goto done;
689 }
690
691 while (rx_slot != NULL && tx_slot != NULL) {
692 // Allocate rx packet
693 kern_packet_t rx_ph = 0;
694 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
695
696 // Advance TX ring
697 tx_pslot = tx_slot;
698 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
699
700 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
701 if (tx_ph == 0) {
702 continue;
703 }
704
705 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
706 if (unlikely(error != 0)) {
707 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
708 STATS_INC(nifs, NETIF_STATS_DROPPED);
709 printf("utun_netif_sync_rx %s: failed to allocate packet\n",
710 pcb->utun_ifp->if_xname);
711 break;
712 }
713
714 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
715 VERIFY(tx_buf != NULL);
716 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
717 VERIFY(tx_baddr != 0);
718 tx_baddr += kern_buflet_get_data_offset(tx_buf);
719
720 // Check packet length
721 size_t header_offset = UTUN_HEADER_SIZE(pcb);
722 uint32_t tx_length = kern_packet_get_data_length(tx_ph);
723 if (tx_length < header_offset) {
724 // Packet is too small
725 kern_pbufpool_free(rx_pp, rx_ph);
726 STATS_INC(nifs, NETIF_STATS_BADLEN);
727 STATS_INC(nifs, NETIF_STATS_DROPPED);
728 printf("utun_netif_sync_rx %s: packet length too short for header %u < %zu\n",
729 pcb->utun_ifp->if_xname, tx_length, header_offset);
730 continue;
731 }
732
733 size_t length = MIN(tx_length - header_offset,
734 UTUN_IF_DEFAULT_SLOT_SIZE);
735
736 tx_ring_stats.kcrsi_slots_transferred++;
737 tx_ring_stats.kcrsi_bytes_transferred += length;
738
739 // Fillout rx packet
740 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
741 VERIFY(rx_buf != NULL);
742 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
743 VERIFY(rx_baddr != NULL);
744
745 // Copy-in data from tx to rx
746 memcpy((void *)rx_baddr, (void *)(tx_baddr + header_offset), length);
747 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
748
749 // Finalize and attach the packet
750 error = kern_buflet_set_data_offset(rx_buf, 0);
751 VERIFY(error == 0);
752 error = kern_buflet_set_data_length(rx_buf, length);
753 VERIFY(error == 0);
754 error = kern_packet_set_link_header_offset(rx_ph, 0);
755 VERIFY(error == 0);
756 error = kern_packet_set_network_header_offset(rx_ph, 0);
757 VERIFY(error == 0);
758 error = kern_packet_finalize(rx_ph);
759 VERIFY(error == 0);
760 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
761 VERIFY(error == 0);
762
763 STATS_INC(nifs, NETIF_STATS_RXPKTS);
764 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
765 bpf_tap_packet_in(pcb->utun_ifp, DLT_RAW, rx_ph, NULL, 0);
766
767 rx_ring_stats.kcrsi_slots_transferred++;
768 rx_ring_stats.kcrsi_bytes_transferred += length;
769
770 rx_pslot = rx_slot;
771 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
772 }
773
774 done:
775 if (rx_pslot) {
776 kern_channel_advance_slot(rx_ring, rx_pslot);
777 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
778 }
779
780 if (tx_pslot) {
781 kern_channel_advance_slot(tx_ring, tx_pslot);
782 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
783 (void)kern_channel_reclaim(tx_ring);
784 }
785
786 // Unlock first, then exit ring
787 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
788 if (tx_ring != NULL) {
789 if (tx_pslot != NULL) {
790 kern_channel_notify(tx_ring, 0);
791 }
792 kr_exit(tx_ring);
793 }
794
795 return 0;
796 }
797
798 static errno_t
799 utun_nexus_ifattach(struct utun_pcb *pcb,
800 struct ifnet_init_eparams *init_params,
801 struct ifnet **ifp)
802 {
803 errno_t err;
804 nexus_controller_t controller = kern_nexus_shared_controller();
805 struct kern_nexus_net_init net_init;
806
807 nexus_name_t provider_name;
808 snprintf((char *)provider_name, sizeof(provider_name),
809 "com.apple.netif.utun%d", pcb->utun_unit);
810
811 struct kern_nexus_provider_init prov_init = {
812 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
813 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
814 .nxpi_pre_connect = utun_nexus_pre_connect,
815 .nxpi_connected = utun_nexus_connected,
816 .nxpi_pre_disconnect = utun_netif_pre_disconnect,
817 .nxpi_disconnected = utun_nexus_disconnected,
818 .nxpi_ring_init = utun_netif_ring_init,
819 .nxpi_ring_fini = utun_netif_ring_fini,
820 .nxpi_slot_init = NULL,
821 .nxpi_slot_fini = NULL,
822 .nxpi_sync_tx = utun_netif_sync_tx,
823 .nxpi_sync_rx = utun_netif_sync_rx,
824 .nxpi_tx_doorbell = utun_netif_tx_doorbell,
825 };
826
827 nexus_attr_t nxa = NULL;
828 err = kern_nexus_attr_create(&nxa);
829 if (err != 0) {
830 printf("%s: kern_nexus_attr_create failed: %d\n",
831 __func__, err);
832 goto failed;
833 }
834
835 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
836 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
837 VERIFY(err == 0);
838
839 // Reset ring size for netif nexus to limit memory usage
840 uint64_t ring_size = if_utun_ring_size;
841 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
842 VERIFY(err == 0);
843 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
844 VERIFY(err == 0);
845
846 pcb->utun_netif_txring_size = ring_size;
847
848 err = kern_nexus_controller_register_provider(controller,
849 utun_nx_dom_prov,
850 provider_name,
851 &prov_init,
852 sizeof(prov_init),
853 nxa,
854 &pcb->utun_nx.if_provider);
855 if (err != 0) {
856 printf("%s register provider failed, error %d\n",
857 __func__, err);
858 goto failed;
859 }
860
861 bzero(&net_init, sizeof(net_init));
862 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
863 net_init.nxneti_flags = 0;
864 net_init.nxneti_eparams = init_params;
865 net_init.nxneti_lladdr = NULL;
866 net_init.nxneti_prepare = utun_netif_prepare;
867 err = kern_nexus_controller_alloc_net_provider_instance(controller,
868 pcb->utun_nx.if_provider,
869 pcb,
870 &pcb->utun_nx.if_instance,
871 &net_init,
872 ifp);
873 if (err != 0) {
874 printf("%s alloc_net_provider_instance failed, %d\n",
875 __func__, err);
876 kern_nexus_controller_deregister_provider(controller,
877 pcb->utun_nx.if_provider);
878 uuid_clear(pcb->utun_nx.if_provider);
879 goto failed;
880 }
881
882 failed:
883 if (nxa) {
884 kern_nexus_attr_destroy(nxa);
885 }
886 return (err);
887 }
888
889 static void
890 utun_detach_provider_and_instance(uuid_t provider, uuid_t instance)
891 {
892 nexus_controller_t controller = kern_nexus_shared_controller();
893 errno_t err;
894
895 if (!uuid_is_null(instance)) {
896 err = kern_nexus_controller_free_provider_instance(controller,
897 instance);
898 if (err != 0) {
899 printf("%s free_provider_instance failed %d\n",
900 __func__, err);
901 }
902 uuid_clear(instance);
903 }
904 if (!uuid_is_null(provider)) {
905 err = kern_nexus_controller_deregister_provider(controller,
906 provider);
907 if (err != 0) {
908 printf("%s deregister_provider %d\n", __func__, err);
909 }
910 uuid_clear(provider);
911 }
912 return;
913 }
914
915 static void
916 utun_nexus_detach(utun_nx_t nx)
917 {
918 nexus_controller_t controller = kern_nexus_shared_controller();
919 errno_t err;
920
921 if (!uuid_is_null(nx->ms_host)) {
922 err = kern_nexus_ifdetach(controller,
923 nx->ms_instance,
924 nx->ms_host);
925 if (err != 0) {
926 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
927 __func__, err);
928 }
929 }
930
931 if (!uuid_is_null(nx->ms_device)) {
932 err = kern_nexus_ifdetach(controller,
933 nx->ms_instance,
934 nx->ms_device);
935 if (err != 0) {
936 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
937 __func__, err);
938 }
939 }
940
941 utun_detach_provider_and_instance(nx->if_provider,
942 nx->if_instance);
943 utun_detach_provider_and_instance(nx->ms_provider,
944 nx->ms_instance);
945
946 memset(nx, 0, sizeof(*nx));
947 }
948
949 static errno_t
950 utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
951 const char *ifname,
952 uuid_t *provider, uuid_t *instance)
953 {
954 nexus_attr_t attr = NULL;
955 nexus_controller_t controller = kern_nexus_shared_controller();
956 uuid_t dom_prov;
957 errno_t err;
958 struct kern_nexus_init init;
959 nexus_name_t provider_name;
960
961 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
962 &dom_prov);
963 if (err != 0) {
964 printf("%s can't get %s provider, error %d\n",
965 __func__, type_name, err);
966 goto failed;
967 }
968
969 err = kern_nexus_attr_create(&attr);
970 if (err != 0) {
971 printf("%s: kern_nexus_attr_create failed: %d\n",
972 __func__, err);
973 goto failed;
974 }
975
976 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
977 VERIFY(err == 0);
978
979 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
980 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
981 VERIFY(err == 0);
982
983 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
984 uint64_t tx_ring_size = if_utun_tx_fsw_ring_size;
985 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
986 VERIFY(err == 0);
987 uint64_t rx_ring_size = if_utun_rx_fsw_ring_size;
988 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
989 VERIFY(err == 0);
990
991 snprintf((char *)provider_name, sizeof(provider_name),
992 "com.apple.%s.%s", type_name, ifname);
993 err = kern_nexus_controller_register_provider(controller,
994 dom_prov,
995 provider_name,
996 NULL,
997 0,
998 attr,
999 provider);
1000 kern_nexus_attr_destroy(attr);
1001 attr = NULL;
1002 if (err != 0) {
1003 printf("%s register %s provider failed, error %d\n",
1004 __func__, type_name, err);
1005 goto failed;
1006 }
1007 bzero(&init, sizeof (init));
1008 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1009 err = kern_nexus_controller_alloc_provider_instance(controller,
1010 *provider,
1011 NULL,
1012 instance, &init);
1013 if (err != 0) {
1014 printf("%s alloc_provider_instance %s failed, %d\n",
1015 __func__, type_name, err);
1016 kern_nexus_controller_deregister_provider(controller,
1017 *provider);
1018 uuid_clear(*provider);
1019 }
1020 failed:
1021 return (err);
1022 }
1023
1024 static errno_t
1025 utun_multistack_attach(struct utun_pcb *pcb)
1026 {
1027 nexus_controller_t controller = kern_nexus_shared_controller();
1028 errno_t err = 0;
1029 utun_nx_t nx = &pcb->utun_nx;
1030
1031 // Allocate multistack flowswitch
1032 err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
1033 "multistack",
1034 pcb->utun_ifp->if_xname,
1035 &nx->ms_provider,
1036 &nx->ms_instance);
1037 if (err != 0) {
1038 printf("%s: failed to create bridge provider and instance\n",
1039 __func__);
1040 goto failed;
1041 }
1042
1043 // Attach multistack to device port
1044 err = kern_nexus_ifattach(controller, nx->ms_instance,
1045 NULL, nx->if_instance,
1046 FALSE, &nx->ms_device);
1047 if (err != 0) {
1048 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1049 goto failed;
1050 }
1051
1052 // Attach multistack to host port
1053 err = kern_nexus_ifattach(controller, nx->ms_instance,
1054 NULL, nx->if_instance,
1055 TRUE, &nx->ms_host);
1056 if (err != 0) {
1057 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1058 goto failed;
1059 }
1060
1061 // Extract the agent UUID and save for later
1062 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1063 if (multistack_nx != NULL) {
1064 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1065 if (flowswitch != NULL) {
1066 FSW_RLOCK(flowswitch);
1067 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1068 if (ms_context != NULL) {
1069 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1070 } else {
1071 printf("utun_multistack_attach - fsw_ms_context is NULL\n");
1072 }
1073 FSW_UNLOCK(flowswitch);
1074 } else {
1075 printf("utun_multistack_attach - flowswitch is NULL\n");
1076 }
1077 nx_release(multistack_nx);
1078 } else {
1079 printf("utun_multistack_attach - unable to find multistack nexus\n");
1080 }
1081
1082 return (0);
1083
1084 failed:
1085 utun_nexus_detach(nx);
1086
1087 errno_t detach_error = 0;
1088 if ((detach_error = ifnet_detach(pcb->utun_ifp)) != 0) {
1089 panic("utun_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1090 /* NOT REACHED */
1091 }
1092
1093 return (err);
1094 }
1095
1096 static errno_t
1097 utun_register_kernel_pipe_nexus(void)
1098 {
1099 nexus_attr_t nxa = NULL;
1100 errno_t result;
1101
1102 lck_mtx_lock(&utun_lock);
1103 if (utun_ncd_refcount++) {
1104 lck_mtx_unlock(&utun_lock);
1105 return 0;
1106 }
1107
1108 result = kern_nexus_controller_create(&utun_ncd);
1109 if (result) {
1110 printf("%s: kern_nexus_controller_create failed: %d\n",
1111 __FUNCTION__, result);
1112 goto done;
1113 }
1114
1115 uuid_t dom_prov;
1116 result = kern_nexus_get_builtin_domain_provider(
1117 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1118 if (result) {
1119 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1120 __FUNCTION__, result);
1121 goto done;
1122 }
1123
1124 struct kern_nexus_provider_init prov_init = {
1125 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1126 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1127 .nxpi_pre_connect = utun_nexus_pre_connect,
1128 .nxpi_connected = utun_nexus_connected,
1129 .nxpi_pre_disconnect = utun_nexus_pre_disconnect,
1130 .nxpi_disconnected = utun_nexus_disconnected,
1131 .nxpi_ring_init = utun_kpipe_ring_init,
1132 .nxpi_ring_fini = utun_kpipe_ring_fini,
1133 .nxpi_slot_init = NULL,
1134 .nxpi_slot_fini = NULL,
1135 .nxpi_sync_tx = utun_kpipe_sync_tx,
1136 .nxpi_sync_rx = utun_kpipe_sync_rx,
1137 .nxpi_tx_doorbell = NULL,
1138 };
1139
1140 result = kern_nexus_attr_create(&nxa);
1141 if (result) {
1142 printf("%s: kern_nexus_attr_create failed: %d\n",
1143 __FUNCTION__, result);
1144 goto done;
1145 }
1146
1147 uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
1148 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1149 VERIFY(result == 0);
1150
1151 // Reset ring size for kernel pipe nexus to limit memory usage
1152 uint64_t ring_size = if_utun_ring_size;
1153 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1154 VERIFY(result == 0);
1155 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1156 VERIFY(result == 0);
1157
1158 result = kern_nexus_controller_register_provider(utun_ncd,
1159 dom_prov,
1160 (const uint8_t *)"com.apple.nexus.utun.kpipe",
1161 &prov_init,
1162 sizeof(prov_init),
1163 nxa,
1164 &utun_kpipe_uuid);
1165 if (result) {
1166 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1167 __FUNCTION__, result);
1168 goto done;
1169 }
1170
1171 done:
1172 if (nxa) {
1173 kern_nexus_attr_destroy(nxa);
1174 }
1175
1176 if (result) {
1177 if (utun_ncd) {
1178 kern_nexus_controller_destroy(utun_ncd);
1179 utun_ncd = NULL;
1180 }
1181 utun_ncd_refcount = 0;
1182 }
1183
1184 lck_mtx_unlock(&utun_lock);
1185
1186 return result;
1187 }
1188
1189 static void
1190 utun_unregister_kernel_pipe_nexus(void)
1191 {
1192 lck_mtx_lock(&utun_lock);
1193
1194 VERIFY(utun_ncd_refcount > 0);
1195
1196 if (--utun_ncd_refcount == 0) {
1197 kern_nexus_controller_destroy(utun_ncd);
1198 utun_ncd = NULL;
1199 }
1200
1201 lck_mtx_unlock(&utun_lock);
1202 }
1203
1204 // For use by socket option, not internally
1205 static errno_t
1206 utun_disable_channel(struct utun_pcb *pcb)
1207 {
1208 errno_t result;
1209 int enabled;
1210 uuid_t uuid;
1211
1212 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1213
1214 enabled = pcb->utun_kpipe_enabled;
1215 uuid_copy(uuid, pcb->utun_kpipe_uuid);
1216
1217 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid) == !enabled);
1218
1219 pcb->utun_kpipe_enabled = 0;
1220 uuid_clear(pcb->utun_kpipe_uuid);
1221
1222 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1223
1224 if (enabled) {
1225 result = kern_nexus_controller_free_provider_instance(utun_ncd, uuid);
1226 } else {
1227 result = ENXIO;
1228 }
1229
1230 if (!result) {
1231 utun_unregister_kernel_pipe_nexus();
1232 }
1233
1234 return result;
1235 }
1236
1237 static errno_t
1238 utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
1239 {
1240 struct kern_nexus_init init;
1241 errno_t result;
1242
1243 result = utun_register_kernel_pipe_nexus();
1244 if (result) {
1245 return result;
1246 }
1247
1248 VERIFY(utun_ncd);
1249
1250 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1251
1252 if (pcb->utun_kpipe_enabled) {
1253 result = EEXIST; // return success instead?
1254 goto done;
1255 }
1256
1257 /*
1258 * Make sure we can fit packets in the channel buffers and
1259 * Allow an extra 4 bytes for the protocol number header in the channel
1260 */
1261 if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) {
1262 result = EOPNOTSUPP;
1263 goto done;
1264 }
1265
1266 VERIFY(uuid_is_null(pcb->utun_kpipe_uuid));
1267 bzero(&init, sizeof (init));
1268 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1269 result = kern_nexus_controller_alloc_provider_instance(utun_ncd,
1270 utun_kpipe_uuid, pcb, &pcb->utun_kpipe_uuid, &init);
1271 if (result) {
1272 goto done;
1273 }
1274
1275 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
1276 result = kern_nexus_controller_bind_provider_instance(utun_ncd,
1277 pcb->utun_kpipe_uuid, &port,
1278 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
1279 if (result) {
1280 kern_nexus_controller_free_provider_instance(utun_ncd,
1281 pcb->utun_kpipe_uuid);
1282 uuid_clear(pcb->utun_kpipe_uuid);
1283 goto done;
1284 }
1285
1286 pcb->utun_kpipe_enabled = 1;
1287
1288 done:
1289 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1290
1291 if (result) {
1292 utun_unregister_kernel_pipe_nexus();
1293 }
1294
1295 return result;
1296 }
1297
1298 #endif // UTUN_NEXUS
1299
1300 errno_t
1301 utun_register_control(void)
1302 {
1303 struct kern_ctl_reg kern_ctl;
1304 errno_t result = 0;
1305
1306 /* Find a unique value for our interface family */
1307 result = mbuf_tag_id_find(UTUN_CONTROL_NAME, &utun_family);
1308 if (result != 0) {
1309 printf("utun_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
1310 return result;
1311 }
1312
1313 utun_pcb_size = sizeof(struct utun_pcb);
1314 utun_pcb_zone = zinit(utun_pcb_size,
1315 UTUN_PCB_ZONE_MAX * utun_pcb_size,
1316 0, UTUN_PCB_ZONE_NAME);
1317 if (utun_pcb_zone == NULL) {
1318 printf("utun_register_control - zinit(utun_pcb) failed");
1319 return ENOMEM;
1320 }
1321
1322 #if UTUN_NEXUS
1323 utun_register_nexus();
1324 #endif // UTUN_NEXUS
1325
1326 TAILQ_INIT(&utun_head);
1327
1328 bzero(&kern_ctl, sizeof(kern_ctl));
1329 strlcpy(kern_ctl.ctl_name, UTUN_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
1330 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
1331 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
1332 kern_ctl.ctl_sendsize = 512 * 1024;
1333 kern_ctl.ctl_recvsize = 512 * 1024;
1334 kern_ctl.ctl_connect = utun_ctl_connect;
1335 kern_ctl.ctl_disconnect = utun_ctl_disconnect;
1336 kern_ctl.ctl_send = utun_ctl_send;
1337 kern_ctl.ctl_setopt = utun_ctl_setopt;
1338 kern_ctl.ctl_getopt = utun_ctl_getopt;
1339 kern_ctl.ctl_rcvd = utun_ctl_rcvd;
1340
1341 result = ctl_register(&kern_ctl, &utun_kctlref);
1342 if (result != 0) {
1343 printf("utun_register_control - ctl_register failed: %d\n", result);
1344 return result;
1345 }
1346
1347 /* Register the protocol plumbers */
1348 if ((result = proto_register_plumber(PF_INET, utun_family,
1349 utun_attach_proto, NULL)) != 0) {
1350 printf("utun_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
1351 utun_family, result);
1352 ctl_deregister(utun_kctlref);
1353 return result;
1354 }
1355
1356 /* Register the protocol plumbers */
1357 if ((result = proto_register_plumber(PF_INET6, utun_family,
1358 utun_attach_proto, NULL)) != 0) {
1359 proto_unregister_plumber(PF_INET, utun_family);
1360 ctl_deregister(utun_kctlref);
1361 printf("utun_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
1362 utun_family, result);
1363 return result;
1364 }
1365
1366 utun_lck_attr = lck_attr_alloc_init();
1367 utun_lck_grp_attr = lck_grp_attr_alloc_init();
1368 utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr);
1369
1370 #if UTUN_NEXUS
1371 lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
1372 #endif // UTUN_NEXUS
1373
1374 return 0;
1375 }
1376
1377 /* Kernel control functions */
1378
1379 static inline void
1380 utun_free_pcb(struct utun_pcb *pcb)
1381 {
1382 #ifdef UTUN_NEXUS
1383 mbuf_freem_list(pcb->utun_input_chain);
1384 lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
1385 #endif // UTUN_NEXUS
1386 lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
1387 lck_mtx_lock(&utun_lock);
1388 TAILQ_REMOVE(&utun_head, pcb, utun_chain);
1389 lck_mtx_unlock(&utun_lock);
1390 zfree(utun_pcb_zone, pcb);
1391 }
1392
1393 static errno_t
1394 utun_ctl_connect(kern_ctl_ref kctlref,
1395 struct sockaddr_ctl *sac,
1396 void **unitinfo)
1397 {
1398 struct ifnet_init_eparams utun_init = {};
1399 errno_t result = 0;
1400
1401 struct utun_pcb *pcb = zalloc(utun_pcb_zone);
1402 memset(pcb, 0, sizeof(*pcb));
1403
1404 *unitinfo = pcb;
1405 pcb->utun_ctlref = kctlref;
1406 pcb->utun_unit = sac->sc_unit;
1407 pcb->utun_max_pending_packets = 1;
1408
1409 lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
1410 lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
1411
1412 lck_mtx_lock(&utun_lock);
1413
1414 /* Find some open interface id */
1415 u_int32_t chosen_unique_id = 1;
1416 struct utun_pcb *next_pcb = TAILQ_LAST(&utun_head, utun_list);
1417 if (next_pcb != NULL) {
1418 /* List was not empty, add one to the last item */
1419 chosen_unique_id = next_pcb->utun_unique_id + 1;
1420 next_pcb = NULL;
1421
1422 /*
1423 * If this wrapped the id number, start looking at
1424 * the front of the list for an unused id.
1425 */
1426 if (chosen_unique_id == 0) {
1427 /* Find the next unused ID */
1428 chosen_unique_id = 1;
1429 TAILQ_FOREACH(next_pcb, &utun_head, utun_chain) {
1430 if (next_pcb->utun_unique_id > chosen_unique_id) {
1431 /* We found a gap */
1432 break;
1433 }
1434
1435 chosen_unique_id = next_pcb->utun_unique_id + 1;
1436 }
1437 }
1438 }
1439
1440 pcb->utun_unique_id = chosen_unique_id;
1441
1442 if (next_pcb != NULL) {
1443 TAILQ_INSERT_BEFORE(next_pcb, pcb, utun_chain);
1444 } else {
1445 TAILQ_INSERT_TAIL(&utun_head, pcb, utun_chain);
1446 }
1447 lck_mtx_unlock(&utun_lock);
1448
1449 snprintf(pcb->utun_if_xname, sizeof(pcb->utun_if_xname), "utun%d", pcb->utun_unit - 1);
1450 snprintf(pcb->utun_unique_name, sizeof(pcb->utun_unique_name), "utunid%d", pcb->utun_unique_id - 1);
1451 printf("utun_ctl_connect: creating interface %s (id %s)\n", pcb->utun_if_xname, pcb->utun_unique_name);
1452
1453 /* Create the interface */
1454 bzero(&utun_init, sizeof(utun_init));
1455 utun_init.ver = IFNET_INIT_CURRENT_VERSION;
1456 utun_init.len = sizeof (utun_init);
1457
1458 #if UTUN_NEXUS
1459 utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
1460 utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
1461 #else // UTUN_NEXUS
1462 utun_init.flags = IFNET_INIT_NX_NOAUTO;
1463 utun_init.start = utun_start;
1464 utun_init.framer_extended = utun_framer;
1465 #endif // UTUN_NEXUS
1466 utun_init.name = "utun";
1467 utun_init.unit = pcb->utun_unit - 1;
1468 utun_init.uniqueid = pcb->utun_unique_name;
1469 utun_init.uniqueid_len = strlen(pcb->utun_unique_name);
1470 utun_init.family = utun_family;
1471 utun_init.subfamily = IFNET_SUBFAMILY_UTUN;
1472 utun_init.type = IFT_OTHER;
1473 utun_init.demux = utun_demux;
1474 utun_init.add_proto = utun_add_proto;
1475 utun_init.del_proto = utun_del_proto;
1476 utun_init.softc = pcb;
1477 utun_init.ioctl = utun_ioctl;
1478 utun_init.detach = utun_detached;
1479
1480 #if UTUN_NEXUS
1481 result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
1482 if (result != 0) {
1483 printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
1484 utun_free_pcb(pcb);
1485 *unitinfo = NULL;
1486 return result;
1487 }
1488
1489 result = utun_multistack_attach(pcb);
1490 if (result != 0) {
1491 printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
1492 *unitinfo = NULL;
1493 return result;
1494 }
1495
1496 #else // UTUN_NEXUS
1497 /*
1498 * Upon success, this holds an ifnet reference which we will
1499 * release via ifnet_release() at final detach time.
1500 */
1501 result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
1502 if (result != 0) {
1503 printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
1504 utun_free_pcb(pcb);
1505 *unitinfo = NULL;
1506 return result;
1507 }
1508
1509 /* Set flags and additional information. */
1510 ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
1511 ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
1512
1513 /* The interface must generate its own IPv6 LinkLocal address,
1514 * if possible following the recommendation of RFC2472 to the 64bit interface ID
1515 */
1516 ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
1517
1518 /* Reset the stats in case as the interface may have been recycled */
1519 struct ifnet_stats_param stats;
1520 bzero(&stats, sizeof(struct ifnet_stats_param));
1521 ifnet_set_stat(pcb->utun_ifp, &stats);
1522
1523 /* Attach the interface */
1524 result = ifnet_attach(pcb->utun_ifp, NULL);
1525 if (result != 0) {
1526 printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
1527 /* Release reference now since attach failed */
1528 ifnet_release(pcb->utun_ifp);
1529 utun_free_pcb(pcb);
1530 *unitinfo = NULL;
1531 return (result);
1532 }
1533 #endif // UTUN_NEXUS
1534
1535 /* Attach to bpf */
1536 bpfattach(pcb->utun_ifp, DLT_RAW, 0);
1537 /* The interfaces resoures allocated, mark it as running */
1538 ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
1539
1540 return result;
1541 }
1542
1543 static errno_t
1544 utun_detach_ip(ifnet_t interface,
1545 protocol_family_t protocol,
1546 socket_t pf_socket)
1547 {
1548 errno_t result = EPROTONOSUPPORT;
1549
1550 /* Attempt a detach */
1551 if (protocol == PF_INET) {
1552 struct ifreq ifr;
1553
1554 bzero(&ifr, sizeof(ifr));
1555 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1556 ifnet_name(interface), ifnet_unit(interface));
1557
1558 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
1559 } else if (protocol == PF_INET6) {
1560 struct in6_ifreq ifr6;
1561
1562 bzero(&ifr6, sizeof(ifr6));
1563 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1564 ifnet_name(interface), ifnet_unit(interface));
1565
1566 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
1567 }
1568
1569 return result;
1570 }
1571
1572 static void
1573 utun_remove_address(ifnet_t interface,
1574 protocol_family_t protocol,
1575 ifaddr_t address,
1576 socket_t pf_socket)
1577 {
1578 errno_t result = 0;
1579
1580 /* Attempt a detach */
1581 if (protocol == PF_INET) {
1582 struct ifreq ifr;
1583
1584 bzero(&ifr, sizeof(ifr));
1585 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
1586 ifnet_name(interface), ifnet_unit(interface));
1587 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
1588 if (result != 0) {
1589 printf("utun_remove_address - ifaddr_address failed: %d", result);
1590 } else {
1591 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
1592 if (result != 0) {
1593 printf("utun_remove_address - SIOCDIFADDR failed: %d", result);
1594 }
1595 }
1596 } else if (protocol == PF_INET6) {
1597 struct in6_ifreq ifr6;
1598
1599 bzero(&ifr6, sizeof(ifr6));
1600 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
1601 ifnet_name(interface), ifnet_unit(interface));
1602 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
1603 sizeof(ifr6.ifr_addr));
1604 if (result != 0) {
1605 printf("utun_remove_address - ifaddr_address failed (v6): %d",
1606 result);
1607 } else {
1608 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
1609 if (result != 0) {
1610 printf("utun_remove_address - SIOCDIFADDR_IN6 failed: %d",
1611 result);
1612 }
1613 }
1614 }
1615 }
1616
1617 static void
1618 utun_cleanup_family(ifnet_t interface,
1619 protocol_family_t protocol)
1620 {
1621 errno_t result = 0;
1622 socket_t pf_socket = NULL;
1623 ifaddr_t *addresses = NULL;
1624 int i;
1625
1626 if (protocol != PF_INET && protocol != PF_INET6) {
1627 printf("utun_cleanup_family - invalid protocol family %d\n", protocol);
1628 return;
1629 }
1630
1631 /* Create a socket for removing addresses and detaching the protocol */
1632 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
1633 if (result != 0) {
1634 if (result != EAFNOSUPPORT)
1635 printf("utun_cleanup_family - failed to create %s socket: %d\n",
1636 protocol == PF_INET ? "IP" : "IPv6", result);
1637 goto cleanup;
1638 }
1639
1640 /* always set SS_PRIV, we want to close and detach regardless */
1641 sock_setpriv(pf_socket, 1);
1642
1643 result = utun_detach_ip(interface, protocol, pf_socket);
1644 if (result == 0 || result == ENXIO) {
1645 /* We are done! We either detached or weren't attached. */
1646 goto cleanup;
1647 } else if (result != EBUSY) {
1648 /* Uh, not really sure what happened here... */
1649 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1650 goto cleanup;
1651 }
1652
1653 /*
1654 * At this point, we received an EBUSY error. This means there are
1655 * addresses attached. We should detach them and then try again.
1656 */
1657 result = ifnet_get_address_list_family(interface, &addresses, protocol);
1658 if (result != 0) {
1659 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
1660 ifnet_name(interface), ifnet_unit(interface),
1661 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
1662 goto cleanup;
1663 }
1664
1665 for (i = 0; addresses[i] != 0; i++) {
1666 utun_remove_address(interface, protocol, addresses[i], pf_socket);
1667 }
1668 ifnet_free_address_list(addresses);
1669 addresses = NULL;
1670
1671 /*
1672 * The addresses should be gone, we should try the remove again.
1673 */
1674 result = utun_detach_ip(interface, protocol, pf_socket);
1675 if (result != 0 && result != ENXIO) {
1676 printf("utun_cleanup_family - utun_detach_ip failed: %d\n", result);
1677 }
1678
1679 cleanup:
1680 if (pf_socket != NULL) {
1681 sock_close(pf_socket);
1682 }
1683
1684 if (addresses != NULL) {
1685 ifnet_free_address_list(addresses);
1686 }
1687 }
1688
1689 static errno_t
1690 utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
1691 __unused u_int32_t unit,
1692 void *unitinfo)
1693 {
1694 struct utun_pcb *pcb = unitinfo;
1695 ifnet_t ifp = NULL;
1696 errno_t result = 0;
1697
1698 if (pcb == NULL) {
1699 return EINVAL;
1700 }
1701
1702 #if UTUN_NEXUS
1703 // Tell the nexus to stop all rings
1704 if (pcb->utun_netif_nexus != NULL) {
1705 kern_nexus_stop(pcb->utun_netif_nexus);
1706 }
1707 #endif // UTUN_NEXUS
1708
1709 lck_rw_lock_exclusive(&pcb->utun_pcb_lock);
1710
1711 #if UTUN_NEXUS
1712 uuid_t kpipe_uuid;
1713 uuid_copy(kpipe_uuid, pcb->utun_kpipe_uuid);
1714 uuid_clear(pcb->utun_kpipe_uuid);
1715 pcb->utun_kpipe_enabled = FALSE;
1716 #endif // UTUN_NEXUS
1717
1718 ifp = pcb->utun_ifp;
1719 VERIFY(ifp != NULL);
1720 pcb->utun_ctlref = NULL;
1721
1722 /*
1723 * Quiesce the interface and flush any pending outbound packets.
1724 */
1725 if_down(ifp);
1726
1727 /* Increment refcnt, but detach interface */
1728 ifnet_incr_iorefcnt(ifp);
1729 if ((result = ifnet_detach(ifp)) != 0) {
1730 panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
1731 }
1732
1733 /*
1734 * We want to do everything in our power to ensure that the interface
1735 * really goes away when the socket is closed. We must remove IP/IPv6
1736 * addresses and detach the protocols. Finally, we can remove and
1737 * release the interface.
1738 */
1739 utun_cleanup_family(ifp, AF_INET);
1740 utun_cleanup_family(ifp, AF_INET6);
1741
1742 lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
1743
1744 #if UTUN_NEXUS
1745 if (!uuid_is_null(kpipe_uuid)) {
1746 if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
1747 utun_unregister_kernel_pipe_nexus();
1748 }
1749 }
1750 utun_nexus_detach(&pcb->utun_nx);
1751 #endif // UTUN_NEXUS
1752
1753 /* Decrement refcnt to finish detaching and freeing */
1754 ifnet_decr_iorefcnt(ifp);
1755
1756 return 0;
1757 }
1758
1759 static errno_t
1760 utun_ctl_send(__unused kern_ctl_ref kctlref,
1761 __unused u_int32_t unit,
1762 void *unitinfo,
1763 mbuf_t m,
1764 __unused int flags)
1765 {
1766 /*
1767 * The userland ABI requires the first four bytes have the protocol family
1768 * in network byte order: swap them
1769 */
1770 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE((struct utun_pcb *)unitinfo)) {
1771 *(protocol_family_t *)mbuf_data(m) = ntohl(*(protocol_family_t *)mbuf_data(m));
1772 } else {
1773 printf("%s - unexpected short mbuf pkt len %d\n", __func__, m_pktlen(m) );
1774 }
1775
1776 return utun_pkt_input((struct utun_pcb *)unitinfo, m);
1777 }
1778
1779 static errno_t
1780 utun_ctl_setopt(__unused kern_ctl_ref kctlref,
1781 __unused u_int32_t unit,
1782 void *unitinfo,
1783 int opt,
1784 void *data,
1785 size_t len)
1786 {
1787 struct utun_pcb *pcb = unitinfo;
1788 errno_t result = 0;
1789 /* check for privileges for privileged options */
1790 switch (opt) {
1791 case UTUN_OPT_FLAGS:
1792 case UTUN_OPT_EXT_IFDATA_STATS:
1793 case UTUN_OPT_SET_DELEGATE_INTERFACE:
1794 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
1795 return EPERM;
1796 }
1797 break;
1798 }
1799
1800 switch (opt) {
1801 case UTUN_OPT_FLAGS:
1802 if (len != sizeof(u_int32_t)) {
1803 result = EMSGSIZE;
1804 } else {
1805 pcb->utun_flags = *(u_int32_t *)data;
1806 }
1807 break;
1808
1809 case UTUN_OPT_EXT_IFDATA_STATS:
1810 if (len != sizeof(int)) {
1811 result = EMSGSIZE;
1812 break;
1813 }
1814 pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
1815 break;
1816
1817 case UTUN_OPT_INC_IFDATA_STATS_IN:
1818 case UTUN_OPT_INC_IFDATA_STATS_OUT: {
1819 struct utun_stats_param *utsp = (struct utun_stats_param *)data;
1820
1821 if (utsp == NULL || len < sizeof(struct utun_stats_param)) {
1822 result = EINVAL;
1823 break;
1824 }
1825 if (!pcb->utun_ext_ifdata_stats) {
1826 result = EINVAL;
1827 break;
1828 }
1829 if (opt == UTUN_OPT_INC_IFDATA_STATS_IN)
1830 ifnet_stat_increment_in(pcb->utun_ifp, utsp->utsp_packets,
1831 utsp->utsp_bytes, utsp->utsp_errors);
1832 else
1833 ifnet_stat_increment_out(pcb->utun_ifp, utsp->utsp_packets,
1834 utsp->utsp_bytes, utsp->utsp_errors);
1835 break;
1836 }
1837 case UTUN_OPT_SET_DELEGATE_INTERFACE: {
1838 ifnet_t del_ifp = NULL;
1839 char name[IFNAMSIZ];
1840
1841 if (len > IFNAMSIZ - 1) {
1842 result = EMSGSIZE;
1843 break;
1844 }
1845 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
1846 bcopy(data, name, len);
1847 name[len] = 0;
1848 result = ifnet_find_by_name(name, &del_ifp);
1849 }
1850 if (result == 0) {
1851 result = ifnet_set_delegate(pcb->utun_ifp, del_ifp);
1852 if (del_ifp)
1853 ifnet_release(del_ifp);
1854 }
1855 break;
1856 }
1857 case UTUN_OPT_MAX_PENDING_PACKETS: {
1858 u_int32_t max_pending_packets = 0;
1859 if (len != sizeof(u_int32_t)) {
1860 result = EMSGSIZE;
1861 break;
1862 }
1863 max_pending_packets = *(u_int32_t *)data;
1864 if (max_pending_packets == 0) {
1865 result = EINVAL;
1866 break;
1867 }
1868 pcb->utun_max_pending_packets = max_pending_packets;
1869 break;
1870 }
1871 #if UTUN_NEXUS
1872 case UTUN_OPT_ENABLE_CHANNEL: {
1873 if (len != sizeof(int)) {
1874 result = EMSGSIZE;
1875 break;
1876 }
1877 if (*(int *)data) {
1878 result = utun_enable_channel(pcb, current_proc());
1879 } else {
1880 result = utun_disable_channel(pcb);
1881 }
1882 break;
1883 }
1884 case UTUN_OPT_ENABLE_FLOWSWITCH: {
1885 if (len != sizeof(int)) {
1886 result = EMSGSIZE;
1887 break;
1888 }
1889 if (!if_enable_netagent) {
1890 result = ENOTSUP;
1891 break;
1892 }
1893 if (uuid_is_null(pcb->utun_nx.ms_agent)) {
1894 result = ENOENT;
1895 break;
1896 }
1897
1898 if (*(int *)data) {
1899 if_add_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
1900 } else {
1901 if_delete_netagent(pcb->utun_ifp, pcb->utun_nx.ms_agent);
1902 }
1903 break;
1904 }
1905 #endif // UTUN_NEXUS
1906 default: {
1907 result = ENOPROTOOPT;
1908 break;
1909 }
1910 }
1911
1912 return result;
1913 }
1914
1915 static errno_t
1916 utun_ctl_getopt(__unused kern_ctl_ref kctlref,
1917 __unused u_int32_t unit,
1918 void *unitinfo,
1919 int opt,
1920 void *data,
1921 size_t *len)
1922 {
1923 struct utun_pcb *pcb = unitinfo;
1924 errno_t result = 0;
1925
1926 switch (opt) {
1927 case UTUN_OPT_FLAGS:
1928 if (*len != sizeof(u_int32_t)) {
1929 result = EMSGSIZE;
1930 } else {
1931 *(u_int32_t *)data = pcb->utun_flags;
1932 }
1933 break;
1934
1935 case UTUN_OPT_EXT_IFDATA_STATS:
1936 if (*len != sizeof(int)) {
1937 result = EMSGSIZE;
1938 } else {
1939 *(int *)data = (pcb->utun_ext_ifdata_stats) ? 1 : 0;
1940 }
1941 break;
1942
1943 case UTUN_OPT_IFNAME:
1944 if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
1945 result = EMSGSIZE;
1946 } else {
1947 *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
1948 }
1949 break;
1950
1951 case UTUN_OPT_MAX_PENDING_PACKETS: {
1952 if (*len != sizeof(u_int32_t)) {
1953 result = EMSGSIZE;
1954 } else {
1955 *((u_int32_t *)data) = pcb->utun_max_pending_packets;
1956 }
1957 break;
1958 }
1959
1960 #if UTUN_NEXUS
1961 case UTUN_OPT_GET_CHANNEL_UUID:
1962 lck_rw_lock_shared(&pcb->utun_pcb_lock);
1963 if (uuid_is_null(pcb->utun_kpipe_uuid)) {
1964 result = ENXIO;
1965 } else if (*len != sizeof(uuid_t)) {
1966 result = EMSGSIZE;
1967 } else {
1968 uuid_copy(data, pcb->utun_kpipe_uuid);
1969 }
1970 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
1971 break;
1972 #endif // UTUN_NEXUS
1973
1974 default:
1975 result = ENOPROTOOPT;
1976 break;
1977 }
1978
1979 return result;
1980 }
1981
1982 static void
1983 utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
1984 {
1985 #pragma unused(flags)
1986 bool reenable_output = false;
1987 struct utun_pcb *pcb = unitinfo;
1988 if (pcb == NULL) {
1989 return;
1990 }
1991 ifnet_lock_exclusive(pcb->utun_ifp);
1992
1993 u_int32_t utun_packet_cnt;
1994 errno_t error_pc = ctl_getenqueuepacketcount(kctlref, unit, &utun_packet_cnt);
1995 if (error_pc != 0) {
1996 printf("utun_ctl_rcvd: ctl_getenqueuepacketcount returned error %d\n", error_pc);
1997 utun_packet_cnt = 0;
1998 }
1999
2000 if (utun_packet_cnt < pcb->utun_max_pending_packets) {
2001 reenable_output = true;
2002 }
2003
2004 if (reenable_output) {
2005 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2006 if (error != 0) {
2007 printf("utun_ctl_rcvd: ifnet_enable_output returned error %d\n", error);
2008 }
2009 }
2010 ifnet_lock_done(pcb->utun_ifp);
2011 }
2012
2013 /* Network Interface functions */
2014 #if !UTUN_NEXUS
2015 static void
2016 utun_start(ifnet_t interface)
2017 {
2018 mbuf_t data;
2019 struct utun_pcb *pcb = ifnet_softc(interface);
2020
2021 VERIFY(pcb != NULL);
2022
2023 #if UTUN_NEXUS
2024 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2025 if (pcb->utun_kpipe_enabled) {
2026 /* It's possible to have channels enabled, but not yet have the channel opened,
2027 * in which case the rxring will not be set
2028 */
2029 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2030 if (pcb->utun_kpipe_rxring != NULL) {
2031 kern_channel_notify(pcb->utun_kpipe_rxring, 0);
2032 }
2033 return;
2034 }
2035 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2036 #endif // UTUN_NEXUS
2037
2038 for (;;) {
2039 bool can_accept_packets = true;
2040 ifnet_lock_shared(pcb->utun_ifp);
2041
2042 u_int32_t utun_packet_cnt;
2043 errno_t error_pc = ctl_getenqueuepacketcount(pcb->utun_ctlref, pcb->utun_unit, &utun_packet_cnt);
2044 if (error_pc != 0) {
2045 printf("utun_start: ctl_getenqueuepacketcount returned error %d\n", error_pc);
2046 utun_packet_cnt = 0;
2047 }
2048
2049 can_accept_packets = (utun_packet_cnt < pcb->utun_max_pending_packets);
2050 if (!can_accept_packets && pcb->utun_ctlref) {
2051 u_int32_t difference = 0;
2052 if (ctl_getenqueuereadable(pcb->utun_ctlref, pcb->utun_unit, &difference) == 0) {
2053 if (difference > 0) {
2054 // If the low-water mark has not yet been reached, we still need to enqueue data
2055 // into the buffer
2056 can_accept_packets = true;
2057 }
2058 }
2059 }
2060 if (!can_accept_packets) {
2061 errno_t error = ifnet_disable_output(interface);
2062 if (error != 0) {
2063 printf("utun_start: ifnet_disable_output returned error %d\n", error);
2064 }
2065 ifnet_lock_done(pcb->utun_ifp);
2066 break;
2067 }
2068 ifnet_lock_done(pcb->utun_ifp);
2069 if (ifnet_dequeue(interface, &data) != 0) {
2070 break;
2071 }
2072 if (utun_output(interface, data) != 0) {
2073 break;
2074 }
2075 }
2076 }
2077 #endif // !UTUN_NEXUS
2078
2079 static errno_t
2080 utun_output(ifnet_t interface,
2081 mbuf_t data)
2082 {
2083 struct utun_pcb *pcb = ifnet_softc(interface);
2084 errno_t result;
2085
2086 VERIFY(interface == pcb->utun_ifp);
2087
2088 if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
2089 /* flush data */
2090 mbuf_freem(data);
2091 return 0;
2092 }
2093
2094 // otherwise, fall thru to ctl_enqueumbuf
2095 if (pcb->utun_ctlref) {
2096 int length;
2097
2098 /*
2099 * The ABI requires the protocol in network byte order
2100 */
2101 if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2102 *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
2103 }
2104
2105 length = mbuf_pkthdr_len(data);
2106 result = ctl_enqueuembuf(pcb->utun_ctlref, pcb->utun_unit, data, CTL_DATA_EOR);
2107 if (result != 0) {
2108 mbuf_freem(data);
2109 printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
2110 #if !UTUN_NEXUS
2111 ifnet_stat_increment_out(interface, 0, 0, 1);
2112 } else {
2113 if (!pcb->utun_ext_ifdata_stats) {
2114 ifnet_stat_increment_out(interface, 1, length, 0);
2115 }
2116 #endif // !UTUN_NEXUS
2117 }
2118 } else {
2119 mbuf_freem(data);
2120 }
2121
2122 return 0;
2123 }
2124
2125 static errno_t
2126 utun_demux(__unused ifnet_t interface,
2127 mbuf_t data,
2128 __unused char *frame_header,
2129 protocol_family_t *protocol)
2130 {
2131
2132 struct ip *ip;
2133 u_int ip_version;
2134
2135 while (data != NULL && mbuf_len(data) < 1) {
2136 data = mbuf_next(data);
2137 }
2138
2139 if (data == NULL)
2140 return ENOENT;
2141
2142 ip = mtod(data, struct ip *);
2143 ip_version = ip->ip_v;
2144
2145 switch(ip_version) {
2146 case 4:
2147 *protocol = PF_INET;
2148 return 0;
2149 case 6:
2150 *protocol = PF_INET6;
2151 return 0;
2152 default:
2153 *protocol = 0;
2154 break;
2155 }
2156
2157 return 0;
2158 }
2159
2160 #if !UTUN_NEXUS
2161 static errno_t
2162 utun_framer(ifnet_t interface,
2163 mbuf_t *packet,
2164 __unused const struct sockaddr *dest,
2165 __unused const char *desk_linkaddr,
2166 const char *frame_type,
2167 u_int32_t *prepend_len,
2168 u_int32_t *postpend_len)
2169 {
2170 struct utun_pcb *pcb = ifnet_softc(interface);
2171 VERIFY(interface == pcb->utun_ifp);
2172
2173 u_int32_t header_length = UTUN_HEADER_SIZE(pcb);
2174 if (mbuf_prepend(packet, header_length, MBUF_DONTWAIT) != 0) {
2175 printf("utun_framer - ifnet_output prepend failed\n");
2176
2177 ifnet_stat_increment_out(interface, 0, 0, 1);
2178
2179 // just return, because the buffer was freed in mbuf_prepend
2180 return EJUSTRETURN;
2181 }
2182 if (prepend_len != NULL) {
2183 *prepend_len = header_length;
2184 }
2185 if (postpend_len != NULL) {
2186 *postpend_len = 0;
2187 }
2188
2189 // place protocol number at the beginning of the mbuf
2190 *(protocol_family_t *)mbuf_data(*packet) = *(protocol_family_t *)(uintptr_t)(size_t)frame_type;
2191
2192
2193 return 0;
2194 }
2195 #endif // !UTUN_NEXUS
2196
2197 static errno_t
2198 utun_add_proto(__unused ifnet_t interface,
2199 protocol_family_t protocol,
2200 __unused const struct ifnet_demux_desc *demux_array,
2201 __unused u_int32_t demux_count)
2202 {
2203 switch(protocol) {
2204 case PF_INET:
2205 return 0;
2206 case PF_INET6:
2207 return 0;
2208 default:
2209 break;
2210 }
2211
2212 return ENOPROTOOPT;
2213 }
2214
2215 static errno_t
2216 utun_del_proto(__unused ifnet_t interface,
2217 __unused protocol_family_t protocol)
2218 {
2219 return 0;
2220 }
2221
2222 static errno_t
2223 utun_ioctl(ifnet_t interface,
2224 u_long command,
2225 void *data)
2226 {
2227 errno_t result = 0;
2228
2229 switch(command) {
2230 case SIOCSIFMTU:
2231 #if UTUN_NEXUS
2232 {
2233 // Make sure we can fit packets in the channel buffers
2234 // Allow for the headroom in the slot
2235 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) {
2236 ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE);
2237 break;
2238 }
2239 }
2240 #endif // UTUN_NEXUS
2241 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
2242 break;
2243
2244 case SIOCSIFFLAGS:
2245 /* ifioctl() takes care of it */
2246 break;
2247
2248 default:
2249 result = EOPNOTSUPP;
2250 }
2251
2252 return result;
2253 }
2254
2255 static void
2256 utun_detached(ifnet_t interface)
2257 {
2258 struct utun_pcb *pcb = ifnet_softc(interface);
2259 (void)ifnet_release(interface);
2260 utun_free_pcb(pcb);
2261 }
2262
2263 /* Protocol Handlers */
2264
2265 static errno_t
2266 utun_proto_input(__unused ifnet_t interface,
2267 protocol_family_t protocol,
2268 mbuf_t m,
2269 __unused char *frame_header)
2270 {
2271 if (proto_input(protocol, m) != 0) {
2272 m_freem(m);
2273 #if !UTUN_NEXUS
2274 ifnet_stat_increment_in(interface, 0, 0, 1);
2275 } else {
2276 ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0);
2277 #endif // UTUN_NEXUS
2278 }
2279
2280 return 0;
2281 }
2282
2283 static errno_t
2284 utun_proto_pre_output(__unused ifnet_t interface,
2285 protocol_family_t protocol,
2286 __unused mbuf_t *packet,
2287 __unused const struct sockaddr *dest,
2288 __unused void *route,
2289 char *frame_type,
2290 __unused char *link_layer_dest)
2291 {
2292 *(protocol_family_t *)(void *)frame_type = protocol;
2293 return 0;
2294 }
2295
2296 static errno_t
2297 utun_attach_proto(ifnet_t interface,
2298 protocol_family_t protocol)
2299 {
2300 struct ifnet_attach_proto_param proto;
2301
2302 bzero(&proto, sizeof(proto));
2303 proto.input = utun_proto_input;
2304 proto.pre_output = utun_proto_pre_output;
2305
2306 errno_t result = ifnet_attach_protocol(interface, protocol, &proto);
2307 if (result != 0 && result != EEXIST) {
2308 printf("utun_attach_inet - ifnet_attach_protocol %d failed: %d\n",
2309 protocol, result);
2310 }
2311
2312 return result;
2313 }
2314
2315 #if UTUN_NEXUS
2316 static errno_t
2317 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
2318 {
2319 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2320
2321 lck_mtx_lock(&pcb->utun_input_chain_lock);
2322 if (pcb->utun_input_chain != NULL) {
2323 pcb->utun_input_chain_last->m_nextpkt = packet;
2324 } else {
2325 pcb->utun_input_chain = packet;
2326 }
2327 while (packet->m_nextpkt) {
2328 VERIFY(packet != packet->m_nextpkt);
2329 packet = packet->m_nextpkt;
2330 }
2331 pcb->utun_input_chain_last = packet;
2332 lck_mtx_unlock(&pcb->utun_input_chain_lock);
2333
2334 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2335 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2336
2337 if (rx_ring != NULL) {
2338 kern_channel_notify(rx_ring, 0);
2339 }
2340
2341 return (0);
2342 }
2343 #else
2344 static errno_t
2345 utun_pkt_input (struct utun_pcb *pcb, mbuf_t m)
2346 {
2347 errno_t result;
2348 protocol_family_t protocol = 0;
2349
2350 mbuf_pkthdr_setrcvif(m, pcb->utun_ifp);
2351
2352 if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
2353 protocol = *(u_int32_t *)mbuf_data(m);
2354
2355 bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0);
2356 }
2357 if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
2358 /* flush data */
2359 mbuf_freem(m);
2360 return 0;
2361 }
2362
2363 if (!pcb->utun_ext_ifdata_stats) {
2364 struct ifnet_stat_increment_param incs;
2365
2366 bzero(&incs, sizeof(incs));
2367 incs.packets_in = 1;
2368 incs.bytes_in = mbuf_pkthdr_len(m);
2369 result = ifnet_input(pcb->utun_ifp, m, &incs);
2370 } else {
2371 result = ifnet_input(pcb->utun_ifp, m, NULL);
2372 }
2373 if (result != 0) {
2374 ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
2375
2376 printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
2377 mbuf_freem(m);
2378 }
2379
2380 return 0;
2381 }
2382 #endif // UTUN_NEXUS
2383
2384
2385 #if UTUN_NEXUS
2386
2387 static errno_t
2388 utun_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
2389 {
2390 return 0;
2391 }
2392
2393 static void
2394 utun_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
2395 {
2396 // Ignore
2397 }
2398
2399 static errno_t
2400 utun_register_nexus(void)
2401 {
2402 const struct kern_nexus_domain_provider_init dp_init = {
2403 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2404 .nxdpi_flags = 0,
2405 .nxdpi_init = utun_nxdp_init,
2406 .nxdpi_fini = utun_nxdp_fini
2407 };
2408 errno_t err = 0;
2409
2410 /* utun_nxdp_init() is called before this function returns */
2411 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
2412 (const uint8_t *) "com.apple.utun",
2413 &dp_init, sizeof(dp_init),
2414 &utun_nx_dom_prov);
2415 if (err != 0) {
2416 printf("%s: failed to register domain provider\n", __func__);
2417 return (err);
2418 }
2419 return (0);
2420 }
2421
2422 static errno_t
2423 utun_ifnet_set_attrs(ifnet_t ifp)
2424 {
2425 /* Set flags and additional information. */
2426 ifnet_set_mtu(ifp, 1500);
2427 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
2428
2429 /* The interface must generate its own IPv6 LinkLocal address,
2430 * if possible following the recommendation of RFC2472 to the 64bit interface ID
2431 */
2432 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
2433
2434 return (0);
2435 }
2436
2437 static errno_t
2438 utun_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
2439 {
2440 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2441 pcb->utun_netif_nexus = nexus;
2442 return (utun_ifnet_set_attrs(ifp));
2443 }
2444
2445 static errno_t
2446 utun_nexus_pre_connect(kern_nexus_provider_t nxprov,
2447 proc_t p, kern_nexus_t nexus,
2448 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
2449 {
2450 #pragma unused(nxprov, p)
2451 #pragma unused(nexus, nexus_port, channel, ch_ctx)
2452 return (0);
2453 }
2454
2455 static errno_t
2456 utun_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2457 kern_channel_t channel)
2458 {
2459 #pragma unused(nxprov, channel)
2460 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2461 boolean_t ok = ifnet_is_attached(pcb->utun_ifp, 1);
2462 return (ok ? 0 : ENXIO);
2463 }
2464
2465 static void
2466 utun_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2467 kern_channel_t channel)
2468 {
2469 #pragma unused(nxprov, nexus, channel)
2470 }
2471
2472 static void
2473 utun_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2474 kern_channel_t channel)
2475 {
2476 #pragma unused(nxprov, nexus, channel)
2477 }
2478
2479 static void
2480 utun_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2481 kern_channel_t channel)
2482 {
2483 #pragma unused(nxprov, channel)
2484 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2485 if (pcb->utun_netif_nexus == nexus) {
2486 pcb->utun_netif_nexus = NULL;
2487 }
2488 ifnet_decr_iorefcnt(pcb->utun_ifp);
2489 }
2490
2491 static errno_t
2492 utun_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2493 kern_channel_t channel, kern_channel_ring_t ring,
2494 boolean_t is_tx_ring, void **ring_ctx)
2495 {
2496 #pragma unused(nxprov)
2497 #pragma unused(channel)
2498 #pragma unused(ring_ctx)
2499 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2500 if (!is_tx_ring) {
2501 VERIFY(pcb->utun_kpipe_rxring == NULL);
2502 pcb->utun_kpipe_rxring = ring;
2503 } else {
2504 VERIFY(pcb->utun_kpipe_txring == NULL);
2505 pcb->utun_kpipe_txring = ring;
2506 }
2507 return 0;
2508 }
2509
2510 static void
2511 utun_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2512 kern_channel_ring_t ring)
2513 {
2514 #pragma unused(nxprov)
2515 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2516 if (pcb->utun_kpipe_rxring == ring) {
2517 pcb->utun_kpipe_rxring = NULL;
2518 } else if (pcb->utun_kpipe_txring == ring) {
2519 pcb->utun_kpipe_txring = NULL;
2520 }
2521 }
2522
2523 static errno_t
2524 utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2525 kern_channel_ring_t tx_ring, uint32_t flags)
2526 {
2527 #pragma unused(nxprov)
2528 #pragma unused(flags)
2529 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2530
2531 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2532 int channel_enabled = pcb->utun_kpipe_enabled;
2533 if (!channel_enabled) {
2534 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2535 return 0;
2536 }
2537
2538 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2539 if (tx_slot == NULL) {
2540 // Nothing to write, bail
2541 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2542 return 0;
2543 }
2544
2545 // Signal the netif ring to read
2546 kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
2547 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2548 if (rx_ring != NULL) {
2549 kern_channel_notify(rx_ring, 0);
2550 }
2551
2552 return 0;
2553 }
2554
2555 static errno_t
2556 utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
2557 kern_channel_ring_t rx_ring, uint32_t flags)
2558 {
2559 #pragma unused(nxprov)
2560 #pragma unused(flags)
2561 struct utun_pcb *pcb = kern_nexus_get_context(nexus);
2562 struct kern_channel_ring_stat_increment rx_ring_stats;
2563
2564 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2565
2566 int channel_enabled = pcb->utun_kpipe_enabled;
2567 if (!channel_enabled) {
2568 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2569 return 0;
2570 }
2571
2572 /* reclaim user-released slots */
2573 (void) kern_channel_reclaim(rx_ring);
2574
2575 uint32_t avail = kern_channel_available_slot_count(rx_ring);
2576 if (avail == 0) {
2577 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2578 return 0;
2579 }
2580
2581 kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
2582 if (tx_ring == NULL ||
2583 pcb->utun_netif_nexus == NULL) {
2584 // Net-If TX ring not set up yet, nothing to read
2585 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2586 return 0;
2587 }
2588
2589 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
2590
2591 // Unlock utun before entering ring
2592 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2593
2594 (void)kr_enter(tx_ring, TRUE);
2595
2596 // Lock again after entering and validate
2597 lck_rw_lock_shared(&pcb->utun_pcb_lock);
2598 if (tx_ring != pcb->utun_netif_txring) {
2599 // Ring no longer valid
2600 // Unlock first, then exit ring
2601 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2602 kr_exit(tx_ring);
2603 return 0;
2604 }
2605
2606 struct kern_channel_ring_stat_increment tx_ring_stats;
2607 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
2608 kern_channel_slot_t tx_pslot = NULL;
2609 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
2610 if (tx_slot == NULL) {
2611 // Nothing to read, don't bother signalling
2612 // Unlock first, then exit ring
2613 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2614 kr_exit(tx_ring);
2615 return 0;
2616 }
2617
2618 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
2619 VERIFY(rx_pp != NULL);
2620 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
2621 kern_channel_slot_t rx_pslot = NULL;
2622 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
2623
2624 while (rx_slot != NULL && tx_slot != NULL) {
2625 size_t length;
2626 kern_buflet_t rx_buf;
2627 void *rx_baddr;
2628
2629 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
2630
2631 // Advance TX ring
2632 tx_pslot = tx_slot;
2633 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
2634
2635 /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
2636 if (tx_ph == 0) {
2637 continue;
2638 }
2639
2640 // Allocate rx packet
2641 kern_packet_t rx_ph = 0;
2642 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
2643 if (unlikely(error != 0)) {
2644 printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
2645 pcb->utun_ifp->if_xname);
2646 break;
2647 }
2648
2649 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
2650 VERIFY(tx_buf != NULL);
2651 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
2652 VERIFY(tx_baddr != NULL);
2653 tx_baddr += kern_buflet_get_data_offset(tx_buf);
2654
2655 bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
2656
2657 length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
2658 UTUN_IF_DEFAULT_SLOT_SIZE);
2659
2660 tx_ring_stats.kcrsi_slots_transferred++;
2661 tx_ring_stats.kcrsi_bytes_transferred += length;
2662
2663 if (length < UTUN_HEADER_SIZE(pcb) ||
2664 length > UTUN_IF_DEFAULT_SLOT_SIZE ||
2665 length > rx_pp->pp_buflet_size ||
2666 (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
2667 /* flush data */
2668 kern_pbufpool_free(rx_pp, rx_ph);
2669 printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
2670 pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
2671 STATS_INC(nifs, NETIF_STATS_BADLEN);
2672 STATS_INC(nifs, NETIF_STATS_DROPPED);
2673 continue;
2674 }
2675
2676 /* fillout packet */
2677 rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
2678 VERIFY(rx_buf != NULL);
2679 rx_baddr = kern_buflet_get_object_address(rx_buf);
2680 VERIFY(rx_baddr != NULL);
2681
2682 // Find family
2683 uint32_t af = 0;
2684 uint8_t vhl = *(uint8_t *)(tx_baddr);
2685 u_int ip_version = (vhl >> 4);
2686 switch (ip_version) {
2687 case 4: {
2688 af = AF_INET;
2689 break;
2690 }
2691 case 6: {
2692 af = AF_INET6;
2693 break;
2694 }
2695 default: {
2696 printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
2697 pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
2698 break;
2699 }
2700 }
2701
2702 // Copy header
2703 af = htonl(af);
2704 memcpy((void *)rx_baddr, &af, sizeof(af));
2705 if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
2706 kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
2707 }
2708
2709 // Copy data from tx to rx
2710 memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
2711 kern_packet_clear_flow_uuid(rx_ph); // zero flow id
2712
2713 /* finalize and attach the packet */
2714 error = kern_buflet_set_data_offset(rx_buf, 0);
2715 VERIFY(error == 0);
2716 error = kern_buflet_set_data_length(rx_buf, length);
2717 VERIFY(error == 0);
2718 error = kern_packet_finalize(rx_ph);
2719 VERIFY(error == 0);
2720 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
2721 VERIFY(error == 0);
2722
2723 STATS_INC(nifs, NETIF_STATS_TXPKTS);
2724 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
2725
2726 rx_ring_stats.kcrsi_slots_transferred++;
2727 rx_ring_stats.kcrsi_bytes_transferred += length;
2728
2729 rx_pslot = rx_slot;
2730 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
2731 }
2732
2733 if (rx_pslot) {
2734 kern_channel_advance_slot(rx_ring, rx_pslot);
2735 kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
2736 }
2737
2738 if (tx_pslot) {
2739 kern_channel_advance_slot(tx_ring, tx_pslot);
2740 kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
2741 (void)kern_channel_reclaim(tx_ring);
2742 }
2743
2744 if (pcb->utun_output_disabled) {
2745 errno_t error = ifnet_enable_output(pcb->utun_ifp);
2746 if (error != 0) {
2747 printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
2748 } else {
2749 pcb->utun_output_disabled = false;
2750 }
2751 }
2752
2753 // Unlock first, then exit ring
2754 lck_rw_unlock_shared(&pcb->utun_pcb_lock);
2755
2756 if (tx_pslot != NULL) {
2757 kern_channel_notify(tx_ring, 0);
2758 }
2759 kr_exit(tx_ring);
2760
2761 return 0;
2762 }
2763
2764 #endif // UTUN_NEXUS
2765
2766
2767 /*
2768 * These are place holders until coreTLS kext stops calling them
2769 */
2770 errno_t utun_ctl_register_dtls (void *reg);
2771 int utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family);
2772 void utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb);
2773
2774 errno_t
2775 utun_ctl_register_dtls (void *reg)
2776 {
2777 #pragma unused(reg)
2778 return 0;
2779 }
2780
2781 int
2782 utun_pkt_dtls_input(struct utun_pcb *pcb, mbuf_t *pkt, protocol_family_t family)
2783 {
2784 #pragma unused(pcb)
2785 #pragma unused(pkt)
2786 #pragma unused(family)
2787 return 0;
2788 }
2789
2790 void
2791 utun_ctl_disable_crypto_dtls(struct utun_pcb *pcb)
2792 {
2793 #pragma unused(pcb)
2794 }
mirror of XNU