]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_ipsec.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.51.1.tar.gz
[apple/xnu.git] / bsd / net / if_ipsec.c
CommitLineData
39236c6e 1/*
a39ff7e2 2 * Copyright (c) 2012-2018 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/systm.h>
30#include <sys/kern_control.h>
31#include <net/kpi_protocol.h>
32#include <net/kpi_interface.h>
33#include <sys/socket.h>
fe8ab488 34#include <sys/socketvar.h>
39236c6e
A		 35#include <net/if.h>
36#include <net/if_types.h>
37#include <net/bpf.h>
38#include <net/if_ipsec.h>
39236c6e
A		 39#include <sys/mbuf.h>
40#include <sys/sockio.h>
41#include <netinet/in.h>
42#include <netinet/ip6.h>
43#include <netinet6/in6_var.h>
44#include <netinet6/ip6_var.h>
45#include <sys/kauth.h>
46#include <netinet6/ipsec.h>
47#include <netinet6/ipsec6.h>
5ba3f43e
A		 48#include <netinet6/esp.h>
49#include <netinet6/esp6.h>
39236c6e
A		 50#include <netinet/ip.h>
51#include <net/flowadv.h>
fe8ab488 52#include <net/necp.h>
39037602
A		 53#include <netkey/key.h>
54#include <net/pktap.h>
5ba3f43e
A		 55#include <kern/zalloc.h>
56
57#define IPSEC_NEXUS 0
39037602
A		 58
59extern int net_qos_policy_restricted;
60extern int net_qos_policy_restrict_avapps;
39236c6e
A		 61
62/* Kernel Control functions */
5c9f4661
A		 63static errno_t ipsec_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
64 void **unitinfo);
39236c6e
A		 65static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
66 void **unitinfo);
67static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
68 void *unitinfo);
69static errno_t ipsec_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
70 void *unitinfo, mbuf_t m, int flags);
71static errno_t ipsec_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
72 int opt, void *data, size_t *len);
73static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
74 int opt, void *data, size_t len);
75
76/* Network Interface functions */
77static void ipsec_start(ifnet_t interface);
78static errno_t ipsec_output(ifnet_t interface, mbuf_t data);
79static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header,
80 protocol_family_t *protocol);
81static errno_t ipsec_add_proto(ifnet_t interface, protocol_family_t protocol,
82 const struct ifnet_demux_desc *demux_array,
83 u_int32_t demux_count);
84static errno_t ipsec_del_proto(ifnet_t interface, protocol_family_t protocol);
85static errno_t ipsec_ioctl(ifnet_t interface, u_long cmd, void *data);
86static void ipsec_detached(ifnet_t interface);
87
88/* Protocol handlers */
89static errno_t ipsec_attach_proto(ifnet_t interface, protocol_family_t proto);
90static errno_t ipsec_proto_input(ifnet_t interface, protocol_family_t protocol,
91 mbuf_t m, char *frame_header);
92static errno_t ipsec_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
93 mbuf_t *packet, const struct sockaddr *dest, void *route,
94 char *frame_type, char *link_layer_dest);
95
5ba3f43e
A		 96static kern_ctl_ref ipsec_kctlref;
97static u_int32_t ipsec_family;
98static lck_attr_t *ipsec_lck_attr;
99static lck_grp_attr_t *ipsec_lck_grp_attr;
100static lck_grp_t *ipsec_lck_grp;
101static lck_mtx_t ipsec_lock;
102
103#if IPSEC_NEXUS
104
105SYSCTL_DECL(_net_ipsec);
106SYSCTL_NODE(_net, OID_AUTO, ipsec, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPsec");
107static int if_ipsec_verify_interface_creation = 0;
108SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG_LOCKED, &if_ipsec_verify_interface_creation, 0, "");
109
a39ff7e2 110#define IPSEC_IF_VERIFY(_e) if (__improbable(if_ipsec_verify_interface_creation)) { VERIFY(_e); }
5ba3f43e 111
5c9f4661 112#define IPSEC_IF_DEFAULT_SLOT_SIZE 2048
5ba3f43e
A		 113#define IPSEC_IF_DEFAULT_RING_SIZE 64
114#define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64
115#define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128
a39ff7e2 116#define IPSEC_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
5ba3f43e
A		 117
118#define IPSEC_IF_MIN_RING_SIZE 16
119#define IPSEC_IF_MAX_RING_SIZE 1024
120
5c9f4661
A		 121#define IPSEC_IF_MIN_SLOT_SIZE 1024
122#define IPSEC_IF_MAX_SLOT_SIZE 4096
123
5ba3f43e
A		 124static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS;
125static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
126static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
127
128static int if_ipsec_ring_size = IPSEC_IF_DEFAULT_RING_SIZE;
129static int if_ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE;
130static int if_ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE;
131
132SYSCTL_PROC(_net_ipsec, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
133 &if_ipsec_ring_size, IPSEC_IF_DEFAULT_RING_SIZE, &sysctl_if_ipsec_ring_size, "I", "");
134SYSCTL_PROC(_net_ipsec, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
135 &if_ipsec_tx_fsw_ring_size, IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_ipsec_tx_fsw_ring_size, "I", "");
136SYSCTL_PROC(_net_ipsec, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
137 &if_ipsec_rx_fsw_ring_size, IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_ipsec_rx_fsw_ring_size, "I", "");
138
139static errno_t
140ipsec_register_nexus(void);
141
142typedef struct ipsec_nx {
143 uuid_t if_provider;
144 uuid_t if_instance;
145 uuid_t ms_provider;
146 uuid_t ms_instance;
147 uuid_t ms_device;
148 uuid_t ms_host;
149 uuid_t ms_agent;
150} *ipsec_nx_t;
151
152static nexus_controller_t ipsec_ncd;
153static int ipsec_ncd_refcount;
154static uuid_t ipsec_kpipe_uuid;
155
156#endif // IPSEC_NEXUS
157
158/* Control block allocated for each kernel control connection */
159struct ipsec_pcb {
160 TAILQ_ENTRY(ipsec_pcb) ipsec_chain;
161 kern_ctl_ref ipsec_ctlref;
162 ifnet_t ipsec_ifp;
163 u_int32_t ipsec_unit;
164 u_int32_t ipsec_unique_id;
165 u_int32_t ipsec_flags;
166 u_int32_t ipsec_input_frag_size;
167 bool ipsec_frag_size_set;
168 int ipsec_ext_ifdata_stats;
169 mbuf_svc_class_t ipsec_output_service_class;
170 char ipsec_if_xname[IFXNAMSIZ];
171 char ipsec_unique_name[IFXNAMSIZ];
172 // PCB lock protects state fields, like ipsec_kpipe_enabled
173 decl_lck_rw_data(, ipsec_pcb_lock);
5ba3f43e
A		 174
175#if IPSEC_NEXUS
176 lck_mtx_t ipsec_input_chain_lock;
177 struct mbuf * ipsec_input_chain;
178 struct mbuf * ipsec_input_chain_last;
179 // Input chain lock protects the list of input mbufs
180 // The input chain lock must be taken AFTER the PCB lock if both are held
181 struct ipsec_nx ipsec_nx;
182 int ipsec_kpipe_enabled;
183 uuid_t ipsec_kpipe_uuid;
184 void * ipsec_kpipe_rxring;
185 void * ipsec_kpipe_txring;
a39ff7e2 186 kern_pbufpool_t ipsec_kpipe_pp;
5ba3f43e
A		 187
188 kern_nexus_t ipsec_netif_nexus;
a39ff7e2 189 kern_pbufpool_t ipsec_netif_pp;
5ba3f43e
A		 190 void * ipsec_netif_rxring;
191 void * ipsec_netif_txring;
192 uint64_t ipsec_netif_txring_size;
5c9f4661
A		 193
194 u_int32_t ipsec_slot_size;
195 u_int32_t ipsec_netif_ring_size;
196 u_int32_t ipsec_tx_fsw_ring_size;
197 u_int32_t ipsec_rx_fsw_ring_size;
198 bool ipsec_use_netif;
199
5ba3f43e
A		 200#endif // IPSEC_NEXUS
201};
202
203TAILQ_HEAD(ipsec_list, ipsec_pcb) ipsec_head;
204
205#define IPSEC_PCB_ZONE_MAX 32
206#define IPSEC_PCB_ZONE_NAME "net.if_ipsec"
207
208static unsigned int ipsec_pcb_size; /* size of zone element */
209static struct zone *ipsec_pcb_zone; /* zone for ipsec_pcb */
210
211#define IPSECQ_MAXLEN 256
212
213#if IPSEC_NEXUS
214static int
215sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS
216{
217#pragma unused(arg1, arg2)
218 int value = if_ipsec_ring_size;
219
220 int error = sysctl_handle_int(oidp, &value, 0, req);
221 if (error || !req->newptr) {
222 return (error);
223 }
224
225 if (value < IPSEC_IF_MIN_RING_SIZE ||
226 value > IPSEC_IF_MAX_RING_SIZE) {
227 return (EINVAL);
228 }
229
230 if_ipsec_ring_size = value;
231
232 return (0);
233}
234
235static int
236sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
237{
238#pragma unused(arg1, arg2)
239 int value = if_ipsec_tx_fsw_ring_size;
240
241 int error = sysctl_handle_int(oidp, &value, 0, req);
242 if (error || !req->newptr) {
243 return (error);
244 }
245
246 if (value < IPSEC_IF_MIN_RING_SIZE ||
247 value > IPSEC_IF_MAX_RING_SIZE) {
248 return (EINVAL);
249 }
250
251 if_ipsec_tx_fsw_ring_size = value;
252
253 return (0);
254}
255
256static int
257sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
258{
259#pragma unused(arg1, arg2)
260 int value = if_ipsec_rx_fsw_ring_size;
261
262 int error = sysctl_handle_int(oidp, &value, 0, req);
263 if (error || !req->newptr) {
264 return (error);
265 }
266
267 if (value < IPSEC_IF_MIN_RING_SIZE ||
268 value > IPSEC_IF_MAX_RING_SIZE) {
269 return (EINVAL);
270 }
271
272 if_ipsec_rx_fsw_ring_size = value;
273
274 return (0);
275}
276#endif // IPSEC_NEXUS
277
278errno_t
279ipsec_register_control(void)
280{
281 struct kern_ctl_reg kern_ctl;
282 errno_t result = 0;
283
284 /* Find a unique value for our interface family */
285 result = mbuf_tag_id_find(IPSEC_CONTROL_NAME, &ipsec_family);
286 if (result != 0) {
287 printf("ipsec_register_control - mbuf_tag_id_find_internal failed: %d\n", result);
288 return result;
289 }
290
291 ipsec_pcb_size = sizeof(struct ipsec_pcb);
292 ipsec_pcb_zone = zinit(ipsec_pcb_size,
293 IPSEC_PCB_ZONE_MAX * ipsec_pcb_size,
294 0, IPSEC_PCB_ZONE_NAME);
295 if (ipsec_pcb_zone == NULL) {
296 printf("ipsec_register_control - zinit(ipsec_pcb) failed");
297 return ENOMEM;
298 }
299
300#if IPSEC_NEXUS
301 ipsec_register_nexus();
302#endif // IPSEC_NEXUS
303
304 TAILQ_INIT(&ipsec_head);
305
306 bzero(&kern_ctl, sizeof(kern_ctl));
307 strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
308 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
309 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */
310 kern_ctl.ctl_sendsize = 64 * 1024;
311 kern_ctl.ctl_recvsize = 64 * 1024;
5c9f4661 312 kern_ctl.ctl_bind = ipsec_ctl_bind;
5ba3f43e
A		 313 kern_ctl.ctl_connect = ipsec_ctl_connect;
314 kern_ctl.ctl_disconnect = ipsec_ctl_disconnect;
315 kern_ctl.ctl_send = ipsec_ctl_send;
316 kern_ctl.ctl_setopt = ipsec_ctl_setopt;
317 kern_ctl.ctl_getopt = ipsec_ctl_getopt;
318
319 result = ctl_register(&kern_ctl, &ipsec_kctlref);
320 if (result != 0) {
321 printf("ipsec_register_control - ctl_register failed: %d\n", result);
322 return result;
323 }
324
325 /* Register the protocol plumbers */
326 if ((result = proto_register_plumber(PF_INET, ipsec_family,
327 ipsec_attach_proto, NULL)) != 0) {
328 printf("ipsec_register_control - proto_register_plumber(PF_INET, %d) failed: %d\n",
329 ipsec_family, result);
330 ctl_deregister(ipsec_kctlref);
331 return result;
332 }
333
334 /* Register the protocol plumbers */
335 if ((result = proto_register_plumber(PF_INET6, ipsec_family,
336 ipsec_attach_proto, NULL)) != 0) {
337 proto_unregister_plumber(PF_INET, ipsec_family);
338 ctl_deregister(ipsec_kctlref);
339 printf("ipsec_register_control - proto_register_plumber(PF_INET6, %d) failed: %d\n",
340 ipsec_family, result);
341 return result;
342 }
343
344 ipsec_lck_attr = lck_attr_alloc_init();
345 ipsec_lck_grp_attr = lck_grp_attr_alloc_init();
346 ipsec_lck_grp = lck_grp_alloc_init("ipsec", ipsec_lck_grp_attr);
347 lck_mtx_init(&ipsec_lock, ipsec_lck_grp, ipsec_lck_attr);
348
349 return 0;
350}
351
352/* Helpers */
353int
354ipsec_interface_isvalid (ifnet_t interface)
355{
356 struct ipsec_pcb *pcb = NULL;
357
358 if (interface == NULL)
359 return 0;
360
361 pcb = ifnet_softc(interface);
362
363 if (pcb == NULL)
364 return 0;
365
366 /* When ctl disconnects, ipsec_unit is set to 0 */
367 if (pcb->ipsec_unit == 0)
368 return 0;
369
370 return 1;
371}
372
373static errno_t
374ipsec_ifnet_set_attrs(ifnet_t ifp)
375{
376 /* Set flags and additional information. */
377 ifnet_set_mtu(ifp, 1500);
378 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
379
380 /* The interface must generate its own IPv6 LinkLocal address,
381 * if possible following the recommendation of RFC2472 to the 64bit interface ID
382 */
383 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
384
385#if !IPSEC_NEXUS
386 /* Reset the stats in case as the interface may have been recycled */
387 struct ifnet_stats_param stats;
388 bzero(&stats, sizeof(struct ifnet_stats_param));
389 ifnet_set_stat(ifp, &stats);
390#endif // !IPSEC_NEXUS
391
392 return (0);
393}
394
395#if IPSEC_NEXUS
396
397static uuid_t ipsec_nx_dom_prov;
398
399static errno_t
400ipsec_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
401{
402 return 0;
403}
404
405static void
406ipsec_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
407{
408 // Ignore
409}
410
411static errno_t
412ipsec_register_nexus(void)
413{
414 const struct kern_nexus_domain_provider_init dp_init = {
415 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
416 .nxdpi_flags = 0,
417 .nxdpi_init = ipsec_nxdp_init,
418 .nxdpi_fini = ipsec_nxdp_fini
419 };
420 errno_t err = 0;
421
422 /* ipsec_nxdp_init() is called before this function returns */
423 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
424 (const uint8_t *) "com.apple.ipsec",
425 &dp_init, sizeof(dp_init),
426 &ipsec_nx_dom_prov);
427 if (err != 0) {
428 printf("%s: failed to register domain provider\n", __func__);
429 return (err);
430 }
431 return (0);
432}
433
434static errno_t
435ipsec_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
436{
437 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
438 pcb->ipsec_netif_nexus = nexus;
439 return (ipsec_ifnet_set_attrs(ifp));
440}
441
442static errno_t
443ipsec_nexus_pre_connect(kern_nexus_provider_t nxprov,
444 proc_t p, kern_nexus_t nexus,
445 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
446{
447#pragma unused(nxprov, p)
448#pragma unused(nexus, nexus_port, channel, ch_ctx)
449 return (0);
450}
451
452static errno_t
453ipsec_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
454 kern_channel_t channel)
455{
456#pragma unused(nxprov, channel)
457 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
458 boolean_t ok = ifnet_is_attached(pcb->ipsec_ifp, 1);
459 return (ok ? 0 : ENXIO);
460}
461
462static void
463ipsec_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
464 kern_channel_t channel)
465{
466#pragma unused(nxprov, nexus, channel)
467}
468
469static void
470ipsec_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
471 kern_channel_t channel)
472{
473#pragma unused(nxprov, nexus, channel)
474}
475
476static void
477ipsec_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
478 kern_channel_t channel)
479{
480#pragma unused(nxprov, channel)
481 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
482 if (pcb->ipsec_netif_nexus == nexus) {
483 pcb->ipsec_netif_nexus = NULL;
484 }
485 ifnet_decr_iorefcnt(pcb->ipsec_ifp);
486}
487
488static errno_t
489ipsec_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
490 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
491 void **ring_ctx)
492{
493#pragma unused(nxprov)
494#pragma unused(channel)
495#pragma unused(ring_ctx)
496 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
497 if (!is_tx_ring) {
498 VERIFY(pcb->ipsec_kpipe_rxring == NULL);
499 pcb->ipsec_kpipe_rxring = ring;
500 } else {
501 VERIFY(pcb->ipsec_kpipe_txring == NULL);
502 pcb->ipsec_kpipe_txring = ring;
503 }
504 return 0;
505}
506
507static void
508ipsec_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
509 kern_channel_ring_t ring)
510{
511#pragma unused(nxprov)
512 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
513 if (pcb->ipsec_kpipe_rxring == ring) {
514 pcb->ipsec_kpipe_rxring = NULL;
515 } else if (pcb->ipsec_kpipe_txring == ring) {
516 pcb->ipsec_kpipe_txring = NULL;
517 }
518}
519
520static errno_t
521ipsec_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
522 kern_channel_ring_t tx_ring, uint32_t flags)
523{
524#pragma unused(nxprov)
525#pragma unused(flags)
526 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
527
528 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
529 int channel_enabled = pcb->ipsec_kpipe_enabled;
530 if (!channel_enabled) {
531 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
532 return 0;
533 }
534
535 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
536 if (tx_slot == NULL) {
537 // Nothing to write, bail
538 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
539 return 0;
540 }
541
542 // Signal the netif ring to read
543 kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring;
544 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
545
546 if (rx_ring != NULL) {
547 kern_channel_notify(rx_ring, 0);
548 }
549 return 0;
550}
551
552static mbuf_t
553ipsec_encrypt_mbuf(ifnet_t interface,
554 mbuf_t data)
555{
556 struct ipsec_output_state ipsec_state;
557 int error = 0;
558 uint32_t af;
559
560 // Make sure this packet isn't looping through the interface
561 if (necp_get_last_interface_index_from_packet(data) == interface->if_index) {
562 error = -1;
563 goto ipsec_output_err;
564 }
565
566 // Mark the interface so NECP can evaluate tunnel policy
567 necp_mark_packet_from_interface(data, interface);
568
569 struct ip *ip = mtod(data, struct ip *);
570 u_int ip_version = ip->ip_v;
571
572 switch (ip_version) {
573 case 4: {
574 af = AF_INET;
575
576 memset(&ipsec_state, 0, sizeof(ipsec_state));
577 ipsec_state.m = data;
578 ipsec_state.dst = (struct sockaddr *)&ip->ip_dst;
579 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
580
581 error = ipsec4_interface_output(&ipsec_state, interface);
582 if (error == 0 && ipsec_state.tunneled == 6) {
583 // Tunneled in IPv6 - packet is gone
584 // TODO: Don't lose mbuf
585 data = NULL;
586 goto done;
587 }
588
589 data = ipsec_state.m;
590 if (error || data == NULL) {
591 if (error) {
592 printf("ipsec_encrypt_mbuf: ipsec4_output error %d\n", error);
593 }
594 goto ipsec_output_err;
595 }
596 goto done;
597 }
598 case 6: {
599 af = AF_INET6;
600
601 data = ipsec6_splithdr(data);
602 if (data == NULL) {
603 printf("ipsec_encrypt_mbuf: ipsec6_splithdr returned NULL\n");
604 goto ipsec_output_err;
605 }
606
607 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
608
609 memset(&ipsec_state, 0, sizeof(ipsec_state));
610 ipsec_state.m = data;
611 ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst;
612 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
613
614 error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m);
615 if (error == 0 && ipsec_state.tunneled == 4) {
616 // Tunneled in IPv4 - packet is gone
617 // TODO: Don't lose mbuf
618 data = NULL;
619 goto done;
620 }
621 data = ipsec_state.m;
622 if (error || data == NULL) {
623 if (error) {
624 printf("ipsec_encrypt_mbuf: ipsec6_output error %d\n", error);
625 }
626 goto ipsec_output_err;
627 }
628 goto done;
629 }
630 default: {
631 printf("ipsec_encrypt_mbuf: Received unknown packet version %d\n", ip_version);
632 error = -1;
633 goto ipsec_output_err;
634 }
635 }
636
637done:
638 return data;
639
640ipsec_output_err:
641 if (data) {
642 mbuf_freem(data);
643 }
644 return NULL;
645}
646
647static errno_t
648ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
649 kern_channel_ring_t rx_ring, uint32_t flags)
650{
651#pragma unused(nxprov)
652#pragma unused(flags)
653 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
654 struct kern_channel_ring_stat_increment rx_ring_stats;
655
656 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
657
658 int channel_enabled = pcb->ipsec_kpipe_enabled;
659 if (!channel_enabled) {
660 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
661 return 0;
662 }
663
664 // Reclaim user-released slots
665 (void) kern_channel_reclaim(rx_ring);
666
667 uint32_t avail = kern_channel_available_slot_count(rx_ring);
668 if (avail == 0) {
669 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
670 return 0;
671 }
672
673 kern_channel_ring_t tx_ring = pcb->ipsec_netif_txring;
674 if (tx_ring == NULL) {
675 // Net-If TX ring not set up yet, nothing to read
676 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
677 return 0;
678 }
679
680 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->ipsec_netif_nexus)->nif_stats;
681
682 // Unlock ipsec before entering ring
683 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
684
685 (void)kr_enter(tx_ring, TRUE);
686
687 // Lock again after entering and validate
688 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
689 if (tx_ring != pcb->ipsec_netif_txring) {
690 // Ring no longer valid
691 // Unlock first, then exit ring
692 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
693 kr_exit(tx_ring);
694 return 0;
695 }
696
697
698 struct kern_channel_ring_stat_increment tx_ring_stats;
699 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
700 kern_channel_slot_t tx_pslot = NULL;
701 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
702 if (tx_slot == NULL) {
703 // Nothing to read, don't bother signalling
704 // Unlock first, then exit ring
705 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
706 kr_exit(tx_ring);
707 return 0;
708 }
709
710 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
711 VERIFY(rx_pp != NULL);
712 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
713 kern_channel_slot_t rx_pslot = NULL;
714 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
715
716 while (rx_slot != NULL && tx_slot != NULL) {
717 size_t length = 0;
718 mbuf_t data = NULL;
719 errno_t error = 0;
720
721 // Allocate rx packet
722 kern_packet_t rx_ph = 0;
723 error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 724 if (__improbable(error != 0)) {
5ba3f43e
A		 725 printf("ipsec_kpipe_sync_rx %s: failed to allocate packet\n",
726 pcb->ipsec_ifp->if_xname);
727 break;
728 }
729
730 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
731
732 // Advance TX ring
733 tx_pslot = tx_slot;
734 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
735
736 if (tx_ph == 0) {
a39ff7e2 737 kern_pbufpool_free(rx_pp, rx_ph);
5ba3f43e
A		 738 continue;
739 }
740
741 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
742 VERIFY(tx_buf != NULL);
743 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
744 VERIFY(tx_baddr != NULL);
745 tx_baddr += kern_buflet_get_data_offset(tx_buf);
746
747 bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
748
749 length = MIN(kern_packet_get_data_length(tx_ph),
5c9f4661 750 pcb->ipsec_slot_size);
5ba3f43e
A		 751
752 // Increment TX stats
753 tx_ring_stats.kcrsi_slots_transferred++;
754 tx_ring_stats.kcrsi_bytes_transferred += length;
755
756 if (length > 0) {
757 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
758 if (error == 0) {
759 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
760 if (error == 0) {
761 // Encrypt and send packet
762 data = ipsec_encrypt_mbuf(pcb->ipsec_ifp, data);
763 } else {
764 printf("ipsec_kpipe_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
765 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
766 STATS_INC(nifs, NETIF_STATS_DROPPED);
767 mbuf_freem(data);
768 data = NULL;
769 }
770 } else {
771 printf("ipsec_kpipe_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
772 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
773 STATS_INC(nifs, NETIF_STATS_DROPPED);
774 }
775 } else {
776 printf("ipsec_kpipe_sync_rx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname);
777 STATS_INC(nifs, NETIF_STATS_BADLEN);
778 STATS_INC(nifs, NETIF_STATS_DROPPED);
779 }
780
781 if (data == NULL) {
782 printf("ipsec_kpipe_sync_rx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname);
783 kern_pbufpool_free(rx_pp, rx_ph);
784 break;
785 }
786
787 length = mbuf_pkthdr_len(data);
788 if (length > rx_pp->pp_buflet_size) {
789 // Flush data
790 mbuf_freem(data);
791 kern_pbufpool_free(rx_pp, rx_ph);
792 printf("ipsec_kpipe_sync_rx %s: encrypted packet length %zu > %u\n",
793 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
794 continue;
795 }
796
797 // Fillout rx packet
798 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
799 VERIFY(rx_buf != NULL);
800 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
801 VERIFY(rx_baddr != NULL);
802
803 // Copy-in data from mbuf to buflet
804 mbuf_copydata(data, 0, length, (void *)rx_baddr);
805 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
806
807 // Finalize and attach the packet
808 error = kern_buflet_set_data_offset(rx_buf, 0);
809 VERIFY(error == 0);
810 error = kern_buflet_set_data_length(rx_buf, length);
811 VERIFY(error == 0);
812 error = kern_packet_finalize(rx_ph);
813 VERIFY(error == 0);
814 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
815 VERIFY(error == 0);
816
817 STATS_INC(nifs, NETIF_STATS_TXPKTS);
818 STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
819
820 rx_ring_stats.kcrsi_slots_transferred++;
821 rx_ring_stats.kcrsi_bytes_transferred += length;
822
823 if (!pcb->ipsec_ext_ifdata_stats) {
824 ifnet_stat_increment_out(pcb->ipsec_ifp, 1, length, 0);
825 }
826
827 mbuf_freem(data);
828
829 rx_pslot = rx_slot;
830 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
831 }
832
833 if (rx_pslot) {
834 kern_channel_advance_slot(rx_ring, rx_pslot);
835 kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats);
836 }
837
838 if (tx_pslot) {
839 kern_channel_advance_slot(tx_ring, tx_pslot);
840 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
841 (void)kern_channel_reclaim(tx_ring);
842 }
843
5c9f4661
A		 844 /* always reenable output */
845 errno_t error = ifnet_enable_output(pcb->ipsec_ifp);
846 if (error != 0) {
847 printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
5ba3f43e
A		 848 }
849
850 // Unlock first, then exit ring
851 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
852
853 if (tx_pslot != NULL) {
854 kern_channel_notify(tx_ring, 0);
855 }
856 kr_exit(tx_ring);
857
858 return 0;
859}
860
861static errno_t
862ipsec_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
863 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
864 void **ring_ctx)
865{
866#pragma unused(nxprov)
867#pragma unused(channel)
868#pragma unused(ring_ctx)
869 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
870 if (!is_tx_ring) {
871 VERIFY(pcb->ipsec_netif_rxring == NULL);
872 pcb->ipsec_netif_rxring = ring;
873 } else {
874 VERIFY(pcb->ipsec_netif_txring == NULL);
875 pcb->ipsec_netif_txring = ring;
876 }
877 return 0;
878}
879
880static void
881ipsec_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
882 kern_channel_ring_t ring)
883{
884#pragma unused(nxprov)
885 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
886 if (pcb->ipsec_netif_rxring == ring) {
887 pcb->ipsec_netif_rxring = NULL;
888 } else if (pcb->ipsec_netif_txring == ring) {
889 pcb->ipsec_netif_txring = NULL;
890 }
891}
892
893static bool
894ipsec_netif_check_policy(mbuf_t data)
895{
896 necp_kernel_policy_result necp_result = 0;
897 necp_kernel_policy_result_parameter necp_result_parameter = {};
898 uint32_t necp_matched_policy_id = 0;
899
900 // This packet has been marked with IP level policy, do not mark again.
901 if (data && data->m_pkthdr.necp_mtag.necp_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) {
902 return (true);
903 }
904
905 size_t length = mbuf_pkthdr_len(data);
906 if (length < sizeof(struct ip)) {
907 return (false);
908 }
909
910 struct ip *ip = mtod(data, struct ip *);
911 u_int ip_version = ip->ip_v;
912 switch (ip_version) {
913 case 4: {
914 necp_matched_policy_id = necp_ip_output_find_policy_match(data, 0, NULL,
915 &necp_result, &necp_result_parameter);
916 break;
917 }
918 case 6: {
919 necp_matched_policy_id = necp_ip6_output_find_policy_match(data, 0, NULL,
920 &necp_result, &necp_result_parameter);
921 break;
922 }
923 default: {
924 return (false);
925 }
926 }
927
928 if (necp_result == NECP_KERNEL_POLICY_RESULT_DROP ||
929 necp_result == NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT) {
930 /* Drop and flow divert packets should be blocked at the IP layer */
931 return (false);
932 }
933
934 necp_mark_packet_from_ip(data, necp_matched_policy_id);
935 return (true);
936}
937
938static errno_t
939ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
940 kern_channel_ring_t tx_ring, uint32_t flags)
941{
942#pragma unused(nxprov)
943#pragma unused(flags)
944 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
945
946 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
947
948 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
949
950 struct kern_channel_ring_stat_increment tx_ring_stats;
951 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
952 kern_channel_slot_t tx_pslot = NULL;
953 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
954
955 STATS_INC(nifs, NETIF_STATS_TXSYNC);
956
957 if (tx_slot == NULL) {
958 // Nothing to write, don't bother signalling
959 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
960 return 0;
961 }
962
963 if (pcb->ipsec_kpipe_enabled) {
964 kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring;
965 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
966
967 // Signal the kernel pipe ring to read
968 if (rx_ring != NULL) {
969 kern_channel_notify(rx_ring, 0);
970 }
971 return 0;
972 }
973
974 // If we're here, we're injecting into the BSD stack
975 while (tx_slot != NULL) {
976 size_t length = 0;
977 mbuf_t data = NULL;
978
979 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
980
981 // Advance TX ring
982 tx_pslot = tx_slot;
983 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
984
985 if (tx_ph == 0) {
986 continue;
987 }
988
989 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
990 VERIFY(tx_buf != NULL);
991 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
992 VERIFY(tx_baddr != 0);
993 tx_baddr += kern_buflet_get_data_offset(tx_buf);
994
995 bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
996
997 length = MIN(kern_packet_get_data_length(tx_ph),
5c9f4661 998 pcb->ipsec_slot_size);
5ba3f43e
A		 999
1000 if (length > 0) {
1001 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
1002 if (error == 0) {
1003 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
1004 if (error == 0) {
1005 // Mark packet from policy
1006 uint32_t policy_id = kern_packet_get_policy_id(tx_ph);
1007 necp_mark_packet_from_ip(data, policy_id);
1008
1009 // Check policy with NECP
1010 if (!ipsec_netif_check_policy(data)) {
1011 printf("ipsec_netif_sync_tx %s - failed policy check\n", pcb->ipsec_ifp->if_xname);
1012 STATS_INC(nifs, NETIF_STATS_DROPPED);
1013 mbuf_freem(data);
1014 data = NULL;
1015 } else {
1016 // Send through encryption
1017 error = ipsec_output(pcb->ipsec_ifp, data);
1018 if (error != 0) {
1019 printf("ipsec_netif_sync_tx %s - ipsec_output error %d\n", pcb->ipsec_ifp->if_xname, error);
1020 }
1021 }
1022 } else {
1023 printf("ipsec_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
1024 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
1025 STATS_INC(nifs, NETIF_STATS_DROPPED);
1026 mbuf_freem(data);
1027 data = NULL;
1028 }
1029 } else {
1030 printf("ipsec_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
1031 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
1032 STATS_INC(nifs, NETIF_STATS_DROPPED);
1033 }
1034 } else {
1035 printf("ipsec_netif_sync_tx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname);
1036 STATS_INC(nifs, NETIF_STATS_BADLEN);
1037 STATS_INC(nifs, NETIF_STATS_DROPPED);
1038 }
1039
1040 if (data == NULL) {
1041 printf("ipsec_netif_sync_tx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname);
1042 break;
1043 }
1044
1045 STATS_INC(nifs, NETIF_STATS_TXPKTS);
1046 STATS_INC(nifs, NETIF_STATS_TXCOPY_MBUF);
1047
1048 tx_ring_stats.kcrsi_slots_transferred++;
1049 tx_ring_stats.kcrsi_bytes_transferred += length;
1050 }
1051
1052 if (tx_pslot) {
1053 kern_channel_advance_slot(tx_ring, tx_pslot);
1054 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
1055 (void)kern_channel_reclaim(tx_ring);
1056 }
1057
1058 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1059
1060 return 0;
1061}
1062
1063static errno_t
1064ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
1065 kern_channel_ring_t ring, __unused uint32_t flags)
1066{
1067#pragma unused(nxprov)
1068 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
5ba3f43e
A		 1069 boolean_t more = false;
1070 errno_t rc = 0;
5ba3f43e 1071
5c9f4661
A		 1072 /*
1073 * Refill and sync the ring; we may be racing against another thread doing
1074 * an RX sync that also wants to do kr_enter(), and so use the blocking
1075 * variant here.
1076 */
1077 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
1078 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
1079 printf("%s, tx refill failed %d\n", __func__, rc);
1080 }
1081
1082 (void) kr_enter(ring, TRUE);
1083 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
1084
1085 if (pcb->ipsec_kpipe_enabled) {
5ba3f43e
A		 1086 uint32_t tx_available = kern_channel_available_slot_count(ring);
1087 if (pcb->ipsec_netif_txring_size > 0 &&
1088 tx_available >= pcb->ipsec_netif_txring_size - 1) {
1089 // No room left in tx ring, disable output for now
1090 errno_t error = ifnet_disable_output(pcb->ipsec_ifp);
1091 if (error != 0) {
1092 printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
5ba3f43e
A		 1093 }
1094 }
1095 }
1096
5c9f4661 1097 if (pcb->ipsec_kpipe_enabled) {
5ba3f43e
A		 1098 kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring;
1099
1100 // Unlock while calling notify
1101 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1102 // Signal the kernel pipe ring to read
1103 if (rx_ring != NULL) {
1104 kern_channel_notify(rx_ring, 0);
1105 }
5ba3f43e
A		 1106 } else {
1107 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1108 }
1109
5c9f4661
A		 1110 kr_exit(ring);
1111
5ba3f43e
A		 1112 return (0);
1113}
1114
1115static errno_t
1116ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
1117 kern_channel_ring_t rx_ring, uint32_t flags)
1118{
1119#pragma unused(nxprov)
1120#pragma unused(flags)
1121 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
1122 struct kern_channel_ring_stat_increment rx_ring_stats;
1123
1124 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
1125
1126 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
1127
1128 // Reclaim user-released slots
1129 (void) kern_channel_reclaim(rx_ring);
1130
1131 STATS_INC(nifs, NETIF_STATS_RXSYNC);
1132
1133 uint32_t avail = kern_channel_available_slot_count(rx_ring);
1134 if (avail == 0) {
1135 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1136 return 0;
1137 }
1138
1139 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
1140 VERIFY(rx_pp != NULL);
1141 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
1142 kern_channel_slot_t rx_pslot = NULL;
1143 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
1144
1145 while (rx_slot != NULL) {
1146 // Check for a waiting packet
1147 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
1148 mbuf_t data = pcb->ipsec_input_chain;
1149 if (data == NULL) {
1150 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1151 break;
1152 }
1153
1154 // Allocate rx packet
1155 kern_packet_t rx_ph = 0;
1156 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 1157 if (__improbable(error != 0)) {
5ba3f43e
A		 1158 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
1159 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 1160 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1161 break;
1162 }
1163
1164 // Advance waiting packets
1165 pcb->ipsec_input_chain = data->m_nextpkt;
1166 data->m_nextpkt = NULL;
1167 if (pcb->ipsec_input_chain == NULL) {
1168 pcb->ipsec_input_chain_last = NULL;
1169 }
1170 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1171
1172 size_t length = mbuf_pkthdr_len(data);
1173
1174 if (length < sizeof(struct ip)) {
1175 // Flush data
1176 mbuf_freem(data);
1177 kern_pbufpool_free(rx_pp, rx_ph);
1178 STATS_INC(nifs, NETIF_STATS_BADLEN);
1179 STATS_INC(nifs, NETIF_STATS_DROPPED);
1180 printf("ipsec_netif_sync_rx %s: legacy decrypted packet length cannot hold IP %zu < %zu\n",
1181 pcb->ipsec_ifp->if_xname, length, sizeof(struct ip));
1182 continue;
1183 }
1184
1185 uint32_t af = 0;
1186 struct ip *ip = mtod(data, struct ip *);
1187 u_int ip_version = ip->ip_v;
1188 switch (ip_version) {
1189 case 4: {
1190 af = AF_INET;
1191 break;
1192 }
1193 case 6: {
1194 af = AF_INET6;
1195 break;
1196 }
1197 default: {
1198 printf("ipsec_netif_sync_rx %s: legacy unknown ip version %u\n",
1199 pcb->ipsec_ifp->if_xname, ip_version);
1200 break;
1201 }
1202 }
1203
1204 if (length > rx_pp->pp_buflet_size ||
1205 (pcb->ipsec_frag_size_set && length > pcb->ipsec_input_frag_size)) {
1206
1207 // We need to fragment to send up into the netif
1208
1209 u_int32_t fragment_mtu = rx_pp->pp_buflet_size;
1210 if (pcb->ipsec_frag_size_set &&
1211 pcb->ipsec_input_frag_size < rx_pp->pp_buflet_size) {
1212 fragment_mtu = pcb->ipsec_input_frag_size;
1213 }
1214
1215 mbuf_t fragment_chain = NULL;
1216 switch (af) {
1217 case AF_INET: {
1218 // ip_fragment expects the length in host order
1219 ip->ip_len = ntohs(ip->ip_len);
1220
1221 // ip_fragment will modify the original data, don't free
1222 int fragment_error = ip_fragment(data, pcb->ipsec_ifp, fragment_mtu, TRUE);
1223 if (fragment_error == 0 && data != NULL) {
1224 fragment_chain = data;
1225 } else {
1226 STATS_INC(nifs, NETIF_STATS_BADLEN);
1227 STATS_INC(nifs, NETIF_STATS_DROPPED);
1228 printf("ipsec_netif_sync_rx %s: failed to fragment IPv4 packet of length %zu (%d)\n",
1229 pcb->ipsec_ifp->if_xname, length, fragment_error);
1230 }
1231 break;
1232 }
1233 case AF_INET6: {
1234 if (length < sizeof(struct ip6_hdr)) {
1235 mbuf_freem(data);
1236 STATS_INC(nifs, NETIF_STATS_BADLEN);
1237 STATS_INC(nifs, NETIF_STATS_DROPPED);
1238 printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu < %zu\n",
1239 pcb->ipsec_ifp->if_xname, length, sizeof(struct ip6_hdr));
1240 } else {
1241
1242 // ip6_do_fragmentation will free the original data on success only
1243 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
1244 struct ip6_exthdrs exthdrs;
1245 memset(&exthdrs, 0, sizeof(exthdrs));
1246
1247 int fragment_error = ip6_do_fragmentation(&data, 0, pcb->ipsec_ifp, sizeof(struct ip6_hdr),
1248 ip6, &exthdrs, fragment_mtu, ip6->ip6_nxt);
1249 if (fragment_error == 0 && data != NULL) {
1250 fragment_chain = data;
1251 } else {
1252 mbuf_freem(data);
1253 STATS_INC(nifs, NETIF_STATS_BADLEN);
1254 STATS_INC(nifs, NETIF_STATS_DROPPED);
1255 printf("ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu (%d)\n",
1256 pcb->ipsec_ifp->if_xname, length, fragment_error);
1257 }
1258 }
1259 break;
1260 }
1261 default: {
1262 // Cannot fragment unknown families
1263 mbuf_freem(data);
1264 STATS_INC(nifs, NETIF_STATS_BADLEN);
1265 STATS_INC(nifs, NETIF_STATS_DROPPED);
1266 printf("ipsec_netif_sync_rx %s: uknown legacy decrypted packet length %zu > %u\n",
1267 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
1268 break;
1269 }
1270 }
1271
1272 if (fragment_chain != NULL) {
1273 // Add fragments to chain before continuing
1274 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
1275 if (pcb->ipsec_input_chain != NULL) {
1276 pcb->ipsec_input_chain_last->m_nextpkt = fragment_chain;
1277 } else {
1278 pcb->ipsec_input_chain = fragment_chain;
1279 }
1280 while (fragment_chain->m_nextpkt) {
1281 VERIFY(fragment_chain != fragment_chain->m_nextpkt);
1282 fragment_chain = fragment_chain->m_nextpkt;
1283 }
1284 pcb->ipsec_input_chain_last = fragment_chain;
1285 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1286 }
1287
1288 // Make sure to free unused rx packet
1289 kern_pbufpool_free(rx_pp, rx_ph);
1290
1291 continue;
1292 }
1293
1294 mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp);
1295
1296 // Fillout rx packet
1297 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
1298 VERIFY(rx_buf != NULL);
1299 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
1300 VERIFY(rx_baddr != NULL);
1301
1302 // Copy-in data from mbuf to buflet
1303 mbuf_copydata(data, 0, length, (void *)rx_baddr);
1304 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
1305
1306 // Finalize and attach the packet
1307 error = kern_buflet_set_data_offset(rx_buf, 0);
1308 VERIFY(error == 0);
1309 error = kern_buflet_set_data_length(rx_buf, length);
1310 VERIFY(error == 0);
1311 error = kern_packet_set_link_header_offset(rx_ph, 0);
1312 VERIFY(error == 0);
1313 error = kern_packet_set_network_header_offset(rx_ph, 0);
1314 VERIFY(error == 0);
1315 error = kern_packet_finalize(rx_ph);
1316 VERIFY(error == 0);
1317 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
1318 VERIFY(error == 0);
1319
1320 STATS_INC(nifs, NETIF_STATS_RXPKTS);
1321 STATS_INC(nifs, NETIF_STATS_RXCOPY_MBUF);
1322 bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0);
1323
1324 rx_ring_stats.kcrsi_slots_transferred++;
1325 rx_ring_stats.kcrsi_bytes_transferred += length;
1326
1327 if (!pcb->ipsec_ext_ifdata_stats) {
1328 ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0);
1329 }
1330
1331 mbuf_freem(data);
1332
1333 // Advance ring
1334 rx_pslot = rx_slot;
1335 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
1336 }
1337
1338 struct kern_channel_ring_stat_increment tx_ring_stats;
1339 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
1340 kern_channel_ring_t tx_ring = pcb->ipsec_kpipe_txring;
1341 kern_channel_slot_t tx_pslot = NULL;
1342 kern_channel_slot_t tx_slot = NULL;
1343 if (tx_ring == NULL) {
1344 // Net-If TX ring not set up yet, nothing to read
1345 goto done;
1346 }
1347
1348
1349 // Unlock ipsec before entering ring
1350 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1351
1352 (void)kr_enter(tx_ring, TRUE);
1353
1354 // Lock again after entering and validate
1355 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
1356
1357 if (tx_ring != pcb->ipsec_kpipe_txring) {
1358 goto done;
1359 }
1360
1361 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
1362 if (tx_slot == NULL) {
1363 // Nothing to read, don't bother signalling
1364 goto done;
1365 }
1366
1367 while (rx_slot != NULL && tx_slot != NULL) {
1368 size_t length = 0;
1369 mbuf_t data = NULL;
1370 errno_t error = 0;
1371 uint32_t af;
1372
1373 // Allocate rx packet
1374 kern_packet_t rx_ph = 0;
1375 error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 1376 if (__improbable(error != 0)) {
5ba3f43e
A		 1377 STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
1378 STATS_INC(nifs, NETIF_STATS_DROPPED);
5ba3f43e
A		 1379 break;
1380 }
1381
1382 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
1383
1384 // Advance TX ring
1385 tx_pslot = tx_slot;
1386 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
1387
1388 if (tx_ph == 0) {
a39ff7e2 1389 kern_pbufpool_free(rx_pp, rx_ph);
5ba3f43e
A		 1390 continue;
1391 }
1392
1393 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
1394 VERIFY(tx_buf != NULL);
1395 uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
1396 VERIFY(tx_baddr != 0);
1397 tx_baddr += kern_buflet_get_data_offset(tx_buf);
1398
1399 length = MIN(kern_packet_get_data_length(tx_ph),
5c9f4661 1400 pcb->ipsec_slot_size);
5ba3f43e
A		 1401
1402 // Increment TX stats
1403 tx_ring_stats.kcrsi_slots_transferred++;
1404 tx_ring_stats.kcrsi_bytes_transferred += length;
1405
1406 if (length >= sizeof(struct ip)) {
1407 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
1408 if (error == 0) {
1409 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
1410 if (error == 0) {
1411 struct ip *ip = mtod(data, struct ip *);
1412 u_int ip_version = ip->ip_v;
1413 switch (ip_version) {
1414 case 4: {
1415 af = AF_INET;
1416 ip->ip_len = ntohs(ip->ip_len) - sizeof(struct ip);
1417 ip->ip_off = ntohs(ip->ip_off);
1418
1419 if (length < ip->ip_len) {
1420 printf("ipsec_netif_sync_rx %s: IPv4 packet length too short (%zu < %u)\n",
1421 pcb->ipsec_ifp->if_xname, length, ip->ip_len);
1422 STATS_INC(nifs, NETIF_STATS_BADLEN);
1423 STATS_INC(nifs, NETIF_STATS_DROPPED);
1424 mbuf_freem(data);
1425 data = NULL;
1426 } else {
1427 data = esp4_input_extended(data, sizeof(struct ip), pcb->ipsec_ifp);
1428 }
1429 break;
1430 }
1431 case 6: {
1432 if (length < sizeof(struct ip6_hdr)) {
1433 printf("ipsec_netif_sync_rx %s: IPv6 packet length too short for header %zu\n",
1434 pcb->ipsec_ifp->if_xname, length);
1435 STATS_INC(nifs, NETIF_STATS_BADLEN);
1436 STATS_INC(nifs, NETIF_STATS_DROPPED);
1437 mbuf_freem(data);
1438 data = NULL;
1439 } else {
1440 af = AF_INET6;
1441 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
1442 const size_t ip6_len = sizeof(*ip6) + ntohs(ip6->ip6_plen);
1443 if (length < ip6_len) {
1444 printf("ipsec_netif_sync_rx %s: IPv6 packet length too short (%zu < %zu)\n",
1445 pcb->ipsec_ifp->if_xname, length, ip6_len);
1446 STATS_INC(nifs, NETIF_STATS_BADLEN);
1447 STATS_INC(nifs, NETIF_STATS_DROPPED);
1448 mbuf_freem(data);
1449 data = NULL;
1450 } else {
1451 int offset = sizeof(struct ip6_hdr);
1452 esp6_input_extended(&data, &offset, ip6->ip6_nxt, pcb->ipsec_ifp);
1453 }
1454 }
1455 break;
1456 }
1457 default: {
1458 printf("ipsec_netif_sync_rx %s: unknown ip version %u\n",
1459 pcb->ipsec_ifp->if_xname, ip_version);
1460 STATS_INC(nifs, NETIF_STATS_DROPPED);
1461 mbuf_freem(data);
1462 data = NULL;
1463 break;
1464 }
1465 }
1466 } else {
1467 printf("ipsec_netif_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
1468 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
1469 STATS_INC(nifs, NETIF_STATS_DROPPED);
1470 mbuf_freem(data);
1471 data = NULL;
1472 }
1473 } else {
1474 printf("ipsec_netif_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
1475 STATS_INC(nifs, NETIF_STATS_NOMEM_MBUF);
1476 STATS_INC(nifs, NETIF_STATS_DROPPED);
1477 }
1478 } else {
1479 printf("ipsec_netif_sync_rx %s - bad packet length %zu\n", pcb->ipsec_ifp->if_xname, length);
1480 STATS_INC(nifs, NETIF_STATS_BADLEN);
1481 STATS_INC(nifs, NETIF_STATS_DROPPED);
1482 }
1483
1484 if (data == NULL) {
1485 // Failed to get decrypted data data
1486 kern_pbufpool_free(rx_pp, rx_ph);
1487 continue;
1488 }
1489
1490 length = mbuf_pkthdr_len(data);
1491 if (length > rx_pp->pp_buflet_size) {
1492 // Flush data
1493 mbuf_freem(data);
1494 kern_pbufpool_free(rx_pp, rx_ph);
1495 STATS_INC(nifs, NETIF_STATS_BADLEN);
1496 STATS_INC(nifs, NETIF_STATS_DROPPED);
1497 printf("ipsec_netif_sync_rx %s: decrypted packet length %zu > %u\n",
1498 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
1499 continue;
1500 }
1501
1502 mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp);
1503
1504 // Fillout rx packet
1505 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
1506 VERIFY(rx_buf != NULL);
1507 void *rx_baddr = kern_buflet_get_object_address(rx_buf);
1508 VERIFY(rx_baddr != NULL);
1509
1510 // Copy-in data from mbuf to buflet
1511 mbuf_copydata(data, 0, length, (void *)rx_baddr);
1512 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
1513
1514 // Finalize and attach the packet
1515 error = kern_buflet_set_data_offset(rx_buf, 0);
1516 VERIFY(error == 0);
1517 error = kern_buflet_set_data_length(rx_buf, length);
1518 VERIFY(error == 0);
1519 error = kern_packet_set_link_header_offset(rx_ph, 0);
1520 VERIFY(error == 0);
1521 error = kern_packet_set_network_header_offset(rx_ph, 0);
1522 VERIFY(error == 0);
1523 error = kern_packet_finalize(rx_ph);
1524 VERIFY(error == 0);
1525 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
1526 VERIFY(error == 0);
1527
1528 STATS_INC(nifs, NETIF_STATS_RXPKTS);
1529 STATS_INC(nifs, NETIF_STATS_RXCOPY_DIRECT);
1530 bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0);
1531
1532 rx_ring_stats.kcrsi_slots_transferred++;
1533 rx_ring_stats.kcrsi_bytes_transferred += length;
1534
1535 if (!pcb->ipsec_ext_ifdata_stats) {
1536 ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0);
1537 }
1538
1539 mbuf_freem(data);
1540
1541 rx_pslot = rx_slot;
1542 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
1543 }
1544
1545done:
1546 if (rx_pslot) {
1547 kern_channel_advance_slot(rx_ring, rx_pslot);
1548 kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats);
1549 }
1550
1551 if (tx_pslot) {
1552 kern_channel_advance_slot(tx_ring, tx_pslot);
1553 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
1554 (void)kern_channel_reclaim(tx_ring);
1555 }
1556
1557 // Unlock first, then exit ring
1558 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1559 if (tx_ring != NULL) {
1560 if (tx_pslot != NULL) {
1561 kern_channel_notify(tx_ring, 0);
1562 }
1563 kr_exit(tx_ring);
1564 }
1565
1566 return 0;
1567}
1568
1569static errno_t
1570ipsec_nexus_ifattach(struct ipsec_pcb *pcb,
1571 struct ifnet_init_eparams *init_params,
1572 struct ifnet **ifp)
1573{
1574 errno_t err;
1575 nexus_controller_t controller = kern_nexus_shared_controller();
1576 struct kern_nexus_net_init net_init;
a39ff7e2 1577 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 1578
1579 nexus_name_t provider_name;
1580 snprintf((char *)provider_name, sizeof(provider_name),
a39ff7e2 1581 "com.apple.netif.%s", pcb->ipsec_if_xname);
5ba3f43e
A		 1582
1583 struct kern_nexus_provider_init prov_init = {
1584 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1585 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1586 .nxpi_pre_connect = ipsec_nexus_pre_connect,
1587 .nxpi_connected = ipsec_nexus_connected,
1588 .nxpi_pre_disconnect = ipsec_netif_pre_disconnect,
1589 .nxpi_disconnected = ipsec_nexus_disconnected,
1590 .nxpi_ring_init = ipsec_netif_ring_init,
1591 .nxpi_ring_fini = ipsec_netif_ring_fini,
1592 .nxpi_slot_init = NULL,
1593 .nxpi_slot_fini = NULL,
1594 .nxpi_sync_tx = ipsec_netif_sync_tx,
1595 .nxpi_sync_rx = ipsec_netif_sync_rx,
1596 .nxpi_tx_doorbell = ipsec_netif_tx_doorbell,
1597 };
1598
1599 nexus_attr_t nxa = NULL;
1600 err = kern_nexus_attr_create(&nxa);
1601 IPSEC_IF_VERIFY(err == 0);
1602 if (err != 0) {
1603 printf("%s: kern_nexus_attr_create failed: %d\n",
1604 __func__, err);
1605 goto failed;
1606 }
1607
5c9f4661 1608 uint64_t slot_buffer_size = pcb->ipsec_slot_size;
5ba3f43e
A		 1609 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1610 VERIFY(err == 0);
1611
1612 // Reset ring size for netif nexus to limit memory usage
5c9f4661 1613 uint64_t ring_size = pcb->ipsec_netif_ring_size;
5ba3f43e
A		 1614 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1615 VERIFY(err == 0);
1616 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1617 VERIFY(err == 0);
1618
1619 pcb->ipsec_netif_txring_size = ring_size;
1620
a39ff7e2
A		 1621 bzero(&pp_init, sizeof (pp_init));
1622 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
1623 pp_init.kbi_packets = pcb->ipsec_netif_ring_size * 2;
1624 pp_init.kbi_bufsize = pcb->ipsec_slot_size;
1625 pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE;
1626 pp_init.kbi_max_frags = 1;
1627 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
1628 "%s", provider_name);
1629
1630 err = kern_pbufpool_create(&pp_init, &pp_init, &pcb->ipsec_netif_pp, NULL);
1631 if (err != 0) {
1632 printf("%s pbufbool create failed, error %d\n", __func__, err);
1633 goto failed;
1634 }
1635
5ba3f43e
A		 1636 err = kern_nexus_controller_register_provider(controller,
1637 ipsec_nx_dom_prov,
1638 provider_name,
1639 &prov_init,
1640 sizeof(prov_init),
1641 nxa,
1642 &pcb->ipsec_nx.if_provider);
1643 IPSEC_IF_VERIFY(err == 0);
1644 if (err != 0) {
1645 printf("%s register provider failed, error %d\n",
1646 __func__, err);
1647 goto failed;
1648 }
1649
1650 bzero(&net_init, sizeof(net_init));
1651 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
1652 net_init.nxneti_flags = 0;
1653 net_init.nxneti_eparams = init_params;
1654 net_init.nxneti_lladdr = NULL;
1655 net_init.nxneti_prepare = ipsec_netif_prepare;
a39ff7e2 1656 net_init.nxneti_tx_pbufpool = pcb->ipsec_netif_pp;
5ba3f43e
A		 1657 err = kern_nexus_controller_alloc_net_provider_instance(controller,
1658 pcb->ipsec_nx.if_provider,
1659 pcb,
1660 &pcb->ipsec_nx.if_instance,
1661 &net_init,
1662 ifp);
1663 IPSEC_IF_VERIFY(err == 0);
1664 if (err != 0) {
1665 printf("%s alloc_net_provider_instance failed, %d\n",
1666 __func__, err);
1667 kern_nexus_controller_deregister_provider(controller,
1668 pcb->ipsec_nx.if_provider);
1669 uuid_clear(pcb->ipsec_nx.if_provider);
1670 goto failed;
1671 }
1672
1673failed:
1674 if (nxa) {
1675 kern_nexus_attr_destroy(nxa);
1676 }
a39ff7e2
A		 1677 if (err && pcb->ipsec_netif_pp != NULL) {
1678 kern_pbufpool_destroy(pcb->ipsec_netif_pp);
1679 pcb->ipsec_netif_pp = NULL;
1680 }
5ba3f43e
A		 1681 return (err);
1682}
1683
1684static void
1685ipsec_detach_provider_and_instance(uuid_t provider, uuid_t instance)
1686{
1687 nexus_controller_t controller = kern_nexus_shared_controller();
1688 errno_t err;
1689
1690 if (!uuid_is_null(instance)) {
1691 err = kern_nexus_controller_free_provider_instance(controller,
1692 instance);
1693 if (err != 0) {
1694 printf("%s free_provider_instance failed %d\n",
1695 __func__, err);
1696 }
1697 uuid_clear(instance);
1698 }
1699 if (!uuid_is_null(provider)) {
1700 err = kern_nexus_controller_deregister_provider(controller,
1701 provider);
1702 if (err != 0) {
1703 printf("%s deregister_provider %d\n", __func__, err);
1704 }
1705 uuid_clear(provider);
1706 }
1707 return;
1708}
1709
1710static void
a39ff7e2 1711ipsec_nexus_detach(struct ipsec_pcb *pcb)
5ba3f43e 1712{
a39ff7e2 1713 ipsec_nx_t nx = &pcb->ipsec_nx;
5ba3f43e
A		 1714 nexus_controller_t controller = kern_nexus_shared_controller();
1715 errno_t err;
1716
1717 if (!uuid_is_null(nx->ms_host)) {
1718 err = kern_nexus_ifdetach(controller,
1719 nx->ms_instance,
1720 nx->ms_host);
1721 if (err != 0) {
1722 printf("%s: kern_nexus_ifdetach ms host failed %d\n",
1723 __func__, err);
1724 }
1725 }
1726
1727 if (!uuid_is_null(nx->ms_device)) {
1728 err = kern_nexus_ifdetach(controller,
1729 nx->ms_instance,
1730 nx->ms_device);
1731 if (err != 0) {
1732 printf("%s: kern_nexus_ifdetach ms device failed %d\n",
1733 __func__, err);
1734 }
1735 }
1736
1737 ipsec_detach_provider_and_instance(nx->if_provider,
1738 nx->if_instance);
1739 ipsec_detach_provider_and_instance(nx->ms_provider,
1740 nx->ms_instance);
1741
a39ff7e2
A		 1742 if (pcb->ipsec_netif_pp != NULL) {
1743 kern_pbufpool_destroy(pcb->ipsec_netif_pp);
1744 pcb->ipsec_netif_pp = NULL;
1745
1746 }
5ba3f43e
A		 1747 memset(nx, 0, sizeof(*nx));
1748}
1749
1750static errno_t
5c9f4661
A		 1751ipsec_create_fs_provider_and_instance(struct ipsec_pcb *pcb,
1752 uint32_t subtype, const char *type_name,
5ba3f43e
A		 1753 const char *ifname,
1754 uuid_t *provider, uuid_t *instance)
1755{
1756 nexus_attr_t attr = NULL;
1757 nexus_controller_t controller = kern_nexus_shared_controller();
1758 uuid_t dom_prov;
1759 errno_t err;
1760 struct kern_nexus_init init;
1761 nexus_name_t provider_name;
1762
1763 err = kern_nexus_get_builtin_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
1764 &dom_prov);
1765 IPSEC_IF_VERIFY(err == 0);
1766 if (err != 0) {
1767 printf("%s can't get %s provider, error %d\n",
1768 __func__, type_name, err);
1769 goto failed;
1770 }
1771
1772 err = kern_nexus_attr_create(&attr);
1773 IPSEC_IF_VERIFY(err == 0);
1774 if (err != 0) {
1775 printf("%s: kern_nexus_attr_create failed: %d\n",
1776 __func__, err);
1777 goto failed;
1778 }
1779
1780 err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
1781 VERIFY(err == 0);
1782
5c9f4661 1783 uint64_t slot_buffer_size = pcb->ipsec_slot_size;
5ba3f43e
A		 1784 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1785 VERIFY(err == 0);
1786
1787 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
5c9f4661 1788 uint64_t tx_ring_size = pcb->ipsec_tx_fsw_ring_size;
5ba3f43e
A		 1789 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
1790 VERIFY(err == 0);
5c9f4661 1791 uint64_t rx_ring_size = pcb->ipsec_rx_fsw_ring_size;
5ba3f43e
A		 1792 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
1793 VERIFY(err == 0);
1794
1795 snprintf((char *)provider_name, sizeof(provider_name),
1796 "com.apple.%s.%s", type_name, ifname);
1797 err = kern_nexus_controller_register_provider(controller,
1798 dom_prov,
1799 provider_name,
1800 NULL,
1801 0,
1802 attr,
1803 provider);
1804 kern_nexus_attr_destroy(attr);
1805 attr = NULL;
1806 IPSEC_IF_VERIFY(err == 0);
1807 if (err != 0) {
1808 printf("%s register %s provider failed, error %d\n",
1809 __func__, type_name, err);
1810 goto failed;
1811 }
1812 bzero(&init, sizeof (init));
1813 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
1814 err = kern_nexus_controller_alloc_provider_instance(controller,
1815 *provider,
1816 NULL,
1817 instance, &init);
1818 IPSEC_IF_VERIFY(err == 0);
1819 if (err != 0) {
1820 printf("%s alloc_provider_instance %s failed, %d\n",
1821 __func__, type_name, err);
1822 kern_nexus_controller_deregister_provider(controller,
1823 *provider);
1824 uuid_clear(*provider);
1825 }
1826failed:
1827 return (err);
1828}
1829
1830static errno_t
1831ipsec_multistack_attach(struct ipsec_pcb *pcb)
1832{
1833 nexus_controller_t controller = kern_nexus_shared_controller();
1834 errno_t err = 0;
1835 ipsec_nx_t nx = &pcb->ipsec_nx;
1836
1837 // Allocate multistack flowswitch
5c9f4661
A		 1838 err = ipsec_create_fs_provider_and_instance(pcb,
1839 NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
5ba3f43e
A		 1840 "multistack",
1841 pcb->ipsec_ifp->if_xname,
1842 &nx->ms_provider,
1843 &nx->ms_instance);
1844 if (err != 0) {
1845 printf("%s: failed to create bridge provider and instance\n",
1846 __func__);
1847 goto failed;
1848 }
1849
1850 // Attach multistack to device port
1851 err = kern_nexus_ifattach(controller, nx->ms_instance,
1852 NULL, nx->if_instance,
1853 FALSE, &nx->ms_device);
1854 if (err != 0) {
1855 printf("%s kern_nexus_ifattach ms device %d\n", __func__, err);
1856 goto failed;
1857 }
1858
1859 // Attach multistack to host port
1860 err = kern_nexus_ifattach(controller, nx->ms_instance,
1861 NULL, nx->if_instance,
1862 TRUE, &nx->ms_host);
1863 if (err != 0) {
1864 printf("%s kern_nexus_ifattach ms host %d\n", __func__, err);
1865 goto failed;
1866 }
1867
1868 // Extract the agent UUID and save for later
1869 struct kern_nexus *multistack_nx = nx_find(nx->ms_instance, false);
1870 if (multistack_nx != NULL) {
1871 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(multistack_nx);
1872 if (flowswitch != NULL) {
1873 FSW_RLOCK(flowswitch);
1874 struct fsw_ms_context *ms_context = (struct fsw_ms_context *)flowswitch->fsw_ops_private;
1875 if (ms_context != NULL) {
1876 uuid_copy(nx->ms_agent, ms_context->mc_agent_uuid);
1877 } else {
1878 printf("ipsec_multistack_attach - fsw_ms_context is NULL\n");
1879 }
1880 FSW_UNLOCK(flowswitch);
1881 } else {
1882 printf("ipsec_multistack_attach - flowswitch is NULL\n");
1883 }
1884 nx_release(multistack_nx);
1885 } else {
1886 printf("ipsec_multistack_attach - unable to find multistack nexus\n");
1887 }
1888
1889 return (0);
1890
1891failed:
a39ff7e2 1892 ipsec_nexus_detach(pcb);
5ba3f43e
A		 1893
1894 errno_t detach_error = 0;
1895 if ((detach_error = ifnet_detach(pcb->ipsec_ifp)) != 0) {
1896 panic("ipsec_multistack_attach - ifnet_detach failed: %d\n", detach_error);
1897 /* NOT REACHED */
1898 }
1899
1900 return (err);
1901}
1902
1903#pragma mark Kernel Pipe Nexus
1904
1905static errno_t
1906ipsec_register_kernel_pipe_nexus(void)
1907{
1908 nexus_attr_t nxa = NULL;
1909 errno_t result;
1910
1911 lck_mtx_lock(&ipsec_lock);
1912 if (ipsec_ncd_refcount++) {
1913 lck_mtx_unlock(&ipsec_lock);
1914 return 0;
1915 }
1916
1917 result = kern_nexus_controller_create(&ipsec_ncd);
1918 if (result) {
1919 printf("%s: kern_nexus_controller_create failed: %d\n",
1920 __FUNCTION__, result);
1921 goto done;
1922 }
1923
1924 uuid_t dom_prov;
1925 result = kern_nexus_get_builtin_domain_provider(
1926 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
1927 if (result) {
1928 printf("%s: kern_nexus_get_builtin_domain_provider failed: %d\n",
1929 __FUNCTION__, result);
1930 goto done;
1931 }
1932
1933 struct kern_nexus_provider_init prov_init = {
1934 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1935 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1936 .nxpi_pre_connect = ipsec_nexus_pre_connect,
1937 .nxpi_connected = ipsec_nexus_connected,
1938 .nxpi_pre_disconnect = ipsec_nexus_pre_disconnect,
1939 .nxpi_disconnected = ipsec_nexus_disconnected,
1940 .nxpi_ring_init = ipsec_kpipe_ring_init,
1941 .nxpi_ring_fini = ipsec_kpipe_ring_fini,
1942 .nxpi_slot_init = NULL,
1943 .nxpi_slot_fini = NULL,
1944 .nxpi_sync_tx = ipsec_kpipe_sync_tx,
1945 .nxpi_sync_rx = ipsec_kpipe_sync_rx,
1946 .nxpi_tx_doorbell = NULL,
1947 };
1948
1949 result = kern_nexus_attr_create(&nxa);
1950 if (result) {
1951 printf("%s: kern_nexus_attr_create failed: %d\n",
1952 __FUNCTION__, result);
1953 goto done;
1954 }
1955
1956 uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
1957 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1958 VERIFY(result == 0);
1959
1960 // Reset ring size for kernel pipe nexus to limit memory usage
1961 uint64_t ring_size = if_ipsec_ring_size;
1962 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1963 VERIFY(result == 0);
1964 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1965 VERIFY(result == 0);
1966
1967 result = kern_nexus_controller_register_provider(ipsec_ncd,
1968 dom_prov,
1969 (const uint8_t *)"com.apple.nexus.ipsec.kpipe",
1970 &prov_init,
1971 sizeof(prov_init),
1972 nxa,
1973 &ipsec_kpipe_uuid);
1974 if (result) {
1975 printf("%s: kern_nexus_controller_register_provider failed: %d\n",
1976 __FUNCTION__, result);
1977 goto done;
1978 }
1979
1980done:
1981 if (nxa) {
1982 kern_nexus_attr_destroy(nxa);
1983 }
1984
1985 if (result) {
1986 if (ipsec_ncd) {
1987 kern_nexus_controller_destroy(ipsec_ncd);
1988 ipsec_ncd = NULL;
1989 }
1990 ipsec_ncd_refcount = 0;
1991 }
1992
1993 lck_mtx_unlock(&ipsec_lock);
1994
1995 return result;
1996}
1997
1998static void
1999ipsec_unregister_kernel_pipe_nexus(void)
2000{
2001 lck_mtx_lock(&ipsec_lock);
2002
2003 VERIFY(ipsec_ncd_refcount > 0);
2004
2005 if (--ipsec_ncd_refcount == 0) {
2006 kern_nexus_controller_destroy(ipsec_ncd);
2007 ipsec_ncd = NULL;
2008 }
2009
2010 lck_mtx_unlock(&ipsec_lock);
2011}
2012
2013// For use by socket option, not internally
2014static errno_t
2015ipsec_disable_channel(struct ipsec_pcb *pcb)
2016{
2017 errno_t result;
2018 int enabled;
2019 uuid_t uuid;
2020
2021 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2022
2023 enabled = pcb->ipsec_kpipe_enabled;
2024 uuid_copy(uuid, pcb->ipsec_kpipe_uuid);
2025
2026 VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid) == !enabled);
39236c6e 2027
5ba3f43e
A		 2028 pcb->ipsec_kpipe_enabled = 0;
2029 uuid_clear(pcb->ipsec_kpipe_uuid);
39236c6e 2030
5ba3f43e
A		 2031 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
2032
2033 if (enabled) {
2034 result = kern_nexus_controller_free_provider_instance(ipsec_ncd, uuid);
2035 } else {
2036 result = ENXIO;
2037 }
2038
2039 if (!result) {
a39ff7e2
A		 2040 if (pcb->ipsec_kpipe_pp != NULL) {
2041 kern_pbufpool_destroy(pcb->ipsec_kpipe_pp);
2042 pcb->ipsec_kpipe_pp = NULL;
2043 }
5ba3f43e
A		 2044 ipsec_unregister_kernel_pipe_nexus();
2045 }
2046
2047 return result;
2048}
2049
2050static errno_t
2051ipsec_enable_channel(struct ipsec_pcb *pcb, struct proc *proc)
39236c6e 2052{
5ba3f43e 2053 struct kern_nexus_init init;
a39ff7e2 2054 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 2055 errno_t result;
2056
2057 result = ipsec_register_kernel_pipe_nexus();
2058 if (result) {
39236c6e
A		 2059 return result;
2060 }
5ba3f43e
A		 2061
2062 VERIFY(ipsec_ncd);
2063
2064 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2065
a39ff7e2
A		 2066 /* ipsec driver doesn't support channels without a netif */
2067 if (!pcb->ipsec_use_netif) {
2068 result = EOPNOTSUPP;
2069 goto done;
2070 }
2071
5ba3f43e
A		 2072 if (pcb->ipsec_kpipe_enabled) {
2073 result = EEXIST; // return success instead?
2074 goto done;
39236c6e 2075 }
5ba3f43e 2076
a39ff7e2
A		 2077 bzero(&pp_init, sizeof (pp_init));
2078 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
2079 pp_init.kbi_packets = pcb->ipsec_netif_ring_size * 2;
2080 pp_init.kbi_bufsize = pcb->ipsec_slot_size;
2081 pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE;
2082 pp_init.kbi_max_frags = 1;
2083 pp_init.kbi_flags |= KBIF_QUANTUM;
2084 (void) snprintf((char *)pp_init.kbi_name, sizeof (pp_init.kbi_name),
2085 "com.apple.kpipe.%s", pcb->ipsec_if_xname);
2086
2087 result = kern_pbufpool_create(&pp_init, &pp_init, &pcb->ipsec_kpipe_pp,
2088 NULL);
2089 if (result != 0) {
2090 printf("%s pbufbool create failed, error %d\n", __func__, result);
2091 goto done;
2092 }
2093
5ba3f43e
A		 2094 VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid));
2095 bzero(&init, sizeof (init));
2096 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
a39ff7e2 2097 init.nxi_tx_pbufpool = pcb->ipsec_kpipe_pp;
5ba3f43e
A		 2098 result = kern_nexus_controller_alloc_provider_instance(ipsec_ncd,
2099 ipsec_kpipe_uuid, pcb, &pcb->ipsec_kpipe_uuid, &init);
2100 if (result) {
2101 goto done;
39236c6e 2102 }
5ba3f43e
A		 2103
2104 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
2105 result = kern_nexus_controller_bind_provider_instance(ipsec_ncd,
2106 pcb->ipsec_kpipe_uuid, &port,
2107 proc_pid(proc), NULL, NULL, 0, NEXUS_BIND_PID);
2108 if (result) {
2109 kern_nexus_controller_free_provider_instance(ipsec_ncd,
2110 pcb->ipsec_kpipe_uuid);
2111 uuid_clear(pcb->ipsec_kpipe_uuid);
2112 goto done;
2113 }
2114
2115 pcb->ipsec_kpipe_enabled = 1;
2116
2117done:
2118 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
39236c6e 2119
5ba3f43e 2120 if (result) {
a39ff7e2
A		 2121 if (pcb->ipsec_kpipe_pp != NULL) {
2122 kern_pbufpool_destroy(pcb->ipsec_kpipe_pp);
2123 pcb->ipsec_kpipe_pp = NULL;
2124 }
5ba3f43e 2125 ipsec_unregister_kernel_pipe_nexus();
39236c6e
A		 2126 }
2127
5ba3f43e 2128 return result;
39236c6e
A		 2129}
2130
5ba3f43e
A		 2131#endif // IPSEC_NEXUS
2132
39236c6e
A		 2133
2134/* Kernel control functions */
2135
5ba3f43e 2136static inline void
5c9f4661 2137ipsec_free_pcb(struct ipsec_pcb *pcb, bool in_list)
5ba3f43e
A		 2138{
2139#if IPSEC_NEXUS
2140 mbuf_freem_list(pcb->ipsec_input_chain);
2141 lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp);
2142#endif // IPSEC_NEXUS
2143 lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp);
5c9f4661
A		 2144 if (in_list) {
2145 lck_mtx_lock(&ipsec_lock);
2146 TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain);
2147 lck_mtx_unlock(&ipsec_lock);
2148 }
5ba3f43e
A		 2149 zfree(ipsec_pcb_zone, pcb);
2150}
2151
39236c6e 2152static errno_t
5c9f4661
A		 2153ipsec_ctl_bind(kern_ctl_ref kctlref,
2154 struct sockaddr_ctl *sac,
2155 void **unitinfo)
39236c6e 2156{
5ba3f43e
A		 2157 struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone);
2158 memset(pcb, 0, sizeof(*pcb));
39037602 2159
39236c6e 2160 /* Setup the protocol control block */
39236c6e
A		 2161 *unitinfo = pcb;
2162 pcb->ipsec_ctlref = kctlref;
2163 pcb->ipsec_unit = sac->sc_unit;
fe8ab488 2164 pcb->ipsec_output_service_class = MBUF_SC_OAM;
5ba3f43e 2165
5c9f4661
A		 2166#if IPSEC_NEXUS
2167 pcb->ipsec_use_netif = false;
2168 pcb->ipsec_slot_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
2169 pcb->ipsec_netif_ring_size = IPSEC_IF_DEFAULT_RING_SIZE;
2170 pcb->ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE;
2171 pcb->ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE;
2172#endif // IPSEC_NEXUS
2173
2174 lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr);
2175#if IPSEC_NEXUS
2176 lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr);
2177#endif // IPSEC_NEXUS
2178
2179 return (0);
2180}
2181
2182static errno_t
2183ipsec_ctl_connect(kern_ctl_ref kctlref,
2184 struct sockaddr_ctl *sac,
2185 void **unitinfo)
2186{
2187 struct ifnet_init_eparams ipsec_init = {};
2188 errno_t result = 0;
2189
2190 if (*unitinfo == NULL) {
2191 (void)ipsec_ctl_bind(kctlref, sac, unitinfo);
2192 }
2193
2194 struct ipsec_pcb *pcb = *unitinfo;
2195
5ba3f43e
A		 2196 lck_mtx_lock(&ipsec_lock);
2197
2198 /* Find some open interface id */
2199 u_int32_t chosen_unique_id = 1;
2200 struct ipsec_pcb *next_pcb = TAILQ_LAST(&ipsec_head, ipsec_list);
2201 if (next_pcb != NULL) {
2202 /* List was not empty, add one to the last item */
2203 chosen_unique_id = next_pcb->ipsec_unique_id + 1;
2204 next_pcb = NULL;
2205
2206 /*
2207 * If this wrapped the id number, start looking at
2208 * the front of the list for an unused id.
2209 */
2210 if (chosen_unique_id == 0) {
2211 /* Find the next unused ID */
2212 chosen_unique_id = 1;
2213 TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) {
2214 if (next_pcb->ipsec_unique_id > chosen_unique_id) {
2215 /* We found a gap */
2216 break;
2217 }
2218
2219 chosen_unique_id = next_pcb->ipsec_unique_id + 1;
2220 }
2221 }
2222 }
2223
2224 pcb->ipsec_unique_id = chosen_unique_id;
2225
2226 if (next_pcb != NULL) {
2227 TAILQ_INSERT_BEFORE(next_pcb, pcb, ipsec_chain);
2228 } else {
2229 TAILQ_INSERT_TAIL(&ipsec_head, pcb, ipsec_chain);
2230 }
2231 lck_mtx_unlock(&ipsec_lock);
2232
2233 snprintf(pcb->ipsec_if_xname, sizeof(pcb->ipsec_if_xname), "ipsec%d", pcb->ipsec_unit - 1);
2234 snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1);
2235 printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name);
2236
39236c6e
A		 2237 /* Create the interface */
2238 bzero(&ipsec_init, sizeof(ipsec_init));
2239 ipsec_init.ver = IFNET_INIT_CURRENT_VERSION;
2240 ipsec_init.len = sizeof (ipsec_init);
5ba3f43e
A		 2241
2242#if IPSEC_NEXUS
5c9f4661
A		 2243 if (pcb->ipsec_use_netif) {
2244 ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
2245 } else
5ba3f43e 2246#endif // IPSEC_NEXUS
5c9f4661
A		 2247 {
2248 ipsec_init.flags = IFNET_INIT_NX_NOAUTO;
2249 ipsec_init.start = ipsec_start;
2250 }
5ba3f43e 2251 ipsec_init.name = "ipsec";
39236c6e 2252 ipsec_init.unit = pcb->ipsec_unit - 1;
5ba3f43e
A		 2253 ipsec_init.uniqueid = pcb->ipsec_unique_name;
2254 ipsec_init.uniqueid_len = strlen(pcb->ipsec_unique_name);
39236c6e 2255 ipsec_init.family = ipsec_family;
5ba3f43e 2256 ipsec_init.subfamily = IFNET_SUBFAMILY_IPSEC;
39236c6e
A		 2257 ipsec_init.type = IFT_OTHER;
2258 ipsec_init.demux = ipsec_demux;
2259 ipsec_init.add_proto = ipsec_add_proto;
2260 ipsec_init.del_proto = ipsec_del_proto;
2261 ipsec_init.softc = pcb;
2262 ipsec_init.ioctl = ipsec_ioctl;
2263 ipsec_init.detach = ipsec_detached;
5ba3f43e
A		 2264
2265#if IPSEC_NEXUS
5c9f4661
A		 2266 if (pcb->ipsec_use_netif) {
2267 result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp);
2268 if (result != 0) {
2269 printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result);
2270 ipsec_free_pcb(pcb, true);
2271 *unitinfo = NULL;
2272 return result;
2273 }
5ba3f43e 2274
5c9f4661
A		 2275 result = ipsec_multistack_attach(pcb);
2276 if (result != 0) {
2277 printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result);
2278 *unitinfo = NULL;
2279 return result;
2280 }
5ba3f43e 2281
5c9f4661
A		 2282 /* Attach to bpf */
2283 bpfattach(pcb->ipsec_ifp, DLT_RAW, 0);
2284 } else
5ba3f43e 2285#endif // IPSEC_NEXUS
5c9f4661
A		 2286 {
2287 result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp);
2288 if (result != 0) {
2289 printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result);
2290 ipsec_free_pcb(pcb, true);
2291 *unitinfo = NULL;
2292 return result;
2293 }
2294 ipsec_ifnet_set_attrs(pcb->ipsec_ifp);
2295
2296 /* Attach the interface */
2297 result = ifnet_attach(pcb->ipsec_ifp, NULL);
2298 if (result != 0) {
2299 printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result);
2300 ifnet_release(pcb->ipsec_ifp);
2301 ipsec_free_pcb(pcb, true);
2302 *unitinfo = NULL;
2303 return (result);
2304 }
5ba3f43e 2305
5c9f4661
A		 2306 /* Attach to bpf */
2307 bpfattach(pcb->ipsec_ifp, DLT_NULL, 0);
2308 }
5ba3f43e
A		 2309
2310 /* The interfaces resoures allocated, mark it as running */
2311 ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING);
2312
2313 return (0);
39236c6e
A		 2314}
2315
2316static errno_t
2317ipsec_detach_ip(ifnet_t interface,
2318 protocol_family_t protocol,
2319 socket_t pf_socket)
2320{
2321 errno_t result = EPROTONOSUPPORT;
2322
2323 /* Attempt a detach */
2324 if (protocol == PF_INET) {
2325 struct ifreq ifr;
2326
2327 bzero(&ifr, sizeof(ifr));
2328 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
2329 ifnet_name(interface), ifnet_unit(interface));
2330
2331 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
2332 }
2333 else if (protocol == PF_INET6) {
2334 struct in6_ifreq ifr6;
2335
2336 bzero(&ifr6, sizeof(ifr6));
2337 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
2338 ifnet_name(interface), ifnet_unit(interface));
2339
2340 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
2341 }
2342
2343 return result;
2344}
2345
2346static void
2347ipsec_remove_address(ifnet_t interface,
2348 protocol_family_t protocol,
2349 ifaddr_t address,
2350 socket_t pf_socket)
2351{
2352 errno_t result = 0;
2353
2354 /* Attempt a detach */
2355 if (protocol == PF_INET) {
2356 struct ifreq ifr;
2357
2358 bzero(&ifr, sizeof(ifr));
2359 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
2360 ifnet_name(interface), ifnet_unit(interface));
2361 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
2362 if (result != 0) {
2363 printf("ipsec_remove_address - ifaddr_address failed: %d", result);
2364 }
2365 else {
2366 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
2367 if (result != 0) {
2368 printf("ipsec_remove_address - SIOCDIFADDR failed: %d", result);
2369 }
2370 }
2371 }
2372 else if (protocol == PF_INET6) {
2373 struct in6_ifreq ifr6;
2374
2375 bzero(&ifr6, sizeof(ifr6));
2376 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
2377 ifnet_name(interface), ifnet_unit(interface));
2378 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
2379 sizeof(ifr6.ifr_addr));
2380 if (result != 0) {
2381 printf("ipsec_remove_address - ifaddr_address failed (v6): %d",
2382 result);
2383 }
2384 else {
2385 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
2386 if (result != 0) {
2387 printf("ipsec_remove_address - SIOCDIFADDR_IN6 failed: %d",
2388 result);
2389 }
2390 }
2391 }
2392}
2393
2394static void
2395ipsec_cleanup_family(ifnet_t interface,
2396 protocol_family_t protocol)
2397{
2398 errno_t result = 0;
2399 socket_t pf_socket = NULL;
2400 ifaddr_t *addresses = NULL;
2401 int i;
2402
2403 if (protocol != PF_INET && protocol != PF_INET6) {
2404 printf("ipsec_cleanup_family - invalid protocol family %d\n", protocol);
2405 return;
2406 }
2407
2408 /* Create a socket for removing addresses and detaching the protocol */
2409 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
2410 if (result != 0) {
2411 if (result != EAFNOSUPPORT)
2412 printf("ipsec_cleanup_family - failed to create %s socket: %d\n",
2413 protocol == PF_INET ? "IP" : "IPv6", result);
2414 goto cleanup;
2415 }
2416
2417 /* always set SS_PRIV, we want to close and detach regardless */
2418 sock_setpriv(pf_socket, 1);
2419
2420 result = ipsec_detach_ip(interface, protocol, pf_socket);
2421 if (result == 0 || result == ENXIO) {
2422 /* We are done! We either detached or weren't attached. */
2423 goto cleanup;
2424 }
2425 else if (result != EBUSY) {
2426 /* Uh, not really sure what happened here... */
2427 printf("ipsec_cleanup_family - ipsec_detach_ip failed: %d\n", result);
2428 goto cleanup;
2429 }
2430
2431 /*
2432 * At this point, we received an EBUSY error. This means there are
2433 * addresses attached. We should detach them and then try again.
2434 */
2435 result = ifnet_get_address_list_family(interface, &addresses, protocol);
2436 if (result != 0) {
2437 printf("fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
2438 ifnet_name(interface), ifnet_unit(interface),
2439 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
2440 goto cleanup;
2441 }
2442
2443 for (i = 0; addresses[i] != 0; i++) {
2444 ipsec_remove_address(interface, protocol, addresses[i], pf_socket);
2445 }
2446 ifnet_free_address_list(addresses);
2447 addresses = NULL;
2448
2449 /*
2450 * The addresses should be gone, we should try the remove again.
2451 */
2452 result = ipsec_detach_ip(interface, protocol, pf_socket);
2453 if (result != 0 && result != ENXIO) {
2454 printf("ipsec_cleanup_family - ipsec_detach_ip failed: %d\n", result);
2455 }
2456
2457cleanup:
2458 if (pf_socket != NULL)
2459 sock_close(pf_socket);
2460
2461 if (addresses != NULL)
2462 ifnet_free_address_list(addresses);
2463}
2464
2465static errno_t
2466ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref,
2467 __unused u_int32_t unit,
2468 void *unitinfo)
2469{
5ba3f43e
A		 2470 struct ipsec_pcb *pcb = unitinfo;
2471 ifnet_t ifp = NULL;
2472 errno_t result = 0;
39037602 2473
5ba3f43e 2474 if (pcb == NULL) {
39037602 2475 return EINVAL;
5ba3f43e
A		 2476 }
2477
2478#if IPSEC_NEXUS
2479 // Tell the nexus to stop all rings
2480 if (pcb->ipsec_netif_nexus != NULL) {
2481 kern_nexus_stop(pcb->ipsec_netif_nexus);
2482 }
2483#endif // IPSEC_NEXUS
2484
2485 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2486
2487#if IPSEC_NEXUS
2488 uuid_t kpipe_uuid;
2489 uuid_copy(kpipe_uuid, pcb->ipsec_kpipe_uuid);
2490 uuid_clear(pcb->ipsec_kpipe_uuid);
2491 pcb->ipsec_kpipe_enabled = FALSE;
2492#endif // IPSEC_NEXUS
39037602 2493
39236c6e 2494 pcb->ipsec_ctlref = NULL;
5ba3f43e 2495
5c9f4661
A		 2496 ifp = pcb->ipsec_ifp;
2497 if (ifp != NULL) {
2498#if IPSEC_NEXUS
2499 if (pcb->ipsec_netif_nexus != NULL) {
2500 /*
2501 * Quiesce the interface and flush any pending outbound packets.
2502 */
2503 if_down(ifp);
2504
2505 /* Increment refcnt, but detach interface */
2506 ifnet_incr_iorefcnt(ifp);
2507 if ((result = ifnet_detach(ifp)) != 0) {
2508 panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
2509 /* NOT REACHED */
2510 }
5ba3f43e 2511
5c9f4661
A		 2512 /*
2513 * We want to do everything in our power to ensure that the interface
2514 * really goes away when the socket is closed. We must remove IP/IPv6
2515 * addresses and detach the protocols. Finally, we can remove and
2516 * release the interface.
2517 */
2518 key_delsp_for_ipsec_if(ifp);
5ba3f43e 2519
5c9f4661
A		 2520 ipsec_cleanup_family(ifp, AF_INET);
2521 ipsec_cleanup_family(ifp, AF_INET6);
5ba3f43e 2522
5c9f4661
A		 2523 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
2524
2525 if (!uuid_is_null(kpipe_uuid)) {
2526 if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 2527 if (pcb->ipsec_kpipe_pp != NULL) {
2528 kern_pbufpool_destroy(pcb->ipsec_kpipe_pp);
2529 pcb->ipsec_kpipe_pp = NULL;
2530 }
5c9f4661
A		 2531 ipsec_unregister_kernel_pipe_nexus();
2532 }
2533 }
a39ff7e2 2534 ipsec_nexus_detach(pcb);
5c9f4661
A		 2535
2536 /* Decrement refcnt to finish detaching and freeing */
2537 ifnet_decr_iorefcnt(ifp);
2538 } else
2539#endif // IPSEC_NEXUS
2540 {
2541 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
5ba3f43e
A		 2542
2543#if IPSEC_NEXUS
5c9f4661
A		 2544 if (!uuid_is_null(kpipe_uuid)) {
2545 if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) {
a39ff7e2
A		 2546 if (pcb->ipsec_kpipe_pp != NULL) {
2547 kern_pbufpool_destroy(pcb->ipsec_kpipe_pp);
2548 pcb->ipsec_kpipe_pp = NULL;
2549 }
5c9f4661
A		 2550 ipsec_unregister_kernel_pipe_nexus();
2551 }
2552 }
5ba3f43e
A		 2553#endif // IPSEC_NEXUS
2554
5c9f4661
A		 2555 /*
2556 * We want to do everything in our power to ensure that the interface
2557 * really goes away when the socket is closed. We must remove IP/IPv6
2558 * addresses and detach the protocols. Finally, we can remove and
2559 * release the interface.
2560 */
2561 key_delsp_for_ipsec_if(ifp);
2562
2563 ipsec_cleanup_family(ifp, AF_INET);
2564 ipsec_cleanup_family(ifp, AF_INET6);
2565
2566 /*
2567 * Detach now; ipsec_detach() will be called asynchronously once
2568 * the I/O reference count drops to 0. There we will invoke
2569 * ifnet_release().
2570 */
2571 if ((result = ifnet_detach(ifp)) != 0) {
2572 printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
2573 }
2574 }
2575 } else {
2576 // Bound, but not connected
2577 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
2578 ipsec_free_pcb(pcb, false);
2579 }
39236c6e
A		 2580
2581 return 0;
2582}
2583
2584static errno_t
2585ipsec_ctl_send(__unused kern_ctl_ref kctlref,
2586 __unused u_int32_t unit,
2587 __unused void *unitinfo,
2588 mbuf_t m,
2589 __unused int flags)
2590{
2591 /* Receive messages from the control socket. Currently unused. */
2592 mbuf_freem(m);
2593 return 0;
2594}
2595
2596static errno_t
2597ipsec_ctl_setopt(__unused kern_ctl_ref kctlref,
2598 __unused u_int32_t unit,
2599 void *unitinfo,
2600 int opt,
2601 void *data,
2602 size_t len)
2603{
2604 struct ipsec_pcb *pcb = unitinfo;
2605 errno_t result = 0;
2606
2607 /* check for privileges for privileged options */
2608 switch (opt) {
2609 case IPSEC_OPT_FLAGS:
2610 case IPSEC_OPT_EXT_IFDATA_STATS:
2611 case IPSEC_OPT_SET_DELEGATE_INTERFACE:
fe8ab488 2612 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS:
39236c6e
A		 2613 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
2614 return EPERM;
2615 }
2616 break;
2617 }
2618
2619 switch (opt) {
2620 case IPSEC_OPT_FLAGS:
5c9f4661 2621 if (len != sizeof(u_int32_t)) {
39236c6e 2622 result = EMSGSIZE;
5c9f4661 2623 } else {
39236c6e 2624 pcb->ipsec_flags = *(u_int32_t *)data;
5c9f4661 2625 }
39236c6e
A		 2626 break;
2627
2628 case IPSEC_OPT_EXT_IFDATA_STATS:
2629 if (len != sizeof(int)) {
2630 result = EMSGSIZE;
2631 break;
2632 }
5c9f4661
A		 2633 if (pcb->ipsec_ifp == NULL) {
2634 // Only can set after connecting
2635 result = EINVAL;
2636 break;
2637 }
39236c6e
A		 2638 pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
2639 break;
2640
2641 case IPSEC_OPT_INC_IFDATA_STATS_IN:
2642 case IPSEC_OPT_INC_IFDATA_STATS_OUT: {
2643 struct ipsec_stats_param *utsp = (struct ipsec_stats_param *)data;
2644
2645 if (utsp == NULL || len < sizeof(struct ipsec_stats_param)) {
2646 result = EINVAL;
2647 break;
2648 }
5c9f4661
A		 2649 if (pcb->ipsec_ifp == NULL) {
2650 // Only can set after connecting
2651 result = EINVAL;
2652 break;
2653 }
39236c6e
A		 2654 if (!pcb->ipsec_ext_ifdata_stats) {
2655 result = EINVAL;
2656 break;
2657 }
2658 if (opt == IPSEC_OPT_INC_IFDATA_STATS_IN)
2659 ifnet_stat_increment_in(pcb->ipsec_ifp, utsp->utsp_packets,
2660 utsp->utsp_bytes, utsp->utsp_errors);
2661 else
2662 ifnet_stat_increment_out(pcb->ipsec_ifp, utsp->utsp_packets,
2663 utsp->utsp_bytes, utsp->utsp_errors);
2664 break;
2665 }
2666
2667 case IPSEC_OPT_SET_DELEGATE_INTERFACE: {
2668 ifnet_t del_ifp = NULL;
2669 char name[IFNAMSIZ];
2670
2671 if (len > IFNAMSIZ - 1) {
2672 result = EMSGSIZE;
2673 break;
2674 }
5c9f4661
A		 2675 if (pcb->ipsec_ifp == NULL) {
2676 // Only can set after connecting
2677 result = EINVAL;
2678 break;
2679 }
39236c6e
A		 2680 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
2681 bcopy(data, name, len);
2682 name[len] = 0;
2683 result = ifnet_find_by_name(name, &del_ifp);
2684 }
2685 if (result == 0) {
39037602
A		 2686 printf("%s IPSEC_OPT_SET_DELEGATE_INTERFACE %s to %s\n",
2687 __func__, pcb->ipsec_ifp->if_xname,
2688 del_ifp->if_xname);
2689
39236c6e
A		 2690 result = ifnet_set_delegate(pcb->ipsec_ifp, del_ifp);
2691 if (del_ifp)
2692 ifnet_release(del_ifp);
2693 }
2694 break;
2695 }
2696
fe8ab488
A		 2697 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: {
2698 if (len != sizeof(int)) {
2699 result = EMSGSIZE;
2700 break;
2701 }
5c9f4661
A		 2702 if (pcb->ipsec_ifp == NULL) {
2703 // Only can set after connecting
2704 result = EINVAL;
2705 break;
2706 }
fe8ab488
A		 2707 mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data);
2708 if (output_service_class == MBUF_SC_UNSPEC) {
2709 pcb->ipsec_output_service_class = MBUF_SC_OAM;
2710 } else {
2711 pcb->ipsec_output_service_class = output_service_class;
2712 }
39037602
A		 2713 printf("%s IPSEC_OPT_OUTPUT_TRAFFIC_CLASS %s svc %d\n",
2714 __func__, pcb->ipsec_ifp->if_xname,
2715 pcb->ipsec_output_service_class);
fe8ab488
A		 2716 break;
2717 }
5ba3f43e
A		 2718
2719#if IPSEC_NEXUS
2720 case IPSEC_OPT_ENABLE_CHANNEL: {
2721 if (len != sizeof(int)) {
2722 result = EMSGSIZE;
2723 break;
2724 }
5c9f4661
A		 2725 if (pcb->ipsec_ifp == NULL) {
2726 // Only can set after connecting
2727 result = EINVAL;
2728 break;
2729 }
5ba3f43e
A		 2730 if (*(int *)data) {
2731 result = ipsec_enable_channel(pcb, current_proc());
2732 } else {
2733 result = ipsec_disable_channel(pcb);
2734 }
2735 break;
2736 }
2737
2738 case IPSEC_OPT_ENABLE_FLOWSWITCH: {
2739 if (len != sizeof(int)) {
2740 result = EMSGSIZE;
2741 break;
2742 }
5c9f4661
A		 2743 if (pcb->ipsec_ifp == NULL) {
2744 // Only can set after connecting
2745 result = EINVAL;
2746 break;
2747 }
a39ff7e2 2748 if (!if_is_netagent_enabled()) {
5ba3f43e
A		 2749 result = ENOTSUP;
2750 break;
2751 }
a39ff7e2
A		 2752 if (uuid_is_null(pcb->ipsec_nx.ms_agent)) {
2753 result = ENOENT;
2754 break;
2755 }
2756
5ba3f43e 2757 if (*(int *)data) {
5ba3f43e 2758 if_add_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent);
5ba3f43e 2759 } else {
5ba3f43e 2760 if_delete_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent);
5ba3f43e
A		 2761 }
2762 break;
2763 }
2764
2765 case IPSEC_OPT_INPUT_FRAG_SIZE: {
2766 if (len != sizeof(u_int32_t)) {
2767 result = EMSGSIZE;
2768 break;
2769 }
2770 u_int32_t input_frag_size = *(u_int32_t *)data;
2771 if (input_frag_size <= sizeof(struct ip6_hdr)) {
2772 pcb->ipsec_frag_size_set = FALSE;
2773 pcb->ipsec_input_frag_size = 0;
2774 } else {
2775 printf("SET FRAG SIZE TO %u\n", input_frag_size);
2776 pcb->ipsec_frag_size_set = TRUE;
2777 pcb->ipsec_input_frag_size = input_frag_size;
2778 }
2779 break;
2780 }
5c9f4661
A		 2781 case IPSEC_OPT_ENABLE_NETIF: {
2782 if (len != sizeof(int)) {
2783 result = EMSGSIZE;
2784 break;
2785 }
2786 if (pcb->ipsec_ifp != NULL) {
2787 // Only can set before connecting
2788 result = EINVAL;
2789 break;
2790 }
a39ff7e2
A		 2791 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2792 pcb->ipsec_use_netif = !!(*(int *)data);
2793 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
5c9f4661
A		 2794 break;
2795 }
2796 case IPSEC_OPT_SLOT_SIZE: {
2797 if (len != sizeof(u_int32_t)) {
2798 result = EMSGSIZE;
2799 break;
2800 }
2801 if (pcb->ipsec_ifp != NULL) {
2802 // Only can set before connecting
2803 result = EINVAL;
2804 break;
2805 }
2806 u_int32_t slot_size = *(u_int32_t *)data;
2807 if (slot_size < IPSEC_IF_MIN_SLOT_SIZE ||
2808 slot_size > IPSEC_IF_MAX_SLOT_SIZE) {
2809 return (EINVAL);
2810 }
2811 pcb->ipsec_slot_size = slot_size;
2812 break;
2813 }
2814 case IPSEC_OPT_NETIF_RING_SIZE: {
2815 if (len != sizeof(u_int32_t)) {
2816 result = EMSGSIZE;
2817 break;
2818 }
2819 if (pcb->ipsec_ifp != NULL) {
2820 // Only can set before connecting
2821 result = EINVAL;
2822 break;
2823 }
2824 u_int32_t ring_size = *(u_int32_t *)data;
2825 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
2826 ring_size > IPSEC_IF_MAX_RING_SIZE) {
2827 return (EINVAL);
2828 }
2829 pcb->ipsec_netif_ring_size = ring_size;
2830 break;
2831 }
2832 case IPSEC_OPT_TX_FSW_RING_SIZE: {
2833 if (len != sizeof(u_int32_t)) {
2834 result = EMSGSIZE;
2835 break;
2836 }
2837 if (pcb->ipsec_ifp != NULL) {
2838 // Only can set before connecting
2839 result = EINVAL;
2840 break;
2841 }
2842 u_int32_t ring_size = *(u_int32_t *)data;
2843 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
2844 ring_size > IPSEC_IF_MAX_RING_SIZE) {
2845 return (EINVAL);
2846 }
2847 pcb->ipsec_tx_fsw_ring_size = ring_size;
2848 break;
2849 }
2850 case IPSEC_OPT_RX_FSW_RING_SIZE: {
2851 if (len != sizeof(u_int32_t)) {
2852 result = EMSGSIZE;
2853 break;
2854 }
2855 if (pcb->ipsec_ifp != NULL) {
2856 // Only can set before connecting
2857 result = EINVAL;
2858 break;
2859 }
2860 u_int32_t ring_size = *(u_int32_t *)data;
2861 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
2862 ring_size > IPSEC_IF_MAX_RING_SIZE) {
2863 return (EINVAL);
2864 }
2865 pcb->ipsec_rx_fsw_ring_size = ring_size;
2866 break;
2867 }
2868
5ba3f43e 2869#endif // IPSEC_NEXUS
fe8ab488 2870
39236c6e
A		 2871 default:
2872 result = ENOPROTOOPT;
2873 break;
2874 }
2875
2876 return result;
2877}
2878
2879static errno_t
5ba3f43e
A		 2880ipsec_ctl_getopt(__unused kern_ctl_ref kctlref,
2881 __unused u_int32_t unit,
2882 void *unitinfo,
2883 int opt,
2884 void *data,
2885 size_t *len)
39236c6e 2886{
5ba3f43e
A		 2887 struct ipsec_pcb *pcb = unitinfo;
2888 errno_t result = 0;
39236c6e
A		 2889
2890 switch (opt) {
5ba3f43e
A		 2891 case IPSEC_OPT_FLAGS: {
2892 if (*len != sizeof(u_int32_t)) {
39236c6e 2893 result = EMSGSIZE;
5ba3f43e 2894 } else {
39236c6e 2895 *(u_int32_t *)data = pcb->ipsec_flags;
5ba3f43e 2896 }
39236c6e 2897 break;
5ba3f43e 2898 }
39236c6e 2899
5ba3f43e
A		 2900 case IPSEC_OPT_EXT_IFDATA_STATS: {
2901 if (*len != sizeof(int)) {
39236c6e 2902 result = EMSGSIZE;
5ba3f43e 2903 } else {
39236c6e 2904 *(int *)data = (pcb->ipsec_ext_ifdata_stats) ? 1 : 0;
5ba3f43e 2905 }
39236c6e 2906 break;
5ba3f43e 2907 }
39236c6e 2908
5ba3f43e
A		 2909 case IPSEC_OPT_IFNAME: {
2910 if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) {
2911 result = EMSGSIZE;
2912 } else {
5c9f4661
A		 2913 if (pcb->ipsec_ifp == NULL) {
2914 // Only can get after connecting
2915 result = EINVAL;
2916 break;
2917 }
5ba3f43e
A		 2918 *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1;
2919 }
39236c6e 2920 break;
5ba3f43e 2921 }
39236c6e 2922
fe8ab488
A		 2923 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: {
2924 if (*len != sizeof(int)) {
2925 result = EMSGSIZE;
5ba3f43e
A		 2926 } else {
2927 *(int *)data = so_svc2tc(pcb->ipsec_output_service_class);
fe8ab488 2928 }
fe8ab488
A		 2929 break;
2930 }
5ba3f43e
A		 2931
2932#if IPSEC_NEXUS
a39ff7e2
A		 2933
2934 case IPSEC_OPT_ENABLE_CHANNEL: {
2935 if (*len != sizeof(int)) {
2936 result = EMSGSIZE;
2937 } else {
2938 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
2939 *(int *)data = pcb->ipsec_kpipe_enabled;
2940 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
2941 }
2942 break;
2943 }
2944
2945 case IPSEC_OPT_ENABLE_FLOWSWITCH: {
2946 if (*len != sizeof(int)) {
2947 result = EMSGSIZE;
2948 } else {
2949 *(int *)data = if_check_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.ms_agent);
2950 }
2951 break;
2952 }
2953
2954 case IPSEC_OPT_ENABLE_NETIF: {
2955 if (*len != sizeof(int)) {
2956 result = EMSGSIZE;
2957 } else {
2958 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
2959 *(int *)data = !!pcb->ipsec_use_netif;
2960 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
2961 }
2962 break;
2963 }
2964
5ba3f43e
A		 2965 case IPSEC_OPT_GET_CHANNEL_UUID: {
2966 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
2967 if (uuid_is_null(pcb->ipsec_kpipe_uuid)) {
2968 result = ENXIO;
2969 } else if (*len != sizeof(uuid_t)) {
2970 result = EMSGSIZE;
2971 } else {
2972 uuid_copy(data, pcb->ipsec_kpipe_uuid);
2973 }
2974 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
2975 break;
2976 }
2977
2978 case IPSEC_OPT_INPUT_FRAG_SIZE: {
2979 if (*len != sizeof(u_int32_t)) {
2980 result = EMSGSIZE;
2981 } else {
2982 *(u_int32_t *)data = pcb->ipsec_input_frag_size;
2983 }
2984 break;
2985 }
5c9f4661
A		 2986 case IPSEC_OPT_SLOT_SIZE: {
2987 if (*len != sizeof(u_int32_t)) {
2988 result = EMSGSIZE;
2989 } else {
2990 *(u_int32_t *)data = pcb->ipsec_slot_size;
2991 }
2992 break;
2993 }
2994 case IPSEC_OPT_NETIF_RING_SIZE: {
2995 if (*len != sizeof(u_int32_t)) {
2996 result = EMSGSIZE;
2997 } else {
2998 *(u_int32_t *)data = pcb->ipsec_netif_ring_size;
2999 }
3000 break;
3001 }
3002 case IPSEC_OPT_TX_FSW_RING_SIZE: {
3003 if (*len != sizeof(u_int32_t)) {
3004 result = EMSGSIZE;
3005 } else {
3006 *(u_int32_t *)data = pcb->ipsec_tx_fsw_ring_size;
3007 }
3008 break;
3009 }
3010 case IPSEC_OPT_RX_FSW_RING_SIZE: {
3011 if (*len != sizeof(u_int32_t)) {
3012 result = EMSGSIZE;
3013 } else {
3014 *(u_int32_t *)data = pcb->ipsec_rx_fsw_ring_size;
3015 }
3016 break;
3017 }
3018
5ba3f43e
A		 3019#endif // IPSEC_NEXUS
3020
3021 default: {
39236c6e
A		 3022 result = ENOPROTOOPT;
3023 break;
5ba3f43e 3024 }
39236c6e
A		 3025 }
3026
3027 return result;
3028}
3029
3030/* Network Interface functions */
3031static errno_t
5ba3f43e
A		 3032ipsec_output(ifnet_t interface,
3033 mbuf_t data)
39236c6e 3034{
5ba3f43e 3035 struct ipsec_pcb *pcb = ifnet_softc(interface);
39236c6e
A		 3036 struct ipsec_output_state ipsec_state;
3037 struct route ro;
3038 struct route_in6 ro6;
3039 int length;
3040 struct ip *ip;
3041 struct ip6_hdr *ip6;
39236c6e
A		 3042 struct ip_out_args ipoa;
3043 struct ip6_out_args ip6oa;
3044 int error = 0;
3045 u_int ip_version = 0;
fe8ab488 3046 int flags = 0;
39236c6e
A		 3047 struct flowadv *adv = NULL;
3048
fe8ab488
A		 3049 // Make sure this packet isn't looping through the interface
3050 if (necp_get_last_interface_index_from_packet(data) == interface->if_index) {
5ba3f43e 3051 error = EINVAL;
fe8ab488
A		 3052 goto ipsec_output_err;
3053 }
3054
3055 // Mark the interface so NECP can evaluate tunnel policy
3056 necp_mark_packet_from_interface(data, interface);
39236c6e 3057
39236c6e
A		 3058 ip = mtod(data, struct ip *);
3059 ip_version = ip->ip_v;
fe8ab488 3060
39236c6e 3061 switch (ip_version) {
5ba3f43e 3062 case 4: {
5c9f4661
A		 3063#if IPSEC_NEXUS
3064 if (!pcb->ipsec_use_netif)
3065#endif // IPSEC_NEXUS
3066 {
3067 int af = AF_INET;
3068 bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
3069 }
3070
39236c6e 3071 /* Apply encryption */
5ba3f43e 3072 memset(&ipsec_state, 0, sizeof(ipsec_state));
39236c6e 3073 ipsec_state.m = data;
fe8ab488 3074 ipsec_state.dst = (struct sockaddr *)&ip->ip_dst;
5ba3f43e 3075 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
39236c6e 3076
3e170ce0
A		 3077 error = ipsec4_interface_output(&ipsec_state, interface);
3078 /* Tunneled in IPv6 - packet is gone */
3079 if (error == 0 && ipsec_state.tunneled == 6) {
3080 goto done;
3081 }
3082
39236c6e
A		 3083 data = ipsec_state.m;
3084 if (error || data == NULL) {
5ba3f43e
A		 3085 if (error) {
3086 printf("ipsec_output: ipsec4_output error %d.\n", error);
3087 }
39236c6e
A		 3088 goto ipsec_output_err;
3089 }
3090
fe8ab488
A		 3091 /* Set traffic class, set flow */
3092 m_set_service_class(data, pcb->ipsec_output_service_class);
39236c6e
A		 3093 data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
3094 data->m_pkthdr.pkt_flowid = interface->if_flowhash;
3095 data->m_pkthdr.pkt_proto = ip->ip_p;
3096 data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
3097
3098 /* Flip endian-ness for ip_output */
3099 ip = mtod(data, struct ip *);
3100 NTOHS(ip->ip_len);
3101 NTOHS(ip->ip_off);
3102
3103 /* Increment statistics */
3104 length = mbuf_pkthdr_len(data);
3105 ifnet_stat_increment_out(interface, 1, length, 0);
3106
3107 /* Send to ip_output */
5ba3f43e 3108 memset(&ro, 0, sizeof(ro));
39236c6e 3109
5ba3f43e
A		 3110 flags = (IP_OUTARGS | /* Passing out args to specify interface */
3111 IP_NOIPSEC); /* To ensure the packet doesn't go through ipsec twice */
39236c6e 3112
5ba3f43e 3113 memset(&ipoa, 0, sizeof(ipoa));
39236c6e
A		 3114 ipoa.ipoa_flowadv.code = 0;
3115 ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR;
fe8ab488
A		 3116 if (ipsec_state.outgoing_if) {
3117 ipoa.ipoa_boundif = ipsec_state.outgoing_if;
39236c6e
A		 3118 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
3119 }
39037602 3120 ipsec_set_ipoa_for_interface(pcb->ipsec_ifp, &ipoa);
39236c6e
A		 3121
3122 adv = &ipoa.ipoa_flowadv;
3123
5ba3f43e 3124 (void)ip_output(data, NULL, &ro, flags, NULL, &ipoa);
39236c6e
A		 3125 data = NULL;
3126
3127 if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) {
3128 error = ENOBUFS;
3129 ifnet_disable_output(interface);
3130 }
3131
3132 goto done;
5ba3f43e
A		 3133 }
3134 case 6: {
5c9f4661
A		 3135#if IPSEC_NEXUS
3136 if (!pcb->ipsec_use_netif)
3137#endif // IPSEC_NEXUS
3138 {
3139 int af = AF_INET6;
3140 bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
3141 }
3142
fe8ab488 3143 data = ipsec6_splithdr(data);
3e170ce0
A		 3144 if (data == NULL) {
3145 printf("ipsec_output: ipsec6_splithdr returned NULL\n");
3146 goto ipsec_output_err;
3147 }
3148
39236c6e 3149 ip6 = mtod(data, struct ip6_hdr *);
fe8ab488 3150
5ba3f43e 3151 memset(&ipsec_state, 0, sizeof(ipsec_state));
fe8ab488
A		 3152 ipsec_state.m = data;
3153 ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst;
5ba3f43e 3154 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
39236c6e 3155
fe8ab488 3156 error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m);
5ba3f43e 3157 if (error == 0 && ipsec_state.tunneled == 4) { /* tunneled in IPv4 - packet is gone */
fe8ab488 3158 goto done;
5ba3f43e 3159 }
39236c6e
A		 3160 data = ipsec_state.m;
3161 if (error || data == NULL) {
5ba3f43e
A		 3162 if (error) {
3163 printf("ipsec_output: ipsec6_output error %d\n", error);
3164 }
39236c6e
A		 3165 goto ipsec_output_err;
3166 }
3167
fe8ab488
A		 3168 /* Set traffic class, set flow */
3169 m_set_service_class(data, pcb->ipsec_output_service_class);
39236c6e
A		 3170 data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
3171 data->m_pkthdr.pkt_flowid = interface->if_flowhash;
fe8ab488 3172 data->m_pkthdr.pkt_proto = ip6->ip6_nxt;
39236c6e
A		 3173 data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
3174
3175 /* Increment statistics */
3176 length = mbuf_pkthdr_len(data);
3177 ifnet_stat_increment_out(interface, 1, length, 0);
3178
3179 /* Send to ip6_output */
5ba3f43e 3180 memset(&ro6, 0, sizeof(ro6));
39236c6e
A		 3181
3182 flags = IPV6_OUTARGS;
3183
5ba3f43e 3184 memset(&ip6oa, 0, sizeof(ip6oa));
39236c6e 3185 ip6oa.ip6oa_flowadv.code = 0;
39037602 3186 ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR;
fe8ab488
A		 3187 if (ipsec_state.outgoing_if) {
3188 ip6oa.ip6oa_boundif = ipsec_state.outgoing_if;
39037602 3189 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
39236c6e 3190 }
39037602 3191 ipsec_set_ip6oa_for_interface(pcb->ipsec_ifp, &ip6oa);
39236c6e
A		 3192
3193 adv = &ip6oa.ip6oa_flowadv;
3194
3195 (void) ip6_output(data, NULL, &ro6, flags, NULL, NULL, &ip6oa);
3196 data = NULL;
3197
3198 if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) {
3199 error = ENOBUFS;
3200 ifnet_disable_output(interface);
3201 }
3202
3203 goto done;
5ba3f43e
A		 3204 }
3205 default: {
39236c6e 3206 printf("ipsec_output: Received unknown packet version %d.\n", ip_version);
5ba3f43e 3207 error = EINVAL;
39236c6e 3208 goto ipsec_output_err;
5ba3f43e 3209 }
39236c6e
A		 3210 }
3211
3212done:
39236c6e
A		 3213 return error;
3214
3215ipsec_output_err:
3216 if (data)
3217 mbuf_freem(data);
3218 goto done;
3219}
3220
3221static void
3222ipsec_start(ifnet_t interface)
3223{
fe8ab488 3224 mbuf_t data;
5ba3f43e 3225 struct ipsec_pcb *pcb = ifnet_softc(interface);
fe8ab488 3226
5ba3f43e 3227 VERIFY(pcb != NULL);
fe8ab488
A		 3228 for (;;) {
3229 if (ifnet_dequeue(interface, &data) != 0)
3230 break;
3231 if (ipsec_output(interface, data) != 0)
3232 break;
3233 }
39236c6e
A		 3234}
3235
3236/* Network Interface functions */
3237static errno_t
3238ipsec_demux(__unused ifnet_t interface,
3239 mbuf_t data,
3240 __unused char *frame_header,
3241 protocol_family_t *protocol)
3242{
3243 struct ip *ip;
3244 u_int ip_version;
3245
3246 while (data != NULL && mbuf_len(data) < 1) {
3247 data = mbuf_next(data);
3248 }
3249
3250 if (data == NULL)
3251 return ENOENT;
3252
3253 ip = mtod(data, struct ip *);
3254 ip_version = ip->ip_v;
3255
3256 switch(ip_version) {
3257 case 4:
3258 *protocol = PF_INET;
3259 return 0;
3260 case 6:
3261 *protocol = PF_INET6;
3262 return 0;
3263 default:
3264 break;
3265 }
3266
3267 return 0;
3268}
3269
3270static errno_t
3271ipsec_add_proto(__unused ifnet_t interface,
3272 protocol_family_t protocol,
3273 __unused const struct ifnet_demux_desc *demux_array,
3274 __unused u_int32_t demux_count)
3275{
3276 switch(protocol) {
3277 case PF_INET:
3278 return 0;
3279 case PF_INET6:
3280 return 0;
3281 default:
3282 break;
3283 }
3284
3285 return ENOPROTOOPT;
3286}
3287
3288static errno_t
3289ipsec_del_proto(__unused ifnet_t interface,
3290 __unused protocol_family_t protocol)
3291{
3292 return 0;
3293}
3294
3295static errno_t
5ba3f43e
A		 3296ipsec_ioctl(ifnet_t interface,
3297 u_long command,
3298 void *data)
39236c6e
A		 3299{
3300 errno_t result = 0;
3301
3302 switch(command) {
5c9f4661 3303 case SIOCSIFMTU: {
5ba3f43e 3304#if IPSEC_NEXUS
a39ff7e2 3305 struct ipsec_pcb *pcb = ifnet_softc(interface);
5c9f4661
A		 3306 if (pcb->ipsec_use_netif) {
3307 // Make sure we can fit packets in the channel buffers
3308 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > pcb->ipsec_slot_size) {
3309 result = EINVAL;
3310 } else {
3311 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
3312 }
3313 } else
5ba3f43e 3314#endif // IPSEC_NEXUS
5c9f4661
A		 3315 {
3316 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
3317 }
39236c6e 3318 break;
5c9f4661 3319 }
39236c6e
A		 3320
3321 case SIOCSIFFLAGS:
3322 /* ifioctl() takes care of it */
3323 break;
3324
3325 default:
3326 result = EOPNOTSUPP;
3327 }
3328
3329 return result;
3330}
3331
3332static void
5ba3f43e 3333ipsec_detached(ifnet_t interface)
39236c6e 3334{
5ba3f43e
A		 3335 struct ipsec_pcb *pcb = ifnet_softc(interface);
3336 (void)ifnet_release(interface);
5c9f4661 3337 ipsec_free_pcb(pcb, true);
39236c6e
A		 3338}
3339
3340/* Protocol Handlers */
3341
3342static errno_t
fe8ab488 3343ipsec_proto_input(ifnet_t interface,
39236c6e 3344 protocol_family_t protocol,
fe8ab488
A		 3345 mbuf_t m,
3346 __unused char *frame_header)
39236c6e 3347{
fe8ab488 3348 mbuf_pkthdr_setrcvif(m, interface);
5c9f4661
A		 3349
3350#if IPSEC_NEXUS
3351 struct ipsec_pcb *pcb = ifnet_softc(interface);
3352 if (!pcb->ipsec_use_netif)
3353#endif // IPSEC_NEXUS
3354 {
3355 uint32_t af = 0;
3356 struct ip *ip = mtod(m, struct ip *);
3357 if (ip->ip_v == 4) {
3358 af = AF_INET;
3359 } else if (ip->ip_v == 6) {
3360 af = AF_INET6;
3361 }
3362 bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af));
a39ff7e2 3363 pktap_input(interface, protocol, m, NULL);
5c9f4661 3364 }
39037602 3365
a39ff7e2 3366 int32_t pktlen = m->m_pkthdr.len;
3e170ce0
A		 3367 if (proto_input(protocol, m) != 0) {
3368 ifnet_stat_increment_in(interface, 0, 0, 1);
39236c6e 3369 m_freem(m);
3e170ce0 3370 } else {
a39ff7e2 3371 ifnet_stat_increment_in(interface, 1, pktlen, 0);
3e170ce0 3372 }
39236c6e
A		 3373
3374 return 0;
3375}
3376
3377static errno_t
3378ipsec_proto_pre_output(__unused ifnet_t interface,
3379 protocol_family_t protocol,
3380 __unused mbuf_t *packet,
3381 __unused const struct sockaddr *dest,
3382 __unused void *route,
3383 __unused char *frame_type,
3384 __unused char *link_layer_dest)
3385{
3386
3387 *(protocol_family_t *)(void *)frame_type = protocol;
3388 return 0;
3389}
3390
3391static errno_t
3392ipsec_attach_proto(ifnet_t interface,
3393 protocol_family_t protocol)
3394{
3395 struct ifnet_attach_proto_param proto;
3396 errno_t result;
3397
3398 bzero(&proto, sizeof(proto));
3399 proto.input = ipsec_proto_input;
3400 proto.pre_output = ipsec_proto_pre_output;
3401
3402 result = ifnet_attach_protocol(interface, protocol, &proto);
3403 if (result != 0 && result != EEXIST) {
3404 printf("ipsec_attach_inet - ifnet_attach_protocol %d failed: %d\n",
3405 protocol, result);
3406 }
3407
3408 return result;
3409}
fe8ab488 3410
5ba3f43e
A		 3411errno_t
3412ipsec_inject_inbound_packet(ifnet_t interface,
3413 mbuf_t packet)
3414{
a39ff7e2 3415#if IPSEC_NEXUS
5ba3f43e
A		 3416 struct ipsec_pcb *pcb = ifnet_softc(interface);
3417
5c9f4661
A		 3418 if (pcb->ipsec_use_netif) {
3419 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 3420
5c9f4661
A		 3421 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
3422 if (pcb->ipsec_input_chain != NULL) {
3423 pcb->ipsec_input_chain_last->m_nextpkt = packet;
3424 } else {
3425 pcb->ipsec_input_chain = packet;
3426 }
3427 while (packet->m_nextpkt) {
3428 VERIFY(packet != packet->m_nextpkt);
3429 packet = packet->m_nextpkt;
3430 }
3431 pcb->ipsec_input_chain_last = packet;
3432 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
5ba3f43e 3433
5c9f4661
A		 3434 kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring;
3435 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 3436
5c9f4661
A		 3437 if (rx_ring != NULL) {
3438 kern_channel_notify(rx_ring, 0);
3439 }
5ba3f43e 3440
5c9f4661
A		 3441 return (0);
3442 } else
3443#endif // IPSEC_NEXUS
3444 {
3445 errno_t error;
3446 protocol_family_t protocol;
3447 if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) {
3448 return error;
3449 }
5ba3f43e 3450
5c9f4661
A		 3451 return ipsec_proto_input(interface, protocol, packet, NULL);
3452 }
fe8ab488
A		 3453}
3454
3455void
3456ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family)
3457{
3458 if (packet != NULL && interface != NULL) {
3459 struct ipsec_pcb *pcb = ifnet_softc(interface);
3460 if (pcb != NULL) {
3461 /* Set traffic class, set flow */
3462 m_set_service_class(packet, pcb->ipsec_output_service_class);
3463 packet->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
3464 packet->m_pkthdr.pkt_flowid = interface->if_flowhash;
3465 if (family == AF_INET) {
3466 struct ip *ip = mtod(packet, struct ip *);
3467 packet->m_pkthdr.pkt_proto = ip->ip_p;
3e170ce0 3468 } else if (family == AF_INET6) {
fe8ab488
A		 3469 struct ip6_hdr *ip6 = mtod(packet, struct ip6_hdr *);
3470 packet->m_pkthdr.pkt_proto = ip6->ip6_nxt;
3471 }
3472 packet->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
3473 }
3474 }
3475}
39037602
A		 3476
3477void
3478ipsec_set_ipoa_for_interface(ifnet_t interface, struct ip_out_args *ipoa)
3479{
3480 struct ipsec_pcb *pcb;
3481
3482 if (interface == NULL || ipoa == NULL)
3483 return;
3484 pcb = ifnet_softc(interface);
3485
3486 if (net_qos_policy_restricted == 0) {
3487 ipoa->ipoa_flags |= IPOAF_QOSMARKING_ALLOWED;
3488 ipoa->ipoa_sotc = so_svc2tc(pcb->ipsec_output_service_class);
3489 } else if (pcb->ipsec_output_service_class != MBUF_SC_VO ||
3490 net_qos_policy_restrict_avapps != 0) {
3491 ipoa->ipoa_flags &= ~IPOAF_QOSMARKING_ALLOWED;
3492 } else {
3493 ipoa->ipoa_flags |= IP6OAF_QOSMARKING_ALLOWED;
3494 ipoa->ipoa_sotc = SO_TC_VO;
3495 }
3496}
3497
3498void
3499ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa)
3500{
3501 struct ipsec_pcb *pcb;
3502
3503 if (interface == NULL || ip6oa == NULL)
3504 return;
3505 pcb = ifnet_softc(interface);
3506
3507 if (net_qos_policy_restricted == 0) {
3508 ip6oa->ip6oa_flags |= IPOAF_QOSMARKING_ALLOWED;
3509 ip6oa->ip6oa_sotc = so_svc2tc(pcb->ipsec_output_service_class);
3510 } else if (pcb->ipsec_output_service_class != MBUF_SC_VO ||
3511 net_qos_policy_restrict_avapps != 0) {
3512 ip6oa->ip6oa_flags &= ~IPOAF_QOSMARKING_ALLOWED;
3513 } else {
3514 ip6oa->ip6oa_flags |= IP6OAF_QOSMARKING_ALLOWED;
3515 ip6oa->ip6oa_sotc = SO_TC_VO;
3516 }
3517}
mirror of XNU