]> git.saurik.com Git - apple/xnu.git/blame - bsd/net/if_ipsec.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.50.7.100.1.tar.gz
[apple/xnu.git] / bsd / net / if_ipsec.c
CommitLineData
39236c6e 1/*
f427ee49 2 * Copyright (c) 2012-2020 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
d9a64523 29
39236c6e
A		 30#include <sys/systm.h>
31#include <sys/kern_control.h>
32#include <net/kpi_protocol.h>
33#include <net/kpi_interface.h>
34#include <sys/socket.h>
fe8ab488 35#include <sys/socketvar.h>
39236c6e
A		 36#include <net/if.h>
37#include <net/if_types.h>
38#include <net/bpf.h>
39#include <net/if_ipsec.h>
39236c6e
A		 40#include <sys/mbuf.h>
41#include <sys/sockio.h>
42#include <netinet/in.h>
43#include <netinet/ip6.h>
44#include <netinet6/in6_var.h>
45#include <netinet6/ip6_var.h>
46#include <sys/kauth.h>
47#include <netinet6/ipsec.h>
48#include <netinet6/ipsec6.h>
5ba3f43e
A		 49#include <netinet6/esp.h>
50#include <netinet6/esp6.h>
39236c6e
A		 51#include <netinet/ip.h>
52#include <net/flowadv.h>
fe8ab488 53#include <net/necp.h>
39037602
A		 54#include <netkey/key.h>
55#include <net/pktap.h>
5ba3f43e 56#include <kern/zalloc.h>
cb323159 57#include <os/log.h>
5ba3f43e
A		 58
59#define IPSEC_NEXUS 0
39037602
A		 60
61extern int net_qos_policy_restricted;
62extern int net_qos_policy_restrict_avapps;
39236c6e
A		 63
64/* Kernel Control functions */
f427ee49 65static errno_t ipsec_ctl_setup(u_int32_t *unit, void **unitinfo);
0a7de745
A		 66static errno_t ipsec_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
67 void **unitinfo);
68static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
69 void **unitinfo);
70static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
71 void *unitinfo);
72static errno_t ipsec_ctl_send(kern_ctl_ref kctlref, u_int32_t unit,
73 void *unitinfo, mbuf_t m, int flags);
74static errno_t ipsec_ctl_getopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
75 int opt, void *data, size_t *len);
76static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
77 int opt, void *data, size_t len);
39236c6e
A		 78
79/* Network Interface functions */
0a7de745
A		 80static void ipsec_start(ifnet_t interface);
81static errno_t ipsec_output(ifnet_t interface, mbuf_t data);
82static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header,
83 protocol_family_t *protocol);
84static errno_t ipsec_add_proto(ifnet_t interface, protocol_family_t protocol,
85 const struct ifnet_demux_desc *demux_array,
86 u_int32_t demux_count);
87static errno_t ipsec_del_proto(ifnet_t interface, protocol_family_t protocol);
88static errno_t ipsec_ioctl(ifnet_t interface, u_long cmd, void *data);
89static void ipsec_detached(ifnet_t interface);
39236c6e
A		 90
91/* Protocol handlers */
0a7de745
A		 92static errno_t ipsec_attach_proto(ifnet_t interface, protocol_family_t proto);
93static errno_t ipsec_proto_input(ifnet_t interface, protocol_family_t protocol,
94 mbuf_t m, char *frame_header);
39236c6e 95static errno_t ipsec_proto_pre_output(ifnet_t interface, protocol_family_t protocol,
0a7de745
A		 96 mbuf_t *packet, const struct sockaddr *dest, void *route,
97 char *frame_type, char *link_layer_dest);
39236c6e 98
0a7de745 99static kern_ctl_ref ipsec_kctlref;
5ba3f43e
A		 100static lck_attr_t *ipsec_lck_attr;
101static lck_grp_attr_t *ipsec_lck_grp_attr;
102static lck_grp_t *ipsec_lck_grp;
103static lck_mtx_t ipsec_lock;
104
105#if IPSEC_NEXUS
106
107SYSCTL_DECL(_net_ipsec);
108SYSCTL_NODE(_net, OID_AUTO, ipsec, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IPsec");
109static int if_ipsec_verify_interface_creation = 0;
110SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG_LOCKED, &if_ipsec_verify_interface_creation, 0, "");
111
0a7de745 112#define IPSEC_IF_VERIFY(_e) if (__improbable(if_ipsec_verify_interface_creation)) { VERIFY(_e); }
5ba3f43e 113
5c9f4661 114#define IPSEC_IF_DEFAULT_SLOT_SIZE 2048
5ba3f43e
A		 115#define IPSEC_IF_DEFAULT_RING_SIZE 64
116#define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64
117#define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128
0a7de745 118#define IPSEC_IF_DEFAULT_BUF_SEG_SIZE skmem_usr_buf_seg_size
5ba3f43e 119
cb323159
A		 120#define IPSEC_IF_WMM_RING_COUNT NEXUS_NUM_WMM_QUEUES
121#define IPSEC_IF_MAX_RING_COUNT IPSEC_IF_WMM_RING_COUNT
122#define IPSEC_NETIF_WMM_TX_RING_COUNT IPSEC_IF_WMM_RING_COUNT
123#define IPSEC_NETIF_WMM_RX_RING_COUNT 1
124#define IPSEC_NETIF_MAX_TX_RING_COUNT IPSEC_NETIF_WMM_TX_RING_COUNT
125#define IPSEC_NETIF_MAX_RX_RING_COUNT IPSEC_NETIF_WMM_RX_RING_COUNT
126
127#define IPSEC_IF_MIN_RING_SIZE 8
5ba3f43e
A		 128#define IPSEC_IF_MAX_RING_SIZE 1024
129
5c9f4661
A		 130#define IPSEC_IF_MIN_SLOT_SIZE 1024
131#define IPSEC_IF_MAX_SLOT_SIZE 4096
132
cb323159
A		 133#define IPSEC_DEFAULT_MAX_PENDING_INPUT_COUNT 512
134
135static int if_ipsec_max_pending_input = IPSEC_DEFAULT_MAX_PENDING_INPUT_COUNT;
136
5ba3f43e
A		 137static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS;
138static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
139static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
140
141static int if_ipsec_ring_size = IPSEC_IF_DEFAULT_RING_SIZE;
142static int if_ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE;
143static int if_ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE;
144
cb323159 145SYSCTL_INT(_net_ipsec, OID_AUTO, max_pending_input, CTLFLAG_LOCKED | CTLFLAG_RW, &if_ipsec_max_pending_input, 0, "");
5ba3f43e 146SYSCTL_PROC(_net_ipsec, OID_AUTO, ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 147 &if_ipsec_ring_size, IPSEC_IF_DEFAULT_RING_SIZE, &sysctl_if_ipsec_ring_size, "I", "");
5ba3f43e 148SYSCTL_PROC(_net_ipsec, OID_AUTO, tx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 149 &if_ipsec_tx_fsw_ring_size, IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE, &sysctl_if_ipsec_tx_fsw_ring_size, "I", "");
5ba3f43e 150SYSCTL_PROC(_net_ipsec, OID_AUTO, rx_fsw_ring_size, CTLTYPE_INT | CTLFLAG_LOCKED | CTLFLAG_RW,
0a7de745 151 &if_ipsec_rx_fsw_ring_size, IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE, &sysctl_if_ipsec_rx_fsw_ring_size, "I", "");
5ba3f43e 152
cb323159
A		 153static int if_ipsec_debug = 0;
154SYSCTL_INT(_net_ipsec, OID_AUTO, debug, CTLFLAG_LOCKED | CTLFLAG_RW, &if_ipsec_debug, 0, "");
155
5ba3f43e
A		 156static errno_t
157ipsec_register_nexus(void);
158
159typedef struct ipsec_nx {
160 uuid_t if_provider;
161 uuid_t if_instance;
cb323159
A		 162 uuid_t fsw_provider;
163 uuid_t fsw_instance;
164 uuid_t fsw_device;
165 uuid_t fsw_host;
166 uuid_t fsw_agent;
5ba3f43e
A		 167} *ipsec_nx_t;
168
169static nexus_controller_t ipsec_ncd;
170static int ipsec_ncd_refcount;
171static uuid_t ipsec_kpipe_uuid;
172
173#endif // IPSEC_NEXUS
174
175/* Control block allocated for each kernel control connection */
176struct ipsec_pcb {
0a7de745
A		 177 TAILQ_ENTRY(ipsec_pcb) ipsec_chain;
178 kern_ctl_ref ipsec_ctlref;
cb323159
A		 179 ifnet_t ipsec_ifp;
180 u_int32_t ipsec_unit;
181 u_int32_t ipsec_unique_id;
182 // These external flags can be set with IPSEC_OPT_FLAGS
183 u_int32_t ipsec_external_flags;
184 // These internal flags are only used within this driver
185 u_int32_t ipsec_internal_flags;
186 u_int32_t ipsec_input_frag_size;
187 bool ipsec_frag_size_set;
188 int ipsec_ext_ifdata_stats;
0a7de745 189 mbuf_svc_class_t ipsec_output_service_class;
cb323159
A		 190 char ipsec_if_xname[IFXNAMSIZ];
191 char ipsec_unique_name[IFXNAMSIZ];
192 // PCB lock protects state fields, like ipsec_kpipe_count
5ba3f43e 193 decl_lck_rw_data(, ipsec_pcb_lock);
cb323159
A		 194 // lock to protect ipsec_pcb_data_move & ipsec_pcb_drainers
195 decl_lck_mtx_data(, ipsec_pcb_data_move_lock);
196 u_int32_t ipsec_pcb_data_move; /* number of data moving contexts */
197 u_int32_t ipsec_pcb_drainers; /* number of threads waiting to drain */
198 u_int32_t ipsec_pcb_data_path_state; /* internal state of interface data path */
5ba3f43e
A		 199
200#if IPSEC_NEXUS
cb323159
A		 201 lck_mtx_t ipsec_input_chain_lock;
202 lck_mtx_t ipsec_kpipe_encrypt_lock;
203 lck_mtx_t ipsec_kpipe_decrypt_lock;
0a7de745
A		 204 struct mbuf * ipsec_input_chain;
205 struct mbuf * ipsec_input_chain_last;
cb323159 206 u_int32_t ipsec_input_chain_count;
5ba3f43e
A		 207 // Input chain lock protects the list of input mbufs
208 // The input chain lock must be taken AFTER the PCB lock if both are held
0a7de745 209 struct ipsec_nx ipsec_nx;
cb323159
A		 210 u_int32_t ipsec_kpipe_count;
211 pid_t ipsec_kpipe_pid;
212 uuid_t ipsec_kpipe_uuid[IPSEC_IF_MAX_RING_COUNT];
213 void * ipsec_kpipe_rxring[IPSEC_IF_MAX_RING_COUNT];
214 void * ipsec_kpipe_txring[IPSEC_IF_MAX_RING_COUNT];
215 kern_pbufpool_t ipsec_kpipe_pp;
216 u_int32_t ipsec_kpipe_tx_ring_size;
217 u_int32_t ipsec_kpipe_rx_ring_size;
0a7de745
A		 218
219 kern_nexus_t ipsec_netif_nexus;
cb323159
A		 220 kern_pbufpool_t ipsec_netif_pp;
221 void * ipsec_netif_rxring[IPSEC_NETIF_MAX_RX_RING_COUNT];
222 void * ipsec_netif_txring[IPSEC_NETIF_MAX_TX_RING_COUNT];
223 uint64_t ipsec_netif_txring_size;
224
225 u_int32_t ipsec_slot_size;
226 u_int32_t ipsec_netif_ring_size;
227 u_int32_t ipsec_tx_fsw_ring_size;
228 u_int32_t ipsec_rx_fsw_ring_size;
229 bool ipsec_use_netif;
230 bool ipsec_needs_netagent;
5ba3f43e
A		 231#endif // IPSEC_NEXUS
232};
233
cb323159
A		 234/* These are internal flags not exposed outside this file */
235#define IPSEC_FLAGS_KPIPE_ALLOCATED 1
236
237/* data movement refcounting functions */
cb323159
A		 238static void ipsec_wait_data_move_drain(struct ipsec_pcb *pcb);
239
240/* Data path states */
241#define IPSEC_PCB_DATA_PATH_READY 0x1
242
243/* Macros to set/clear/test data path states */
244#define IPSEC_SET_DATA_PATH_READY(_pcb) ((_pcb)->ipsec_pcb_data_path_state |= IPSEC_PCB_DATA_PATH_READY)
245#define IPSEC_CLR_DATA_PATH_READY(_pcb) ((_pcb)->ipsec_pcb_data_path_state &= ~IPSEC_PCB_DATA_PATH_READY)
246#define IPSEC_IS_DATA_PATH_READY(_pcb) (((_pcb)->ipsec_pcb_data_path_state & IPSEC_PCB_DATA_PATH_READY) != 0)
247
248#if IPSEC_NEXUS
249/* Macros to clear/set/test flags. */
250static inline void
251ipsec_flag_set(struct ipsec_pcb *pcb, uint32_t flag)
252{
253 pcb->ipsec_internal_flags |= flag;
254}
255static inline void
256ipsec_flag_clr(struct ipsec_pcb *pcb, uint32_t flag)
257{
258 pcb->ipsec_internal_flags &= ~flag;
259}
260
261static inline bool
262ipsec_flag_isset(struct ipsec_pcb *pcb, uint32_t flag)
263{
264 return !!(pcb->ipsec_internal_flags & flag);
265}
266#endif // IPSEC_NEXUS
267
5ba3f43e
A		 268TAILQ_HEAD(ipsec_list, ipsec_pcb) ipsec_head;
269
f427ee49
A		 270static ZONE_DECLARE(ipsec_pcb_zone, "net.if_ipsec",
271 sizeof(struct ipsec_pcb), ZC_ZFREE_CLEARMEM);
5ba3f43e
A		 272
273#define IPSECQ_MAXLEN 256
274
275#if IPSEC_NEXUS
276static int
277sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS
278{
279#pragma unused(arg1, arg2)
280 int value = if_ipsec_ring_size;
281
282 int error = sysctl_handle_int(oidp, &value, 0, req);
283 if (error || !req->newptr) {
0a7de745 284 return error;
5ba3f43e
A		 285 }
286
287 if (value < IPSEC_IF_MIN_RING_SIZE ||
0a7de745
A		 288 value > IPSEC_IF_MAX_RING_SIZE) {
289 return EINVAL;
5ba3f43e
A		 290 }
291
292 if_ipsec_ring_size = value;
293
0a7de745 294 return 0;
5ba3f43e
A		 295}
296
297static int
298sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS
299{
300#pragma unused(arg1, arg2)
301 int value = if_ipsec_tx_fsw_ring_size;
302
303 int error = sysctl_handle_int(oidp, &value, 0, req);
304 if (error || !req->newptr) {
0a7de745 305 return error;
5ba3f43e
A		 306 }
307
308 if (value < IPSEC_IF_MIN_RING_SIZE ||
0a7de745
A		 309 value > IPSEC_IF_MAX_RING_SIZE) {
310 return EINVAL;
5ba3f43e
A		 311 }
312
313 if_ipsec_tx_fsw_ring_size = value;
314
0a7de745 315 return 0;
5ba3f43e
A		 316}
317
318static int
319sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS
320{
321#pragma unused(arg1, arg2)
322 int value = if_ipsec_rx_fsw_ring_size;
323
324 int error = sysctl_handle_int(oidp, &value, 0, req);
325 if (error || !req->newptr) {
0a7de745 326 return error;
5ba3f43e
A		 327 }
328
329 if (value < IPSEC_IF_MIN_RING_SIZE ||
0a7de745
A		 330 value > IPSEC_IF_MAX_RING_SIZE) {
331 return EINVAL;
5ba3f43e
A		 332 }
333
334 if_ipsec_rx_fsw_ring_size = value;
335
0a7de745 336 return 0;
5ba3f43e 337}
cb323159
A		 338
339
340static inline bool
341ipsec_in_wmm_mode(struct ipsec_pcb *pcb)
342{
343 return pcb->ipsec_kpipe_count == IPSEC_IF_WMM_RING_COUNT;
344}
345
5ba3f43e
A		 346#endif // IPSEC_NEXUS
347
348errno_t
349ipsec_register_control(void)
350{
0a7de745 351 struct kern_ctl_reg kern_ctl;
f427ee49 352 errno_t result = 0;
5ba3f43e
A		 353
354#if IPSEC_NEXUS
355 ipsec_register_nexus();
356#endif // IPSEC_NEXUS
357
358 TAILQ_INIT(&ipsec_head);
0a7de745 359
5ba3f43e
A		 360 bzero(&kern_ctl, sizeof(kern_ctl));
361 strlcpy(kern_ctl.ctl_name, IPSEC_CONTROL_NAME, sizeof(kern_ctl.ctl_name));
362 kern_ctl.ctl_name[sizeof(kern_ctl.ctl_name) - 1] = 0;
f427ee49 363 kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_SETUP; /* Require root */
5ba3f43e
A		 364 kern_ctl.ctl_sendsize = 64 * 1024;
365 kern_ctl.ctl_recvsize = 64 * 1024;
f427ee49 366 kern_ctl.ctl_setup = ipsec_ctl_setup;
5c9f4661 367 kern_ctl.ctl_bind = ipsec_ctl_bind;
5ba3f43e
A		 368 kern_ctl.ctl_connect = ipsec_ctl_connect;
369 kern_ctl.ctl_disconnect = ipsec_ctl_disconnect;
370 kern_ctl.ctl_send = ipsec_ctl_send;
371 kern_ctl.ctl_setopt = ipsec_ctl_setopt;
372 kern_ctl.ctl_getopt = ipsec_ctl_getopt;
0a7de745 373
5ba3f43e
A		 374 result = ctl_register(&kern_ctl, &ipsec_kctlref);
375 if (result != 0) {
cb323159 376 os_log_error(OS_LOG_DEFAULT, "ipsec_register_control - ctl_register failed: %d\n", result);
5ba3f43e
A		 377 return result;
378 }
0a7de745 379
5ba3f43e 380 /* Register the protocol plumbers */
cb323159 381 if ((result = proto_register_plumber(PF_INET, IFNET_FAMILY_IPSEC,
0a7de745 382 ipsec_attach_proto, NULL)) != 0) {
cb323159
A		 383 os_log_error(OS_LOG_DEFAULT, "ipsec_register_control - proto_register_plumber(PF_INET, IFNET_FAMILY_IPSEC) failed: %d\n",
384 result);
5ba3f43e
A		 385 ctl_deregister(ipsec_kctlref);
386 return result;
387 }
0a7de745 388
5ba3f43e 389 /* Register the protocol plumbers */
cb323159 390 if ((result = proto_register_plumber(PF_INET6, IFNET_FAMILY_IPSEC,
0a7de745 391 ipsec_attach_proto, NULL)) != 0) {
cb323159 392 proto_unregister_plumber(PF_INET, IFNET_FAMILY_IPSEC);
5ba3f43e 393 ctl_deregister(ipsec_kctlref);
cb323159
A		 394 os_log_error(OS_LOG_DEFAULT, "ipsec_register_control - proto_register_plumber(PF_INET6, IFNET_FAMILY_IPSEC) failed: %d\n",
395 result);
5ba3f43e
A		 396 return result;
397 }
398
399 ipsec_lck_attr = lck_attr_alloc_init();
400 ipsec_lck_grp_attr = lck_grp_attr_alloc_init();
401 ipsec_lck_grp = lck_grp_alloc_init("ipsec", ipsec_lck_grp_attr);
402 lck_mtx_init(&ipsec_lock, ipsec_lck_grp, ipsec_lck_attr);
0a7de745 403
5ba3f43e
A		 404 return 0;
405}
406
407/* Helpers */
408int
0a7de745 409ipsec_interface_isvalid(ifnet_t interface)
5ba3f43e 410{
0a7de745
A		 411 struct ipsec_pcb *pcb = NULL;
412
413 if (interface == NULL) {
414 return 0;
415 }
416
417 pcb = ifnet_softc(interface);
418
419 if (pcb == NULL) {
420 return 0;
421 }
422
423 /* When ctl disconnects, ipsec_unit is set to 0 */
424 if (pcb->ipsec_unit == 0) {
425 return 0;
426 }
427
428 return 1;
5ba3f43e
A		 429}
430
d9a64523 431#if IPSEC_NEXUS
9d749ea3
A		 432boolean_t
433ipsec_interface_needs_netagent(ifnet_t interface)
434{
435 struct ipsec_pcb *pcb = NULL;
436
437 if (interface == NULL) {
0a7de745 438 return FALSE;
9d749ea3
A		 439 }
440
441 pcb = ifnet_softc(interface);
442
443 if (pcb == NULL) {
0a7de745 444 return FALSE;
9d749ea3
A		 445 }
446
0a7de745 447 return pcb->ipsec_needs_netagent == true;
9d749ea3 448}
d9a64523 449#endif // IPSEC_NEXUS
9d749ea3 450
5ba3f43e
A		 451static errno_t
452ipsec_ifnet_set_attrs(ifnet_t ifp)
453{
454 /* Set flags and additional information. */
455 ifnet_set_mtu(ifp, 1500);
456 ifnet_set_flags(ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
457
458 /* The interface must generate its own IPv6 LinkLocal address,
459 * if possible following the recommendation of RFC2472 to the 64bit interface ID
460 */
461 ifnet_set_eflags(ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
462
463#if !IPSEC_NEXUS
464 /* Reset the stats in case as the interface may have been recycled */
465 struct ifnet_stats_param stats;
466 bzero(&stats, sizeof(struct ifnet_stats_param));
467 ifnet_set_stat(ifp, &stats);
468#endif // !IPSEC_NEXUS
469
0a7de745 470 return 0;
5ba3f43e
A		 471}
472
473#if IPSEC_NEXUS
474
475static uuid_t ipsec_nx_dom_prov;
476
477static errno_t
478ipsec_nxdp_init(__unused kern_nexus_domain_provider_t domprov)
479{
480 return 0;
481}
482
483static void
484ipsec_nxdp_fini(__unused kern_nexus_domain_provider_t domprov)
485{
486 // Ignore
487}
488
489static errno_t
490ipsec_register_nexus(void)
491{
492 const struct kern_nexus_domain_provider_init dp_init = {
493 .nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
494 .nxdpi_flags = 0,
495 .nxdpi_init = ipsec_nxdp_init,
496 .nxdpi_fini = ipsec_nxdp_fini
497 };
498 errno_t err = 0;
499
500 /* ipsec_nxdp_init() is called before this function returns */
501 err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
0a7de745
A		 502 (const uint8_t *) "com.apple.ipsec",
503 &dp_init, sizeof(dp_init),
504 &ipsec_nx_dom_prov);
5ba3f43e 505 if (err != 0) {
cb323159 506 os_log_error(OS_LOG_DEFAULT, "%s: failed to register domain provider\n", __func__);
0a7de745 507 return err;
5ba3f43e 508 }
0a7de745 509 return 0;
5ba3f43e
A		 510}
511
512static errno_t
513ipsec_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
514{
515 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
516 pcb->ipsec_netif_nexus = nexus;
0a7de745 517 return ipsec_ifnet_set_attrs(ifp);
5ba3f43e
A		 518}
519
520static errno_t
521ipsec_nexus_pre_connect(kern_nexus_provider_t nxprov,
0a7de745
A		 522 proc_t p, kern_nexus_t nexus,
523 nexus_port_t nexus_port, kern_channel_t channel, void **ch_ctx)
5ba3f43e
A		 524{
525#pragma unused(nxprov, p)
526#pragma unused(nexus, nexus_port, channel, ch_ctx)
0a7de745 527 return 0;
5ba3f43e
A		 528}
529
530static errno_t
531ipsec_nexus_connected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 532 kern_channel_t channel)
5ba3f43e
A		 533{
534#pragma unused(nxprov, channel)
535 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
536 boolean_t ok = ifnet_is_attached(pcb->ipsec_ifp, 1);
cb323159
A		 537 /* Mark the data path as ready */
538 if (ok) {
539 lck_mtx_lock(&pcb->ipsec_pcb_data_move_lock);
540 IPSEC_SET_DATA_PATH_READY(pcb);
541 lck_mtx_unlock(&pcb->ipsec_pcb_data_move_lock);
542 }
0a7de745 543 return ok ? 0 : ENXIO;
5ba3f43e
A		 544}
545
546static void
547ipsec_nexus_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 548 kern_channel_t channel)
5ba3f43e 549{
cb323159
A		 550#pragma unused(nxprov, channel)
551 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
552
553 VERIFY(pcb->ipsec_kpipe_count != 0);
554
555 /* Wait until all threads in the data paths are done. */
556 ipsec_wait_data_move_drain(pcb);
5ba3f43e
A		 557}
558
559static void
560ipsec_netif_pre_disconnect(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 561 kern_channel_t channel)
5ba3f43e 562{
cb323159
A		 563#pragma unused(nxprov, channel)
564 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
565
566 /* Wait until all threads in the data paths are done. */
567 ipsec_wait_data_move_drain(pcb);
5ba3f43e
A		 568}
569
570static void
571ipsec_nexus_disconnected(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 572 kern_channel_t channel)
5ba3f43e
A		 573{
574#pragma unused(nxprov, channel)
575 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
576 if (pcb->ipsec_netif_nexus == nexus) {
577 pcb->ipsec_netif_nexus = NULL;
578 }
579 ifnet_decr_iorefcnt(pcb->ipsec_ifp);
580}
581
582static errno_t
583ipsec_kpipe_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 584 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
585 void **ring_ctx)
5ba3f43e
A		 586{
587#pragma unused(nxprov)
588#pragma unused(channel)
5ba3f43e 589 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
cb323159
A		 590 uint8_t ring_idx;
591
592 for (ring_idx = 0; ring_idx < pcb->ipsec_kpipe_count; ring_idx++) {
593 if (!uuid_compare(channel->ch_info->cinfo_nx_uuid, pcb->ipsec_kpipe_uuid[ring_idx])) {
594 break;
595 }
596 }
597
598 if (ring_idx == pcb->ipsec_kpipe_count) {
599 uuid_string_t uuidstr;
600 uuid_unparse(channel->ch_info->cinfo_nx_uuid, uuidstr);
601 os_log_error(OS_LOG_DEFAULT, "%s: %s cannot find channel %s\n", __func__, pcb->ipsec_if_xname, uuidstr);
602 return ENOENT;
603 }
604
605 *ring_ctx = (void *)(uintptr_t)ring_idx;
606
5ba3f43e 607 if (!is_tx_ring) {
cb323159
A		 608 VERIFY(pcb->ipsec_kpipe_rxring[ring_idx] == NULL);
609 pcb->ipsec_kpipe_rxring[ring_idx] = ring;
5ba3f43e 610 } else {
cb323159
A		 611 VERIFY(pcb->ipsec_kpipe_txring[ring_idx] == NULL);
612 pcb->ipsec_kpipe_txring[ring_idx] = ring;
5ba3f43e
A		 613 }
614 return 0;
615}
616
617static void
618ipsec_kpipe_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 619 kern_channel_ring_t ring)
5ba3f43e
A		 620{
621#pragma unused(nxprov)
cb323159 622 bool found = false;
5ba3f43e 623 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
cb323159
A		 624
625 for (unsigned int i = 0; i < pcb->ipsec_kpipe_count; i++) {
626 if (pcb->ipsec_kpipe_rxring[i] == ring) {
627 pcb->ipsec_kpipe_rxring[i] = NULL;
628 found = true;
629 } else if (pcb->ipsec_kpipe_txring[i] == ring) {
630 pcb->ipsec_kpipe_txring[i] = NULL;
631 found = true;
632 }
5ba3f43e 633 }
cb323159 634 VERIFY(found);
5ba3f43e
A		 635}
636
637static errno_t
638ipsec_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 639 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 640{
641#pragma unused(nxprov)
642#pragma unused(flags)
643 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
644
cb323159
A		 645 if (!ipsec_data_move_begin(pcb)) {
646 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__, if_name(pcb->ipsec_ifp));
647 return 0;
648 }
649
5ba3f43e 650 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
cb323159
A		 651
652 if (!ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED)) {
5ba3f43e 653 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 654 ipsec_data_move_end(pcb);
5ba3f43e
A		 655 return 0;
656 }
657
cb323159
A		 658 VERIFY(pcb->ipsec_kpipe_count);
659
5ba3f43e
A		 660 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
661 if (tx_slot == NULL) {
662 // Nothing to write, bail
663 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 664 ipsec_data_move_end(pcb);
5ba3f43e
A		 665 return 0;
666 }
667
668 // Signal the netif ring to read
cb323159 669 kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring[0];
5ba3f43e
A		 670 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
671
672 if (rx_ring != NULL) {
673 kern_channel_notify(rx_ring, 0);
674 }
cb323159
A		 675
676 ipsec_data_move_end(pcb);
5ba3f43e
A		 677 return 0;
678}
679
680static mbuf_t
681ipsec_encrypt_mbuf(ifnet_t interface,
0a7de745 682 mbuf_t data)
5ba3f43e
A		 683{
684 struct ipsec_output_state ipsec_state;
685 int error = 0;
686 uint32_t af;
687
688 // Make sure this packet isn't looping through the interface
689 if (necp_get_last_interface_index_from_packet(data) == interface->if_index) {
690 error = -1;
691 goto ipsec_output_err;
692 }
693
694 // Mark the interface so NECP can evaluate tunnel policy
695 necp_mark_packet_from_interface(data, interface);
696
697 struct ip *ip = mtod(data, struct ip *);
698 u_int ip_version = ip->ip_v;
699
700 switch (ip_version) {
0a7de745
A		 701 case 4: {
702 af = AF_INET;
703
704 memset(&ipsec_state, 0, sizeof(ipsec_state));
705 ipsec_state.m = data;
706 ipsec_state.dst = (struct sockaddr *)&ip->ip_dst;
707 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
708
709 error = ipsec4_interface_output(&ipsec_state, interface);
710 if (error == 0 && ipsec_state.tunneled == 6) {
711 // Tunneled in IPv6 - packet is gone
712 // TODO: Don't lose mbuf
713 data = NULL;
5ba3f43e
A		 714 goto done;
715 }
5ba3f43e 716
0a7de745
A		 717 data = ipsec_state.m;
718 if (error || data == NULL) {
719 if (error) {
cb323159 720 os_log_error(OS_LOG_DEFAULT, "ipsec_encrypt_mbuf: ipsec4_output error %d\n", error);
5ba3f43e 721 }
0a7de745
A		 722 goto ipsec_output_err;
723 }
724 goto done;
725 }
726 case 6: {
727 af = AF_INET6;
5ba3f43e 728
0a7de745
A		 729 data = ipsec6_splithdr(data);
730 if (data == NULL) {
cb323159 731 os_log_error(OS_LOG_DEFAULT, "ipsec_encrypt_mbuf: ipsec6_splithdr returned NULL\n");
0a7de745
A		 732 goto ipsec_output_err;
733 }
5ba3f43e 734
0a7de745 735 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
5ba3f43e 736
0a7de745
A		 737 memset(&ipsec_state, 0, sizeof(ipsec_state));
738 ipsec_state.m = data;
739 ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst;
740 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
741
742 error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m);
743 if (error == 0 && ipsec_state.tunneled == 4) {
744 // Tunneled in IPv4 - packet is gone
745 // TODO: Don't lose mbuf
746 data = NULL;
5ba3f43e
A		 747 goto done;
748 }
0a7de745
A		 749 data = ipsec_state.m;
750 if (error || data == NULL) {
751 if (error) {
cb323159 752 os_log_error(OS_LOG_DEFAULT, "ipsec_encrypt_mbuf: ipsec6_output error %d\n", error);
0a7de745 753 }
5ba3f43e
A		 754 goto ipsec_output_err;
755 }
0a7de745
A		 756 goto done;
757 }
758 default: {
cb323159 759 os_log_error(OS_LOG_DEFAULT, "ipsec_encrypt_mbuf: Received unknown packet version %d\n", ip_version);
0a7de745
A		 760 error = -1;
761 goto ipsec_output_err;
762 }
5ba3f43e
A		 763 }
764
765done:
766 return data;
767
768ipsec_output_err:
769 if (data) {
770 mbuf_freem(data);
771 }
772 return NULL;
773}
774
775static errno_t
776ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 777 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 778{
779#pragma unused(nxprov)
780#pragma unused(flags)
781 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
782 struct kern_channel_ring_stat_increment rx_ring_stats;
cb323159
A		 783 uint8_t ring_idx = (uint8_t)(uintptr_t)kern_channel_ring_get_context(rx_ring);
784
785 if (!ipsec_data_move_begin(pcb)) {
786 os_log_error(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__, if_name(pcb->ipsec_ifp));
787 return 0;
788 }
5ba3f43e
A		 789
790 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
791
cb323159 792 if (!ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED)) {
5ba3f43e 793 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 794 ipsec_data_move_end(pcb);
5ba3f43e
A		 795 return 0;
796 }
797
cb323159
A		 798 VERIFY(pcb->ipsec_kpipe_count);
799 VERIFY(ring_idx <= pcb->ipsec_kpipe_count);
800
5ba3f43e
A		 801 // Reclaim user-released slots
802 (void) kern_channel_reclaim(rx_ring);
803
804 uint32_t avail = kern_channel_available_slot_count(rx_ring);
805 if (avail == 0) {
806 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159
A		 807 os_log_error(OS_LOG_DEFAULT, "%s: %s ring %s index %d no room in rx_ring\n", __func__,
808 pcb->ipsec_if_xname, rx_ring->ckr_name, ring_idx);
809 ipsec_data_move_end(pcb);
5ba3f43e
A		 810 return 0;
811 }
812
cb323159 813 kern_channel_ring_t tx_ring = pcb->ipsec_netif_txring[ring_idx];
5ba3f43e
A		 814 if (tx_ring == NULL) {
815 // Net-If TX ring not set up yet, nothing to read
816 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159
A		 817 os_log_error(OS_LOG_DEFAULT, "%s: %s ring %s index %d bad netif_txring 1\n", __func__,
818 pcb->ipsec_if_xname, rx_ring->ckr_name, ring_idx);
819 ipsec_data_move_end(pcb);
5ba3f43e
A		 820 return 0;
821 }
822
823 struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->ipsec_netif_nexus)->nif_stats;
824
825 // Unlock ipsec before entering ring
826 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
827
828 (void)kr_enter(tx_ring, TRUE);
829
830 // Lock again after entering and validate
831 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
cb323159 832 if (tx_ring != pcb->ipsec_netif_txring[ring_idx]) {
5ba3f43e
A		 833 // Ring no longer valid
834 // Unlock first, then exit ring
835 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
836 kr_exit(tx_ring);
cb323159
A		 837 os_log_error(OS_LOG_DEFAULT, "%s: %s ring %s index %d bad netif_txring 2\n", __func__,
838 pcb->ipsec_if_xname, rx_ring->ckr_name, ring_idx);
839 ipsec_data_move_end(pcb);
5ba3f43e
A		 840 return 0;
841 }
842
5ba3f43e
A		 843 struct kern_channel_ring_stat_increment tx_ring_stats;
844 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
845 kern_channel_slot_t tx_pslot = NULL;
846 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
847 if (tx_slot == NULL) {
848 // Nothing to read, don't bother signalling
849 // Unlock first, then exit ring
850 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
851 kr_exit(tx_ring);
cb323159 852 ipsec_data_move_end(pcb);
5ba3f43e
A		 853 return 0;
854 }
855
856 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
857 VERIFY(rx_pp != NULL);
858 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
859 kern_channel_slot_t rx_pslot = NULL;
860 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
861
862 while (rx_slot != NULL && tx_slot != NULL) {
863 size_t length = 0;
864 mbuf_t data = NULL;
865 errno_t error = 0;
866
867 // Allocate rx packet
868 kern_packet_t rx_ph = 0;
869 error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 870 if (__improbable(error != 0)) {
cb323159 871 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s: failed to allocate packet\n",
0a7de745 872 pcb->ipsec_ifp->if_xname);
5ba3f43e
A		 873 break;
874 }
875
876 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
877
878 // Advance TX ring
879 tx_pslot = tx_slot;
880 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
881
882 if (tx_ph == 0) {
a39ff7e2 883 kern_pbufpool_free(rx_pp, rx_ph);
5ba3f43e
A		 884 continue;
885 }
0a7de745 886
5ba3f43e
A		 887 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
888 VERIFY(tx_buf != NULL);
f427ee49 889 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
5ba3f43e
A		 890 VERIFY(tx_baddr != NULL);
891 tx_baddr += kern_buflet_get_data_offset(tx_buf);
892
893 bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
894
895 length = MIN(kern_packet_get_data_length(tx_ph),
0a7de745 896 pcb->ipsec_slot_size);
5ba3f43e
A		 897
898 // Increment TX stats
899 tx_ring_stats.kcrsi_slots_transferred++;
900 tx_ring_stats.kcrsi_bytes_transferred += length;
901
902 if (length > 0) {
903 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
904 if (error == 0) {
905 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
906 if (error == 0) {
907 // Encrypt and send packet
cb323159 908 lck_mtx_lock(&pcb->ipsec_kpipe_encrypt_lock);
5ba3f43e 909 data = ipsec_encrypt_mbuf(pcb->ipsec_ifp, data);
cb323159 910 lck_mtx_unlock(&pcb->ipsec_kpipe_encrypt_lock);
5ba3f43e 911 } else {
cb323159
A		 912 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
913 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
914 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 915 mbuf_freem(data);
916 data = NULL;
917 }
918 } else {
cb323159
A		 919 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
920 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
921 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 922 }
923 } else {
cb323159
A		 924 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname);
925 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
926 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 927 }
928
929 if (data == NULL) {
cb323159 930 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname);
5ba3f43e
A		 931 kern_pbufpool_free(rx_pp, rx_ph);
932 break;
933 }
934
935 length = mbuf_pkthdr_len(data);
936 if (length > rx_pp->pp_buflet_size) {
937 // Flush data
938 mbuf_freem(data);
939 kern_pbufpool_free(rx_pp, rx_ph);
cb323159 940 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx %s: encrypted packet length %zu > %u\n",
0a7de745 941 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
5ba3f43e
A		 942 continue;
943 }
944
945 // Fillout rx packet
946 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
947 VERIFY(rx_buf != NULL);
f427ee49 948 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
5ba3f43e
A		 949 VERIFY(rx_baddr != NULL);
950
951 // Copy-in data from mbuf to buflet
952 mbuf_copydata(data, 0, length, (void *)rx_baddr);
0a7de745 953 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 954
955 // Finalize and attach the packet
956 error = kern_buflet_set_data_offset(rx_buf, 0);
957 VERIFY(error == 0);
958 error = kern_buflet_set_data_length(rx_buf, length);
959 VERIFY(error == 0);
960 error = kern_packet_finalize(rx_ph);
961 VERIFY(error == 0);
962 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
963 VERIFY(error == 0);
964
cb323159
A		 965 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
966 STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
5ba3f43e
A		 967
968 rx_ring_stats.kcrsi_slots_transferred++;
969 rx_ring_stats.kcrsi_bytes_transferred += length;
970
971 if (!pcb->ipsec_ext_ifdata_stats) {
972 ifnet_stat_increment_out(pcb->ipsec_ifp, 1, length, 0);
973 }
974
975 mbuf_freem(data);
976
977 rx_pslot = rx_slot;
978 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
979 }
980
981 if (rx_pslot) {
982 kern_channel_advance_slot(rx_ring, rx_pslot);
983 kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats);
984 }
985
986 if (tx_pslot) {
987 kern_channel_advance_slot(tx_ring, tx_pslot);
988 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
989 (void)kern_channel_reclaim(tx_ring);
990 }
991
5c9f4661
A		 992 /* always reenable output */
993 errno_t error = ifnet_enable_output(pcb->ipsec_ifp);
994 if (error != 0) {
cb323159 995 os_log_error(OS_LOG_DEFAULT, "ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
5ba3f43e
A		 996 }
997
998 // Unlock first, then exit ring
999 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1000
1001 if (tx_pslot != NULL) {
1002 kern_channel_notify(tx_ring, 0);
1003 }
1004 kr_exit(tx_ring);
1005
cb323159 1006 ipsec_data_move_end(pcb);
5ba3f43e
A		 1007 return 0;
1008}
1009
cb323159
A		 1010static uint8_t
1011ipsec_find_tx_ring_by_svc(kern_packet_svc_class_t svc_class)
1012{
1013 switch (svc_class) {
1014 case KPKT_SC_VO: {
1015 return 0;
1016 }
1017 case KPKT_SC_VI: {
1018 return 1;
1019 }
1020 case KPKT_SC_BE: {
1021 return 2;
1022 }
1023 case KPKT_SC_BK: {
1024 return 3;
1025 }
1026 default: {
1027 VERIFY(0);
1028 return 0;
1029 }
1030 }
1031}
1032
5ba3f43e
A		 1033static errno_t
1034ipsec_netif_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745
A		 1035 kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
1036 void **ring_ctx)
5ba3f43e
A		 1037{
1038#pragma unused(nxprov)
1039#pragma unused(channel)
5ba3f43e 1040 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
cb323159 1041
5ba3f43e 1042 if (!is_tx_ring) {
cb323159
A		 1043 VERIFY(pcb->ipsec_netif_rxring[0] == NULL);
1044 pcb->ipsec_netif_rxring[0] = ring;
5ba3f43e 1045 } else {
cb323159
A		 1046 uint8_t ring_idx = 0;
1047 if (ipsec_in_wmm_mode(pcb)) {
1048 int err;
1049 kern_packet_svc_class_t svc_class;
1050 err = kern_channel_get_service_class(ring, &svc_class);
1051 VERIFY(err == 0);
1052 ring_idx = ipsec_find_tx_ring_by_svc(svc_class);
1053 VERIFY(ring_idx < IPSEC_IF_WMM_RING_COUNT);
1054 }
1055
1056 *ring_ctx = (void *)(uintptr_t)ring_idx;
1057
1058 VERIFY(pcb->ipsec_netif_txring[ring_idx] == NULL);
1059 pcb->ipsec_netif_txring[ring_idx] = ring;
5ba3f43e
A		 1060 }
1061 return 0;
1062}
1063
1064static void
1065ipsec_netif_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 1066 kern_channel_ring_t ring)
5ba3f43e
A		 1067{
1068#pragma unused(nxprov)
1069 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
cb323159
A		 1070 bool found = false;
1071
1072 for (int i = 0; i < IPSEC_NETIF_MAX_RX_RING_COUNT; i++) {
1073 if (pcb->ipsec_netif_rxring[i] == ring) {
1074 pcb->ipsec_netif_rxring[i] = NULL;
1075 VERIFY(!found);
1076 found = true;
1077 }
1078 }
1079 for (int i = 0; i < IPSEC_NETIF_MAX_TX_RING_COUNT; i++) {
1080 if (pcb->ipsec_netif_txring[i] == ring) {
1081 pcb->ipsec_netif_txring[i] = NULL;
1082 VERIFY(!found);
1083 found = true;
1084 }
5ba3f43e 1085 }
cb323159 1086 VERIFY(found);
5ba3f43e
A		 1087}
1088
1089static bool
f427ee49 1090ipsec_netif_check_policy(ifnet_t interface, mbuf_t data)
5ba3f43e
A		 1091{
1092 necp_kernel_policy_result necp_result = 0;
1093 necp_kernel_policy_result_parameter necp_result_parameter = {};
1094 uint32_t necp_matched_policy_id = 0;
f427ee49
A		 1095 struct ip_out_args args4 = { };
1096 struct ip6_out_args args6 = { };
5ba3f43e
A		 1097
1098 // This packet has been marked with IP level policy, do not mark again.
1099 if (data && data->m_pkthdr.necp_mtag.necp_policy_id >= NECP_KERNEL_POLICY_ID_FIRST_VALID_IP) {
0a7de745 1100 return true;
5ba3f43e
A		 1101 }
1102
1103 size_t length = mbuf_pkthdr_len(data);
1104 if (length < sizeof(struct ip)) {
0a7de745 1105 return false;
5ba3f43e
A		 1106 }
1107
1108 struct ip *ip = mtod(data, struct ip *);
1109 u_int ip_version = ip->ip_v;
1110 switch (ip_version) {
0a7de745 1111 case 4: {
f427ee49
A		 1112 if (interface != NULL) {
1113 args4.ipoa_flags |= IPOAF_BOUND_IF;
1114 args4.ipoa_boundif = interface->if_index;
1115 }
1116 necp_matched_policy_id = necp_ip_output_find_policy_match(data, IP_OUTARGS, &args4, NULL,
0a7de745
A		 1117 &necp_result, &necp_result_parameter);
1118 break;
1119 }
1120 case 6: {
f427ee49
A		 1121 if (interface != NULL) {
1122 args6.ip6oa_flags |= IP6OAF_BOUND_IF;
1123 args6.ip6oa_boundif = interface->if_index;
1124 }
1125 necp_matched_policy_id = necp_ip6_output_find_policy_match(data, IPV6_OUTARGS, &args6, NULL,
0a7de745
A		 1126 &necp_result, &necp_result_parameter);
1127 break;
1128 }
1129 default: {
1130 return false;
1131 }
5ba3f43e
A		 1132 }
1133
1134 if (necp_result == NECP_KERNEL_POLICY_RESULT_DROP ||
0a7de745 1135 necp_result == NECP_KERNEL_POLICY_RESULT_SOCKET_DIVERT) {
5ba3f43e 1136 /* Drop and flow divert packets should be blocked at the IP layer */
0a7de745 1137 return false;
5ba3f43e
A		 1138 }
1139
1140 necp_mark_packet_from_ip(data, necp_matched_policy_id);
0a7de745 1141 return true;
5ba3f43e
A		 1142}
1143
1144static errno_t
1145ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 1146 kern_channel_ring_t tx_ring, uint32_t flags)
5ba3f43e
A		 1147{
1148#pragma unused(nxprov)
1149#pragma unused(flags)
1150 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
1151
1152 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
1153
cb323159
A		 1154 if (!ipsec_data_move_begin(pcb)) {
1155 os_log_error(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__, if_name(pcb->ipsec_ifp));
1156 return 0;
1157 }
1158
5ba3f43e
A		 1159 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
1160
1161 struct kern_channel_ring_stat_increment tx_ring_stats;
1162 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
1163 kern_channel_slot_t tx_pslot = NULL;
1164 kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
1165
cb323159 1166 STATS_INC(nifs, NETIF_STATS_TX_SYNC);
5ba3f43e
A		 1167
1168 if (tx_slot == NULL) {
1169 // Nothing to write, don't bother signalling
1170 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 1171 ipsec_data_move_end(pcb);
5ba3f43e
A		 1172 return 0;
1173 }
1174
cb323159
A		 1175 if (pcb->ipsec_kpipe_count &&
1176 ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED)) {
1177 // Select the corresponding kpipe rx ring
1178 uint8_t ring_idx = (uint8_t)(uintptr_t)kern_channel_ring_get_context(tx_ring);
1179 VERIFY(ring_idx < IPSEC_IF_MAX_RING_COUNT);
1180 kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring[ring_idx];
1181
1182 // Unlock while calling notify
5ba3f43e
A		 1183 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1184
1185 // Signal the kernel pipe ring to read
1186 if (rx_ring != NULL) {
1187 kern_channel_notify(rx_ring, 0);
1188 }
cb323159
A		 1189
1190 ipsec_data_move_end(pcb);
5ba3f43e
A		 1191 return 0;
1192 }
1193
1194 // If we're here, we're injecting into the BSD stack
1195 while (tx_slot != NULL) {
1196 size_t length = 0;
1197 mbuf_t data = NULL;
1198
1199 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
1200
1201 // Advance TX ring
1202 tx_pslot = tx_slot;
1203 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
1204
1205 if (tx_ph == 0) {
1206 continue;
1207 }
1208
1209 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
1210 VERIFY(tx_buf != NULL);
f427ee49 1211 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
5ba3f43e
A		 1212 VERIFY(tx_baddr != 0);
1213 tx_baddr += kern_buflet_get_data_offset(tx_buf);
1214
1215 bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
1216
1217 length = MIN(kern_packet_get_data_length(tx_ph),
0a7de745 1218 pcb->ipsec_slot_size);
5ba3f43e
A		 1219
1220 if (length > 0) {
1221 errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
1222 if (error == 0) {
1223 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
1224 if (error == 0) {
1225 // Mark packet from policy
1226 uint32_t policy_id = kern_packet_get_policy_id(tx_ph);
1227 necp_mark_packet_from_ip(data, policy_id);
1228
1229 // Check policy with NECP
f427ee49 1230 if (!ipsec_netif_check_policy(pcb->ipsec_ifp, data)) {
cb323159
A		 1231 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s - failed policy check\n", pcb->ipsec_ifp->if_xname);
1232 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1233 mbuf_freem(data);
1234 data = NULL;
1235 } else {
1236 // Send through encryption
1237 error = ipsec_output(pcb->ipsec_ifp, data);
1238 if (error != 0) {
cb323159 1239 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s - ipsec_output error %d\n", pcb->ipsec_ifp->if_xname, error);
5ba3f43e
A		 1240 }
1241 }
1242 } else {
cb323159
A		 1243 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
1244 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
1245 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1246 mbuf_freem(data);
1247 data = NULL;
1248 }
1249 } else {
cb323159
A		 1250 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
1251 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
1252 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1253 }
1254 } else {
cb323159
A		 1255 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s - 0 length packet\n", pcb->ipsec_ifp->if_xname);
1256 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1257 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1258 }
1259
1260 if (data == NULL) {
cb323159 1261 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_tx %s: no encrypted packet to send\n", pcb->ipsec_ifp->if_xname);
5ba3f43e
A		 1262 break;
1263 }
1264
cb323159
A		 1265 STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
1266 STATS_INC(nifs, NETIF_STATS_TX_COPY_MBUF);
5ba3f43e
A		 1267
1268 tx_ring_stats.kcrsi_slots_transferred++;
1269 tx_ring_stats.kcrsi_bytes_transferred += length;
1270 }
1271
1272 if (tx_pslot) {
1273 kern_channel_advance_slot(tx_ring, tx_pslot);
1274 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
1275 (void)kern_channel_reclaim(tx_ring);
1276 }
1277
1278 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 1279 ipsec_data_move_end(pcb);
5ba3f43e
A		 1280
1281 return 0;
1282}
1283
1284static errno_t
cb323159
A		 1285ipsec_netif_tx_doorbell_one(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
1286 kern_channel_ring_t ring, uint32_t flags, uint8_t ring_idx)
5ba3f43e
A		 1287{
1288#pragma unused(nxprov)
1289 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
5ba3f43e
A		 1290 boolean_t more = false;
1291 errno_t rc = 0;
5ba3f43e 1292
cb323159
A		 1293 VERIFY((flags & KERN_NEXUS_TXDOORBELLF_ASYNC_REFILL) == 0);
1294
5c9f4661
A		 1295 /*
1296 * Refill and sync the ring; we may be racing against another thread doing
1297 * an RX sync that also wants to do kr_enter(), and so use the blocking
1298 * variant here.
1299 */
1300 rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
1301 if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
cb323159
A		 1302 os_log_error(OS_LOG_DEFAULT, "%s: %s ring %s tx refill failed %d\n", __func__,
1303 pcb->ipsec_if_xname, ring->ckr_name, rc);
5c9f4661
A		 1304 }
1305
1306 (void) kr_enter(ring, TRUE);
1307 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
cb323159
A		 1308 if (ring != pcb->ipsec_netif_txring[ring_idx]) {
1309 // ring no longer valid
1310 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1311 kr_exit(ring);
1312 os_log_error(OS_LOG_DEFAULT, "%s: %s ring %s index %d bad netif_txring 3\n", __func__,
1313 pcb->ipsec_if_xname, ring->ckr_name, ring_idx);
1314 return ENXIO;
1315 }
5c9f4661 1316
cb323159 1317 if (pcb->ipsec_kpipe_count) {
5ba3f43e
A		 1318 uint32_t tx_available = kern_channel_available_slot_count(ring);
1319 if (pcb->ipsec_netif_txring_size > 0 &&
0a7de745 1320 tx_available >= pcb->ipsec_netif_txring_size - 1) {
5ba3f43e
A		 1321 // No room left in tx ring, disable output for now
1322 errno_t error = ifnet_disable_output(pcb->ipsec_ifp);
1323 if (error != 0) {
cb323159 1324 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
5ba3f43e
A		 1325 }
1326 }
1327 }
1328
cb323159
A		 1329 if (pcb->ipsec_kpipe_count) {
1330 kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring[ring_idx];
5ba3f43e
A		 1331
1332 // Unlock while calling notify
1333 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1334 // Signal the kernel pipe ring to read
1335 if (rx_ring != NULL) {
1336 kern_channel_notify(rx_ring, 0);
1337 }
5ba3f43e
A		 1338 } else {
1339 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1340 }
1341
5c9f4661
A		 1342 kr_exit(ring);
1343
0a7de745 1344 return 0;
5ba3f43e
A		 1345}
1346
cb323159
A		 1347static errno_t
1348ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
1349 kern_channel_ring_t ring, __unused uint32_t flags)
1350{
1351 errno_t ret = 0;
1352 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
1353
1354 if (!ipsec_data_move_begin(pcb)) {
1355 os_log_error(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__, if_name(pcb->ipsec_ifp));
1356 return 0;
1357 }
1358
1359 if (ipsec_in_wmm_mode(pcb)) {
1360 for (uint8_t i = 0; i < IPSEC_IF_WMM_RING_COUNT; i++) {
1361 kern_channel_ring_t nring = pcb->ipsec_netif_txring[i];
1362 ret = ipsec_netif_tx_doorbell_one(nxprov, nexus, nring, flags, i);
1363 if (ret) {
1364 break;
1365 }
1366 }
1367 } else {
1368 ret = ipsec_netif_tx_doorbell_one(nxprov, nexus, ring, flags, 0);
1369 }
1370
1371 ipsec_data_move_end(pcb);
1372 return ret;
1373}
1374
5ba3f43e
A		 1375static errno_t
1376ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
0a7de745 1377 kern_channel_ring_t rx_ring, uint32_t flags)
5ba3f43e
A		 1378{
1379#pragma unused(nxprov)
1380#pragma unused(flags)
1381 struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
1382 struct kern_channel_ring_stat_increment rx_ring_stats;
1383
1384 struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
1385
cb323159
A		 1386 if (!ipsec_data_move_begin(pcb)) {
1387 os_log_error(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__, if_name(pcb->ipsec_ifp));
1388 return 0;
1389 }
1390
5ba3f43e
A		 1391 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
1392
1393 // Reclaim user-released slots
1394 (void) kern_channel_reclaim(rx_ring);
1395
cb323159 1396 STATS_INC(nifs, NETIF_STATS_RX_SYNC);
5ba3f43e
A		 1397
1398 uint32_t avail = kern_channel_available_slot_count(rx_ring);
1399 if (avail == 0) {
1400 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
cb323159 1401 ipsec_data_move_end(pcb);
5ba3f43e
A		 1402 return 0;
1403 }
1404
1405 struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
1406 VERIFY(rx_pp != NULL);
1407 bzero(&rx_ring_stats, sizeof(rx_ring_stats));
1408 kern_channel_slot_t rx_pslot = NULL;
1409 kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
1410
1411 while (rx_slot != NULL) {
1412 // Check for a waiting packet
1413 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
1414 mbuf_t data = pcb->ipsec_input_chain;
1415 if (data == NULL) {
1416 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1417 break;
1418 }
1419
1420 // Allocate rx packet
1421 kern_packet_t rx_ph = 0;
1422 errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
a39ff7e2 1423 if (__improbable(error != 0)) {
cb323159
A		 1424 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
1425 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1426 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1427 break;
1428 }
1429
1430 // Advance waiting packets
cb323159
A		 1431 if (pcb->ipsec_input_chain_count > 0) {
1432 pcb->ipsec_input_chain_count--;
1433 }
5ba3f43e
A		 1434 pcb->ipsec_input_chain = data->m_nextpkt;
1435 data->m_nextpkt = NULL;
1436 if (pcb->ipsec_input_chain == NULL) {
1437 pcb->ipsec_input_chain_last = NULL;
1438 }
1439 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1440
1441 size_t length = mbuf_pkthdr_len(data);
1442
1443 if (length < sizeof(struct ip)) {
1444 // Flush data
1445 mbuf_freem(data);
1446 kern_pbufpool_free(rx_pp, rx_ph);
cb323159
A		 1447 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1448 STATS_INC(nifs, NETIF_STATS_DROP);
1449 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: legacy decrypted packet length cannot hold IP %zu < %zu\n",
0a7de745 1450 pcb->ipsec_ifp->if_xname, length, sizeof(struct ip));
5ba3f43e
A		 1451 continue;
1452 }
1453
1454 uint32_t af = 0;
1455 struct ip *ip = mtod(data, struct ip *);
1456 u_int ip_version = ip->ip_v;
1457 switch (ip_version) {
0a7de745
A		 1458 case 4: {
1459 af = AF_INET;
1460 break;
1461 }
1462 case 6: {
1463 af = AF_INET6;
1464 break;
1465 }
1466 default: {
cb323159 1467 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: legacy unknown ip version %u\n",
0a7de745
A		 1468 pcb->ipsec_ifp->if_xname, ip_version);
1469 break;
1470 }
5ba3f43e
A		 1471 }
1472
1473 if (length > rx_pp->pp_buflet_size ||
0a7de745 1474 (pcb->ipsec_frag_size_set && length > pcb->ipsec_input_frag_size)) {
5ba3f43e
A		 1475 // We need to fragment to send up into the netif
1476
1477 u_int32_t fragment_mtu = rx_pp->pp_buflet_size;
1478 if (pcb->ipsec_frag_size_set &&
0a7de745 1479 pcb->ipsec_input_frag_size < rx_pp->pp_buflet_size) {
5ba3f43e
A		 1480 fragment_mtu = pcb->ipsec_input_frag_size;
1481 }
1482
1483 mbuf_t fragment_chain = NULL;
1484 switch (af) {
0a7de745
A		 1485 case AF_INET: {
1486 // ip_fragment expects the length in host order
1487 ip->ip_len = ntohs(ip->ip_len);
1488
1489 // ip_fragment will modify the original data, don't free
1490 int fragment_error = ip_fragment(data, pcb->ipsec_ifp, fragment_mtu, TRUE);
1491 if (fragment_error == 0 && data != NULL) {
1492 fragment_chain = data;
1493 } else {
cb323159
A		 1494 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1495 STATS_INC(nifs, NETIF_STATS_DROP);
1496 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: failed to fragment IPv4 packet of length %zu (%d)\n",
0a7de745
A		 1497 pcb->ipsec_ifp->if_xname, length, fragment_error);
1498 }
1499 break;
1500 }
1501 case AF_INET6: {
1502 if (length < sizeof(struct ip6_hdr)) {
1503 mbuf_freem(data);
cb323159
A		 1504 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1505 STATS_INC(nifs, NETIF_STATS_DROP);
1506 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu < %zu\n",
0a7de745
A		 1507 pcb->ipsec_ifp->if_xname, length, sizeof(struct ip6_hdr));
1508 } else {
1509 // ip6_do_fragmentation will free the original data on success only
1510 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
5ba3f43e 1511
0a7de745 1512 int fragment_error = ip6_do_fragmentation(&data, 0, pcb->ipsec_ifp, sizeof(struct ip6_hdr),
cb323159 1513 ip6, NULL, fragment_mtu, ip6->ip6_nxt, htonl(ip6_randomid()));
5ba3f43e
A		 1514 if (fragment_error == 0 && data != NULL) {
1515 fragment_chain = data;
1516 } else {
5ba3f43e 1517 mbuf_freem(data);
cb323159
A		 1518 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1519 STATS_INC(nifs, NETIF_STATS_DROP);
1520 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: failed to fragment IPv6 packet of length %zu (%d)\n",
0a7de745 1521 pcb->ipsec_ifp->if_xname, length, fragment_error);
5ba3f43e 1522 }
5ba3f43e 1523 }
0a7de745
A		 1524 break;
1525 }
1526 default: {
1527 // Cannot fragment unknown families
1528 mbuf_freem(data);
cb323159
A		 1529 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1530 STATS_INC(nifs, NETIF_STATS_DROP);
1531 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: uknown legacy decrypted packet length %zu > %u\n",
0a7de745
A		 1532 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
1533 break;
1534 }
5ba3f43e
A		 1535 }
1536
1537 if (fragment_chain != NULL) {
1538 // Add fragments to chain before continuing
1539 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
1540 if (pcb->ipsec_input_chain != NULL) {
1541 pcb->ipsec_input_chain_last->m_nextpkt = fragment_chain;
1542 } else {
1543 pcb->ipsec_input_chain = fragment_chain;
1544 }
cb323159 1545 pcb->ipsec_input_chain_count++;
5ba3f43e
A		 1546 while (fragment_chain->m_nextpkt) {
1547 VERIFY(fragment_chain != fragment_chain->m_nextpkt);
1548 fragment_chain = fragment_chain->m_nextpkt;
cb323159 1549 pcb->ipsec_input_chain_count++;
5ba3f43e
A		 1550 }
1551 pcb->ipsec_input_chain_last = fragment_chain;
1552 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
1553 }
1554
1555 // Make sure to free unused rx packet
1556 kern_pbufpool_free(rx_pp, rx_ph);
1557
1558 continue;
1559 }
1560
1561 mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp);
1562
1563 // Fillout rx packet
1564 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
1565 VERIFY(rx_buf != NULL);
f427ee49 1566 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
5ba3f43e
A		 1567 VERIFY(rx_baddr != NULL);
1568
1569 // Copy-in data from mbuf to buflet
1570 mbuf_copydata(data, 0, length, (void *)rx_baddr);
0a7de745 1571 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
5ba3f43e
A		 1572
1573 // Finalize and attach the packet
1574 error = kern_buflet_set_data_offset(rx_buf, 0);
1575 VERIFY(error == 0);
1576 error = kern_buflet_set_data_length(rx_buf, length);
1577 VERIFY(error == 0);
cb323159 1578 error = kern_packet_set_headroom(rx_ph, 0);
5ba3f43e
A		 1579 VERIFY(error == 0);
1580 error = kern_packet_finalize(rx_ph);
1581 VERIFY(error == 0);
1582 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
1583 VERIFY(error == 0);
1584
cb323159
A		 1585 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
1586 STATS_INC(nifs, NETIF_STATS_RX_COPY_MBUF);
5ba3f43e
A		 1587 bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0);
1588
1589 rx_ring_stats.kcrsi_slots_transferred++;
1590 rx_ring_stats.kcrsi_bytes_transferred += length;
1591
1592 if (!pcb->ipsec_ext_ifdata_stats) {
1593 ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0);
1594 }
1595
1596 mbuf_freem(data);
1597
1598 // Advance ring
1599 rx_pslot = rx_slot;
1600 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
1601 }
1602
cb323159
A		 1603 for (uint8_t ring_idx = 0; ring_idx < pcb->ipsec_kpipe_count; ring_idx++) {
1604 struct kern_channel_ring_stat_increment tx_ring_stats;
1605 bzero(&tx_ring_stats, sizeof(tx_ring_stats));
1606 kern_channel_ring_t tx_ring = pcb->ipsec_kpipe_txring[ring_idx];
1607 kern_channel_slot_t tx_pslot = NULL;
1608 kern_channel_slot_t tx_slot = NULL;
1609 if (tx_ring == NULL) {
1610 // Net-If TX ring not set up yet, nothing to read
1611 goto done;
1612 }
5ba3f43e 1613
5ba3f43e 1614
cb323159
A		 1615 // Unlock ipsec before entering ring
1616 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 1617
cb323159 1618 (void)kr_enter(tx_ring, TRUE);
5ba3f43e 1619
cb323159
A		 1620 // Lock again after entering and validate
1621 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 1622
cb323159
A		 1623 if (tx_ring != pcb->ipsec_kpipe_txring[ring_idx]) {
1624 goto done;
1625 }
5ba3f43e 1626
cb323159
A		 1627 tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
1628 if (tx_slot == NULL) {
1629 // Nothing to read, don't bother signalling
1630 goto done;
5ba3f43e
A		 1631 }
1632
cb323159
A		 1633 while (rx_slot != NULL && tx_slot != NULL) {
1634 size_t length = 0;
1635 mbuf_t data = NULL;
1636 errno_t error = 0;
1637 uint32_t af;
1638
1639 // Allocate rx packet
1640 kern_packet_t rx_ph = 0;
1641 error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
1642 if (__improbable(error != 0)) {
1643 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_PKT);
1644 STATS_INC(nifs, NETIF_STATS_DROP);
1645 break;
1646 }
5ba3f43e 1647
cb323159 1648 kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
5ba3f43e 1649
cb323159
A		 1650 // Advance TX ring
1651 tx_pslot = tx_slot;
1652 tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
5ba3f43e 1653
cb323159
A		 1654 if (tx_ph == 0) {
1655 kern_pbufpool_free(rx_pp, rx_ph);
1656 continue;
1657 }
5ba3f43e 1658
cb323159
A		 1659 kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
1660 VERIFY(tx_buf != NULL);
f427ee49 1661 uint8_t *tx_baddr = kern_buflet_get_data_address(tx_buf);
cb323159
A		 1662 VERIFY(tx_baddr != 0);
1663 tx_baddr += kern_buflet_get_data_offset(tx_buf);
5ba3f43e 1664
cb323159
A		 1665 length = MIN(kern_packet_get_data_length(tx_ph),
1666 pcb->ipsec_slot_size);
5ba3f43e 1667
cb323159
A		 1668 // Increment TX stats
1669 tx_ring_stats.kcrsi_slots_transferred++;
1670 tx_ring_stats.kcrsi_bytes_transferred += length;
1671
1672 if (length >= sizeof(struct ip)) {
1673 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
5ba3f43e 1674 if (error == 0) {
cb323159
A		 1675 error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_DONTWAIT);
1676 if (error == 0) {
1677 lck_mtx_lock(&pcb->ipsec_kpipe_decrypt_lock);
1678 struct ip *ip = mtod(data, struct ip *);
1679 u_int ip_version = ip->ip_v;
1680 switch (ip_version) {
1681 case 4: {
1682 af = AF_INET;
1683 ip->ip_len = ntohs(ip->ip_len) - sizeof(struct ip);
1684 ip->ip_off = ntohs(ip->ip_off);
1685
1686 if (length < ip->ip_len) {
1687 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: IPv4 packet length too short (%zu < %u)\n",
1688 pcb->ipsec_ifp->if_xname, length, ip->ip_len);
1689 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1690 STATS_INC(nifs, NETIF_STATS_DROP);
1691 mbuf_freem(data);
1692 data = NULL;
1693 } else {
1694 data = esp4_input_extended(data, sizeof(struct ip), pcb->ipsec_ifp);
1695 }
1696 break;
5ba3f43e 1697 }
cb323159
A		 1698 case 6: {
1699 if (length < sizeof(struct ip6_hdr)) {
1700 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: IPv6 packet length too short for header %zu\n",
1701 pcb->ipsec_ifp->if_xname, length);
1702 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1703 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1704 mbuf_freem(data);
1705 data = NULL;
1706 } else {
cb323159
A		 1707 af = AF_INET6;
1708 struct ip6_hdr *ip6 = mtod(data, struct ip6_hdr *);
1709 const size_t ip6_len = sizeof(*ip6) + ntohs(ip6->ip6_plen);
1710 if (length < ip6_len) {
1711 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: IPv6 packet length too short (%zu < %zu)\n",
1712 pcb->ipsec_ifp->if_xname, length, ip6_len);
1713 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1714 STATS_INC(nifs, NETIF_STATS_DROP);
1715 mbuf_freem(data);
1716 data = NULL;
1717 } else {
1718 int offset = sizeof(struct ip6_hdr);
1719 esp6_input_extended(&data, &offset, ip6->ip6_nxt, pcb->ipsec_ifp);
1720 }
5ba3f43e 1721 }
cb323159 1722 break;
5ba3f43e 1723 }
cb323159
A		 1724 default: {
1725 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: unknown ip version %u\n",
1726 pcb->ipsec_ifp->if_xname, ip_version);
1727 STATS_INC(nifs, NETIF_STATS_DROP);
1728 mbuf_freem(data);
1729 data = NULL;
1730 break;
1731 }
1732 }
1733 lck_mtx_unlock(&pcb->ipsec_kpipe_decrypt_lock);
1734 } else {
1735 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s - mbuf_copyback(%zu) error %d\n", pcb->ipsec_ifp->if_xname, length, error);
1736 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
1737 STATS_INC(nifs, NETIF_STATS_DROP);
0a7de745
A		 1738 mbuf_freem(data);
1739 data = NULL;
5ba3f43e
A		 1740 }
1741 } else {
cb323159
A		 1742 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s - mbuf_gethdr error %d\n", pcb->ipsec_ifp->if_xname, error);
1743 STATS_INC(nifs, NETIF_STATS_DROP_NOMEM_MBUF);
1744 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e
A		 1745 }
1746 } else {
cb323159
A		 1747 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s - bad packet length %zu\n", pcb->ipsec_ifp->if_xname, length);
1748 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1749 STATS_INC(nifs, NETIF_STATS_DROP);
5ba3f43e 1750 }
5ba3f43e 1751
cb323159
A		 1752 if (data == NULL) {
1753 // Failed to get decrypted data data
1754 kern_pbufpool_free(rx_pp, rx_ph);
1755 continue;
1756 }
5ba3f43e 1757
cb323159
A		 1758 length = mbuf_pkthdr_len(data);
1759 if (length > rx_pp->pp_buflet_size) {
1760 // Flush data
1761 mbuf_freem(data);
1762 kern_pbufpool_free(rx_pp, rx_ph);
1763 STATS_INC(nifs, NETIF_STATS_DROP_BADLEN);
1764 STATS_INC(nifs, NETIF_STATS_DROP);
1765 os_log_error(OS_LOG_DEFAULT, "ipsec_netif_sync_rx %s: decrypted packet length %zu > %u\n",
1766 pcb->ipsec_ifp->if_xname, length, rx_pp->pp_buflet_size);
1767 continue;
1768 }
5ba3f43e 1769
cb323159
A		 1770 mbuf_pkthdr_setrcvif(data, pcb->ipsec_ifp);
1771
1772 // Fillout rx packet
1773 kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
1774 VERIFY(rx_buf != NULL);
f427ee49 1775 void *rx_baddr = kern_buflet_get_data_address(rx_buf);
cb323159
A		 1776 VERIFY(rx_baddr != NULL);
1777
1778 // Copy-in data from mbuf to buflet
1779 mbuf_copydata(data, 0, length, (void *)rx_baddr);
1780 kern_packet_clear_flow_uuid(rx_ph); // Zero flow id
1781
1782 // Finalize and attach the packet
1783 error = kern_buflet_set_data_offset(rx_buf, 0);
1784 VERIFY(error == 0);
1785 error = kern_buflet_set_data_length(rx_buf, length);
1786 VERIFY(error == 0);
1787 error = kern_packet_set_link_header_offset(rx_ph, 0);
1788 VERIFY(error == 0);
1789 error = kern_packet_set_network_header_offset(rx_ph, 0);
1790 VERIFY(error == 0);
1791 error = kern_packet_finalize(rx_ph);
1792 VERIFY(error == 0);
1793 error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
1794 VERIFY(error == 0);
1795
1796 STATS_INC(nifs, NETIF_STATS_RX_PACKETS);
1797 STATS_INC(nifs, NETIF_STATS_RX_COPY_DIRECT);
1798 bpf_tap_packet_in(pcb->ipsec_ifp, DLT_RAW, rx_ph, NULL, 0);
1799
1800 rx_ring_stats.kcrsi_slots_transferred++;
1801 rx_ring_stats.kcrsi_bytes_transferred += length;
1802
1803 if (!pcb->ipsec_ext_ifdata_stats) {
1804 ifnet_stat_increment_in(pcb->ipsec_ifp, 1, length, 0);
1805 }
5ba3f43e 1806
cb323159 1807 mbuf_freem(data);
5ba3f43e 1808
cb323159
A		 1809 rx_pslot = rx_slot;
1810 rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
1811 }
5ba3f43e 1812
cb323159
A		 1813done:
1814 if (tx_pslot) {
1815 kern_channel_advance_slot(tx_ring, tx_pslot);
1816 kern_channel_increment_ring_net_stats(tx_ring, pcb->ipsec_ifp, &tx_ring_stats);
1817 (void)kern_channel_reclaim(tx_ring);
5ba3f43e
A		 1818 }
1819
cb323159
A		 1820 // Unlock first, then exit ring
1821 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
1822 if (tx_ring != NULL) {
1823 if (tx_pslot != NULL) {
1824 kern_channel_notify(tx_ring, 0);
1825 }
1826 kr_exit(tx_ring);
1827 }
5ba3f43e 1828
cb323159 1829 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e
A		 1830 }
1831
5ba3f43e
A		 1832 if (rx_pslot) {
1833 kern_channel_advance_slot(rx_ring, rx_pslot);
1834 kern_channel_increment_ring_net_stats(rx_ring, pcb->ipsec_ifp, &rx_ring_stats);
1835 }
1836
5ba3f43e 1837
5ba3f43e 1838 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 1839
cb323159 1840 ipsec_data_move_end(pcb);
5ba3f43e
A		 1841 return 0;
1842}
1843
1844static errno_t
1845ipsec_nexus_ifattach(struct ipsec_pcb *pcb,
0a7de745
A		 1846 struct ifnet_init_eparams *init_params,
1847 struct ifnet **ifp)
5ba3f43e
A		 1848{
1849 errno_t err;
1850 nexus_controller_t controller = kern_nexus_shared_controller();
1851 struct kern_nexus_net_init net_init;
a39ff7e2 1852 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 1853
1854 nexus_name_t provider_name;
1855 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 1856 "com.apple.netif.%s", pcb->ipsec_if_xname);
5ba3f43e
A		 1857
1858 struct kern_nexus_provider_init prov_init = {
1859 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
1860 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
1861 .nxpi_pre_connect = ipsec_nexus_pre_connect,
1862 .nxpi_connected = ipsec_nexus_connected,
1863 .nxpi_pre_disconnect = ipsec_netif_pre_disconnect,
1864 .nxpi_disconnected = ipsec_nexus_disconnected,
1865 .nxpi_ring_init = ipsec_netif_ring_init,
1866 .nxpi_ring_fini = ipsec_netif_ring_fini,
1867 .nxpi_slot_init = NULL,
1868 .nxpi_slot_fini = NULL,
1869 .nxpi_sync_tx = ipsec_netif_sync_tx,
1870 .nxpi_sync_rx = ipsec_netif_sync_rx,
1871 .nxpi_tx_doorbell = ipsec_netif_tx_doorbell,
1872 };
1873
1874 nexus_attr_t nxa = NULL;
1875 err = kern_nexus_attr_create(&nxa);
1876 IPSEC_IF_VERIFY(err == 0);
1877 if (err != 0) {
cb323159 1878 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 1879 __func__, err);
5ba3f43e
A		 1880 goto failed;
1881 }
1882
5c9f4661 1883 uint64_t slot_buffer_size = pcb->ipsec_slot_size;
5ba3f43e
A		 1884 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
1885 VERIFY(err == 0);
1886
1887 // Reset ring size for netif nexus to limit memory usage
5c9f4661 1888 uint64_t ring_size = pcb->ipsec_netif_ring_size;
5ba3f43e
A		 1889 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
1890 VERIFY(err == 0);
1891 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
1892 VERIFY(err == 0);
1893
cb323159
A		 1894 assert(err == 0);
1895
1896 if (ipsec_in_wmm_mode(pcb)) {
1897 os_log(OS_LOG_DEFAULT, "%s: %s enabling wmm mode\n",
1898 __func__, pcb->ipsec_if_xname);
1899
1900 init_params->output_sched_model = IFNET_SCHED_MODEL_DRIVER_MANAGED;
1901
1902 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_RINGS,
1903 IPSEC_NETIF_WMM_TX_RING_COUNT);
1904 VERIFY(err == 0);
1905 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_RINGS,
1906 IPSEC_NETIF_WMM_RX_RING_COUNT);
1907 VERIFY(err == 0);
1908
1909 err = kern_nexus_attr_set(nxa, NEXUS_ATTR_QMAP, NEXUS_QMAP_TYPE_WMM);
1910 VERIFY(err == 0);
1911 }
1912
5ba3f43e
A		 1913 pcb->ipsec_netif_txring_size = ring_size;
1914
0a7de745 1915 bzero(&pp_init, sizeof(pp_init));
a39ff7e2 1916 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
cb323159
A		 1917 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
1918 // Note: we need more packets than can be held in the tx and rx rings because
1919 // packets can also be in the AQM queue(s)
1920 pp_init.kbi_packets = pcb->ipsec_netif_ring_size * (2 * pcb->ipsec_kpipe_count + 1);
a39ff7e2
A		 1921 pp_init.kbi_bufsize = pcb->ipsec_slot_size;
1922 pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE;
1923 pp_init.kbi_max_frags = 1;
0a7de745 1924 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2 1925 "%s", provider_name);
cb323159
A		 1926 pp_init.kbi_ctx = NULL;
1927 pp_init.kbi_ctx_retain = NULL;
1928 pp_init.kbi_ctx_release = NULL;
a39ff7e2 1929
cb323159 1930 err = kern_pbufpool_create(&pp_init, &pcb->ipsec_netif_pp, NULL);
a39ff7e2 1931 if (err != 0) {
cb323159 1932 os_log_error(OS_LOG_DEFAULT, "%s pbufbool create failed, error %d\n", __func__, err);
a39ff7e2
A		 1933 goto failed;
1934 }
1935
5ba3f43e 1936 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 1937 ipsec_nx_dom_prov,
1938 provider_name,
1939 &prov_init,
1940 sizeof(prov_init),
1941 nxa,
1942 &pcb->ipsec_nx.if_provider);
5ba3f43e
A		 1943 IPSEC_IF_VERIFY(err == 0);
1944 if (err != 0) {
cb323159 1945 os_log_error(OS_LOG_DEFAULT, "%s register provider failed, error %d\n",
0a7de745 1946 __func__, err);
5ba3f43e
A		 1947 goto failed;
1948 }
1949
1950 bzero(&net_init, sizeof(net_init));
1951 net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
1952 net_init.nxneti_flags = 0;
1953 net_init.nxneti_eparams = init_params;
1954 net_init.nxneti_lladdr = NULL;
1955 net_init.nxneti_prepare = ipsec_netif_prepare;
f427ee49 1956 net_init.nxneti_rx_pbufpool = pcb->ipsec_netif_pp;
a39ff7e2 1957 net_init.nxneti_tx_pbufpool = pcb->ipsec_netif_pp;
5ba3f43e 1958 err = kern_nexus_controller_alloc_net_provider_instance(controller,
0a7de745
A		 1959 pcb->ipsec_nx.if_provider,
1960 pcb,
1961 &pcb->ipsec_nx.if_instance,
1962 &net_init,
1963 ifp);
5ba3f43e
A		 1964 IPSEC_IF_VERIFY(err == 0);
1965 if (err != 0) {
cb323159 1966 os_log_error(OS_LOG_DEFAULT, "%s alloc_net_provider_instance failed, %d\n",
0a7de745 1967 __func__, err);
5ba3f43e 1968 kern_nexus_controller_deregister_provider(controller,
0a7de745 1969 pcb->ipsec_nx.if_provider);
5ba3f43e
A		 1970 uuid_clear(pcb->ipsec_nx.if_provider);
1971 goto failed;
1972 }
1973
1974failed:
1975 if (nxa) {
1976 kern_nexus_attr_destroy(nxa);
1977 }
a39ff7e2
A		 1978 if (err && pcb->ipsec_netif_pp != NULL) {
1979 kern_pbufpool_destroy(pcb->ipsec_netif_pp);
1980 pcb->ipsec_netif_pp = NULL;
1981 }
0a7de745 1982 return err;
5ba3f43e
A		 1983}
1984
1985static void
1986ipsec_detach_provider_and_instance(uuid_t provider, uuid_t instance)
1987{
1988 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 1989 errno_t err;
5ba3f43e
A		 1990
1991 if (!uuid_is_null(instance)) {
1992 err = kern_nexus_controller_free_provider_instance(controller,
0a7de745 1993 instance);
5ba3f43e 1994 if (err != 0) {
cb323159 1995 os_log_error(OS_LOG_DEFAULT, "%s free_provider_instance failed %d\n",
0a7de745 1996 __func__, err);
5ba3f43e
A		 1997 }
1998 uuid_clear(instance);
1999 }
2000 if (!uuid_is_null(provider)) {
2001 err = kern_nexus_controller_deregister_provider(controller,
0a7de745 2002 provider);
5ba3f43e 2003 if (err != 0) {
cb323159 2004 os_log_error(OS_LOG_DEFAULT, "%s deregister_provider %d\n", __func__, err);
5ba3f43e
A		 2005 }
2006 uuid_clear(provider);
2007 }
2008 return;
2009}
2010
2011static void
a39ff7e2 2012ipsec_nexus_detach(struct ipsec_pcb *pcb)
5ba3f43e 2013{
a39ff7e2 2014 ipsec_nx_t nx = &pcb->ipsec_nx;
5ba3f43e 2015 nexus_controller_t controller = kern_nexus_shared_controller();
0a7de745 2016 errno_t err;
5ba3f43e 2017
cb323159 2018 if (!uuid_is_null(nx->fsw_host)) {
5ba3f43e 2019 err = kern_nexus_ifdetach(controller,
cb323159
A		 2020 nx->fsw_instance,
2021 nx->fsw_host);
5ba3f43e 2022 if (err != 0) {
cb323159 2023 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms host failed %d\n",
0a7de745 2024 __func__, err);
5ba3f43e
A		 2025 }
2026 }
2027
cb323159 2028 if (!uuid_is_null(nx->fsw_device)) {
5ba3f43e 2029 err = kern_nexus_ifdetach(controller,
cb323159
A		 2030 nx->fsw_instance,
2031 nx->fsw_device);
5ba3f43e 2032 if (err != 0) {
cb323159 2033 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_ifdetach ms device failed %d\n",
0a7de745 2034 __func__, err);
5ba3f43e
A		 2035 }
2036 }
2037
2038 ipsec_detach_provider_and_instance(nx->if_provider,
0a7de745 2039 nx->if_instance);
cb323159
A		 2040 ipsec_detach_provider_and_instance(nx->fsw_provider,
2041 nx->fsw_instance);
5ba3f43e 2042
a39ff7e2
A		 2043 if (pcb->ipsec_netif_pp != NULL) {
2044 kern_pbufpool_destroy(pcb->ipsec_netif_pp);
2045 pcb->ipsec_netif_pp = NULL;
a39ff7e2 2046 }
5ba3f43e
A		 2047 memset(nx, 0, sizeof(*nx));
2048}
2049
2050static errno_t
5c9f4661 2051ipsec_create_fs_provider_and_instance(struct ipsec_pcb *pcb,
cb323159 2052 const char *type_name,
0a7de745
A		 2053 const char *ifname,
2054 uuid_t *provider, uuid_t *instance)
5ba3f43e
A		 2055{
2056 nexus_attr_t attr = NULL;
2057 nexus_controller_t controller = kern_nexus_shared_controller();
2058 uuid_t dom_prov;
2059 errno_t err;
2060 struct kern_nexus_init init;
0a7de745 2061 nexus_name_t provider_name;
5ba3f43e 2062
cb323159 2063 err = kern_nexus_get_default_domain_provider(NEXUS_TYPE_FLOW_SWITCH,
0a7de745 2064 &dom_prov);
5ba3f43e
A		 2065 IPSEC_IF_VERIFY(err == 0);
2066 if (err != 0) {
cb323159 2067 os_log_error(OS_LOG_DEFAULT, "%s can't get %s provider, error %d\n",
0a7de745 2068 __func__, type_name, err);
5ba3f43e
A		 2069 goto failed;
2070 }
2071
2072 err = kern_nexus_attr_create(&attr);
2073 IPSEC_IF_VERIFY(err == 0);
2074 if (err != 0) {
cb323159 2075 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 2076 __func__, err);
5ba3f43e
A		 2077 goto failed;
2078 }
2079
5c9f4661 2080 uint64_t slot_buffer_size = pcb->ipsec_slot_size;
5ba3f43e
A		 2081 err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
2082 VERIFY(err == 0);
2083
2084 // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
5c9f4661 2085 uint64_t tx_ring_size = pcb->ipsec_tx_fsw_ring_size;
5ba3f43e
A		 2086 err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
2087 VERIFY(err == 0);
5c9f4661 2088 uint64_t rx_ring_size = pcb->ipsec_rx_fsw_ring_size;
5ba3f43e
A		 2089 err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
2090 VERIFY(err == 0);
f427ee49
A		 2091 /*
2092 * Configure flowswitch to use super-packet (multi-buflet).
2093 * This allows flowswitch to perform intra-stack packet aggregation.
2094 */
2095 err = kern_nexus_attr_set(attr, NEXUS_ATTR_MAX_FRAGS,
2096 sk_fsw_rx_agg_tcp ? NX_PBUF_FRAGS_MAX : 1);
2097 VERIFY(err == 0);
5ba3f43e
A		 2098
2099 snprintf((char *)provider_name, sizeof(provider_name),
0a7de745 2100 "com.apple.%s.%s", type_name, ifname);
5ba3f43e 2101 err = kern_nexus_controller_register_provider(controller,
0a7de745
A		 2102 dom_prov,
2103 provider_name,
2104 NULL,
2105 0,
2106 attr,
2107 provider);
5ba3f43e
A		 2108 kern_nexus_attr_destroy(attr);
2109 attr = NULL;
2110 IPSEC_IF_VERIFY(err == 0);
2111 if (err != 0) {
cb323159 2112 os_log_error(OS_LOG_DEFAULT, "%s register %s provider failed, error %d\n",
0a7de745 2113 __func__, type_name, err);
5ba3f43e
A		 2114 goto failed;
2115 }
0a7de745 2116 bzero(&init, sizeof(init));
5ba3f43e
A		 2117 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
2118 err = kern_nexus_controller_alloc_provider_instance(controller,
0a7de745
A		 2119 *provider,
2120 NULL,
2121 instance, &init);
5ba3f43e
A		 2122 IPSEC_IF_VERIFY(err == 0);
2123 if (err != 0) {
cb323159 2124 os_log_error(OS_LOG_DEFAULT, "%s alloc_provider_instance %s failed, %d\n",
0a7de745 2125 __func__, type_name, err);
5ba3f43e 2126 kern_nexus_controller_deregister_provider(controller,
0a7de745 2127 *provider);
5ba3f43e
A		 2128 uuid_clear(*provider);
2129 }
2130failed:
0a7de745 2131 return err;
5ba3f43e
A		 2132}
2133
2134static errno_t
cb323159 2135ipsec_flowswitch_attach(struct ipsec_pcb *pcb)
5ba3f43e
A		 2136{
2137 nexus_controller_t controller = kern_nexus_shared_controller();
2138 errno_t err = 0;
2139 ipsec_nx_t nx = &pcb->ipsec_nx;
2140
cb323159 2141 // Allocate flowswitch
5c9f4661 2142 err = ipsec_create_fs_provider_and_instance(pcb,
cb323159 2143 "flowswitch",
0a7de745 2144 pcb->ipsec_ifp->if_xname,
cb323159
A		 2145 &nx->fsw_provider,
2146 &nx->fsw_instance);
5ba3f43e 2147 if (err != 0) {
cb323159 2148 os_log_error(OS_LOG_DEFAULT, "%s: failed to create bridge provider and instance\n",
0a7de745 2149 __func__);
5ba3f43e
A		 2150 goto failed;
2151 }
2152
cb323159
A		 2153 // Attach flowswitch to device port
2154 err = kern_nexus_ifattach(controller, nx->fsw_instance,
0a7de745 2155 NULL, nx->if_instance,
cb323159 2156 FALSE, &nx->fsw_device);
5ba3f43e 2157 if (err != 0) {
cb323159 2158 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms device %d\n", __func__, err);
5ba3f43e
A		 2159 goto failed;
2160 }
2161
cb323159
A		 2162 // Attach flowswitch to host port
2163 err = kern_nexus_ifattach(controller, nx->fsw_instance,
0a7de745 2164 NULL, nx->if_instance,
cb323159 2165 TRUE, &nx->fsw_host);
5ba3f43e 2166 if (err != 0) {
cb323159 2167 os_log_error(OS_LOG_DEFAULT, "%s kern_nexus_ifattach ms host %d\n", __func__, err);
5ba3f43e
A		 2168 goto failed;
2169 }
2170
2171 // Extract the agent UUID and save for later
cb323159
A		 2172 struct kern_nexus *flowswitch_nx = nx_find(nx->fsw_instance, false);
2173 if (flowswitch_nx != NULL) {
2174 struct nx_flowswitch *flowswitch = NX_FSW_PRIVATE(flowswitch_nx);
5ba3f43e
A		 2175 if (flowswitch != NULL) {
2176 FSW_RLOCK(flowswitch);
cb323159 2177 uuid_copy(nx->fsw_agent, flowswitch->fsw_agent_uuid);
5ba3f43e
A		 2178 FSW_UNLOCK(flowswitch);
2179 } else {
cb323159 2180 os_log_error(OS_LOG_DEFAULT, "ipsec_flowswitch_attach - flowswitch is NULL\n");
5ba3f43e 2181 }
cb323159 2182 nx_release(flowswitch_nx);
5ba3f43e 2183 } else {
cb323159 2184 os_log_error(OS_LOG_DEFAULT, "ipsec_flowswitch_attach - unable to find flowswitch nexus\n");
5ba3f43e
A		 2185 }
2186
0a7de745 2187 return 0;
5ba3f43e
A		 2188
2189failed:
a39ff7e2 2190 ipsec_nexus_detach(pcb);
5ba3f43e
A		 2191
2192 errno_t detach_error = 0;
2193 if ((detach_error = ifnet_detach(pcb->ipsec_ifp)) != 0) {
cb323159 2194 panic("ipsec_flowswitch_attach - ifnet_detach failed: %d\n", detach_error);
5ba3f43e
A		 2195 /* NOT REACHED */
2196 }
2197
0a7de745 2198 return err;
5ba3f43e
A		 2199}
2200
2201#pragma mark Kernel Pipe Nexus
2202
2203static errno_t
cb323159 2204ipsec_register_kernel_pipe_nexus(struct ipsec_pcb *pcb)
5ba3f43e
A		 2205{
2206 nexus_attr_t nxa = NULL;
2207 errno_t result;
2208
2209 lck_mtx_lock(&ipsec_lock);
2210 if (ipsec_ncd_refcount++) {
2211 lck_mtx_unlock(&ipsec_lock);
2212 return 0;
2213 }
2214
2215 result = kern_nexus_controller_create(&ipsec_ncd);
2216 if (result) {
cb323159 2217 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_create failed: %d\n",
0a7de745 2218 __FUNCTION__, result);
5ba3f43e
A		 2219 goto done;
2220 }
2221
2222 uuid_t dom_prov;
cb323159 2223 result = kern_nexus_get_default_domain_provider(
0a7de745 2224 NEXUS_TYPE_KERNEL_PIPE, &dom_prov);
5ba3f43e 2225 if (result) {
cb323159 2226 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_get_default_domain_provider failed: %d\n",
0a7de745 2227 __FUNCTION__, result);
5ba3f43e
A		 2228 goto done;
2229 }
2230
2231 struct kern_nexus_provider_init prov_init = {
2232 .nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
2233 .nxpi_flags = NXPIF_VIRTUAL_DEVICE,
2234 .nxpi_pre_connect = ipsec_nexus_pre_connect,
2235 .nxpi_connected = ipsec_nexus_connected,
2236 .nxpi_pre_disconnect = ipsec_nexus_pre_disconnect,
2237 .nxpi_disconnected = ipsec_nexus_disconnected,
2238 .nxpi_ring_init = ipsec_kpipe_ring_init,
2239 .nxpi_ring_fini = ipsec_kpipe_ring_fini,
2240 .nxpi_slot_init = NULL,
2241 .nxpi_slot_fini = NULL,
2242 .nxpi_sync_tx = ipsec_kpipe_sync_tx,
2243 .nxpi_sync_rx = ipsec_kpipe_sync_rx,
2244 .nxpi_tx_doorbell = NULL,
2245 };
2246
2247 result = kern_nexus_attr_create(&nxa);
2248 if (result) {
cb323159 2249 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_attr_create failed: %d\n",
0a7de745 2250 __FUNCTION__, result);
5ba3f43e
A		 2251 goto done;
2252 }
2253
2254 uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
2255 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
2256 VERIFY(result == 0);
2257
2258 // Reset ring size for kernel pipe nexus to limit memory usage
cb323159
A		 2259 // Note: It's better to have less on slots on the kpipe TX ring than the netif
2260 // so back pressure is applied at the AQM layer
2261 uint64_t ring_size =
2262 pcb->ipsec_kpipe_tx_ring_size != 0 ? pcb->ipsec_kpipe_tx_ring_size :
2263 pcb->ipsec_netif_ring_size != 0 ? pcb->ipsec_netif_ring_size :
2264 if_ipsec_ring_size;
5ba3f43e
A		 2265 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
2266 VERIFY(result == 0);
cb323159
A		 2267
2268 ring_size =
2269 pcb->ipsec_kpipe_rx_ring_size != 0 ? pcb->ipsec_kpipe_rx_ring_size :
2270 pcb->ipsec_netif_ring_size != 0 ? pcb->ipsec_netif_ring_size :
2271 if_ipsec_ring_size;
5ba3f43e
A		 2272 result = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
2273 VERIFY(result == 0);
2274
2275 result = kern_nexus_controller_register_provider(ipsec_ncd,
0a7de745
A		 2276 dom_prov,
2277 (const uint8_t *)"com.apple.nexus.ipsec.kpipe",
2278 &prov_init,
2279 sizeof(prov_init),
2280 nxa,
2281 &ipsec_kpipe_uuid);
5ba3f43e 2282 if (result) {
cb323159 2283 os_log_error(OS_LOG_DEFAULT, "%s: kern_nexus_controller_register_provider failed: %d\n",
0a7de745 2284 __FUNCTION__, result);
5ba3f43e
A		 2285 goto done;
2286 }
2287
2288done:
2289 if (nxa) {
2290 kern_nexus_attr_destroy(nxa);
2291 }
2292
2293 if (result) {
2294 if (ipsec_ncd) {
2295 kern_nexus_controller_destroy(ipsec_ncd);
2296 ipsec_ncd = NULL;
2297 }
2298 ipsec_ncd_refcount = 0;
2299 }
2300
2301 lck_mtx_unlock(&ipsec_lock);
2302
2303 return result;
2304}
2305
2306static void
2307ipsec_unregister_kernel_pipe_nexus(void)
2308{
2309 lck_mtx_lock(&ipsec_lock);
2310
2311 VERIFY(ipsec_ncd_refcount > 0);
2312
2313 if (--ipsec_ncd_refcount == 0) {
2314 kern_nexus_controller_destroy(ipsec_ncd);
2315 ipsec_ncd = NULL;
2316 }
2317
2318 lck_mtx_unlock(&ipsec_lock);
2319}
2320
cb323159
A		 2321/* This structure only holds onto kpipe channels that need to be
2322 * freed in the future, but are cleared from the pcb under lock
2323 */
2324struct ipsec_detached_channels {
2325 int count;
2326 kern_pbufpool_t pp;
2327 uuid_t uuids[IPSEC_IF_MAX_RING_COUNT];
2328};
5ba3f43e 2329
cb323159
A		 2330static void
2331ipsec_detach_channels(struct ipsec_pcb *pcb, struct ipsec_detached_channels *dc)
2332{
2333 LCK_RW_ASSERT(&pcb->ipsec_pcb_lock, LCK_RW_TYPE_EXCLUSIVE);
5ba3f43e 2334
cb323159
A		 2335 if (!ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED)) {
2336 for (int i = 0; i < IPSEC_IF_MAX_RING_COUNT; i++) {
2337 VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid[i]));
2338 }
2339 dc->count = 0;
2340 return;
2341 }
5ba3f43e 2342
cb323159 2343 dc->count = pcb->ipsec_kpipe_count;
39236c6e 2344
cb323159
A		 2345 VERIFY(dc->count >= 0);
2346 VERIFY(dc->count <= IPSEC_IF_MAX_RING_COUNT);
39236c6e 2347
cb323159
A		 2348 for (int i = 0; i < dc->count; i++) {
2349 VERIFY(!uuid_is_null(pcb->ipsec_kpipe_uuid[i]));
2350 uuid_copy(dc->uuids[i], pcb->ipsec_kpipe_uuid[i]);
2351 uuid_clear(pcb->ipsec_kpipe_uuid[i]);
2352 }
2353 for (int i = dc->count; i < IPSEC_IF_MAX_RING_COUNT; i++) {
2354 VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid[i]));
2355 }
5ba3f43e 2356
cb323159
A		 2357 if (dc->count) {
2358 VERIFY(pcb->ipsec_kpipe_pp);
5ba3f43e 2359 } else {
cb323159 2360 VERIFY(!pcb->ipsec_kpipe_pp);
5ba3f43e
A		 2361 }
2362
cb323159
A		 2363 dc->pp = pcb->ipsec_kpipe_pp;
2364
2365 pcb->ipsec_kpipe_pp = NULL;
2366
2367 ipsec_flag_clr(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED);
2368}
2369
2370static void
2371ipsec_free_channels(struct ipsec_detached_channels *dc)
2372{
2373 if (!dc->count) {
2374 return;
5ba3f43e
A		 2375 }
2376
cb323159
A		 2377 for (int i = 0; i < dc->count; i++) {
2378 errno_t result;
2379 result = kern_nexus_controller_free_provider_instance(ipsec_ncd, dc->uuids[i]);
2380 VERIFY(!result);
2381 }
2382
2383 VERIFY(dc->pp);
2384 kern_pbufpool_destroy(dc->pp);
2385
2386 ipsec_unregister_kernel_pipe_nexus();
2387
2388 memset(dc, 0, sizeof(*dc));
5ba3f43e
A		 2389}
2390
2391static errno_t
2392ipsec_enable_channel(struct ipsec_pcb *pcb, struct proc *proc)
39236c6e 2393{
5ba3f43e 2394 struct kern_nexus_init init;
a39ff7e2 2395 struct kern_pbufpool_init pp_init;
5ba3f43e
A		 2396 errno_t result;
2397
d9a64523
A		 2398 kauth_cred_t cred = kauth_cred_get();
2399 result = priv_check_cred(cred, PRIV_SKYWALK_REGISTER_KERNEL_PIPE, 0);
2400 if (result) {
2401 return result;
2402 }
2403
cb323159
A		 2404 VERIFY(pcb->ipsec_kpipe_count);
2405 VERIFY(!ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED));
5ba3f43e 2406
cb323159 2407 result = ipsec_register_kernel_pipe_nexus(pcb);
5ba3f43e
A		 2408
2409 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2410
cb323159
A		 2411 if (result) {
2412 os_log_error(OS_LOG_DEFAULT, "%s: %s failed to register kernel pipe nexus\n",
2413 __func__, pcb->ipsec_if_xname);
a39ff7e2
A		 2414 goto done;
2415 }
2416
cb323159 2417 VERIFY(ipsec_ncd);
5ba3f43e 2418
0a7de745 2419 bzero(&pp_init, sizeof(pp_init));
a39ff7e2 2420 pp_init.kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
cb323159
A		 2421 pp_init.kbi_flags |= KBIF_VIRTUAL_DEVICE;
2422 // Note: We only needs are many packets as can be held in the tx and rx rings
2423 pp_init.kbi_packets = pcb->ipsec_netif_ring_size * 2 * pcb->ipsec_kpipe_count;
a39ff7e2
A		 2424 pp_init.kbi_bufsize = pcb->ipsec_slot_size;
2425 pp_init.kbi_buf_seg_size = IPSEC_IF_DEFAULT_BUF_SEG_SIZE;
2426 pp_init.kbi_max_frags = 1;
2427 pp_init.kbi_flags |= KBIF_QUANTUM;
0a7de745 2428 (void) snprintf((char *)pp_init.kbi_name, sizeof(pp_init.kbi_name),
a39ff7e2 2429 "com.apple.kpipe.%s", pcb->ipsec_if_xname);
cb323159
A		 2430 pp_init.kbi_ctx = NULL;
2431 pp_init.kbi_ctx_retain = NULL;
2432 pp_init.kbi_ctx_release = NULL;
a39ff7e2 2433
cb323159 2434 result = kern_pbufpool_create(&pp_init, &pcb->ipsec_kpipe_pp,
a39ff7e2
A		 2435 NULL);
2436 if (result != 0) {
cb323159
A		 2437 os_log_error(OS_LOG_DEFAULT, "%s: %s pbufbool create failed, error %d\n",
2438 __func__, pcb->ipsec_if_xname, result);
a39ff7e2
A		 2439 goto done;
2440 }
2441
0a7de745 2442 bzero(&init, sizeof(init));
5ba3f43e 2443 init.nxi_version = KERN_NEXUS_CURRENT_VERSION;
a39ff7e2 2444 init.nxi_tx_pbufpool = pcb->ipsec_kpipe_pp;
5ba3f43e 2445
cb323159
A		 2446 for (unsigned int i = 0; i < pcb->ipsec_kpipe_count; i++) {
2447 VERIFY(uuid_is_null(pcb->ipsec_kpipe_uuid[i]));
2448 result = kern_nexus_controller_alloc_provider_instance(ipsec_ncd,
2449 ipsec_kpipe_uuid, pcb, &pcb->ipsec_kpipe_uuid[i], &init);
2450
2451 if (result == 0) {
2452 nexus_port_t port = NEXUS_PORT_KERNEL_PIPE_CLIENT;
2453 pid_t pid = pcb->ipsec_kpipe_pid;
2454 if (!pid) {
2455 pid = proc_pid(proc);
2456 }
2457 result = kern_nexus_controller_bind_provider_instance(ipsec_ncd,
2458 pcb->ipsec_kpipe_uuid[i], &port,
2459 pid, NULL, NULL, 0, NEXUS_BIND_PID);
2460 }
5ba3f43e 2461
cb323159
A		 2462 if (result) {
2463 /* Unwind all of them on error */
2464 for (int j = 0; j < IPSEC_IF_MAX_RING_COUNT; j++) {
2465 if (!uuid_is_null(pcb->ipsec_kpipe_uuid[j])) {
2466 kern_nexus_controller_free_provider_instance(ipsec_ncd,
2467 pcb->ipsec_kpipe_uuid[j]);
2468 uuid_clear(pcb->ipsec_kpipe_uuid[j]);
2469 }
2470 }
2471 goto done;
2472 }
2473 }
5ba3f43e
A		 2474
2475done:
2476 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
0a7de745 2477
5ba3f43e 2478 if (result) {
a39ff7e2
A		 2479 if (pcb->ipsec_kpipe_pp != NULL) {
2480 kern_pbufpool_destroy(pcb->ipsec_kpipe_pp);
2481 pcb->ipsec_kpipe_pp = NULL;
2482 }
5ba3f43e 2483 ipsec_unregister_kernel_pipe_nexus();
cb323159
A		 2484 } else {
2485 ipsec_flag_set(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED);
39236c6e 2486 }
0a7de745 2487
5ba3f43e 2488 return result;
39236c6e
A		 2489}
2490
5ba3f43e
A		 2491#endif // IPSEC_NEXUS
2492
39236c6e
A		 2493
2494/* Kernel control functions */
2495
f427ee49
A		 2496static inline int
2497ipsec_find_by_unit(u_int32_t unit)
2498{
2499 struct ipsec_pcb *next_pcb = NULL;
2500 int found = 0;
2501
2502 TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) {
2503 if (next_pcb->ipsec_unit == unit) {
2504 found = 1;
2505 break;
2506 }
2507 }
2508
2509 return found;
2510}
2511
5ba3f43e 2512static inline void
f427ee49 2513ipsec_free_pcb(struct ipsec_pcb *pcb, bool locked)
5ba3f43e
A		 2514{
2515#if IPSEC_NEXUS
2516 mbuf_freem_list(pcb->ipsec_input_chain);
cb323159 2517 pcb->ipsec_input_chain_count = 0;
5ba3f43e 2518 lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp);
cb323159
A		 2519 lck_mtx_destroy(&pcb->ipsec_kpipe_encrypt_lock, ipsec_lck_grp);
2520 lck_mtx_destroy(&pcb->ipsec_kpipe_decrypt_lock, ipsec_lck_grp);
5ba3f43e 2521#endif // IPSEC_NEXUS
cb323159 2522 lck_mtx_destroy(&pcb->ipsec_pcb_data_move_lock, ipsec_lck_grp);
5ba3f43e 2523 lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp);
f427ee49 2524 if (!locked) {
5c9f4661 2525 lck_mtx_lock(&ipsec_lock);
f427ee49
A		 2526 }
2527 TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain);
2528 if (!locked) {
5c9f4661
A		 2529 lck_mtx_unlock(&ipsec_lock);
2530 }
5ba3f43e
A		 2531 zfree(ipsec_pcb_zone, pcb);
2532}
2533
f427ee49
A		 2534static errno_t
2535ipsec_ctl_setup(u_int32_t *unit, void **unitinfo)
2536{
2537 if (unit == NULL || unitinfo == NULL) {
2538 return EINVAL;
2539 }
2540
2541 lck_mtx_lock(&ipsec_lock);
2542
2543 /* Find next available unit */
2544 if (*unit == 0) {
2545 *unit = 1;
2546 while (*unit != ctl_maxunit) {
2547 if (ipsec_find_by_unit(*unit)) {
2548 (*unit)++;
2549 } else {
2550 break;
2551 }
2552 }
2553 if (*unit == ctl_maxunit) {
2554 lck_mtx_unlock(&ipsec_lock);
2555 return EBUSY;
2556 }
2557 } else if (ipsec_find_by_unit(*unit)) {
2558 lck_mtx_unlock(&ipsec_lock);
2559 return EBUSY;
2560 }
2561
2562 /* Find some open interface id */
2563 u_int32_t chosen_unique_id = 1;
2564 struct ipsec_pcb *next_pcb = TAILQ_LAST(&ipsec_head, ipsec_list);
2565 if (next_pcb != NULL) {
2566 /* List was not empty, add one to the last item */
2567 chosen_unique_id = next_pcb->ipsec_unique_id + 1;
2568 next_pcb = NULL;
2569
2570 /*
2571 * If this wrapped the id number, start looking at
2572 * the front of the list for an unused id.
2573 */
2574 if (chosen_unique_id == 0) {
2575 /* Find the next unused ID */
2576 chosen_unique_id = 1;
2577 TAILQ_FOREACH(next_pcb, &ipsec_head, ipsec_chain) {
2578 if (next_pcb->ipsec_unique_id > chosen_unique_id) {
2579 /* We found a gap */
2580 break;
2581 }
2582
2583 chosen_unique_id = next_pcb->ipsec_unique_id + 1;
2584 }
2585 }
2586 }
2587
2588 struct ipsec_pcb *pcb = zalloc_flags(ipsec_pcb_zone, Z_WAITOK | Z_ZERO);
2589
2590 *unitinfo = pcb;
2591 pcb->ipsec_unit = *unit;
2592 pcb->ipsec_unique_id = chosen_unique_id;
2593
2594 if (next_pcb != NULL) {
2595 TAILQ_INSERT_BEFORE(next_pcb, pcb, ipsec_chain);
2596 } else {
2597 TAILQ_INSERT_TAIL(&ipsec_head, pcb, ipsec_chain);
2598 }
2599
2600 lck_mtx_unlock(&ipsec_lock);
2601
2602 return 0;
2603}
2604
39236c6e 2605static errno_t
5c9f4661 2606ipsec_ctl_bind(kern_ctl_ref kctlref,
0a7de745
A		 2607 struct sockaddr_ctl *sac,
2608 void **unitinfo)
39236c6e 2609{
f427ee49
A		 2610 if (*unitinfo == NULL) {
2611 u_int32_t unit = 0;
2612 (void)ipsec_ctl_setup(&unit, unitinfo);
2613 }
2614
2615 struct ipsec_pcb *pcb = (struct ipsec_pcb *)*unitinfo;
2616 if (pcb == NULL) {
2617 return EINVAL;
2618 }
39037602 2619
39236c6e 2620 /* Setup the protocol control block */
39236c6e
A		 2621 pcb->ipsec_ctlref = kctlref;
2622 pcb->ipsec_unit = sac->sc_unit;
fe8ab488 2623 pcb->ipsec_output_service_class = MBUF_SC_OAM;
5ba3f43e 2624
5c9f4661
A		 2625#if IPSEC_NEXUS
2626 pcb->ipsec_use_netif = false;
2627 pcb->ipsec_slot_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
cb323159
A		 2628 pcb->ipsec_netif_ring_size = if_ipsec_ring_size;
2629 pcb->ipsec_tx_fsw_ring_size = if_ipsec_tx_fsw_ring_size;
2630 pcb->ipsec_rx_fsw_ring_size = if_ipsec_rx_fsw_ring_size;
5c9f4661
A		 2631#endif // IPSEC_NEXUS
2632
2633 lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr);
cb323159 2634 lck_mtx_init(&pcb->ipsec_pcb_data_move_lock, ipsec_lck_grp, ipsec_lck_attr);
5c9f4661 2635#if IPSEC_NEXUS
cb323159 2636 pcb->ipsec_input_chain_count = 0;
5c9f4661 2637 lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr);
cb323159
A		 2638 lck_mtx_init(&pcb->ipsec_kpipe_encrypt_lock, ipsec_lck_grp, ipsec_lck_attr);
2639 lck_mtx_init(&pcb->ipsec_kpipe_decrypt_lock, ipsec_lck_grp, ipsec_lck_attr);
5c9f4661
A		 2640#endif // IPSEC_NEXUS
2641
0a7de745 2642 return 0;
5c9f4661
A		 2643}
2644
2645static errno_t
2646ipsec_ctl_connect(kern_ctl_ref kctlref,
0a7de745
A		 2647 struct sockaddr_ctl *sac,
2648 void **unitinfo)
5c9f4661
A		 2649{
2650 struct ifnet_init_eparams ipsec_init = {};
2651 errno_t result = 0;
2652
2653 if (*unitinfo == NULL) {
2654 (void)ipsec_ctl_bind(kctlref, sac, unitinfo);
2655 }
2656
2657 struct ipsec_pcb *pcb = *unitinfo;
94ff46dc
A		 2658 if (pcb == NULL) {
2659 return EINVAL;
2660 }
5c9f4661 2661
f427ee49
A		 2662 /* Handle case where ipsec_ctl_setup() was called, but ipsec_ctl_bind() was not */
2663 if (pcb->ipsec_ctlref == NULL) {
2664 (void)ipsec_ctl_bind(kctlref, sac, unitinfo);
5ba3f43e 2665 }
5ba3f43e
A		 2666
2667 snprintf(pcb->ipsec_if_xname, sizeof(pcb->ipsec_if_xname), "ipsec%d", pcb->ipsec_unit - 1);
2668 snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1);
cb323159 2669 os_log(OS_LOG_DEFAULT, "ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name);
5ba3f43e 2670
39236c6e
A		 2671 /* Create the interface */
2672 bzero(&ipsec_init, sizeof(ipsec_init));
2673 ipsec_init.ver = IFNET_INIT_CURRENT_VERSION;
0a7de745 2674 ipsec_init.len = sizeof(ipsec_init);
5ba3f43e
A		 2675
2676#if IPSEC_NEXUS
5c9f4661
A		 2677 if (pcb->ipsec_use_netif) {
2678 ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
2679 } else
5ba3f43e 2680#endif // IPSEC_NEXUS
5c9f4661
A		 2681 {
2682 ipsec_init.flags = IFNET_INIT_NX_NOAUTO;
2683 ipsec_init.start = ipsec_start;
2684 }
5ba3f43e 2685 ipsec_init.name = "ipsec";
39236c6e 2686 ipsec_init.unit = pcb->ipsec_unit - 1;
5ba3f43e
A		 2687 ipsec_init.uniqueid = pcb->ipsec_unique_name;
2688 ipsec_init.uniqueid_len = strlen(pcb->ipsec_unique_name);
cb323159 2689 ipsec_init.family = IFNET_FAMILY_IPSEC;
39236c6e
A		 2690 ipsec_init.type = IFT_OTHER;
2691 ipsec_init.demux = ipsec_demux;
2692 ipsec_init.add_proto = ipsec_add_proto;
2693 ipsec_init.del_proto = ipsec_del_proto;
2694 ipsec_init.softc = pcb;
2695 ipsec_init.ioctl = ipsec_ioctl;
f427ee49 2696 ipsec_init.free = ipsec_detached;
5ba3f43e
A		 2697
2698#if IPSEC_NEXUS
cb323159
A		 2699 /* We don't support kpipes without a netif */
2700 if (pcb->ipsec_kpipe_count && !pcb->ipsec_use_netif) {
2701 result = ENOTSUP;
2702 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_connect - kpipe requires netif: failed %d\n", result);
f427ee49 2703 ipsec_free_pcb(pcb, false);
cb323159
A		 2704 *unitinfo = NULL;
2705 return result;
2706 }
2707
2708 if (if_ipsec_debug != 0) {
2709 printf("%s: %s%d use_netif %d kpipe_count %d slot_size %u ring_size %u "
2710 "kpipe_tx_ring_size %u kpipe_rx_ring_size %u\n",
2711 __func__,
2712 ipsec_init.name, ipsec_init.unit,
2713 pcb->ipsec_use_netif,
2714 pcb->ipsec_kpipe_count,
2715 pcb->ipsec_slot_size,
2716 pcb->ipsec_netif_ring_size,
2717 pcb->ipsec_kpipe_tx_ring_size,
2718 pcb->ipsec_kpipe_rx_ring_size);
2719 }
5c9f4661 2720 if (pcb->ipsec_use_netif) {
cb323159
A		 2721 if (pcb->ipsec_kpipe_count) {
2722 result = ipsec_enable_channel(pcb, current_proc());
2723 if (result) {
2724 os_log_error(OS_LOG_DEFAULT, "%s: %s failed to enable channels\n",
2725 __func__, pcb->ipsec_if_xname);
f427ee49 2726 ipsec_free_pcb(pcb, false);
cb323159
A		 2727 *unitinfo = NULL;
2728 return result;
2729 }
2730 }
2731
5c9f4661
A		 2732 result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp);
2733 if (result != 0) {
cb323159 2734 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result);
f427ee49 2735 ipsec_free_pcb(pcb, false);
5c9f4661
A		 2736 *unitinfo = NULL;
2737 return result;
2738 }
5ba3f43e 2739
cb323159 2740 result = ipsec_flowswitch_attach(pcb);
5c9f4661 2741 if (result != 0) {
cb323159
A		 2742 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_connect - ipsec_flowswitch_attach failed: %d\n", result);
2743 // Do not call ipsec_free_pcb(). We will be attached already, and will be freed later
2744 // in ipsec_detached().
5c9f4661
A		 2745 *unitinfo = NULL;
2746 return result;
2747 }
5ba3f43e 2748
5c9f4661
A		 2749 /* Attach to bpf */
2750 bpfattach(pcb->ipsec_ifp, DLT_RAW, 0);
2751 } else
5ba3f43e 2752#endif // IPSEC_NEXUS
5c9f4661
A		 2753 {
2754 result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp);
2755 if (result != 0) {
cb323159 2756 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_connect - ifnet_allocate failed: %d\n", result);
f427ee49 2757 ipsec_free_pcb(pcb, false);
5c9f4661
A		 2758 *unitinfo = NULL;
2759 return result;
2760 }
2761 ipsec_ifnet_set_attrs(pcb->ipsec_ifp);
2762
2763 /* Attach the interface */
2764 result = ifnet_attach(pcb->ipsec_ifp, NULL);
2765 if (result != 0) {
cb323159 2766 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_connect - ifnet_attach failed: %d\n", result);
5c9f4661 2767 ifnet_release(pcb->ipsec_ifp);
f427ee49 2768 ipsec_free_pcb(pcb, false);
5c9f4661 2769 *unitinfo = NULL;
0a7de745 2770 return result;
5c9f4661 2771 }
5ba3f43e 2772
5c9f4661
A		 2773 /* Attach to bpf */
2774 bpfattach(pcb->ipsec_ifp, DLT_NULL, 0);
2775 }
5ba3f43e 2776
c6bf4f31 2777#if IPSEC_NEXUS
cb323159
A		 2778 /*
2779 * Mark the data path as ready.
2780 * If kpipe nexus is being used then the data path is marked ready only when a kpipe channel is connected.
2781 */
2782 if (pcb->ipsec_kpipe_count == 0) {
2783 lck_mtx_lock(&pcb->ipsec_pcb_data_move_lock);
2784 IPSEC_SET_DATA_PATH_READY(pcb);
2785 lck_mtx_unlock(&pcb->ipsec_pcb_data_move_lock);
2786 }
c6bf4f31 2787#endif
cb323159 2788
5ba3f43e
A		 2789 /* The interfaces resoures allocated, mark it as running */
2790 ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING);
2791
0a7de745 2792 return 0;
39236c6e
A		 2793}
2794
2795static errno_t
0a7de745
A		 2796ipsec_detach_ip(ifnet_t interface,
2797 protocol_family_t protocol,
2798 socket_t pf_socket)
39236c6e
A		 2799{
2800 errno_t result = EPROTONOSUPPORT;
0a7de745 2801
39236c6e
A		 2802 /* Attempt a detach */
2803 if (protocol == PF_INET) {
0a7de745
A		 2804 struct ifreq ifr;
2805
39236c6e
A		 2806 bzero(&ifr, sizeof(ifr));
2807 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745
A		 2808 ifnet_name(interface), ifnet_unit(interface));
2809
39236c6e 2810 result = sock_ioctl(pf_socket, SIOCPROTODETACH, &ifr);
0a7de745
A		 2811 } else if (protocol == PF_INET6) {
2812 struct in6_ifreq ifr6;
2813
39236c6e
A		 2814 bzero(&ifr6, sizeof(ifr6));
2815 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745
A		 2816 ifnet_name(interface), ifnet_unit(interface));
2817
39236c6e
A		 2818 result = sock_ioctl(pf_socket, SIOCPROTODETACH_IN6, &ifr6);
2819 }
0a7de745 2820
39236c6e
A		 2821 return result;
2822}
2823
2824static void
0a7de745
A		 2825ipsec_remove_address(ifnet_t interface,
2826 protocol_family_t protocol,
2827 ifaddr_t address,
2828 socket_t pf_socket)
39236c6e
A		 2829{
2830 errno_t result = 0;
0a7de745 2831
39236c6e
A		 2832 /* Attempt a detach */
2833 if (protocol == PF_INET) {
0a7de745
A		 2834 struct ifreq ifr;
2835
39236c6e
A		 2836 bzero(&ifr, sizeof(ifr));
2837 snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d",
0a7de745 2838 ifnet_name(interface), ifnet_unit(interface));
39236c6e
A		 2839 result = ifaddr_address(address, &ifr.ifr_addr, sizeof(ifr.ifr_addr));
2840 if (result != 0) {
cb323159 2841 os_log_error(OS_LOG_DEFAULT, "ipsec_remove_address - ifaddr_address failed: %d", result);
0a7de745 2842 } else {
39236c6e
A		 2843 result = sock_ioctl(pf_socket, SIOCDIFADDR, &ifr);
2844 if (result != 0) {
cb323159 2845 os_log_error(OS_LOG_DEFAULT, "ipsec_remove_address - SIOCDIFADDR failed: %d", result);
39236c6e
A		 2846 }
2847 }
0a7de745
A		 2848 } else if (protocol == PF_INET6) {
2849 struct in6_ifreq ifr6;
2850
39236c6e
A		 2851 bzero(&ifr6, sizeof(ifr6));
2852 snprintf(ifr6.ifr_name, sizeof(ifr6.ifr_name), "%s%d",
0a7de745 2853 ifnet_name(interface), ifnet_unit(interface));
39236c6e 2854 result = ifaddr_address(address, (struct sockaddr*)&ifr6.ifr_addr,
0a7de745 2855 sizeof(ifr6.ifr_addr));
39236c6e 2856 if (result != 0) {
cb323159 2857 os_log_error(OS_LOG_DEFAULT, "ipsec_remove_address - ifaddr_address failed (v6): %d",
0a7de745
A		 2858 result);
2859 } else {
39236c6e
A		 2860 result = sock_ioctl(pf_socket, SIOCDIFADDR_IN6, &ifr6);
2861 if (result != 0) {
cb323159 2862 os_log_error(OS_LOG_DEFAULT, "ipsec_remove_address - SIOCDIFADDR_IN6 failed: %d",
0a7de745 2863 result);
39236c6e
A		 2864 }
2865 }
2866 }
2867}
2868
2869static void
0a7de745
A		 2870ipsec_cleanup_family(ifnet_t interface,
2871 protocol_family_t protocol)
39236c6e 2872{
0a7de745
A		 2873 errno_t result = 0;
2874 socket_t pf_socket = NULL;
2875 ifaddr_t *addresses = NULL;
2876 int i;
2877
39236c6e 2878 if (protocol != PF_INET && protocol != PF_INET6) {
cb323159 2879 os_log_error(OS_LOG_DEFAULT, "ipsec_cleanup_family - invalid protocol family %d\n", protocol);
39236c6e
A		 2880 return;
2881 }
0a7de745 2882
39236c6e
A		 2883 /* Create a socket for removing addresses and detaching the protocol */
2884 result = sock_socket(protocol, SOCK_DGRAM, 0, NULL, NULL, &pf_socket);
2885 if (result != 0) {
0a7de745 2886 if (result != EAFNOSUPPORT) {
cb323159 2887 os_log_error(OS_LOG_DEFAULT, "ipsec_cleanup_family - failed to create %s socket: %d\n",
0a7de745
A		 2888 protocol == PF_INET ? "IP" : "IPv6", result);
2889 }
39236c6e
A		 2890 goto cleanup;
2891 }
0a7de745 2892
39236c6e
A		 2893 /* always set SS_PRIV, we want to close and detach regardless */
2894 sock_setpriv(pf_socket, 1);
0a7de745 2895
39236c6e
A		 2896 result = ipsec_detach_ip(interface, protocol, pf_socket);
2897 if (result == 0 || result == ENXIO) {
2898 /* We are done! We either detached or weren't attached. */
2899 goto cleanup;
0a7de745 2900 } else if (result != EBUSY) {
39236c6e 2901 /* Uh, not really sure what happened here... */
cb323159 2902 os_log_error(OS_LOG_DEFAULT, "ipsec_cleanup_family - ipsec_detach_ip failed: %d\n", result);
39236c6e
A		 2903 goto cleanup;
2904 }
0a7de745 2905
39236c6e
A		 2906 /*
2907 * At this point, we received an EBUSY error. This means there are
2908 * addresses attached. We should detach them and then try again.
2909 */
2910 result = ifnet_get_address_list_family(interface, &addresses, protocol);
2911 if (result != 0) {
cb323159 2912 os_log_error(OS_LOG_DEFAULT, "fnet_get_address_list_family(%s%d, 0xblah, %s) - failed: %d\n",
0a7de745
A		 2913 ifnet_name(interface), ifnet_unit(interface),
2914 protocol == PF_INET ? "PF_INET" : "PF_INET6", result);
39236c6e
A		 2915 goto cleanup;
2916 }
0a7de745 2917
39236c6e
A		 2918 for (i = 0; addresses[i] != 0; i++) {
2919 ipsec_remove_address(interface, protocol, addresses[i], pf_socket);
2920 }
2921 ifnet_free_address_list(addresses);
2922 addresses = NULL;
0a7de745 2923
39236c6e
A		 2924 /*
2925 * The addresses should be gone, we should try the remove again.
2926 */
2927 result = ipsec_detach_ip(interface, protocol, pf_socket);
2928 if (result != 0 && result != ENXIO) {
cb323159 2929 os_log_error(OS_LOG_DEFAULT, "ipsec_cleanup_family - ipsec_detach_ip failed: %d\n", result);
39236c6e 2930 }
0a7de745 2931
39236c6e 2932cleanup:
0a7de745 2933 if (pf_socket != NULL) {
39236c6e 2934 sock_close(pf_socket);
0a7de745
A		 2935 }
2936
2937 if (addresses != NULL) {
39236c6e 2938 ifnet_free_address_list(addresses);
0a7de745 2939 }
39236c6e
A		 2940}
2941
2942static errno_t
0a7de745
A		 2943ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref,
2944 __unused u_int32_t unit,
2945 void *unitinfo)
39236c6e 2946{
5ba3f43e
A		 2947 struct ipsec_pcb *pcb = unitinfo;
2948 ifnet_t ifp = NULL;
2949 errno_t result = 0;
39037602 2950
5ba3f43e 2951 if (pcb == NULL) {
39037602 2952 return EINVAL;
5ba3f43e
A		 2953 }
2954
cb323159
A		 2955 /* Wait until all threads in the data paths are done. */
2956 ipsec_wait_data_move_drain(pcb);
2957
5ba3f43e
A		 2958#if IPSEC_NEXUS
2959 // Tell the nexus to stop all rings
2960 if (pcb->ipsec_netif_nexus != NULL) {
2961 kern_nexus_stop(pcb->ipsec_netif_nexus);
2962 }
2963#endif // IPSEC_NEXUS
2964
2965 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
2966
2967#if IPSEC_NEXUS
cb323159
A		 2968 if (if_ipsec_debug != 0) {
2969 printf("ipsec_ctl_disconnect: detaching interface %s (id %s)\n",
2970 pcb->ipsec_if_xname, pcb->ipsec_unique_name);
2971 }
2972
2973 struct ipsec_detached_channels dc;
2974 ipsec_detach_channels(pcb, &dc);
5ba3f43e 2975#endif // IPSEC_NEXUS
39037602 2976
39236c6e 2977 pcb->ipsec_ctlref = NULL;
5ba3f43e 2978
5c9f4661
A		 2979 ifp = pcb->ipsec_ifp;
2980 if (ifp != NULL) {
2981#if IPSEC_NEXUS
2982 if (pcb->ipsec_netif_nexus != NULL) {
2983 /*
2984 * Quiesce the interface and flush any pending outbound packets.
2985 */
2986 if_down(ifp);
2987
2988 /* Increment refcnt, but detach interface */
2989 ifnet_incr_iorefcnt(ifp);
2990 if ((result = ifnet_detach(ifp)) != 0) {
2991 panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
2992 /* NOT REACHED */
2993 }
5ba3f43e 2994
5c9f4661
A		 2995 /*
2996 * We want to do everything in our power to ensure that the interface
2997 * really goes away when the socket is closed. We must remove IP/IPv6
2998 * addresses and detach the protocols. Finally, we can remove and
2999 * release the interface.
3000 */
3001 key_delsp_for_ipsec_if(ifp);
5ba3f43e 3002
5c9f4661
A		 3003 ipsec_cleanup_family(ifp, AF_INET);
3004 ipsec_cleanup_family(ifp, AF_INET6);
5ba3f43e 3005
5c9f4661
A		 3006 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
3007
cb323159
A		 3008 ipsec_free_channels(&dc);
3009
a39ff7e2 3010 ipsec_nexus_detach(pcb);
5c9f4661
A		 3011
3012 /* Decrement refcnt to finish detaching and freeing */
3013 ifnet_decr_iorefcnt(ifp);
3014 } else
3015#endif // IPSEC_NEXUS
3016 {
3017 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
5ba3f43e
A		 3018
3019#if IPSEC_NEXUS
cb323159 3020 ipsec_free_channels(&dc);
5ba3f43e
A		 3021#endif // IPSEC_NEXUS
3022
5c9f4661
A		 3023 /*
3024 * We want to do everything in our power to ensure that the interface
3025 * really goes away when the socket is closed. We must remove IP/IPv6
3026 * addresses and detach the protocols. Finally, we can remove and
3027 * release the interface.
3028 */
3029 key_delsp_for_ipsec_if(ifp);
3030
3031 ipsec_cleanup_family(ifp, AF_INET);
3032 ipsec_cleanup_family(ifp, AF_INET6);
3033
3034 /*
3035 * Detach now; ipsec_detach() will be called asynchronously once
3036 * the I/O reference count drops to 0. There we will invoke
3037 * ifnet_release().
3038 */
3039 if ((result = ifnet_detach(ifp)) != 0) {
cb323159 3040 os_log_error(OS_LOG_DEFAULT, "ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
5c9f4661
A		 3041 }
3042 }
3043 } else {
3044 // Bound, but not connected
3045 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
3046 ipsec_free_pcb(pcb, false);
3047 }
0a7de745 3048
39236c6e
A		 3049 return 0;
3050}
3051
3052static errno_t
0a7de745
A		 3053ipsec_ctl_send(__unused kern_ctl_ref kctlref,
3054 __unused u_int32_t unit,
3055 __unused void *unitinfo,
3056 mbuf_t m,
3057 __unused int flags)
39236c6e 3058{
0a7de745
A		 3059 /* Receive messages from the control socket. Currently unused. */
3060 mbuf_freem(m);
39236c6e
A		 3061 return 0;
3062}
3063
3064static errno_t
0a7de745
A		 3065ipsec_ctl_setopt(__unused kern_ctl_ref kctlref,
3066 __unused u_int32_t unit,
3067 void *unitinfo,
3068 int opt,
3069 void *data,
3070 size_t len)
39236c6e 3071{
0a7de745 3072 errno_t result = 0;
94ff46dc
A		 3073 struct ipsec_pcb *pcb = unitinfo;
3074 if (pcb == NULL) {
3075 return EINVAL;
3076 }
0a7de745 3077
39236c6e
A		 3078 /* check for privileges for privileged options */
3079 switch (opt) {
0a7de745
A		 3080 case IPSEC_OPT_FLAGS:
3081 case IPSEC_OPT_EXT_IFDATA_STATS:
3082 case IPSEC_OPT_SET_DELEGATE_INTERFACE:
3083 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS:
3084 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
3085 return EPERM;
3086 }
3087 break;
39236c6e 3088 }
0a7de745 3089
39236c6e 3090 switch (opt) {
cb323159 3091 case IPSEC_OPT_FLAGS: {
0a7de745
A		 3092 if (len != sizeof(u_int32_t)) {
3093 result = EMSGSIZE;
3094 } else {
cb323159 3095 pcb->ipsec_external_flags = *(u_int32_t *)data;
0a7de745
A		 3096 }
3097 break;
cb323159 3098 }
0a7de745 3099
cb323159 3100 case IPSEC_OPT_EXT_IFDATA_STATS: {
0a7de745
A		 3101 if (len != sizeof(int)) {
3102 result = EMSGSIZE;
39236c6e 3103 break;
0a7de745
A		 3104 }
3105 if (pcb->ipsec_ifp == NULL) {
3106 // Only can set after connecting
3107 result = EINVAL;
39236c6e 3108 break;
0a7de745
A		 3109 }
3110 pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
3111 break;
cb323159 3112 }
0a7de745
A		 3113
3114 case IPSEC_OPT_INC_IFDATA_STATS_IN:
3115 case IPSEC_OPT_INC_IFDATA_STATS_OUT: {
3116 struct ipsec_stats_param *utsp = (struct ipsec_stats_param *)data;
3117
3118 if (utsp == NULL || len < sizeof(struct ipsec_stats_param)) {
3119 result = EINVAL;
39236c6e
A		 3120 break;
3121 }
0a7de745
A		 3122 if (pcb->ipsec_ifp == NULL) {
3123 // Only can set after connecting
3124 result = EINVAL;
39236c6e
A		 3125 break;
3126 }
0a7de745
A		 3127 if (!pcb->ipsec_ext_ifdata_stats) {
3128 result = EINVAL;
fe8ab488
A		 3129 break;
3130 }
0a7de745
A		 3131 if (opt == IPSEC_OPT_INC_IFDATA_STATS_IN) {
3132 ifnet_stat_increment_in(pcb->ipsec_ifp, utsp->utsp_packets,
3133 utsp->utsp_bytes, utsp->utsp_errors);
3134 } else {
3135 ifnet_stat_increment_out(pcb->ipsec_ifp, utsp->utsp_packets,
3136 utsp->utsp_bytes, utsp->utsp_errors);
3137 }
3138 break;
3139 }
5ba3f43e 3140
0a7de745 3141 case IPSEC_OPT_SET_DELEGATE_INTERFACE: {
cb323159
A		 3142 ifnet_t del_ifp = NULL;
3143 char name[IFNAMSIZ];
0a7de745
A		 3144
3145 if (len > IFNAMSIZ - 1) {
3146 result = EMSGSIZE;
5ba3f43e
A		 3147 break;
3148 }
0a7de745
A		 3149 if (pcb->ipsec_ifp == NULL) {
3150 // Only can set after connecting
3151 result = EINVAL;
3152 break;
3153 }
cb323159 3154 if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */
0a7de745
A		 3155 bcopy(data, name, len);
3156 name[len] = 0;
3157 result = ifnet_find_by_name(name, &del_ifp);
3158 }
3159 if (result == 0) {
cb323159 3160 os_log_error(OS_LOG_DEFAULT, "%s IPSEC_OPT_SET_DELEGATE_INTERFACE %s to %s\n",
0a7de745
A		 3161 __func__, pcb->ipsec_ifp->if_xname,
3162 del_ifp ? del_ifp->if_xname : "NULL");
5ba3f43e 3163
0a7de745
A		 3164 result = ifnet_set_delegate(pcb->ipsec_ifp, del_ifp);
3165 if (del_ifp) {
3166 ifnet_release(del_ifp);
a39ff7e2 3167 }
0a7de745
A		 3168 }
3169 break;
3170 }
a39ff7e2 3171
0a7de745
A		 3172 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: {
3173 if (len != sizeof(int)) {
3174 result = EMSGSIZE;
3175 break;
3176 }
3177 if (pcb->ipsec_ifp == NULL) {
3178 // Only can set after connecting
3179 result = EINVAL;
5ba3f43e
A		 3180 break;
3181 }
0a7de745
A		 3182 mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data);
3183 if (output_service_class == MBUF_SC_UNSPEC) {
3184 pcb->ipsec_output_service_class = MBUF_SC_OAM;
3185 } else {
3186 pcb->ipsec_output_service_class = output_service_class;
3187 }
cb323159 3188 os_log_error(OS_LOG_DEFAULT, "%s IPSEC_OPT_OUTPUT_TRAFFIC_CLASS %s svc %d\n",
0a7de745
A		 3189 __func__, pcb->ipsec_ifp->if_xname,
3190 pcb->ipsec_output_service_class);
3191 break;
3192 }
5ba3f43e 3193
0a7de745
A		 3194#if IPSEC_NEXUS
3195 case IPSEC_OPT_ENABLE_CHANNEL: {
3196 if (len != sizeof(int)) {
3197 result = EMSGSIZE;
5ba3f43e
A		 3198 break;
3199 }
cb323159
A		 3200 if (pcb->ipsec_ifp != NULL) {
3201 // Only can set before connecting
0a7de745 3202 result = EINVAL;
5c9f4661
A		 3203 break;
3204 }
cb323159
A		 3205 if ((*(int *)data) != 0 &&
3206 (*(int *)data) != 1 &&
3207 (*(int *)data) != IPSEC_IF_WMM_RING_COUNT) {
3208 result = EINVAL;
3209 break;
0a7de745 3210 }
cb323159
A		 3211 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
3212 pcb->ipsec_kpipe_count = *(int *)data;
3213 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
3214 break;
3215 }
3216
3217 case IPSEC_OPT_CHANNEL_BIND_PID: {
3218 if (len != sizeof(pid_t)) {
3219 result = EMSGSIZE;
3220 break;
3221 }
3222 if (pcb->ipsec_ifp != NULL) {
3223 // Only can set before connecting
3224 result = EINVAL;
3225 break;
3226 }
3227 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
3228 pcb->ipsec_kpipe_pid = *(pid_t *)data;
3229 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
0a7de745
A		 3230 break;
3231 }
3232
3233 case IPSEC_OPT_ENABLE_FLOWSWITCH: {
3234 if (len != sizeof(int)) {
3235 result = EMSGSIZE;
5c9f4661
A		 3236 break;
3237 }
0a7de745
A		 3238 if (pcb->ipsec_ifp == NULL) {
3239 // Only can set after connecting
3240 result = EINVAL;
5c9f4661
A		 3241 break;
3242 }
cb323159 3243 if (!if_is_fsw_transport_netagent_enabled()) {
0a7de745 3244 result = ENOTSUP;
5c9f4661
A		 3245 break;
3246 }
cb323159 3247 if (uuid_is_null(pcb->ipsec_nx.fsw_agent)) {
0a7de745 3248 result = ENOENT;
5c9f4661
A		 3249 break;
3250 }
3251
cb323159
A		 3252 uint32_t flags = netagent_get_flags(pcb->ipsec_nx.fsw_agent);
3253
0a7de745 3254 if (*(int *)data) {
cb323159
A		 3255 flags |= (NETAGENT_FLAG_NEXUS_PROVIDER |
3256 NETAGENT_FLAG_NEXUS_LISTENER);
3257 result = netagent_set_flags(pcb->ipsec_nx.fsw_agent, flags);
0a7de745
A		 3258 pcb->ipsec_needs_netagent = true;
3259 } else {
3260 pcb->ipsec_needs_netagent = false;
cb323159
A		 3261 flags &= ~(NETAGENT_FLAG_NEXUS_PROVIDER |
3262 NETAGENT_FLAG_NEXUS_LISTENER);
3263 result = netagent_set_flags(pcb->ipsec_nx.fsw_agent, flags);
0a7de745
A		 3264 }
3265 break;
3266 }
3267
3268 case IPSEC_OPT_INPUT_FRAG_SIZE: {
3269 if (len != sizeof(u_int32_t)) {
3270 result = EMSGSIZE;
3271 break;
3272 }
3273 u_int32_t input_frag_size = *(u_int32_t *)data;
3274 if (input_frag_size <= sizeof(struct ip6_hdr)) {
3275 pcb->ipsec_frag_size_set = FALSE;
3276 pcb->ipsec_input_frag_size = 0;
3277 } else {
0a7de745
A		 3278 pcb->ipsec_frag_size_set = TRUE;
3279 pcb->ipsec_input_frag_size = input_frag_size;
3280 }
3281 break;
3282 }
3283 case IPSEC_OPT_ENABLE_NETIF: {
3284 if (len != sizeof(int)) {
3285 result = EMSGSIZE;
3286 break;
3287 }
3288 if (pcb->ipsec_ifp != NULL) {
3289 // Only can set before connecting
3290 result = EINVAL;
3291 break;
3292 }
3293 lck_rw_lock_exclusive(&pcb->ipsec_pcb_lock);
3294 pcb->ipsec_use_netif = !!(*(int *)data);
3295 lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
3296 break;
3297 }
3298 case IPSEC_OPT_SLOT_SIZE: {
3299 if (len != sizeof(u_int32_t)) {
3300 result = EMSGSIZE;
3301 break;
3302 }
3303 if (pcb->ipsec_ifp != NULL) {
3304 // Only can set before connecting
3305 result = EINVAL;
39236c6e 3306 break;
0a7de745
A		 3307 }
3308 u_int32_t slot_size = *(u_int32_t *)data;
3309 if (slot_size < IPSEC_IF_MIN_SLOT_SIZE ||
3310 slot_size > IPSEC_IF_MAX_SLOT_SIZE) {
3311 return EINVAL;
3312 }
3313 pcb->ipsec_slot_size = slot_size;
cb323159
A		 3314 if (if_ipsec_debug != 0) {
3315 printf("%s: IPSEC_OPT_SLOT_SIZE %u\n", __func__, slot_size);
3316 }
0a7de745 3317 break;
39236c6e 3318 }
0a7de745
A		 3319 case IPSEC_OPT_NETIF_RING_SIZE: {
3320 if (len != sizeof(u_int32_t)) {
3321 result = EMSGSIZE;
3322 break;
3323 }
3324 if (pcb->ipsec_ifp != NULL) {
3325 // Only can set before connecting
3326 result = EINVAL;
3327 break;
3328 }
3329 u_int32_t ring_size = *(u_int32_t *)data;
3330 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
3331 ring_size > IPSEC_IF_MAX_RING_SIZE) {
3332 return EINVAL;
3333 }
3334 pcb->ipsec_netif_ring_size = ring_size;
cb323159
A		 3335 if (if_ipsec_debug != 0) {
3336 printf("%s: IPSEC_OPT_NETIF_RING_SIZE %u\n", __func__, ring_size);
3337 }
0a7de745
A		 3338 break;
3339 }
3340 case IPSEC_OPT_TX_FSW_RING_SIZE: {
3341 if (len != sizeof(u_int32_t)) {
3342 result = EMSGSIZE;
3343 break;
3344 }
3345 if (pcb->ipsec_ifp != NULL) {
3346 // Only can set before connecting
3347 result = EINVAL;
3348 break;
3349 }
3350 u_int32_t ring_size = *(u_int32_t *)data;
3351 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
3352 ring_size > IPSEC_IF_MAX_RING_SIZE) {
3353 return EINVAL;
3354 }
3355 pcb->ipsec_tx_fsw_ring_size = ring_size;
cb323159
A		 3356 if (if_ipsec_debug != 0) {
3357 printf("%s: IPSEC_OPT_TX_FSW_RING_SIZE %u\n", __func__, ring_size);
3358 }
0a7de745
A		 3359 break;
3360 }
3361 case IPSEC_OPT_RX_FSW_RING_SIZE: {
3362 if (len != sizeof(u_int32_t)) {
3363 result = EMSGSIZE;
3364 break;
3365 }
3366 if (pcb->ipsec_ifp != NULL) {
3367 // Only can set before connecting
3368 result = EINVAL;
3369 break;
3370 }
3371 u_int32_t ring_size = *(u_int32_t *)data;
3372 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
3373 ring_size > IPSEC_IF_MAX_RING_SIZE) {
3374 return EINVAL;
3375 }
3376 pcb->ipsec_rx_fsw_ring_size = ring_size;
cb323159
A		 3377 if (if_ipsec_debug != 0) {
3378 printf("%s: IPSEC_OPT_TX_FSW_RING_SIZE %u\n", __func__, ring_size);
3379 }
3380 break;
3381 }
3382 case IPSEC_OPT_KPIPE_TX_RING_SIZE: {
3383 if (len != sizeof(u_int32_t)) {
3384 result = EMSGSIZE;
3385 break;
3386 }
3387 if (pcb->ipsec_ifp != NULL) {
3388 // Only can set before connecting
3389 result = EINVAL;
3390 break;
3391 }
3392 u_int32_t ring_size = *(u_int32_t *)data;
3393 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
3394 ring_size > IPSEC_IF_MAX_RING_SIZE) {
3395 return EINVAL;
3396 }
3397 pcb->ipsec_kpipe_tx_ring_size = ring_size;
3398 if (if_ipsec_debug != 0) {
3399 printf("%s: IPSEC_OPT_KPIPE_TX_RING_SIZE %u\n", __func__, ring_size);
3400 }
3401 break;
3402 }
3403 case IPSEC_OPT_KPIPE_RX_RING_SIZE: {
3404 if (len != sizeof(u_int32_t)) {
3405 result = EMSGSIZE;
3406 break;
3407 }
3408 if (pcb->ipsec_ifp != NULL) {
3409 // Only can set before connecting
3410 result = EINVAL;
3411 break;
3412 }
3413 u_int32_t ring_size = *(u_int32_t *)data;
3414 if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
3415 ring_size > IPSEC_IF_MAX_RING_SIZE) {
3416 return EINVAL;
3417 }
3418 pcb->ipsec_kpipe_rx_ring_size = ring_size;
3419 if (if_ipsec_debug != 0) {
3420 printf("%s: IPSEC_OPT_KPIPE_RX_RING_SIZE %u\n", __func__, ring_size);
3421 }
0a7de745
A		 3422 break;
3423 }
3424
3425#endif // IPSEC_NEXUS
3426
cb323159 3427 default: {
0a7de745
A		 3428 result = ENOPROTOOPT;
3429 break;
3430 }
cb323159 3431 }
0a7de745 3432
39236c6e
A		 3433 return result;
3434}
3435
3436static errno_t
5ba3f43e 3437ipsec_ctl_getopt(__unused kern_ctl_ref kctlref,
0a7de745
A		 3438 __unused u_int32_t unit,
3439 void *unitinfo,
3440 int opt,
3441 void *data,
3442 size_t *len)
39236c6e 3443{
5ba3f43e 3444 errno_t result = 0;
94ff46dc
A		 3445 struct ipsec_pcb *pcb = unitinfo;
3446 if (pcb == NULL) {
3447 return EINVAL;
3448 }
0a7de745 3449
39236c6e 3450 switch (opt) {
0a7de745
A		 3451 case IPSEC_OPT_FLAGS: {
3452 if (*len != sizeof(u_int32_t)) {
3453 result = EMSGSIZE;
3454 } else {
cb323159 3455 *(u_int32_t *)data = pcb->ipsec_external_flags;
5ba3f43e 3456 }
0a7de745
A		 3457 break;
3458 }
3459
3460 case IPSEC_OPT_EXT_IFDATA_STATS: {
3461 if (*len != sizeof(int)) {
3462 result = EMSGSIZE;
3463 } else {
3464 *(int *)data = (pcb->ipsec_ext_ifdata_stats) ? 1 : 0;
5ba3f43e 3465 }
0a7de745
A		 3466 break;
3467 }
3468
3469 case IPSEC_OPT_IFNAME: {
3470 if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) {
3471 result = EMSGSIZE;
3472 } else {
3473 if (pcb->ipsec_ifp == NULL) {
3474 // Only can get after connecting
3475 result = EINVAL;
3476 break;
5ba3f43e 3477 }
4ba76501 3478 *len = scnprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1;
5ba3f43e 3479 }
0a7de745
A		 3480 break;
3481 }
3482
3483 case IPSEC_OPT_OUTPUT_TRAFFIC_CLASS: {
3484 if (*len != sizeof(int)) {
3485 result = EMSGSIZE;
3486 } else {
3487 *(int *)data = so_svc2tc(pcb->ipsec_output_service_class);
fe8ab488 3488 }
0a7de745
A		 3489 break;
3490 }
5ba3f43e
A		 3491
3492#if IPSEC_NEXUS
a39ff7e2 3493
0a7de745
A		 3494 case IPSEC_OPT_ENABLE_CHANNEL: {
3495 if (*len != sizeof(int)) {
3496 result = EMSGSIZE;
3497 } else {
3498 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
cb323159
A		 3499 *(int *)data = pcb->ipsec_kpipe_count;
3500 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
3501 }
3502 break;
3503 }
3504
3505 case IPSEC_OPT_CHANNEL_BIND_PID: {
3506 if (*len != sizeof(pid_t)) {
3507 result = EMSGSIZE;
3508 } else {
3509 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
3510 *(pid_t *)data = pcb->ipsec_kpipe_pid;
0a7de745 3511 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
a39ff7e2 3512 }
0a7de745
A		 3513 break;
3514 }
a39ff7e2 3515
0a7de745
A		 3516 case IPSEC_OPT_ENABLE_FLOWSWITCH: {
3517 if (*len != sizeof(int)) {
3518 result = EMSGSIZE;
3519 } else {
cb323159 3520 *(int *)data = if_check_netagent(pcb->ipsec_ifp, pcb->ipsec_nx.fsw_agent);
a39ff7e2 3521 }
0a7de745
A		 3522 break;
3523 }
a39ff7e2 3524
0a7de745
A		 3525 case IPSEC_OPT_ENABLE_NETIF: {
3526 if (*len != sizeof(int)) {
3527 result = EMSGSIZE;
3528 } else {
5ba3f43e 3529 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
0a7de745 3530 *(int *)data = !!pcb->ipsec_use_netif;
5ba3f43e 3531 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 3532 }
0a7de745
A		 3533 break;
3534 }
5ba3f43e 3535
0a7de745
A		 3536 case IPSEC_OPT_GET_CHANNEL_UUID: {
3537 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
cb323159 3538 if (!ipsec_flag_isset(pcb, IPSEC_FLAGS_KPIPE_ALLOCATED)) {
0a7de745 3539 result = ENXIO;
cb323159 3540 } else if (*len != sizeof(uuid_t) * pcb->ipsec_kpipe_count) {
0a7de745
A		 3541 result = EMSGSIZE;
3542 } else {
cb323159
A		 3543 for (unsigned int i = 0; i < pcb->ipsec_kpipe_count; i++) {
3544 uuid_copy(((uuid_t *)data)[i], pcb->ipsec_kpipe_uuid[i]);
3545 }
5ba3f43e 3546 }
0a7de745
A		 3547 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
3548 break;
3549 }
3550
3551 case IPSEC_OPT_INPUT_FRAG_SIZE: {
3552 if (*len != sizeof(u_int32_t)) {
3553 result = EMSGSIZE;
3554 } else {
3555 *(u_int32_t *)data = pcb->ipsec_input_frag_size;
5c9f4661 3556 }
0a7de745
A		 3557 break;
3558 }
3559 case IPSEC_OPT_SLOT_SIZE: {
3560 if (*len != sizeof(u_int32_t)) {
3561 result = EMSGSIZE;
3562 } else {
3563 *(u_int32_t *)data = pcb->ipsec_slot_size;
5c9f4661 3564 }
0a7de745
A		 3565 break;
3566 }
3567 case IPSEC_OPT_NETIF_RING_SIZE: {
3568 if (*len != sizeof(u_int32_t)) {
3569 result = EMSGSIZE;
3570 } else {
3571 *(u_int32_t *)data = pcb->ipsec_netif_ring_size;
5c9f4661 3572 }
0a7de745
A		 3573 break;
3574 }
3575 case IPSEC_OPT_TX_FSW_RING_SIZE: {
3576 if (*len != sizeof(u_int32_t)) {
3577 result = EMSGSIZE;
3578 } else {
3579 *(u_int32_t *)data = pcb->ipsec_tx_fsw_ring_size;
3580 }
3581 break;
3582 }
3583 case IPSEC_OPT_RX_FSW_RING_SIZE: {
3584 if (*len != sizeof(u_int32_t)) {
3585 result = EMSGSIZE;
3586 } else {
3587 *(u_int32_t *)data = pcb->ipsec_rx_fsw_ring_size;
5c9f4661 3588 }
0a7de745
A		 3589 break;
3590 }
cb323159
A		 3591 case IPSEC_OPT_KPIPE_TX_RING_SIZE: {
3592 if (*len != sizeof(u_int32_t)) {
3593 result = EMSGSIZE;
3594 } else {
3595 *(u_int32_t *)data = pcb->ipsec_kpipe_tx_ring_size;
3596 }
3597 break;
3598 }
3599 case IPSEC_OPT_KPIPE_RX_RING_SIZE: {
3600 if (*len != sizeof(u_int32_t)) {
3601 result = EMSGSIZE;
3602 } else {
3603 *(u_int32_t *)data = pcb->ipsec_kpipe_rx_ring_size;
3604 }
3605 break;
3606 }
5c9f4661 3607
5ba3f43e
A		 3608#endif // IPSEC_NEXUS
3609
0a7de745
A		 3610 default: {
3611 result = ENOPROTOOPT;
3612 break;
3613 }
39236c6e 3614 }
0a7de745 3615
39236c6e
A		 3616 return result;
3617}
3618
3619/* Network Interface functions */
3620static errno_t
5ba3f43e 3621ipsec_output(ifnet_t interface,
0a7de745 3622 mbuf_t data)
39236c6e 3623{
5ba3f43e 3624 struct ipsec_pcb *pcb = ifnet_softc(interface);
0a7de745
A		 3625 struct ipsec_output_state ipsec_state;
3626 struct route ro;
3627 struct route_in6 ro6;
3628 int length;
f427ee49
A		 3629 struct ip *ip = NULL;
3630 struct ip6_hdr *ip6 = NULL;
0a7de745
A		 3631 struct ip_out_args ipoa;
3632 struct ip6_out_args ip6oa;
3633 int error = 0;
3634 u_int ip_version = 0;
3635 int flags = 0;
3636 struct flowadv *adv = NULL;
3637
fe8ab488
A		 3638 // Make sure this packet isn't looping through the interface
3639 if (necp_get_last_interface_index_from_packet(data) == interface->if_index) {
5ba3f43e 3640 error = EINVAL;
fe8ab488
A		 3641 goto ipsec_output_err;
3642 }
0a7de745 3643
fe8ab488
A		 3644 // Mark the interface so NECP can evaluate tunnel policy
3645 necp_mark_packet_from_interface(data, interface);
0a7de745 3646
f427ee49
A		 3647 if (data->m_len < sizeof(*ip)) {
3648 os_log_error(OS_LOG_DEFAULT, "ipsec_output: first mbuf length shorter than IP header length: %d.\n", data->m_len);
3649 IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
3650 error = EINVAL;
3651 goto ipsec_output_err;
3652 }
3653
0a7de745
A		 3654 ip = mtod(data, struct ip *);
3655 ip_version = ip->ip_v;
3656
3657 switch (ip_version) {
3658 case 4: {
f427ee49
A		 3659 u_int8_t ip_hlen = 0;
3660#ifdef _IP_VHL
3661 ip_hlen = _IP_VHL_HL(ip->ip_vhl) << 2;
3662#else
3663 ip_hlen = ip->ip_hl << 2;
3664#endif
3665 if (ip_hlen < sizeof(*ip)) {
3666 os_log_error(OS_LOG_DEFAULT, "ipsec_output: Bad ip header length %d.\n", ip_hlen);
3667 IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
3668 error = EINVAL;
3669 goto ipsec_output_err;
3670 }
5c9f4661 3671#if IPSEC_NEXUS
0a7de745 3672 if (!pcb->ipsec_use_netif)
5c9f4661 3673#endif // IPSEC_NEXUS
0a7de745
A		 3674 {
3675 int af = AF_INET;
3676 bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
3677 }
3678
3679 /* Apply encryption */
3680 memset(&ipsec_state, 0, sizeof(ipsec_state));
3681 ipsec_state.m = data;
3682 ipsec_state.dst = (struct sockaddr *)&ip->ip_dst;
3683 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
3684
3685 error = ipsec4_interface_output(&ipsec_state, interface);
3686 /* Tunneled in IPv6 - packet is gone */
3687 if (error == 0 && ipsec_state.tunneled == 6) {
3688 goto done;
3689 }
3690
3691 data = ipsec_state.m;
3692 if (error || data == NULL) {
3693 if (error) {
cb323159 3694 os_log_error(OS_LOG_DEFAULT, "ipsec_output: ipsec4_output error %d.\n", error);
5c9f4661 3695 }
0a7de745
A		 3696 goto ipsec_output_err;
3697 }
5c9f4661 3698
0a7de745
A		 3699 /* Set traffic class, set flow */
3700 m_set_service_class(data, pcb->ipsec_output_service_class);
3701 data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
3702 data->m_pkthdr.pkt_flowid = interface->if_flowhash;
3703 data->m_pkthdr.pkt_proto = ip->ip_p;
3704 data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
3705
3706 /* Flip endian-ness for ip_output */
3707 ip = mtod(data, struct ip *);
3708 NTOHS(ip->ip_len);
3709 NTOHS(ip->ip_off);
3710
3711 /* Increment statistics */
3712 length = mbuf_pkthdr_len(data);
3713 ifnet_stat_increment_out(interface, 1, length, 0);
3714
3715 /* Send to ip_output */
3716 memset(&ro, 0, sizeof(ro));
3717
3718 flags = (IP_OUTARGS | /* Passing out args to specify interface */
3719 IP_NOIPSEC); /* To ensure the packet doesn't go through ipsec twice */
3720
3721 memset(&ipoa, 0, sizeof(ipoa));
3722 ipoa.ipoa_flowadv.code = 0;
3723 ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR;
3724 if (ipsec_state.outgoing_if) {
3725 ipoa.ipoa_boundif = ipsec_state.outgoing_if;
3726 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
5ba3f43e 3727 }
0a7de745
A		 3728 ipsec_set_ipoa_for_interface(pcb->ipsec_ifp, &ipoa);
3729
3730 adv = &ipoa.ipoa_flowadv;
3731
3732 (void)ip_output(data, NULL, &ro, flags, NULL, &ipoa);
3733 data = NULL;
3734
3735 if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) {
3736 error = ENOBUFS;
3737 ifnet_disable_output(interface);
3738 }
3739
3740 goto done;
3741 }
3742 case 6: {
f427ee49
A		 3743 if (data->m_len < sizeof(*ip6)) {
3744 os_log_error(OS_LOG_DEFAULT, "ipsec_output: first mbuf length shorter than IPv6 header length: %d.\n", data->m_len);
3745 IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
3746 error = EINVAL;
3747 goto ipsec_output_err;
3748 }
5c9f4661 3749#if IPSEC_NEXUS
0a7de745 3750 if (!pcb->ipsec_use_netif)
5c9f4661 3751#endif // IPSEC_NEXUS
0a7de745
A		 3752 {
3753 int af = AF_INET6;
3754 bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
3755 }
5c9f4661 3756
0a7de745
A		 3757 data = ipsec6_splithdr(data);
3758 if (data == NULL) {
cb323159 3759 os_log_error(OS_LOG_DEFAULT, "ipsec_output: ipsec6_splithdr returned NULL\n");
0a7de745
A		 3760 goto ipsec_output_err;
3761 }
3e170ce0 3762
0a7de745
A		 3763 ip6 = mtod(data, struct ip6_hdr *);
3764
3765 memset(&ipsec_state, 0, sizeof(ipsec_state));
3766 ipsec_state.m = data;
3767 ipsec_state.dst = (struct sockaddr *)&ip6->ip6_dst;
3768 memset(&ipsec_state.ro, 0, sizeof(ipsec_state.ro));
3769
3770 error = ipsec6_interface_output(&ipsec_state, interface, &ip6->ip6_nxt, ipsec_state.m);
3771 if (error == 0 && ipsec_state.tunneled == 4) { /* tunneled in IPv4 - packet is gone */
3772 goto done;
3773 }
3774 data = ipsec_state.m;
3775 if (error || data == NULL) {
3776 if (error) {
cb323159 3777 os_log_error(OS_LOG_DEFAULT, "ipsec_output: ipsec6_output error %d\n", error);
5ba3f43e 3778 }
0a7de745 3779 goto ipsec_output_err;
5ba3f43e 3780 }
0a7de745
A		 3781
3782 /* Set traffic class, set flow */
3783 m_set_service_class(data, pcb->ipsec_output_service_class);
3784 data->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
3785 data->m_pkthdr.pkt_flowid = interface->if_flowhash;
3786 data->m_pkthdr.pkt_proto = ip6->ip6_nxt;
3787 data->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
3788
3789 /* Increment statistics */
3790 length = mbuf_pkthdr_len(data);
3791 ifnet_stat_increment_out(interface, 1, length, 0);
3792
3793 /* Send to ip6_output */
3794 memset(&ro6, 0, sizeof(ro6));
3795
3796 flags = IPV6_OUTARGS;
3797
3798 memset(&ip6oa, 0, sizeof(ip6oa));
3799 ip6oa.ip6oa_flowadv.code = 0;
3800 ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR;
3801 if (ipsec_state.outgoing_if) {
3802 ip6oa.ip6oa_boundif = ipsec_state.outgoing_if;
3803 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
5ba3f43e 3804 }
0a7de745
A		 3805 ipsec_set_ip6oa_for_interface(pcb->ipsec_ifp, &ip6oa);
3806
3807 adv = &ip6oa.ip6oa_flowadv;
3808
3809 (void) ip6_output(data, NULL, &ro6, flags, NULL, NULL, &ip6oa);
3810 data = NULL;
3811
3812 if (adv->code == FADV_FLOW_CONTROLLED || adv->code == FADV_SUSPENDED) {
3813 error = ENOBUFS;
3814 ifnet_disable_output(interface);
3815 }
3816
3817 goto done;
3818 }
3819 default: {
cb323159 3820 os_log_error(OS_LOG_DEFAULT, "ipsec_output: Received unknown packet version %d.\n", ip_version);
0a7de745
A		 3821 error = EINVAL;
3822 goto ipsec_output_err;
3823 }
3824 }
3825
39236c6e 3826done:
0a7de745
A		 3827 return error;
3828
39236c6e 3829ipsec_output_err:
0a7de745
A		 3830 if (data) {
3831 mbuf_freem(data);
3832 }
39236c6e
A		 3833 goto done;
3834}
3835
3836static void
0a7de745 3837ipsec_start(ifnet_t interface)
39236c6e 3838{
fe8ab488 3839 mbuf_t data;
5ba3f43e 3840 struct ipsec_pcb *pcb = ifnet_softc(interface);
fe8ab488 3841
5ba3f43e 3842 VERIFY(pcb != NULL);
fe8ab488 3843 for (;;) {
0a7de745 3844 if (ifnet_dequeue(interface, &data) != 0) {
fe8ab488 3845 break;
0a7de745
A		 3846 }
3847 if (ipsec_output(interface, data) != 0) {
fe8ab488 3848 break;
0a7de745 3849 }
fe8ab488 3850 }
39236c6e
A		 3851}
3852
3853/* Network Interface functions */
3854static errno_t
0a7de745
A		 3855ipsec_demux(__unused ifnet_t interface,
3856 mbuf_t data,
3857 __unused char *frame_header,
3858 protocol_family_t *protocol)
39236c6e 3859{
0a7de745
A		 3860 struct ip *ip;
3861 u_int ip_version;
3862
39236c6e
A		 3863 while (data != NULL && mbuf_len(data) < 1) {
3864 data = mbuf_next(data);
3865 }
0a7de745
A		 3866
3867 if (data == NULL) {
39236c6e 3868 return ENOENT;
39236c6e 3869 }
0a7de745
A		 3870
3871 ip = mtod(data, struct ip *);
3872 ip_version = ip->ip_v;
3873
3874 switch (ip_version) {
3875 case 4:
3876 *protocol = PF_INET;
3877 return 0;
3878 case 6:
3879 *protocol = PF_INET6;
3880 return 0;
3881 default:
3882 break;
3883 }
3884
39236c6e
A		 3885 return 0;
3886}
3887
3888static errno_t
0a7de745
A		 3889ipsec_add_proto(__unused ifnet_t interface,
3890 protocol_family_t protocol,
3891 __unused const struct ifnet_demux_desc *demux_array,
3892 __unused u_int32_t demux_count)
39236c6e 3893{
0a7de745
A		 3894 switch (protocol) {
3895 case PF_INET:
3896 return 0;
3897 case PF_INET6:
3898 return 0;
3899 default:
3900 break;
39236c6e 3901 }
0a7de745 3902
39236c6e
A		 3903 return ENOPROTOOPT;
3904}
3905
3906static errno_t
0a7de745
A		 3907ipsec_del_proto(__unused ifnet_t interface,
3908 __unused protocol_family_t protocol)
39236c6e
A		 3909{
3910 return 0;
3911}
3912
3913static errno_t
5ba3f43e 3914ipsec_ioctl(ifnet_t interface,
0a7de745
A		 3915 u_long command,
3916 void *data)
39236c6e 3917{
d9a64523
A		 3918#if IPSEC_NEXUS
3919 struct ipsec_pcb *pcb = ifnet_softc(interface);
3920#endif
0a7de745
A		 3921 errno_t result = 0;
3922
3923 switch (command) {
3924 case SIOCSIFMTU: {
5ba3f43e 3925#if IPSEC_NEXUS
0a7de745
A		 3926 if (pcb->ipsec_use_netif) {
3927 // Make sure we can fit packets in the channel buffers
3928 if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > pcb->ipsec_slot_size) {
3929 result = EINVAL;
3930 } else {
3931 ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
5c9f4661 3932 }
0a7de745
A		 3933 } else
3934#endif // IPSEC_NEXUS
3935 {
3936 ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
5c9f4661 3937 }
0a7de745 3938 break;
39236c6e 3939 }
0a7de745
A		 3940
3941 case SIOCSIFFLAGS:
3942 /* ifioctl() takes care of it */
3943 break;
3944
cb323159
A		 3945 case SIOCSIFSUBFAMILY: {
3946 uint32_t subfamily;
3947
3948 subfamily = ((struct ifreq*)data)->ifr_type.ift_subfamily;
3949 switch (subfamily) {
3950 case IFRTYPE_SUBFAMILY_BLUETOOTH:
3951 interface->if_subfamily = IFNET_SUBFAMILY_BLUETOOTH;
3952 break;
3953 case IFRTYPE_SUBFAMILY_WIFI:
3954 interface->if_subfamily = IFNET_SUBFAMILY_WIFI;
3955 break;
3956 case IFRTYPE_SUBFAMILY_QUICKRELAY:
3957 interface->if_subfamily = IFNET_SUBFAMILY_QUICKRELAY;
3958 break;
3959 case IFRTYPE_SUBFAMILY_DEFAULT:
3960 interface->if_subfamily = IFNET_SUBFAMILY_DEFAULT;
3961 break;
3962 default:
3963 result = EINVAL;
3964 break;
3965 }
3966 break;
3967 }
3968
0a7de745
A		 3969 default:
3970 result = EOPNOTSUPP;
3971 }
3972
39236c6e
A		 3973 return result;
3974}
3975
3976static void
5ba3f43e 3977ipsec_detached(ifnet_t interface)
39236c6e 3978{
5ba3f43e 3979 struct ipsec_pcb *pcb = ifnet_softc(interface);
cb323159 3980
5ba3f43e 3981 (void)ifnet_release(interface);
f427ee49 3982 lck_mtx_lock(&ipsec_lock);
5c9f4661 3983 ipsec_free_pcb(pcb, true);
f427ee49
A		 3984 (void)ifnet_dispose(interface);
3985 lck_mtx_unlock(&ipsec_lock);
39236c6e
A		 3986}
3987
3988/* Protocol Handlers */
3989
3990static errno_t
fe8ab488 3991ipsec_proto_input(ifnet_t interface,
0a7de745
A		 3992 protocol_family_t protocol,
3993 mbuf_t m,
3994 __unused char *frame_header)
39236c6e 3995{
fe8ab488 3996 mbuf_pkthdr_setrcvif(m, interface);
5c9f4661
A		 3997
3998#if IPSEC_NEXUS
3999 struct ipsec_pcb *pcb = ifnet_softc(interface);
4000 if (!pcb->ipsec_use_netif)
4001#endif // IPSEC_NEXUS
4002 {
4003 uint32_t af = 0;
4004 struct ip *ip = mtod(m, struct ip *);
4005 if (ip->ip_v == 4) {
4006 af = AF_INET;
4007 } else if (ip->ip_v == 6) {
4008 af = AF_INET6;
4009 }
4010 bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af));
a39ff7e2 4011 pktap_input(interface, protocol, m, NULL);
5c9f4661 4012 }
39037602 4013
0a7de745 4014 int32_t pktlen = m->m_pkthdr.len;
3e170ce0
A		 4015 if (proto_input(protocol, m) != 0) {
4016 ifnet_stat_increment_in(interface, 0, 0, 1);
39236c6e 4017 m_freem(m);
3e170ce0 4018 } else {
a39ff7e2 4019 ifnet_stat_increment_in(interface, 1, pktlen, 0);
3e170ce0 4020 }
0a7de745 4021
39236c6e
A		 4022 return 0;
4023}
4024
4025static errno_t
0a7de745
A		 4026ipsec_proto_pre_output(__unused ifnet_t interface,
4027 protocol_family_t protocol,
4028 __unused mbuf_t *packet,
4029 __unused const struct sockaddr *dest,
4030 __unused void *route,
4031 __unused char *frame_type,
4032 __unused char *link_layer_dest)
39236c6e 4033{
39236c6e
A		 4034 *(protocol_family_t *)(void *)frame_type = protocol;
4035 return 0;
4036}
4037
4038static errno_t
0a7de745
A		 4039ipsec_attach_proto(ifnet_t interface,
4040 protocol_family_t protocol)
39236c6e 4041{
0a7de745
A		 4042 struct ifnet_attach_proto_param proto;
4043 errno_t result;
4044
39236c6e
A		 4045 bzero(&proto, sizeof(proto));
4046 proto.input = ipsec_proto_input;
4047 proto.pre_output = ipsec_proto_pre_output;
0a7de745 4048
39236c6e
A		 4049 result = ifnet_attach_protocol(interface, protocol, &proto);
4050 if (result != 0 && result != EEXIST) {
cb323159 4051 os_log_error(OS_LOG_DEFAULT, "ipsec_attach_inet - ifnet_attach_protocol %d failed: %d\n",
0a7de745 4052 protocol, result);
39236c6e 4053 }
0a7de745 4054
39236c6e
A		 4055 return result;
4056}
fe8ab488 4057
5ba3f43e 4058errno_t
0a7de745
A		 4059ipsec_inject_inbound_packet(ifnet_t interface,
4060 mbuf_t packet)
5ba3f43e 4061{
a39ff7e2 4062#if IPSEC_NEXUS
5ba3f43e
A		 4063 struct ipsec_pcb *pcb = ifnet_softc(interface);
4064
5c9f4661 4065 if (pcb->ipsec_use_netif) {
cb323159
A		 4066 if (!ipsec_data_move_begin(pcb)) {
4067 os_log_info(OS_LOG_DEFAULT, "%s: data path stopped for %s\n", __func__,
4068 if_name(pcb->ipsec_ifp));
4069 return ENXIO;
4070 }
4071
5c9f4661 4072 lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 4073
5c9f4661 4074 lck_mtx_lock(&pcb->ipsec_input_chain_lock);
cb323159
A		 4075
4076 if (pcb->ipsec_input_chain_count > (u_int32_t)if_ipsec_max_pending_input) {
4077 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
4078 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
4079 ipsec_data_move_end(pcb);
4080 return ENOSPC;
4081 }
4082
5c9f4661
A		 4083 if (pcb->ipsec_input_chain != NULL) {
4084 pcb->ipsec_input_chain_last->m_nextpkt = packet;
4085 } else {
4086 pcb->ipsec_input_chain = packet;
4087 }
cb323159 4088 pcb->ipsec_input_chain_count++;
5c9f4661
A		 4089 while (packet->m_nextpkt) {
4090 VERIFY(packet != packet->m_nextpkt);
4091 packet = packet->m_nextpkt;
cb323159 4092 pcb->ipsec_input_chain_count++;
5c9f4661
A		 4093 }
4094 pcb->ipsec_input_chain_last = packet;
4095 lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
5ba3f43e 4096
cb323159 4097 kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring[0];
5c9f4661 4098 lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
5ba3f43e 4099
5c9f4661
A		 4100 if (rx_ring != NULL) {
4101 kern_channel_notify(rx_ring, 0);
4102 }
5ba3f43e 4103
cb323159 4104 ipsec_data_move_end(pcb);
0a7de745 4105 return 0;
5c9f4661
A		 4106 } else
4107#endif // IPSEC_NEXUS
4108 {
4109 errno_t error;
4110 protocol_family_t protocol;
4111 if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) {
4112 return error;
4113 }
5ba3f43e 4114
5c9f4661
A		 4115 return ipsec_proto_input(interface, protocol, packet, NULL);
4116 }
fe8ab488
A		 4117}
4118
4119void
4120ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family)
4121{
4122 if (packet != NULL && interface != NULL) {
4123 struct ipsec_pcb *pcb = ifnet_softc(interface);
4124 if (pcb != NULL) {
4125 /* Set traffic class, set flow */
4126 m_set_service_class(packet, pcb->ipsec_output_service_class);
4127 packet->m_pkthdr.pkt_flowsrc = FLOWSRC_IFNET;
4128 packet->m_pkthdr.pkt_flowid = interface->if_flowhash;
4129 if (family == AF_INET) {
4130 struct ip *ip = mtod(packet, struct ip *);
4131 packet->m_pkthdr.pkt_proto = ip->ip_p;
3e170ce0 4132 } else if (family == AF_INET6) {
fe8ab488
A		 4133 struct ip6_hdr *ip6 = mtod(packet, struct ip6_hdr *);
4134 packet->m_pkthdr.pkt_proto = ip6->ip6_nxt;
4135 }
4136 packet->m_pkthdr.pkt_flags = (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC);
4137 }
4138 }
4139}
39037602
A		 4140
4141void
4142ipsec_set_ipoa_for_interface(ifnet_t interface, struct ip_out_args *ipoa)
4143{
4144 struct ipsec_pcb *pcb;
0a7de745
A		 4145
4146 if (interface == NULL || ipoa == NULL) {
39037602 4147 return;
0a7de745 4148 }
39037602 4149 pcb = ifnet_softc(interface);
0a7de745 4150
39037602
A		 4151 if (net_qos_policy_restricted == 0) {
4152 ipoa->ipoa_flags |= IPOAF_QOSMARKING_ALLOWED;
4153 ipoa->ipoa_sotc = so_svc2tc(pcb->ipsec_output_service_class);
4154 } else if (pcb->ipsec_output_service_class != MBUF_SC_VO ||
0a7de745 4155 net_qos_policy_restrict_avapps != 0) {
39037602
A		 4156 ipoa->ipoa_flags &= ~IPOAF_QOSMARKING_ALLOWED;
4157 } else {
4158 ipoa->ipoa_flags |= IP6OAF_QOSMARKING_ALLOWED;
4159 ipoa->ipoa_sotc = SO_TC_VO;
4160 }
4161}
4162
4163void
4164ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa)
4165{
4166 struct ipsec_pcb *pcb;
0a7de745
A		 4167
4168 if (interface == NULL || ip6oa == NULL) {
39037602 4169 return;
0a7de745 4170 }
39037602 4171 pcb = ifnet_softc(interface);
0a7de745 4172
39037602
A		 4173 if (net_qos_policy_restricted == 0) {
4174 ip6oa->ip6oa_flags |= IPOAF_QOSMARKING_ALLOWED;
4175 ip6oa->ip6oa_sotc = so_svc2tc(pcb->ipsec_output_service_class);
4176 } else if (pcb->ipsec_output_service_class != MBUF_SC_VO ||
0a7de745 4177 net_qos_policy_restrict_avapps != 0) {
39037602
A		 4178 ip6oa->ip6oa_flags &= ~IPOAF_QOSMARKING_ALLOWED;
4179 } else {
4180 ip6oa->ip6oa_flags |= IP6OAF_QOSMARKING_ALLOWED;
4181 ip6oa->ip6oa_sotc = SO_TC_VO;
4182 }
4183}
cb323159 4184
cb323159
A		 4185
4186static void
4187ipsec_data_move_drain(struct ipsec_pcb *pcb)
4188{
4189 lck_mtx_lock(&pcb->ipsec_pcb_data_move_lock);
4190 /* data path must already be marked as not ready */
4191 VERIFY(!IPSEC_IS_DATA_PATH_READY(pcb));
4192 pcb->ipsec_pcb_drainers++;
4193 while (pcb->ipsec_pcb_data_move != 0) {
4194 (void)msleep(&(pcb->ipsec_pcb_data_move), &pcb->ipsec_pcb_data_move_lock,
4195 (PZERO - 1), __func__, NULL);
4196 }
4197 VERIFY(!IPSEC_IS_DATA_PATH_READY(pcb));
4198 VERIFY(pcb->ipsec_pcb_drainers > 0);
4199 pcb->ipsec_pcb_drainers--;
4200 lck_mtx_unlock(&pcb->ipsec_pcb_data_move_lock);
4201}
4202
4203static void
4204ipsec_wait_data_move_drain(struct ipsec_pcb *pcb)
4205{
4206 /*
4207 * Mark the data path as not usable.
4208 */
4209 lck_mtx_lock(&pcb->ipsec_pcb_data_move_lock);
4210 IPSEC_CLR_DATA_PATH_READY(pcb);
4211 lck_mtx_unlock(&pcb->ipsec_pcb_data_move_lock);
4212
4213 /* Wait until all threads in the data paths are done. */
4214 ipsec_data_move_drain(pcb);
4215}
mirror of XNU