]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.101.1.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
c3c9b80d 2 * Copyright (c) 2012-2017, 2020, 2021 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
39236c6e
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
39236c6e
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
39236c6e
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
39236c6e
A		 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
cb323159
A		 48#include <kern/task.h>
49#include <mach/task_info.h>
39236c6e
A		 50#include <net/if_var.h>
51#include <net/route.h>
52#include <net/flowhash.h>
53#include <net/ntstat.h>
ea3f0419 54#include <net/content_filter.h>
f427ee49 55#include <net/necp.h>
39236c6e
A		 56#include <netinet/in.h>
57#include <netinet/in_var.h>
58#include <netinet/tcp.h>
59#include <netinet/tcp_var.h>
60#include <netinet/tcp_fsm.h>
61#include <netinet/flow_divert.h>
62#include <netinet/flow_divert_proto.h>
3e170ce0 63#include <netinet6/in6_pcb.h>
39236c6e 64#include <netinet6/ip6protosw.h>
39236c6e
A		 65#include <dev/random/randomdev.h>
66#include <libkern/crypto/sha1.h>
67#include <libkern/crypto/crypto_internal.h>
39037602 68#include <os/log.h>
cb323159
A		 69#include <corecrypto/cc.h>
70#if CONTENT_FILTER
71#include <net/content_filter.h>
72#endif /* CONTENT_FILTER */
39236c6e 73
0a7de745
A		 74#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75#define FLOW_DIVERT_READ_CLOSED 0x00000002
76#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
490019cf 79#define FLOW_DIVERT_HAS_HMAC 0x00000040
f427ee49
A		 80#define FLOW_DIVERT_NOTIFY_ON_RECEIVED 0x00000080
81#define FLOW_DIVERT_IMPLICIT_CONNECT 0x00000100
82#define FLOW_DIVERT_DID_SET_LOCAL_ADDR 0x00000200
c3c9b80d
A		 83#define FLOW_DIVERT_HAS_TOKEN 0x00000400
84#define FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR 0x00000800
39236c6e 85
39037602
A		 86#define FDLOG(level, pcb, format, ...) \
87 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
39236c6e 88
39037602
A		 89#define FDLOG0(level, pcb, msg) \
90 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
39236c6e 91
0a7de745
A		 92#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
93#define FDRELEASE(pcb) \
94 do { \
95 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
96 flow_divert_pcb_destroy(pcb); \
97 } \
39236c6e
A		 98 } while (0)
99
0a7de745
A		 100#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
101#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
39236c6e 102
0a7de745
A		 103#define FD_CTL_SENDBUFF_SIZE (128 * 1024)
104#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
39236c6e 105
0a7de745 106#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
39236c6e 107
f427ee49 108#define GROUP_COUNT_MAX 31
0a7de745
A		 109#define FLOW_DIVERT_MAX_NAME_SIZE 4096
110#define FLOW_DIVERT_MAX_KEY_SIZE 1024
111#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e 112
0a7de745 113struct flow_divert_trie_node {
39236c6e
A		 114 uint16_t start;
115 uint16_t length;
116 uint16_t child_map;
39236c6e
A		 117};
118
0a7de745
A		 119#define CHILD_MAP_SIZE 256
120#define NULL_TRIE_IDX 0xffff
121#define TRIE_NODE(t, i) ((t)->nodes[(i)])
122#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
123#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
39236c6e 124
0a7de745 125static struct flow_divert_pcb nil_pcb;
39236c6e
A		 126
127decl_lck_rw_data(static, g_flow_divert_group_lck);
0a7de745
A		 128static struct flow_divert_group **g_flow_divert_groups = NULL;
129static uint32_t g_active_group_count = 0;
39236c6e 130
0a7de745
A		 131static lck_grp_attr_t *flow_divert_grp_attr = NULL;
132static lck_attr_t *flow_divert_mtx_attr = NULL;
133static lck_grp_t *flow_divert_mtx_grp = NULL;
134static errno_t g_init_result = 0;
39236c6e 135
0a7de745 136static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
39236c6e 137
0a7de745
A		 138static struct protosw g_flow_divert_in_protosw;
139static struct pr_usrreqs g_flow_divert_in_usrreqs;
140static struct protosw g_flow_divert_in_udp_protosw;
141static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
0a7de745
A		 142static struct ip6protosw g_flow_divert_in6_protosw;
143static struct pr_usrreqs g_flow_divert_in6_usrreqs;
144static struct ip6protosw g_flow_divert_in6_udp_protosw;
145static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
39236c6e 146
0a7de745
A		 147static struct protosw *g_tcp_protosw = NULL;
148static struct ip6protosw *g_tcp6_protosw = NULL;
149static struct protosw *g_udp_protosw = NULL;
150static struct ip6protosw *g_udp6_protosw = NULL;
3e170ce0 151
f427ee49
A		 152ZONE_DECLARE(flow_divert_group_zone, "flow_divert_group",
153 sizeof(struct flow_divert_group), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
154ZONE_DECLARE(flow_divert_pcb_zone, "flow_divert_pcb",
155 sizeof(struct flow_divert_pcb), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
3e170ce0
A		 156
157static errno_t
f427ee49 158flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
3e170ce0
A		 159
160static boolean_t
161flow_divert_is_sockaddr_valid(struct sockaddr *addr);
162
163static int
164flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
165
166struct sockaddr *
167flow_divert_get_buffered_target_address(mbuf_t buffer);
168
3e170ce0
A		 169static void
170flow_divert_disconnect_socket(struct socket *so);
39236c6e 171
39037602
A		 172static inline uint8_t
173flow_divert_syslog_type_to_oslog_type(int syslog_type)
174{
175 switch (syslog_type) {
0a7de745
A		 176 case LOG_ERR: return OS_LOG_TYPE_ERROR;
177 case LOG_INFO: return OS_LOG_TYPE_INFO;
178 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
179 default: return OS_LOG_TYPE_DEFAULT;
39037602
A		 180 }
181}
182
39236c6e
A		 183static inline int
184flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
185{
186 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
187}
188
189RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
190RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
191
192static const char *
193flow_divert_packet_type2str(uint8_t packet_type)
194{
195 switch (packet_type) {
0a7de745
A		 196 case FLOW_DIVERT_PKT_CONNECT:
197 return "connect";
198 case FLOW_DIVERT_PKT_CONNECT_RESULT:
199 return "connect result";
200 case FLOW_DIVERT_PKT_DATA:
201 return "data";
202 case FLOW_DIVERT_PKT_CLOSE:
203 return "close";
204 case FLOW_DIVERT_PKT_READ_NOTIFY:
205 return "read notification";
206 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
207 return "properties update";
208 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
209 return "app map create";
210 default:
211 return "unknown";
39236c6e
A		 212 }
213}
214
215static struct flow_divert_pcb *
216flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
217{
0a7de745
A		 218 struct flow_divert_pcb key_item;
219 struct flow_divert_pcb *fd_cb = NULL;
39236c6e
A		 220
221 key_item.hash = hash;
222
223 lck_rw_lock_shared(&group->lck);
224 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
225 FDRETAIN(fd_cb);
226 lck_rw_done(&group->lck);
227
228 return fd_cb;
229}
230
231static errno_t
232flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
233{
0a7de745
A		 234 errno_t error = 0;
235 struct flow_divert_pcb *exist = NULL;
236 struct flow_divert_group *group;
237 static uint32_t g_nextkey = 1;
238 static uint32_t g_hash_seed = 0;
239 int try_count = 0;
39236c6e
A		 240
241 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
242 return EINVAL;
243 }
244
245 socket_unlock(fd_cb->so, 0);
246 lck_rw_lock_shared(&g_flow_divert_group_lck);
247
248 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
249 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
250 error = ENETUNREACH;
251 goto done;
252 }
253
254 group = g_flow_divert_groups[ctl_unit];
255 if (group == NULL) {
256 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
257 error = ENETUNREACH;
258 goto done;
259 }
260
261 socket_lock(fd_cb->so, 0);
262
263 do {
0a7de745
A		 264 uint32_t key[2];
265 uint32_t idx;
39236c6e
A		 266
267 key[0] = g_nextkey++;
268 key[1] = RandomULong();
269
270 if (g_hash_seed == 0) {
271 g_hash_seed = RandomULong();
272 }
273
274 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
275
276 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
277 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
278 if (curr_group != NULL && curr_group != group) {
279 lck_rw_lock_shared(&curr_group->lck);
280 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
281 lck_rw_done(&curr_group->lck);
282 if (exist != NULL) {
283 break;
284 }
285 }
286 }
287
288 if (exist == NULL) {
289 lck_rw_lock_exclusive(&group->lck);
290 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
291 lck_rw_done(&group->lck);
292 }
293 } while (exist != NULL && try_count++ < 3);
294
295 if (exist == NULL) {
296 fd_cb->group = group;
0a7de745 297 FDRETAIN(fd_cb); /* The group now has a reference */
39236c6e
A		 298 } else {
299 fd_cb->hash = 0;
3e170ce0 300 error = EEXIST;
39236c6e
A		 301 }
302
303 socket_unlock(fd_cb->so, 0);
304
305done:
306 lck_rw_done(&g_flow_divert_group_lck);
307 socket_lock(fd_cb->so, 0);
308
3e170ce0 309 return error;
39236c6e
A		 310}
311
312static struct flow_divert_pcb *
313flow_divert_pcb_create(socket_t so)
314{
f427ee49 315 struct flow_divert_pcb *new_pcb = NULL;
39236c6e 316
f427ee49 317 new_pcb = zalloc_flags(flow_divert_pcb_zone, Z_WAITOK | Z_ZERO);
39236c6e
A		 318 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
319 new_pcb->so = so;
320 new_pcb->log_level = nil_pcb.log_level;
321
0a7de745 322 FDRETAIN(new_pcb); /* Represents the socket's reference */
39236c6e
A		 323
324 return new_pcb;
325}
326
327static void
328flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
329{
f427ee49
A		 330 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u",
331 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
39236c6e 332
39236c6e
A		 333 if (fd_cb->connect_token != NULL) {
334 mbuf_freem(fd_cb->connect_token);
335 }
490019cf
A		 336 if (fd_cb->connect_packet != NULL) {
337 mbuf_freem(fd_cb->connect_packet);
338 }
339 if (fd_cb->app_data != NULL) {
340 FREE(fd_cb->app_data, M_TEMP);
341 }
f427ee49
A		 342 if (fd_cb->original_remote_endpoint != NULL) {
343 FREE(fd_cb->original_remote_endpoint, M_SONAME);
344 }
345 zfree(flow_divert_pcb_zone, fd_cb);
39236c6e
A		 346}
347
348static void
349flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
350{
351 if (fd_cb->group != NULL) {
352 struct flow_divert_group *group = fd_cb->group;
353 lck_rw_lock_exclusive(&group->lck);
354 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
355 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
356 fd_cb->group = NULL;
0a7de745 357 FDRELEASE(fd_cb); /* Release the group's reference */
39236c6e
A		 358 lck_rw_done(&group->lck);
359 }
360}
361
362static int
363flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
364{
0a7de745
A		 365 struct flow_divert_packet_header hdr;
366 int error = 0;
39236c6e
A		 367
368 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
369 if (error) {
370 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
371 return error;
372 }
373
374 hdr.packet_type = packet_type;
375 hdr.conn_id = htonl(fd_cb->hash);
376
377 /* Lay down the header */
378 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
379 if (error) {
380 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
381 mbuf_freem(*packet);
382 *packet = NULL;
383 return error;
384 }
385
386 return 0;
387}
388
389static int
3e170ce0 390flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 391{
0a7de745
A		 392 uint32_t net_length = htonl(length);
393 int error = 0;
39236c6e
A		 394
395 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
396 if (error) {
397 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
398 return error;
399 }
400
401 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
402 if (error) {
39037602 403 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
39236c6e
A		 404 return error;
405 }
406
407 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
408 if (error) {
409 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
410 return error;
411 }
412
413 return error;
414}
415
416static int
417flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
418{
0a7de745
A		 419 size_t cursor = offset;
420 int error = 0;
421 uint32_t curr_length;
422 uint8_t curr_type;
39236c6e
A		 423
424 *err = 0;
425
426 do {
427 if (!next) {
428 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
429 if (error) {
430 *err = ENOENT;
431 return -1;
432 }
433 } else {
434 next = 0;
435 curr_type = FLOW_DIVERT_TLV_NIL;
436 }
437
438 if (curr_type != type) {
439 cursor += sizeof(curr_type);
440 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
441 if (error) {
442 *err = error;
443 return -1;
444 }
445
446 cursor += (sizeof(curr_length) + ntohl(curr_length));
447 }
448 } while (curr_type != type);
449
f427ee49 450 return (int)cursor;
39236c6e
A		 451}
452
453static int
3e170ce0 454flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 455{
0a7de745
A		 456 int error = 0;
457 uint32_t length;
458 int tlv_offset;
39236c6e
A		 459
460 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
461 if (tlv_offset < 0) {
462 return error;
463 }
464
465 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
466 if (error) {
467 return error;
468 }
469
470 length = ntohl(length);
471
cb323159
A		 472 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
473
474 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
475 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
476 return EINVAL;
477 }
478
39236c6e
A		 479 if (val_size != NULL) {
480 *val_size = length;
481 }
482
483 if (buff != NULL && buff_len > 0) {
cb323159 484 memset(buff, 0, buff_len);
39236c6e 485 size_t to_copy = (length < buff_len) ? length : buff_len;
cb323159 486 error = mbuf_copydata(packet, data_offset, to_copy, buff);
39236c6e
A		 487 if (error) {
488 return error;
489 }
490 }
491
492 return 0;
493}
494
495static int
496flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
497{
0a7de745 498 mbuf_t curr_mbuf = packet;
39236c6e
A		 499
500 if (g_crypto_funcs == NULL || group->token_key == NULL) {
501 return ENOPROTOOPT;
502 }
503
504 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
505 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
506
507 while (curr_mbuf != NULL) {
508 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
509 curr_mbuf = mbuf_next(curr_mbuf);
510 }
511
512 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
513
514 return 0;
515}
516
517static int
518flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
519{
0a7de745
A		 520 int error = 0;
521 struct flow_divert_group *group = NULL;
522 int hmac_offset;
523 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
524 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
525 mbuf_t tail;
39236c6e
A		 526
527 lck_rw_lock_shared(&g_flow_divert_group_lck);
528
529 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
530 group = g_flow_divert_groups[ctl_unit];
531 }
532
533 if (group == NULL) {
534 lck_rw_done(&g_flow_divert_group_lck);
535 return ENOPROTOOPT;
536 }
537
538 lck_rw_lock_shared(&group->lck);
539
540 if (group->token_key == NULL) {
541 error = ENOPROTOOPT;
542 goto done;
543 }
544
545 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
546 if (hmac_offset < 0) {
547 goto done;
548 }
549
550 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
551 if (error) {
552 goto done;
553 }
554
555 /* Chop off the HMAC TLV */
556 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
557 if (error) {
558 goto done;
559 }
560
561 mbuf_free(tail);
562
563 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
564 if (error) {
565 goto done;
566 }
567
cb323159 568 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
39236c6e
A		 569 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
570 error = EINVAL;
571 goto done;
572 }
573
574done:
575 lck_rw_done(&group->lck);
576 lck_rw_done(&g_flow_divert_group_lck);
577 return error;
578}
579
580static void
f427ee49 581flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send)
39236c6e
A		 582{
583 struct inpcb *inp = NULL;
584 struct ifnet *ifp = NULL;
585 Boolean cell = FALSE;
586 Boolean wifi = FALSE;
fe8ab488 587 Boolean wired = FALSE;
0a7de745 588
39236c6e
A		 589 inp = sotoinpcb(fd_cb->so);
590 if (inp == NULL) {
591 return;
592 }
593
f427ee49
A		 594 if (inp->inp_vflag & INP_IPV4) {
595 ifp = inp->inp_last_outifp;
596 } else if (inp->inp_vflag & INP_IPV6) {
597 ifp = inp->in6p_last_outifp;
598 }
39236c6e
A		 599 if (ifp != NULL) {
600 cell = IFNET_IS_CELLULAR(ifp);
601 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 602 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e 603 }
0a7de745 604
39236c6e 605 if (send) {
fe8ab488
A		 606 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
607 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 608 } else {
fe8ab488
A		 609 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
610 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e 611 }
5ba3f43e 612 inp_set_activity_bitmap(inp);
39236c6e
A		 613}
614
615static errno_t
616flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
617{
f427ee49
A		 618 struct inpcb *inp = sotoinpcb(fd_cb->so);
619 if (INP_NO_CELLULAR(inp)) {
620 struct ifnet *ifp = NULL;
621 if (inp->inp_vflag & INP_IPV4) {
622 ifp = inp->inp_last_outifp;
623 } else if (inp->inp_vflag & INP_IPV6) {
624 ifp = inp->in6p_last_outifp;
625 }
626 if (ifp != NULL && IFNET_IS_CELLULAR(ifp)) {
627 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied");
628 return EHOSTUNREACH;
629 }
0a7de745 630 }
fe8ab488
A		 631 return 0;
632}
633
634static errno_t
635flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
636{
f427ee49
A		 637 struct inpcb *inp = sotoinpcb(fd_cb->so);
638 if (INP_NO_EXPENSIVE(inp)) {
639 struct ifnet *ifp = NULL;
640 if (inp->inp_vflag & INP_IPV4) {
641 ifp = inp->inp_last_outifp;
642 } else if (inp->inp_vflag & INP_IPV6) {
643 ifp = inp->in6p_last_outifp;
644 }
645 if (ifp != NULL && IFNET_IS_EXPENSIVE(ifp)) {
646 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied");
647 return EHOSTUNREACH;
648 }
0a7de745 649 }
39236c6e
A		 650 return 0;
651}
652
cb323159
A		 653static errno_t
654flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
655{
f427ee49
A		 656 struct inpcb *inp = sotoinpcb(fd_cb->so);
657 if (INP_NO_CONSTRAINED(inp)) {
658 struct ifnet *ifp = NULL;
659 if (inp->inp_vflag & INP_IPV4) {
660 ifp = inp->inp_last_outifp;
661 } else if (inp->inp_vflag & INP_IPV6) {
662 ifp = inp->in6p_last_outifp;
663 }
664 if (ifp != NULL && IFNET_IS_CONSTRAINED(ifp)) {
665 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied");
666 return EHOSTUNREACH;
667 }
cb323159 668 }
cb323159
A		 669 return 0;
670}
671
39236c6e
A		 672static void
673flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
674{
675 if (how != SHUT_RD) {
676 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
677 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
678 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
679 /* If the tunnel is not accepting writes any more, then flush the send buffer */
680 sbflush(&fd_cb->so->so_snd);
681 }
682 }
683 if (how != SHUT_WR) {
684 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
685 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
686 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
687 }
688 }
689}
690
691static uint16_t
692trie_node_alloc(struct flow_divert_trie *trie)
693{
694 if (trie->nodes_free_next < trie->nodes_count) {
695 uint16_t node_idx = trie->nodes_free_next++;
696 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
697 return node_idx;
698 } else {
699 return NULL_TRIE_IDX;
700 }
701}
702
703static uint16_t
704trie_child_map_alloc(struct flow_divert_trie *trie)
705{
706 if (trie->child_maps_free_next < trie->child_maps_count) {
707 return trie->child_maps_free_next++;
708 } else {
709 return NULL_TRIE_IDX;
710 }
711}
712
713static uint16_t
714trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
715{
716 uint16_t start = trie->bytes_free_next;
717 if (start + bytes_size <= trie->bytes_count) {
718 if (start != bytes_idx) {
719 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
720 }
721 trie->bytes_free_next += bytes_size;
722 return start;
723 } else {
724 return NULL_TRIE_IDX;
725 }
726}
727
728static uint16_t
729flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
730{
731 uint16_t current = trie->root;
732 uint16_t child = trie->root;
f427ee49 733 uint16_t string_end = string_start + (uint16_t)string_len;
39236c6e 734 uint16_t string_idx = string_start;
f427ee49 735 uint16_t string_remainder = (uint16_t)string_len;
39236c6e
A		 736
737 while (child != NULL_TRIE_IDX) {
738 uint16_t parent = current;
739 uint16_t node_idx;
740 uint16_t current_end;
741
742 current = child;
743 child = NULL_TRIE_IDX;
744
745 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
746
747 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 748 node_idx < current_end &&
749 string_idx < string_end &&
750 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
751 node_idx++, string_idx++) {
752 ;
753 }
39236c6e
A		 754
755 string_remainder = string_end - string_idx;
756
757 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
758 /*
759 * We did not reach the end of the current node's string.
760 * We need to split the current node into two:
761 * 1. A new node that contains the prefix of the node that matches
762 * the prefix of the string being inserted.
763 * 2. The current node modified to point to the remainder
764 * of the current node's string.
765 */
766 uint16_t prefix = trie_node_alloc(trie);
767 if (prefix == NULL_TRIE_IDX) {
768 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
769 return NULL_TRIE_IDX;
770 }
771
772 /*
773 * Prefix points to the portion of the current nodes's string that has matched
774 * the input string thus far.
775 */
776 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
777 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
778
779 /*
780 * Prefix has the current node as the child corresponding to the first byte
781 * after the split.
782 */
783 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
784 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
785 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
786 return NULL_TRIE_IDX;
787 }
788 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
789
790 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
791 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
792
793 /* Current node is adjusted to point to the remainder */
794 TRIE_NODE(trie, current).start = node_idx;
795 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
796
797 /* We want to insert the new leaf (if any) as a child of the prefix */
798 current = prefix;
799 }
800
801 if (string_remainder > 0) {
802 /*
803 * We still have bytes in the string that have not been matched yet.
804 * If the current node has children, iterate to the child corresponding
805 * to the next byte in the string.
806 */
807 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
808 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
809 }
810 }
811 } /* while (child != NULL_TRIE_IDX) */
812
813 if (string_remainder > 0) {
814 /* Add a new leaf containing the remainder of the string */
815 uint16_t leaf = trie_node_alloc(trie);
816 if (leaf == NULL_TRIE_IDX) {
817 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
818 return NULL_TRIE_IDX;
819 }
820
821 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
822 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
823 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
824 return NULL_TRIE_IDX;
825 }
826 TRIE_NODE(trie, leaf).length = string_remainder;
827
828 /* Set the new leaf as the child of the current node */
829 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
830 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
831 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
832 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
833 return NULL_TRIE_IDX;
834 }
835 }
836 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
837 current = leaf;
838 } /* else duplicate or this string is a prefix of one of the existing strings */
839
840 return current;
841}
842
0a7de745 843#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 844static uint16_t
ea3f0419 845flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
39236c6e
A		 846{
847 uint16_t current = trie->root;
848 uint16_t string_idx = 0;
849
850 while (current != NULL_TRIE_IDX) {
851 uint16_t next = NULL_TRIE_IDX;
852 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
853 uint16_t node_idx;
854
855 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 856 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
857 node_idx++, string_idx++) {
858 ;
859 }
39236c6e
A		 860
861 if (node_idx == node_end) {
862 if (string_bytes[string_idx] == '\0') {
863 return current; /* Got an exact match */
3e170ce0 864 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
0a7de745 865 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
3e170ce0 866 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 867 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
868 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
869 }
870 }
871 current = next;
872 }
873
874 return NULL_TRIE_IDX;
875}
876
490019cf
A		 877struct uuid_search_info {
878 uuid_t target_uuid;
879 char *found_signing_id;
880 boolean_t found_multiple_signing_ids;
881 proc_t found_proc;
882};
883
884static int
885flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
886{
887 struct uuid_search_info *info = (struct uuid_search_info *)arg;
888 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
889
890 if (info->found_signing_id != NULL) {
891 if (!info->found_multiple_signing_ids) {
892 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
893 info->found_proc = p;
894 result = PROC_CLAIMED_DONE;
895 } else {
896 uuid_string_t uuid_str;
897 uuid_unparse(info->target_uuid, uuid_str);
898 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
899 }
900 FREE(info->found_signing_id, M_TEMP);
901 info->found_signing_id = NULL;
902 }
903
904 if (result == PROC_RETURNED_DONE) {
905 uuid_string_t uuid_str;
906 uuid_unparse(info->target_uuid, uuid_str);
907 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
908 }
909
910 return result;
911}
912
913static int
914flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
915{
916 struct uuid_search_info *info = (struct uuid_search_info *)arg;
917 int include = 0;
918
919 if (info->found_multiple_signing_ids) {
920 return include;
921 }
922
923 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
924 if (include) {
925 const char *signing_id = cs_identity_get(p);
926 if (signing_id != NULL) {
927 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
928 size_t signing_id_size = strlen(signing_id) + 1;
929 if (info->found_signing_id == NULL) {
930 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
931 memcpy(info->found_signing_id, signing_id, signing_id_size);
932 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
933 info->found_multiple_signing_ids = TRUE;
934 }
935 } else {
936 info->found_multiple_signing_ids = TRUE;
937 }
938 include = !info->found_multiple_signing_ids;
939 }
940
941 return include;
942}
943
944static proc_t
945flow_divert_find_proc_by_uuid(uuid_t uuid)
946{
947 struct uuid_search_info info;
948
949 if (LOG_INFO <= nil_pcb.log_level) {
950 uuid_string_t uuid_str;
951 uuid_unparse(uuid, uuid_str);
952 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
953 }
954
955 memset(&info, 0, sizeof(info));
956 info.found_proc = PROC_NULL;
957 uuid_copy(info.target_uuid, uuid);
958
959 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
960
961 return info.found_proc;
962}
963
39236c6e 964static int
ea3f0419 965flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
39236c6e 966{
ea3f0419 967 int error = 0;
f427ee49 968 uint8_t *cdhash = NULL;
ea3f0419
A		 969 audit_token_t audit_token = {};
970 const char *proc_cs_id = signing_id;
39236c6e 971
ea3f0419
A		 972 proc_lock(proc);
973
974 if (proc_cs_id == NULL) {
975 if (proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
976 proc_cs_id = cs_identity_get(proc);
977 } else {
978 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
490019cf 979 }
39236c6e
A		 980 }
981
ea3f0419
A		 982 if (is_effective) {
983 lck_rw_lock_shared(&fd_cb->group->lck);
984 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
985 if (proc_cs_id != NULL) {
986 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
987 if (result == NULL_TRIE_IDX) {
988 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
989 error = EPERM;
990 } else {
991 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
992 }
993 } else {
994 error = EPERM;
39236c6e 995 }
ea3f0419
A		 996 }
997 lck_rw_done(&fd_cb->group->lck);
998 }
999
1000 if (error != 0) {
1001 goto done;
1002 }
1003
1004 /*
1005 * If signing_id is not NULL then it came from the flow divert token and will be added
1006 * as part of the token, so there is no need to add it here.
1007 */
1008 if (signing_id == NULL && proc_cs_id != NULL) {
1009 error = flow_divert_packet_append_tlv(connect_packet,
1010 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
f427ee49 1011 (uint32_t)strlen(proc_cs_id),
ea3f0419
A		 1012 proc_cs_id);
1013 if (error != 0) {
1014 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1015 goto done;
1016 }
1017 }
1018
f427ee49
A		 1019 cdhash = cs_get_cdhash(proc);
1020 if (cdhash != NULL) {
ea3f0419
A		 1021 error = flow_divert_packet_append_tlv(connect_packet,
1022 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
f427ee49 1023 SHA1_RESULTLEN,
ea3f0419
A		 1024 cdhash);
1025 if (error) {
1026 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1027 goto done;
1028 }
1029 } else {
f427ee49 1030 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash");
ea3f0419
A		 1031 }
1032
1033 task_t task = proc_task(proc);
1034 if (task != TASK_NULL) {
1035 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1036 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1037 if (rc == KERN_SUCCESS) {
1038 int append_error = flow_divert_packet_append_tlv(connect_packet,
1039 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1040 sizeof(audit_token_t),
1041 &audit_token);
1042 if (append_error) {
1043 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1044 }
1045 }
1046 }
1047
1048done:
1049 proc_unlock(proc);
1050
1051 return error;
1052}
1053
1054static int
1055flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1056{
1057 int error = 0;
1058 proc_t effective_proc = PROC_NULL;
1059 proc_t responsible_proc = PROC_NULL;
1060 proc_t real_proc = proc_find(so->last_pid);
1061 bool release_real_proc = true;
1062
1063 proc_t src_proc = PROC_NULL;
1064 proc_t real_src_proc = PROC_NULL;
1065
1066 if (real_proc == PROC_NULL) {
1067 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1068 release_real_proc = false;
1069 real_proc = proc;
1070 if (real_proc == PROC_NULL) {
1071 real_proc = current_proc();
1072 }
1073 }
1074
1075 if (so->so_flags & SOF_DELEGATED) {
1076 if (real_proc->p_pid != so->e_pid) {
1077 effective_proc = proc_find(so->e_pid);
1078 } else if (uuid_compare(real_proc->p_uuid, so->e_uuid)) {
1079 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
39236c6e
A		 1080 }
1081 }
1082
ea3f0419
A		 1083#if defined(XNU_TARGET_OS_OSX)
1084 lck_rw_lock_shared(&fd_cb->group->lck);
1085 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1086 if (so->so_rpid > 0) {
1087 responsible_proc = proc_find(so->so_rpid);
1088 }
1089 }
1090 lck_rw_done(&fd_cb->group->lck);
1091#endif
1092
1093 real_src_proc = real_proc;
1094
1095 if (responsible_proc != PROC_NULL) {
1096 src_proc = responsible_proc;
1097 if (effective_proc != NULL) {
1098 real_src_proc = effective_proc;
1099 }
1100 } else if (effective_proc != PROC_NULL) {
1101 src_proc = effective_proc;
1102 } else {
1103 src_proc = real_proc;
1104 }
1105
1106 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1107 if (error != 0) {
1108 goto done;
1109 }
1110
1111 if (real_src_proc != NULL && real_src_proc != src_proc) {
1112 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1113 if (error != 0) {
1114 goto done;
1115 }
1116 }
1117
1118done:
1119 if (responsible_proc != PROC_NULL) {
1120 proc_rele(responsible_proc);
1121 }
1122
1123 if (effective_proc != PROC_NULL) {
1124 proc_rele(effective_proc);
1125 }
1126
1127 if (real_proc != PROC_NULL && release_real_proc) {
1128 proc_rele(real_proc);
1129 }
1130
1131 return error;
39236c6e
A		 1132}
1133
1134static int
1135flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1136{
0a7de745 1137 int error;
39236c6e
A		 1138
1139 if (fd_cb->group == NULL) {
1140 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 1141 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1142 return ECONNABORTED;
1143 }
1144
1145 lck_rw_lock_shared(&fd_cb->group->lck);
1146
1147 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1148 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1149 } else {
1150 error = ENOBUFS;
1151 }
1152
1153 if (error == ENOBUFS) {
1154 if (enqueue) {
1155 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1156 lck_rw_lock_exclusive(&fd_cb->group->lck);
1157 }
1158 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1159 error = 0;
1160 }
1161 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1162 }
1163
1164 lck_rw_done(&fd_cb->group->lck);
1165
1166 return error;
1167}
1168
1169static int
490019cf 1170flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 1171{
ea3f0419
A		 1172 int error = 0;
1173 int flow_type = 0;
0a7de745 1174 char *signing_id = NULL;
0a7de745 1175 mbuf_t connect_packet = NULL;
ea3f0419
A		 1176 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1177 const void *cfil_id = NULL;
1178 size_t cfil_id_size = 0;
1179 struct inpcb *inp = sotoinpcb(so);
f427ee49 1180 struct ifnet *ifp = NULL;
2a1bd2d3 1181 uint32_t flags = 0;
490019cf
A		 1182
1183 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1184 if (error) {
1185 goto done;
1186 }
1187
490019cf
A		 1188 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1189 uint32_t sid_size = 0;
1190 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1191 if (find_error == 0 && sid_size > 0) {
1192 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1193 if (signing_id != NULL) {
1194 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1195 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
490019cf
A		 1196 }
1197 }
1198 }
1199
1200 socket_unlock(so, 0);
39037602 1201
ea3f0419 1202 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
490019cf 1203
490019cf
A		 1204 socket_lock(so, 0);
1205
ea3f0419 1206 if (signing_id != NULL) {
490019cf
A		 1207 FREE(signing_id, M_TEMP);
1208 }
1209
1210 if (error) {
ea3f0419 1211 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
490019cf
A		 1212 goto done;
1213 }
39236c6e 1214
39236c6e 1215 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1216 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1217 sizeof(fd_cb->so->so_traffic_class),
1218 &fd_cb->so->so_traffic_class);
39236c6e
A		 1219 if (error) {
1220 goto done;
1221 }
1222
3e170ce0
A		 1223 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1224 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1225 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1226 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1227 } else {
1228 error = EINVAL;
1229 goto done;
1230 }
1231 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1232 FLOW_DIVERT_TLV_FLOW_TYPE,
1233 sizeof(flow_type),
1234 &flow_type);
3e170ce0
A		 1235
1236 if (error) {
1237 goto done;
1238 }
1239
39236c6e
A		 1240 if (fd_cb->connect_token != NULL) {
1241 unsigned int token_len = m_length(fd_cb->connect_token);
1242 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1243 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1244 fd_cb->connect_token = NULL;
1245 } else {
3e170ce0 1246 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1247 if (error) {
1248 goto done;
1249 }
39236c6e
A		 1250 }
1251
f427ee49
A		 1252 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1253 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
490019cf
A		 1254 if (error) {
1255 goto done;
1256 }
1257 }
3e170ce0 1258
f427ee49 1259 if (inp->inp_vflag & INP_IPV4) {
ea3f0419 1260 ifp = inp->inp_last_outifp;
f427ee49
A		 1261 } else if (inp->inp_vflag & INP_IPV6) {
1262 ifp = inp->in6p_last_outifp;
ea3f0419 1263 }
ea3f0419
A		 1264 if (ifp != NULL) {
1265 uint32_t flow_if_index = ifp->if_index;
1266 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1267 sizeof(flow_if_index), &flow_if_index);
1268 if (error) {
1269 goto done;
1270 }
1271 }
1272
490019cf 1273 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
2a1bd2d3
A		 1274 flags |= FLOW_DIVERT_TOKEN_FLAG_TFO;
1275 }
1276
1277 if ((inp->inp_flags & INP_BOUND_IF) ||
1278 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) ||
1279 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
1280 flags |= FLOW_DIVERT_TOKEN_FLAG_BOUND;
1281 }
1282
1283 if (flags != 0) {
490019cf
A		 1284 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1285 if (error) {
1286 goto done;
1287 }
39236c6e
A		 1288 }
1289
f427ee49
A		 1290 if (SOCK_TYPE(so) == SOCK_DGRAM) {
1291 cfil_sock_id = cfil_sock_id_from_datagram_socket(so, NULL, to);
1292 } else {
1293 cfil_sock_id = cfil_sock_id_from_socket(so);
1294 }
1295
ea3f0419
A		 1296 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1297 cfil_id = &cfil_sock_id;
1298 cfil_id_size = sizeof(cfil_sock_id);
1299 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1300 cfil_id = &inp->necp_client_uuid;
1301 cfil_id_size = sizeof(inp->necp_client_uuid);
1302 }
1303
1304 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
f427ee49 1305 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, (uint32_t)cfil_id_size, cfil_id);
ea3f0419
A		 1306 if (error) {
1307 goto done;
1308 }
1309 }
1310
39236c6e 1311done:
490019cf
A		 1312 if (!error) {
1313 *out_connect_packet = connect_packet;
1314 } else if (connect_packet != NULL) {
1315 mbuf_freem(connect_packet);
1316 }
1317
39236c6e
A		 1318 return error;
1319}
1320
f427ee49
A		 1321static int
1322flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb)
1323{
1324 int error = 0;
1325 mbuf_t connect_packet = fd_cb->connect_packet;
1326 mbuf_t saved_connect_packet = NULL;
1327
1328 if (connect_packet != NULL) {
1329 error = mbuf_copym(connect_packet, 0, mbuf_pkthdr_len(connect_packet), MBUF_DONTWAIT, &saved_connect_packet);
1330 if (error) {
1331 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet");
1332 goto done;
1333 }
1334
1335 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
1336 if (error) {
1337 goto done;
1338 }
1339
1340 fd_cb->connect_packet = saved_connect_packet;
1341 saved_connect_packet = NULL;
1342 } else {
1343 error = ENOENT;
1344 }
1345done:
1346 if (saved_connect_packet != NULL) {
1347 mbuf_freem(saved_connect_packet);
1348 }
1349
1350 return error;
1351}
1352
39236c6e
A		 1353static int
1354flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1355{
0a7de745
A		 1356 int error = 0;
1357 mbuf_t packet = NULL;
1358 int rbuff_space = 0;
39236c6e
A		 1359
1360 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1361 if (error) {
1362 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1363 goto done;
1364 }
1365
3e170ce0 1366 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1367 if (rbuff_space < 0) {
1368 rbuff_space = 0;
1369 }
1370 rbuff_space = htonl(rbuff_space);
1371 error = flow_divert_packet_append_tlv(packet,
0a7de745
A		 1372 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1373 sizeof(rbuff_space),
1374 &rbuff_space);
39236c6e
A		 1375 if (error) {
1376 goto done;
1377 }
1378
c3c9b80d
A		 1379 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1380 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1381 if (error) {
1382 goto done;
1383 }
1384 }
1385
39236c6e
A		 1386 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1387 if (error) {
1388 goto done;
1389 }
1390
1391done:
1392 if (error && packet != NULL) {
3e170ce0 1393 mbuf_freem(packet);
39236c6e
A		 1394 }
1395
1396 return error;
1397}
1398
1399static int
1400flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1401{
0a7de745
A		 1402 int error = 0;
1403 mbuf_t packet = NULL;
1404 uint32_t zero = 0;
39236c6e
A		 1405
1406 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1407 if (error) {
1408 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1409 goto done;
1410 }
1411
1412 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1413 if (error) {
1414 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1415 goto done;
1416 }
1417
1418 how = htonl(how);
1419 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1420 if (error) {
1421 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1422 goto done;
1423 }
1424
1425 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1426 if (error) {
1427 goto done;
1428 }
1429
1430done:
1431 if (error && packet != NULL) {
1432 mbuf_free(packet);
1433 }
1434
1435 return error;
1436}
1437
1438static int
1439flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1440{
0a7de745
A		 1441 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1442 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
39236c6e
A		 1443 return SHUT_RDWR;
1444 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1445 return SHUT_RD;
1446 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1447 return SHUT_WR;
1448 }
1449
1450 return -1;
1451}
1452
1453/*
1454 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1455 * writes. Returns FALSE otherwise.
1456 */
1457static void
1458flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1459{
0a7de745 1460 int how = -1;
39236c6e
A		 1461
1462 /* Do not send any close messages if there is still data in the send buffer */
1463 if (fd_cb->so->so_snd.sb_cc == 0) {
0a7de745 1464 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
39236c6e
A		 1465 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1466 how = SHUT_RD;
1467 }
0a7de745 1468 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
39236c6e
A		 1469 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1470 if (how == SHUT_RD) {
1471 how = SHUT_RDWR;
1472 } else {
1473 how = SHUT_WR;
1474 }
1475 }
1476 }
1477
1478 if (how != -1) {
1479 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1480 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1481 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1482 if (how != SHUT_RD) {
1483 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1484 }
1485 if (how != SHUT_WR) {
1486 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1487 }
1488 }
1489 }
1490
1491 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1492 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1493 }
1494}
1495
1496static errno_t
3e170ce0 1497flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e 1498{
eb6b6ca3
A		 1499 mbuf_t packet = NULL;
1500 mbuf_t last = NULL;
0a7de745 1501 int error = 0;
39236c6e
A		 1502
1503 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
eb6b6ca3 1504 if (error || packet == NULL) {
39236c6e 1505 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
eb6b6ca3 1506 goto done;
39236c6e
A		 1507 }
1508
3e170ce0
A		 1509 if (toaddr != NULL) {
1510 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1511 if (error) {
1512 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
eb6b6ca3 1513 goto done;
3e170ce0
A		 1514 }
1515 }
1516
f427ee49 1517 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
5ba3f43e
A		 1518 last = m_last(packet);
1519 mbuf_setnext(last, data);
f427ee49
A		 1520 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1521 } else {
1522 data_len = 0;
5ba3f43e 1523 }
39236c6e 1524 error = flow_divert_send_packet(fd_cb, packet, force);
eb6b6ca3 1525 if (error == 0 && data_len > 0) {
39236c6e
A		 1526 fd_cb->bytes_sent += data_len;
1527 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1528 }
1529
eb6b6ca3
A		 1530done:
1531 if (error) {
1532 if (last != NULL) {
1533 mbuf_setnext(last, NULL);
1534 }
1535 if (packet != NULL) {
1536 mbuf_freem(packet);
1537 }
1538 }
1539
39236c6e
A		 1540 return error;
1541}
1542
1543static void
1544flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1545{
0a7de745
A		 1546 size_t to_send;
1547 size_t sent = 0;
1548 int error = 0;
1549 mbuf_t buffer;
39236c6e
A		 1550
1551 to_send = fd_cb->so->so_snd.sb_cc;
1552 buffer = fd_cb->so->so_snd.sb_mb;
1553
1554 if (buffer == NULL && to_send > 0) {
1555 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1556 return;
1557 }
1558
1559 /* Ignore the send window if force is enabled */
1560 if (!force && (to_send > fd_cb->send_window)) {
1561 to_send = fd_cb->send_window;
1562 }
1563
3e170ce0
A		 1564 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1565 while (sent < to_send) {
0a7de745
A		 1566 mbuf_t data;
1567 size_t data_len;
39236c6e 1568
3e170ce0
A		 1569 data_len = to_send - sent;
1570 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1571 data_len = FLOW_DIVERT_CHUNK_SIZE;
1572 }
39236c6e 1573
3e170ce0
A		 1574 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1575 if (error) {
1576 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1577 break;
1578 }
39236c6e 1579
3e170ce0
A		 1580 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1581 if (error) {
f427ee49
A		 1582 if (data != NULL) {
1583 mbuf_freem(data);
1584 }
3e170ce0
A		 1585 break;
1586 }
39236c6e 1587
3e170ce0
A		 1588 sent += data_len;
1589 }
f427ee49 1590 sbdrop(&fd_cb->so->so_snd, (int)sent);
3e170ce0
A		 1591 sowwakeup(fd_cb->so);
1592 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1593 mbuf_t data;
1594 mbuf_t m;
1595 size_t data_len;
1596
0a7de745 1597 while (buffer) {
3e170ce0
A		 1598 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1599
1600 m = buffer;
1601 if (toaddr != NULL) {
1602 /* look for data in the chain */
1603 do {
1604 m = m->m_next;
1605 if (m != NULL && m->m_type == MT_DATA) {
1606 break;
1607 }
0a7de745 1608 } while (m);
3e170ce0
A		 1609 if (m == NULL) {
1610 /* unexpected */
1611 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1612 goto move_on;
1613 }
1614 }
1615 data_len = mbuf_pkthdr_len(m);
5ba3f43e
A		 1616 if (data_len > 0) {
1617 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1618 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1619 if (error) {
1620 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1621 break;
1622 }
1623 } else {
1624 data = NULL;
3e170ce0
A		 1625 }
1626 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1627 if (error) {
f427ee49
A		 1628 if (data != NULL) {
1629 mbuf_freem(data);
1630 }
3e170ce0
A		 1631 break;
1632 }
1633 sent += data_len;
1634move_on:
1635 buffer = buffer->m_nextpkt;
1636 (void) sbdroprecord(&(fd_cb->so->so_snd));
1637 }
39236c6e
A		 1638 }
1639
1640 if (sent > 0) {
1641 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1642 if (fd_cb->send_window >= sent) {
1643 fd_cb->send_window -= sent;
1644 } else {
1645 fd_cb->send_window = 0;
1646 }
39236c6e
A		 1647 }
1648}
1649
1650static int
3e170ce0 1651flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e 1652{
0a7de745
A		 1653 size_t to_send = mbuf_pkthdr_len(data);
1654 int error = 0;
39236c6e
A		 1655
1656 if (to_send > fd_cb->send_window) {
1657 to_send = fd_cb->send_window;
1658 }
1659
1660 if (fd_cb->so->so_snd.sb_cc > 0) {
0a7de745 1661 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
39236c6e
A		 1662 }
1663
3e170ce0 1664 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
0a7de745
A		 1665 size_t sent = 0;
1666 mbuf_t remaining_data = data;
1667 mbuf_t pkt_data = NULL;
490019cf 1668 while (sent < to_send && remaining_data != NULL) {
0a7de745 1669 size_t pkt_data_len;
3e170ce0
A		 1670
1671 pkt_data = remaining_data;
39236c6e 1672
3e170ce0
A		 1673 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1674 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1675 } else {
1676 pkt_data_len = to_send - sent;
1677 }
1678
1679 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1680 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1681 if (error) {
1682 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1683 pkt_data = NULL;
1684 break;
1685 }
1686 } else {
1687 remaining_data = NULL;
1688 }
1689
1690 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1691
39236c6e 1692 if (error) {
39236c6e
A		 1693 break;
1694 }
39236c6e 1695
3e170ce0
A		 1696 pkt_data = NULL;
1697 sent += pkt_data_len;
39236c6e
A		 1698 }
1699
3e170ce0 1700 fd_cb->send_window -= sent;
39236c6e 1701
3e170ce0 1702 error = 0;
39236c6e 1703
3e170ce0
A		 1704 if (pkt_data != NULL) {
1705 if (sbspace(&fd_cb->so->so_snd) > 0) {
1706 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1707 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
0a7de745 1708 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1709 }
1710 } else {
f427ee49 1711 mbuf_freem(pkt_data);
3e170ce0 1712 error = ENOBUFS;
39236c6e 1713 }
39236c6e 1714 }
39236c6e 1715
3e170ce0
A		 1716 if (remaining_data != NULL) {
1717 if (sbspace(&fd_cb->so->so_snd) > 0) {
1718 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1719 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
0a7de745 1720 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1721 }
1722 } else {
f427ee49 1723 mbuf_freem(remaining_data);
3e170ce0
A		 1724 error = ENOBUFS;
1725 }
1726 }
1727 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
5ba3f43e 1728 if (to_send || mbuf_pkthdr_len(data) == 0) {
3e170ce0
A		 1729 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1730 if (error) {
39037602 1731 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
f427ee49
A		 1732 if (data != NULL) {
1733 mbuf_freem(data);
1734 }
3e170ce0
A		 1735 } else {
1736 fd_cb->send_window -= to_send;
39236c6e
A		 1737 }
1738 } else {
3e170ce0
A		 1739 /* buffer it */
1740 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1741 if (toaddr != NULL) {
1742 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1743 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1744 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1745 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0 1746 }
bca245ac 1747 error = 0;
3e170ce0
A		 1748 } else {
1749 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1750 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1751 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1752 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0
A		 1753 }
1754 }
1755 } else {
f427ee49
A		 1756 if (data != NULL) {
1757 mbuf_freem(data);
1758 }
3e170ce0
A		 1759 error = ENOBUFS;
1760 }
39236c6e
A		 1761 }
1762 }
1763
1764 return error;
1765}
1766
1767static int
f427ee49 1768flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb)
39236c6e 1769{
f427ee49
A		 1770 int error = 0;
1771 mbuf_t packet = NULL;
39236c6e
A		 1772
1773 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1774 if (error) {
1775 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1776 goto done;
1777 }
1778
39236c6e
A		 1779 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1780 if (error) {
1781 goto done;
1782 }
1783
1784done:
1785 if (error && packet != NULL) {
1786 mbuf_free(packet);
1787 }
1788
1789 return error;
1790}
1791
1792static int
1793flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1794{
0a7de745
A		 1795 int error = 0;
1796 mbuf_t packet = NULL;
39236c6e
A		 1797
1798 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1799 if (error) {
1800 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1801 goto done;
1802 }
1803
1804 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1805 if (error) {
1806 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1807 goto done;
1808 }
1809
1810 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1811 if (error) {
1812 goto done;
1813 }
1814
1815done:
1816 if (error && packet != NULL) {
1817 mbuf_free(packet);
1818 }
1819
1820 return error;
1821}
1822
f427ee49 1823static void
c3c9b80d 1824flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint)
f427ee49
A		 1825{
1826 struct inpcb *inp = sotoinpcb(fd_cb->so);
1827
1828 if (local_endpoint->sa_family == AF_INET6) {
c3c9b80d 1829 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
f427ee49
A		 1830 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1831 inp->in6p_laddr = (satosin6(local_endpoint))->sin6_addr;
1832 }
1833 if (inp->inp_lport == 0) {
1834 inp->inp_lport = (satosin6(local_endpoint))->sin6_port;
1835 }
1836 } else if (local_endpoint->sa_family == AF_INET) {
c3c9b80d 1837 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
f427ee49
A		 1838 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1839 inp->inp_laddr = (satosin(local_endpoint))->sin_addr;
1840 }
1841 if (inp->inp_lport == 0) {
1842 inp->inp_lport = (satosin(local_endpoint))->sin_port;
1843 }
1844 }
1845}
1846
1847static void
1848flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint)
1849{
1850 struct inpcb *inp = sotoinpcb(fd_cb->so);
1851
1852 if (remote_endpoint->sa_family == AF_INET6) {
1853 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
1854 inp->in6p_faddr = (satosin6(remote_endpoint))->sin6_addr;
1855 }
1856 if (inp->inp_fport == 0) {
1857 inp->inp_fport = (satosin6(remote_endpoint))->sin6_port;
1858 }
1859 } else if (remote_endpoint->sa_family == AF_INET) {
1860 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1861 inp->inp_faddr = (satosin(remote_endpoint))->sin_addr;
1862 }
1863 if (inp->inp_fport == 0) {
1864 inp->inp_fport = (satosin(remote_endpoint))->sin_port;
1865 }
1866 }
1867}
1868
1869static uint32_t
1870flow_divert_derive_kernel_control_unit(uint32_t ctl_unit, uint32_t *aggregate_unit)
1871{
1872 if (aggregate_unit != NULL && *aggregate_unit != 0) {
1873 uint32_t counter;
1874 for (counter = 0; counter < (GROUP_COUNT_MAX - 1); counter++) {
1875 if ((*aggregate_unit) & (1 << counter)) {
1876 break;
1877 }
1878 }
1879 if (counter < (GROUP_COUNT_MAX - 1)) {
1880 *aggregate_unit &= ~(1 << counter);
1881 return counter + 1;
1882 } else {
1883 return ctl_unit;
1884 }
1885 } else {
1886 return ctl_unit;
1887 }
1888}
1889
1890static int
1891flow_divert_try_next(struct flow_divert_pcb *fd_cb)
1892{
1893 uint32_t current_ctl_unit = 0;
1894 uint32_t next_ctl_unit = 0;
1895 struct flow_divert_group *current_group = NULL;
1896 struct flow_divert_group *next_group = NULL;
1897 int error = 0;
1898
1899 next_ctl_unit = flow_divert_derive_kernel_control_unit(fd_cb->policy_control_unit, &(fd_cb->aggregate_unit));
1900 current_ctl_unit = fd_cb->control_group_unit;
1901
1902 if (current_ctl_unit == next_ctl_unit) {
1903 FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling flow divert");
1904 error = EALREADY;
1905 goto done;
1906 }
1907
1908 if (next_ctl_unit == 0 || next_ctl_unit >= GROUP_COUNT_MAX) {
1909 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert");
1910 error = ENOENT;
1911 goto done;
1912 }
1913
1914 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1915 FDLOG0(LOG_NOTICE, fd_cb, "No active groups, disabling flow divert");
1916 error = ENOENT;
1917 goto done;
1918 }
1919
1920 next_group = g_flow_divert_groups[next_ctl_unit];
1921 if (next_group == NULL) {
1922 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit);
1923 error = ENOENT;
1924 goto done;
1925 }
1926
1927 current_group = fd_cb->group;
1928
1929 lck_rw_lock_exclusive(&(current_group->lck));
1930 lck_rw_lock_exclusive(&(next_group->lck));
1931
1932 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", current_ctl_unit, next_ctl_unit);
1933
1934 RB_REMOVE(fd_pcb_tree, &(current_group->pcb_tree), fd_cb);
1935 if (RB_INSERT(fd_pcb_tree, &(next_group->pcb_tree), fd_cb) != NULL) {
1936 panic("group with unit %u already contains a connection with hash %u", next_ctl_unit, fd_cb->hash);
1937 }
1938
1939 fd_cb->group = next_group;
1940 fd_cb->control_group_unit = next_ctl_unit;
1941
1942 lck_rw_done(&(next_group->lck));
1943 lck_rw_done(&(current_group->lck));
1944
1945 error = flow_divert_send_connect_packet(fd_cb);
1946 if (error) {
1947 FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", next_ctl_unit);
1948 error = ENOENT;
1949 goto done;
1950 }
1951
1952done:
1953 return error;
1954}
1955
1956static void
1957flow_divert_disable(struct flow_divert_pcb *fd_cb)
1958{
1959 struct socket *so = NULL;
1960 mbuf_t buffer;
1961 int error = 0;
1962 proc_t last_proc = NULL;
1963 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint;
1964 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT);
1965 struct inpcb *inp = NULL;
1966
1967 so = fd_cb->so;
1968 if (so == NULL) {
1969 goto done;
1970 }
1971
1972 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert");
1973
1974 /* Restore the IP state */
1975 inp = sotoinpcb(so);
1976 inp->inp_vflag = fd_cb->original_vflag;
1977 inp->inp_faddr.s_addr = INADDR_ANY;
1978 inp->inp_fport = 0;
1979 memset(&(inp->in6p_faddr), 0, sizeof(inp->in6p_faddr));
1980 inp->in6p_fport = 0;
1981 /* If flow divert set the local address, clear it out */
1982 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) {
1983 inp->inp_laddr.s_addr = INADDR_ANY;
1984 memset(&(inp->in6p_laddr), 0, sizeof(inp->in6p_laddr));
1985 }
1986 inp->inp_last_outifp = fd_cb->original_last_outifp;
1987 inp->in6p_last_outifp = fd_cb->original_last_outifp6;
1988
1989 /* Dis-associate the socket */
1990 so->so_flags &= ~SOF_FLOW_DIVERT;
1991 so->so_flags1 |= SOF1_FLOW_DIVERT_SKIP;
1992 so->so_fd_pcb = NULL;
1993 fd_cb->so = NULL;
1994
1995 /* Remove from the group */
1996 flow_divert_pcb_remove(fd_cb);
1997
1998 FDRELEASE(fd_cb); /* Release the socket's reference */
1999
2000 /* Revert back to the original protocol */
2001 so->so_proto = pffindproto(SOCK_DOM(so), SOCK_PROTO(so), SOCK_TYPE(so));
2002
2003 last_proc = proc_find(so->last_pid);
2004
2005 if (do_connect) {
2006 /* Connect using the original protocol */
2007 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, remote_endpoint, (last_proc != NULL ? last_proc : current_proc()));
2008 if (error) {
2009 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error);
2010 goto done;
2011 }
2012 }
2013
2014 buffer = so->so_snd.sb_mb;
2015 if (buffer == NULL) {
2016 /* No buffered data, done */
2017 goto done;
2018 }
2019
2020 /* Send any buffered data using the original protocol */
2021 if (SOCK_TYPE(so) == SOCK_STREAM) {
2022 mbuf_t data_to_send = NULL;
2023 size_t data_len = so->so_snd.sb_cc;
2024
2025 error = mbuf_copym(buffer, 0, data_len, MBUF_DONTWAIT, &data_to_send);
2026 if (error) {
2027 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer");
2028 goto done;
2029 }
2030
2031 sbflush(&so->so_snd);
2032
2033 if (data_to_send->m_flags & M_PKTHDR) {
2034 mbuf_pkthdr_setlen(data_to_send, data_len);
2035 }
2036
2037 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2038 0,
2039 data_to_send,
2040 NULL,
2041 NULL,
2042 (last_proc != NULL ? last_proc : current_proc()));
2043
c3c9b80d 2044 if (error && error != EWOULDBLOCK) {
f427ee49 2045 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
c3c9b80d
A		 2046 } else {
2047 error = 0;
f427ee49
A		 2048 }
2049 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
2050 struct sockbuf *sb = &so->so_snd;
2051 MBUFQ_HEAD(send_queue_head) send_queue;
2052 MBUFQ_INIT(&send_queue);
2053
2054 /* Flush the send buffer, moving all records to a temporary queue */
2055 while (sb->sb_mb != NULL) {
2056 mbuf_t record = sb->sb_mb;
2057 mbuf_t m = record;
2058 sb->sb_mb = sb->sb_mb->m_nextpkt;
2059 while (m != NULL) {
2060 sbfree(sb, m);
2061 m = m->m_next;
2062 }
2063 record->m_nextpkt = NULL;
2064 MBUFQ_ENQUEUE(&send_queue, record);
2065 }
2066 SB_EMPTY_FIXUP(sb);
2067
2068 while (!MBUFQ_EMPTY(&send_queue)) {
2069 mbuf_t next_record = MBUFQ_FIRST(&send_queue);
2070 mbuf_t addr = NULL;
2071 mbuf_t control = NULL;
2072 mbuf_t last_control = NULL;
2073 mbuf_t data = NULL;
2074 mbuf_t m = next_record;
2075 struct sockaddr *to_endpoint = NULL;
2076
2077 MBUFQ_DEQUEUE(&send_queue, next_record);
2078
2079 while (m != NULL) {
2080 if (m->m_type == MT_SONAME) {
2081 addr = m;
2082 } else if (m->m_type == MT_CONTROL) {
2083 if (control == NULL) {
2084 control = m;
2085 }
2086 last_control = m;
2087 } else if (m->m_type == MT_DATA) {
2088 data = m;
2089 break;
2090 }
2091 m = m->m_next;
2092 }
2093
2094 if (addr != NULL) {
2095 to_endpoint = flow_divert_get_buffered_target_address(addr);
2096 if (to_endpoint == NULL) {
2097 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer");
2098 }
2099 }
2100
2101 if (data == NULL) {
2102 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data");
2103 mbuf_freem(next_record);
2104 continue;
2105 }
2106
2107 if (!(data->m_flags & M_PKTHDR)) {
2108 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header");
2109 mbuf_freem(next_record);
2110 continue;
2111 }
2112
2113 if (addr != NULL) {
2114 addr->m_next = NULL;
2115 }
2116
2117 if (last_control != NULL) {
2118 last_control->m_next = NULL;
2119 }
2120
2121 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2122 0,
2123 data,
2124 to_endpoint,
2125 control,
2126 (last_proc != NULL ? last_proc : current_proc()));
2127
2128 if (addr != NULL) {
2129 mbuf_freem(addr);
2130 }
2131
2132 if (error) {
2133 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2134 }
2135 }
2136 }
2137done:
2138 if (last_proc != NULL) {
2139 proc_rele(last_proc);
2140 }
2141
2142 if (error) {
2143 so->so_error = (uint16_t)error;
2144 flow_divert_disconnect_socket(so);
2145 }
2146}
2147
c3c9b80d
A		 2148static void
2149flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address)
2150{
2151 struct socket *so = NULL;
2152 struct inpcb *inp = NULL;
2153 struct ifnet *current_ifp = NULL;
2154 struct ifnet *new_ifp = NULL;
2155 int error = 0;
2156
2157 so = fd_cb->so;
2158 if (so == NULL) {
2159 return;
2160 }
2161
2162 inp = sotoinpcb(so);
2163
2164 if (out_if_index <= 0) {
2165 return;
2166 }
2167
2168 if (inp->inp_vflag & INP_IPV6) {
2169 current_ifp = inp->in6p_last_outifp;
2170 } else {
2171 current_ifp = inp->inp_last_outifp;
2172 }
2173
2174 if (current_ifp != NULL) {
2175 if (current_ifp->if_index == out_if_index) {
2176 /* No change */
2177 return;
2178 }
2179
2180 /* Scope the socket to the given interface */
2181 error = inp_bindif(inp, out_if_index, &new_ifp);
2182 if (error != 0) {
2183 FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error);
2184 return;
2185 }
2186
2187 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) {
2188 /* Get the appropriate address for the given interface */
2189 if (inp->inp_vflag & INP_IPV6) {
2190 inp->in6p_laddr = sa6_any.sin6_addr;
2191 error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr), NULL);
2192 } else {
2193 inp->inp_laddr.s_addr = INADDR_ANY;
2194 error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, NULL, 0);
2195 }
2196
2197 if (error != 0) {
2198 FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr returned %d", out_if_index, error);
2199 }
2200 }
2201 } else {
2202 ifnet_head_lock_shared();
2203 if (out_if_index <= if_index) {
2204 new_ifp = ifindex2ifnet[out_if_index];
2205 }
2206 ifnet_head_done();
2207 }
2208
2209 /* Update the "last interface" of the socket */
2210 if (new_ifp != NULL) {
2211 if (inp->inp_vflag & INP_IPV6) {
2212 inp->in6p_last_outifp = new_ifp;
2213 } else {
2214 inp->inp_last_outifp = new_ifp;
2215 }
2216
2217 }
2218}
2219
39236c6e
A		 2220static void
2221flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2222{
f427ee49 2223 uint32_t connect_error = 0;
0a7de745
A		 2224 uint32_t ctl_unit = 0;
2225 int error = 0;
2226 struct flow_divert_group *grp = NULL;
f427ee49
A		 2227 union sockaddr_in_4_6 local_endpoint = {};
2228 union sockaddr_in_4_6 remote_endpoint = {};
0a7de745 2229 int out_if_index = 0;
0a7de745
A		 2230 uint32_t send_window;
2231 uint32_t app_data_length = 0;
39236c6e 2232
f427ee49
A		 2233 memset(&local_endpoint, 0, sizeof(local_endpoint));
2234 memset(&remote_endpoint, 0, sizeof(remote_endpoint));
39236c6e
A		 2235
2236 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
2237 if (error) {
2238 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
2239 return;
2240 }
2241
f427ee49 2242 connect_error = ntohl(connect_error);
39236c6e
A		 2243 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
2244
2245 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
2246 if (error) {
2247 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
2248 return;
2249 }
2250
2251 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
2252 if (error) {
39037602 2253 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
39236c6e
A		 2254 }
2255
f427ee49 2256 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_endpoint), &(local_endpoint.sa), NULL);
39236c6e 2257 if (error) {
39037602 2258 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
39236c6e
A		 2259 }
2260
f427ee49 2261 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_endpoint), &(remote_endpoint.sa), NULL);
39236c6e 2262 if (error) {
39037602 2263 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
39236c6e
A		 2264 }
2265
2266 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2267 if (error) {
39037602
A		 2268 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
2269 }
2270
2271 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2272 if (error) {
2273 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
39236c6e
A		 2274 }
2275
39037602 2276 error = 0;
0a7de745 2277 ctl_unit = ntohl(ctl_unit);
39236c6e
A		 2278
2279 lck_rw_lock_shared(&g_flow_divert_group_lck);
2280
39037602
A		 2281 if (connect_error == 0 && ctl_unit > 0) {
2282 if (ctl_unit >= GROUP_COUNT_MAX) {
39236c6e
A		 2283 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
2284 error = EINVAL;
2285 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
2286 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
2287 error = EINVAL;
2288 } else {
2289 grp = g_flow_divert_groups[ctl_unit];
2290 if (grp == NULL) {
2291 error = ECONNRESET;
2292 }
2293 }
2294 }
2295
2296 FDLOCK(fd_cb);
2297 if (fd_cb->so != NULL) {
0a7de745 2298 struct inpcb *inp = NULL;
0a7de745 2299 struct flow_divert_group *old_group;
f427ee49 2300 struct socket *so = fd_cb->so;
c3c9b80d 2301 bool local_address_is_valid = false;
39236c6e 2302
f427ee49 2303 socket_lock(so, 0);
39236c6e 2304
c3c9b80d
A		 2305 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
2306 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result");
2307 goto done;
2308 }
2309
f427ee49
A		 2310 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_state & SS_ISCONNECTING)) {
2311 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result");
39236c6e
A		 2312 goto done;
2313 }
2314
f427ee49 2315 inp = sotoinpcb(so);
39236c6e
A		 2316
2317 if (connect_error || error) {
2318 goto set_socket_state;
2319 }
2320
f427ee49
A		 2321 if (flow_divert_is_sockaddr_valid(&(local_endpoint.sa))) {
2322 if (local_endpoint.sa.sa_family == AF_INET) {
2323 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
c3c9b80d
A		 2324 if ((inp->inp_vflag & INP_IPV4) && local_endpoint.sin.sin_addr.s_addr != INADDR_ANY) {
2325 local_address_is_valid = true;
2326 fd_cb->local_endpoint = local_endpoint;
2327 inp->inp_laddr.s_addr = INADDR_ANY;
2328 } else {
2329 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port;
2330 }
f427ee49
A		 2331 } else if (local_endpoint.sa.sa_family == AF_INET6) {
2332 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
c3c9b80d
A		 2333 if ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&local_endpoint.sin6.sin6_addr)) {
2334 local_address_is_valid = true;
2335 fd_cb->local_endpoint = local_endpoint;
2336 inp->in6p_laddr = sa6_any.sin6_addr;
2337 } else {
2338 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port;
2339 }
cb323159
A		 2340 }
2341 }
39236c6e 2342
c3c9b80d
A		 2343 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid);
2344 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
2345
f427ee49
A		 2346 if (flow_divert_is_sockaddr_valid(&(remote_endpoint.sa)) && SOCK_TYPE(so) == SOCK_STREAM) {
2347 if (remote_endpoint.sa.sa_family == AF_INET) {
2348 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2349 } else if (remote_endpoint.sa.sa_family == AF_INET6) {
2350 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
cb323159 2351 }
f427ee49 2352 flow_divert_set_remote_endpoint(fd_cb, &(remote_endpoint.sa));
39236c6e
A		 2353 }
2354
39037602 2355 if (app_data_length > 0) {
0a7de745 2356 uint8_t *app_data = NULL;
39037602
A		 2357 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2358 if (app_data != NULL) {
2359 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2360 if (error == 0) {
2361 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
2362 if (fd_cb->app_data != NULL) {
2363 FREE(fd_cb->app_data, M_TEMP);
2364 }
2365 fd_cb->app_data = app_data;
2366 fd_cb->app_data_length = app_data_length;
2367 } else {
2368 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
2369 FREE(app_data, M_TEMP);
2370 }
2371 } else {
2372 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
2373 }
2374 }
2375
39236c6e
A		 2376 if (error) {
2377 goto set_socket_state;
2378 }
2379
2380 if (fd_cb->group == NULL) {
2381 error = EINVAL;
2382 goto set_socket_state;
2383 }
2384
39037602
A		 2385 if (grp != NULL) {
2386 old_group = fd_cb->group;
39236c6e 2387
39037602
A		 2388 lck_rw_lock_exclusive(&old_group->lck);
2389 lck_rw_lock_exclusive(&grp->lck);
39236c6e 2390
39037602
A		 2391 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
2392 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
2393 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
2394 }
39236c6e 2395
39037602 2396 fd_cb->group = grp;
39236c6e 2397
39037602
A		 2398 lck_rw_done(&grp->lck);
2399 lck_rw_done(&old_group->lck);
2400 }
39236c6e
A		 2401
2402 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 2403
2404set_socket_state:
2405 if (!connect_error && !error) {
2406 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
2407 error = flow_divert_send_connect_result(fd_cb);
2408 }
2409
2410 if (connect_error || error) {
f427ee49
A		 2411 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) {
2412 error = flow_divert_try_next(fd_cb);
2413 if (error) {
2414 flow_divert_disable(fd_cb);
2415 }
2416 goto done;
2417 }
2418
2419 if (!connect_error) {
2420 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2421 so->so_error = (uint16_t)error;
2422 flow_divert_send_close_if_needed(fd_cb);
2423 } else {
39236c6e 2424 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
f427ee49 2425 so->so_error = (uint16_t)connect_error;
39236c6e 2426 }
f427ee49 2427 flow_divert_disconnect_socket(so);
39236c6e 2428 } else {
cb323159
A		 2429#if NECP
2430 /* Update NECP client with connected five-tuple */
2431 if (!uuid_is_null(inp->necp_client_uuid)) {
f427ee49
A		 2432 socket_unlock(so, 0);
2433 necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp);
2434 socket_lock(so, 0);
cb323159
A		 2435 }
2436#endif /* NECP */
2437
490019cf 2438 flow_divert_send_buffered_data(fd_cb, FALSE);
f427ee49
A		 2439 soisconnected(so);
2440 }
2441
2442 /* We don't need the connect packet any more */
2443 if (fd_cb->connect_packet != NULL) {
2444 mbuf_freem(fd_cb->connect_packet);
2445 fd_cb->connect_packet = NULL;
39236c6e
A		 2446 }
2447
f427ee49
A		 2448 /* We don't need the original remote endpoint any more */
2449 if (fd_cb->original_remote_endpoint != NULL) {
2450 FREE(fd_cb->original_remote_endpoint, M_SONAME);
2451 fd_cb->original_remote_endpoint = NULL;
2452 }
39236c6e 2453done:
f427ee49 2454 socket_unlock(so, 0);
39236c6e
A		 2455 }
2456 FDUNLOCK(fd_cb);
2457
2458 lck_rw_done(&g_flow_divert_group_lck);
2459}
2460
2461static void
2462flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2463{
0a7de745
A		 2464 uint32_t close_error;
2465 int error = 0;
2466 int how;
39236c6e
A		 2467
2468 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2469 if (error) {
2470 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2471 return;
2472 }
2473
2474 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2475 if (error) {
2476 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2477 return;
2478 }
2479
2480 how = ntohl(how);
2481
2482 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2483
2484 FDLOCK(fd_cb);
2485 if (fd_cb->so != NULL) {
2486 socket_lock(fd_cb->so, 0);
2487
c3c9b80d
A		 2488 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2489 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider");
2490 goto done;
2491 }
2492
f427ee49 2493 fd_cb->so->so_error = (uint16_t)ntohl(close_error);
39236c6e
A		 2494
2495 flow_divert_update_closed_state(fd_cb, how, TRUE);
0a7de745 2496
39236c6e
A		 2497 how = flow_divert_tunnel_how_closed(fd_cb);
2498 if (how == SHUT_RDWR) {
3e170ce0 2499 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2500 } else if (how == SHUT_RD) {
2501 socantrcvmore(fd_cb->so);
2502 } else if (how == SHUT_WR) {
2503 socantsendmore(fd_cb->so);
2504 }
c3c9b80d 2505done:
39236c6e
A		 2506 socket_unlock(fd_cb->so, 0);
2507 }
2508 FDUNLOCK(fd_cb);
2509}
2510
3e170ce0 2511static mbuf_t
f427ee49 2512flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 2513{
cb323159 2514 struct inpcb *inp = sotoinpcb(fd_cb->so);
f427ee49
A		 2515 bool is_cfil_enabled = false;
2516#if CONTENT_FILTER
2517 /* Content Filter needs to see the local address */
2518 is_cfil_enabled = (inp->inp_socket && inp->inp_socket->so_cfil_db != NULL);
2519#endif
2520 if ((inp->inp_vflag & INP_IPV4) &&
2521 fd_cb->local_endpoint.sa.sa_family == AF_INET &&
2522 ((inp->inp_flags & INP_RECVDSTADDR) || is_cfil_enabled)) {
2523 return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2524 } else if ((inp->inp_vflag & INP_IPV6) &&
2525 fd_cb->local_endpoint.sa.sa_family == AF_INET6 &&
2526 ((inp->inp_flags & IN6P_PKTINFO) || is_cfil_enabled)) {
cb323159
A		 2527 struct in6_pktinfo pi6;
2528 memset(&pi6, 0, sizeof(pi6));
f427ee49 2529 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr;
cb323159
A		 2530
2531 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 2532 }
0a7de745 2533 return NULL;
3e170ce0 2534}
39236c6e 2535
f427ee49 2536static int
3e170ce0
A		 2537flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2538{
f427ee49
A		 2539 int error = 0;
2540
39236c6e
A		 2541 FDLOCK(fd_cb);
2542 if (fd_cb->so != NULL) {
0a7de745
A		 2543 mbuf_t data = NULL;
2544 size_t data_size;
3e170ce0
A		 2545 struct sockaddr_storage remote_address;
2546 boolean_t got_remote_sa = FALSE;
f427ee49
A		 2547 boolean_t appended = FALSE;
2548 boolean_t append_success = FALSE;
3e170ce0 2549
39236c6e 2550 socket_lock(fd_cb->so, 0);
3e170ce0 2551
c3c9b80d
A		 2552 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2553 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data");
2554 goto done;
2555 }
2556
f427ee49
A		 2557 if (sbspace(&fd_cb->so->so_rcv) == 0) {
2558 error = ENOBUFS;
2559 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED;
2560 FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some data");
2561 goto done;
2562 }
2563
3e170ce0
A		 2564 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2565 uint32_t val_size = 0;
2566
2567 /* check if we got remote address with data */
2568 memset(&remote_address, 0, sizeof(remote_address));
f427ee49 2569 error = flow_divert_packet_get_tlv(packet, (int)offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
3e170ce0
A		 2570 if (error || val_size > sizeof(remote_address)) {
2571 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2572 error = 0;
39236c6e 2573 } else {
bca245ac
A		 2574 if (remote_address.ss_len > sizeof(remote_address)) {
2575 remote_address.ss_len = sizeof(remote_address);
2576 }
3e170ce0
A		 2577 /* validate the address */
2578 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2579 got_remote_sa = TRUE;
f427ee49
A		 2580 } else {
2581 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid");
3e170ce0
A		 2582 }
2583 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2584 }
2585 }
2586
2587 data_size = (mbuf_pkthdr_len(packet) - offset);
2588
f427ee49
A		 2589 if (fd_cb->so->so_state & SS_CANTRCVMORE) {
2590 FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size);
2591 goto done;
2592 }
2593
2594 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) {
2595 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so));
2596 goto done;
2597 }
2598
3e170ce0
A		 2599 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2600
2601 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2602 if (error || data == NULL) {
2603 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
f427ee49
A		 2604 goto done;
2605 }
2606
2607 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2608 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0);
2609 append_success = TRUE;
3e170ce0 2610 } else {
f427ee49
A		 2611 struct sockaddr *append_sa = NULL;
2612 mbuf_t mctl;
3e170ce0 2613
f427ee49
A		 2614 if (got_remote_sa == TRUE) {
2615 error = flow_divert_dup_addr(remote_address.ss_family, (struct sockaddr *)&remote_address, &append_sa);
2616 } else {
2617 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) {
2618 error = in6_mapped_peeraddr(fd_cb->so, &append_sa);
2619 } else {
2620 error = in_getpeeraddr(fd_cb->so, &append_sa);
3e170ce0 2621 }
39236c6e 2622 }
f427ee49
A		 2623 if (error) {
2624 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2625 }
2626
2627 mctl = flow_divert_create_control_mbuf(fd_cb);
2628 int append_error = 0;
2629 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error) || append_error == EJUSTRETURN) {
2630 append_success = TRUE;
2631 appended = (append_error == 0);
2632 } else {
2633 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error);
2634 }
2635
2636 if (append_sa != NULL) {
2637 FREE(append_sa, M_SONAME);
2638 }
39236c6e 2639 }
f427ee49
A		 2640
2641 if (append_success) {
2642 fd_cb->bytes_received += data_size;
2643 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2644 }
2645
2646 if (appended) {
2647 sorwakeup(fd_cb->so);
2648 }
2649done:
39236c6e 2650 socket_unlock(fd_cb->so, 0);
39236c6e 2651 }
3e170ce0 2652 FDUNLOCK(fd_cb);
f427ee49
A		 2653
2654 return error;
39236c6e
A		 2655}
2656
2657static void
2658flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2659{
0a7de745
A		 2660 uint32_t read_count;
2661 int error = 0;
39236c6e
A		 2662
2663 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2664 if (error) {
2665 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2666 return;
2667 }
2668
3e170ce0 2669 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2670
2671 FDLOCK(fd_cb);
2672 if (fd_cb->so != NULL) {
2673 socket_lock(fd_cb->so, 0);
c3c9b80d
A		 2674
2675 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2676 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification");
2677 goto done;
2678 }
2679
39236c6e
A		 2680 fd_cb->send_window += ntohl(read_count);
2681 flow_divert_send_buffered_data(fd_cb, FALSE);
c3c9b80d 2682done:
39236c6e
A		 2683 socket_unlock(fd_cb->so, 0);
2684 }
2685 FDUNLOCK(fd_cb);
2686}
2687
2688static void
2689flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2690{
2691 int error = 0;
3e170ce0 2692 uint32_t key_size = 0;
39236c6e 2693 int log_level;
5ba3f43e 2694 uint32_t flags = 0;
39236c6e
A		 2695
2696 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2697 if (error) {
2698 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2699 return;
2700 }
2701
2702 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
39037602 2703 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
39236c6e
A		 2704 return;
2705 }
2706
2707 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2708 if (!error) {
f427ee49 2709 nil_pcb.log_level = (uint8_t)log_level;
39236c6e
A		 2710 }
2711
2712 lck_rw_lock_exclusive(&group->lck);
2713
cb323159
A		 2714 if (group->token_key != NULL) {
2715 FREE(group->token_key, M_TEMP);
2716 group->token_key = NULL;
2717 }
2718
39236c6e
A		 2719 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2720 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2721 if (error) {
2722 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2723 FREE(group->token_key, M_TEMP);
2724 group->token_key = NULL;
2725 lck_rw_done(&group->lck);
2726 return;
2727 }
2728
2729 group->token_key_size = key_size;
2730
5ba3f43e
A		 2731 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2732 if (!error) {
2733 group->flags = flags;
2734 }
2735
39236c6e
A		 2736 lck_rw_done(&group->lck);
2737}
2738
2739static void
2740flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2741{
0a7de745 2742 int error = 0;
0a7de745 2743 int out_if_index = 0;
0a7de745 2744 uint32_t app_data_length = 0;
39236c6e
A		 2745
2746 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2747
39236c6e
A		 2748 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2749 if (error) {
490019cf
A		 2750 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2751 }
2752
2753 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2754 if (error) {
2755 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2756 }
2757
2758 FDLOCK(fd_cb);
2759 if (fd_cb->so != NULL) {
39236c6e
A		 2760 socket_lock(fd_cb->so, 0);
2761
c3c9b80d
A		 2762 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2763 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update");
2764 goto done;
2765 }
490019cf 2766
c3c9b80d
A		 2767 if (out_if_index > 0) {
2768 flow_divert_scope(fd_cb, out_if_index, true);
2769 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
39236c6e
A		 2770 }
2771
490019cf 2772 if (app_data_length > 0) {
0a7de745 2773 uint8_t *app_data = NULL;
490019cf
A		 2774 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2775 if (app_data != NULL) {
2776 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2777 if (error == 0) {
2778 if (fd_cb->app_data != NULL) {
2779 FREE(fd_cb->app_data, M_TEMP);
2780 }
2781 fd_cb->app_data = app_data;
2782 fd_cb->app_data_length = app_data_length;
2783 } else {
2784 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2785 FREE(app_data, M_TEMP);
2786 }
2787 } else {
2788 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2789 }
39236c6e 2790 }
c3c9b80d 2791done:
39236c6e
A		 2792 socket_unlock(fd_cb->so, 0);
2793 }
2794 FDUNLOCK(fd_cb);
2795}
2796
2797static void
39037602 2798flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
39236c6e
A		 2799{
2800 size_t bytes_mem_size;
2801 size_t child_maps_mem_size;
f427ee49
A		 2802 size_t nodes_mem_size;
2803 size_t trie_memory_size = 0;
39236c6e
A		 2804 int cursor;
2805 int error = 0;
2806 struct flow_divert_trie new_trie;
2807 int insert_error = 0;
c6bf4f31 2808 int prefix_count = -1;
39236c6e 2809 int signing_id_count = 0;
f427ee49
A		 2810 size_t bytes_count = 0;
2811 size_t nodes_count = 0;
2812 size_t maps_count = 0;
39236c6e 2813
39037602 2814 lck_rw_lock_exclusive(&group->lck);
39236c6e
A		 2815
2816 /* Re-set the current trie */
39037602
A		 2817 if (group->signing_id_trie.memory != NULL) {
2818 FREE(group->signing_id_trie.memory, M_TEMP);
39236c6e 2819 }
39037602
A		 2820 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2821 group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 2822
2823 memset(&new_trie, 0, sizeof(new_trie));
2824
2825 /* Get the number of shared prefixes in the new set of signing ID strings */
c6bf4f31 2826 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
39236c6e 2827
c6bf4f31
A		 2828 if (prefix_count < 0 || error) {
2829 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
39037602 2830 lck_rw_done(&group->lck);
ecc0ceb4
A		 2831 return;
2832 }
2833
39236c6e
A		 2834 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2835 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2836 cursor >= 0;
2837 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2838 uint32_t sid_size = 0;
c6bf4f31
A		 2839 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2840 if (error || sid_size == 0) {
2841 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
2842 signing_id_count = 0;
2843 break;
2844 }
f427ee49
A		 2845 if (os_add_overflow(bytes_count, sid_size, &bytes_count)) {
2846 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while incrementing number of bytes");
2847 signing_id_count = 0;
2848 break;
2849 }
39236c6e
A		 2850 signing_id_count++;
2851 }
2852
2853 if (signing_id_count == 0) {
39037602 2854 lck_rw_done(&group->lck);
f427ee49 2855 FDLOG0(LOG_NOTICE, &nil_pcb, "No signing identifiers");
39236c6e
A		 2856 return;
2857 }
2858
f427ee49
A		 2859 if (os_add3_overflow(prefix_count, signing_id_count, 1, &nodes_count)) { /* + 1 for the root node */
2860 lck_rw_done(&group->lck);
2861 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of nodes");
2862 return;
2863 }
2864
2865 if (os_add_overflow(prefix_count, 1, &maps_count)) { /* + 1 for the root node */
2866 lck_rw_done(&group->lck);
2867 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of maps");
2868 return;
2869 }
2870
2871 if (bytes_count > UINT16_MAX || nodes_count > UINT16_MAX || maps_count > UINT16_MAX) {
2872 lck_rw_done(&group->lck);
2873 FDLOG(LOG_NOTICE, &nil_pcb, "Invalid bytes count (%lu), nodes count (%lu) or maps count (%lu)", bytes_count, nodes_count, maps_count);
2874 return;
2875 }
39236c6e
A		 2876
2877 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
f427ee49 2878 nodes_count, maps_count, bytes_count);
39236c6e 2879
f427ee49
A		 2880 if (os_mul_overflow(sizeof(*new_trie.nodes), (size_t)nodes_count, &nodes_mem_size) ||
2881 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, (size_t)maps_count, &child_maps_mem_size) ||
2882 os_mul_overflow(sizeof(*new_trie.bytes), (size_t)bytes_count, &bytes_mem_size) ||
4ba76501
A		 2883 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2884 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2885 lck_rw_done(&group->lck);
2886 return;
2887 }
39236c6e 2888
ecc0ceb4 2889 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
39037602
A		 2890 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2891 lck_rw_done(&group->lck);
ecc0ceb4
A		 2892 return;
2893 }
2894
2895 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2896 if (new_trie.memory == NULL) {
2897 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
0a7de745 2898 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
39037602 2899 lck_rw_done(&group->lck);
39236c6e
A		 2900 return;
2901 }
2902
f427ee49
A		 2903 new_trie.bytes_count = (uint16_t)bytes_count;
2904 new_trie.nodes_count = (uint16_t)nodes_count;
2905 new_trie.child_maps_count = (uint16_t)maps_count;
2906
39236c6e
A		 2907 /* Initialize the free lists */
2908 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2909 new_trie.nodes_free_next = 0;
2910 memset(new_trie.nodes, 0, nodes_mem_size);
2911
2912 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2913 new_trie.child_maps_free_next = 0;
2914 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2915
2916 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2917 new_trie.bytes_free_next = 0;
c6bf4f31 2918 memset(new_trie.bytes, 0, bytes_mem_size);
39236c6e
A		 2919
2920 /* The root is an empty node */
2921 new_trie.root = trie_node_alloc(&new_trie);
2922
2923 /* Add each signing ID to the trie */
2924 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2925 cursor >= 0;
2926 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2927 uint32_t sid_size = 0;
c6bf4f31
A		 2928 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2929 if (error || sid_size == 0) {
2930 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
2931 insert_error = EINVAL;
2932 break;
2933 }
f427ee49 2934 if (sid_size <= UINT16_MAX && new_trie.bytes_free_next + (uint16_t)sid_size <= new_trie.bytes_count) {
39236c6e 2935 uint16_t new_node_idx;
c6bf4f31
A		 2936 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2937 if (error) {
2938 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
2939 insert_error = EINVAL;
2940 break;
2941 }
39236c6e 2942 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
39037602 2943 if (new_node_idx == NULL_TRIE_IDX) {
39236c6e
A		 2944 insert_error = EINVAL;
2945 break;
2946 }
2947 } else {
2948 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2949 insert_error = ENOBUFS;
2950 break;
2951 }
2952 }
2953
2954 if (!insert_error) {
39037602 2955 group->signing_id_trie = new_trie;
39236c6e
A		 2956 } else {
2957 FREE(new_trie.memory, M_TEMP);
2958 }
2959
39236c6e 2960 lck_rw_done(&group->lck);
39236c6e
A		 2961}
2962
2963static int
2964flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2965{
0a7de745
A		 2966 struct flow_divert_packet_header hdr;
2967 int error = 0;
2968 struct flow_divert_pcb *fd_cb;
39236c6e
A		 2969
2970 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2971 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2972 error = EINVAL;
2973 goto done;
2974 }
2975
3e170ce0 2976 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
39037602 2977 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
3e170ce0
A		 2978 error = EINVAL;
2979 goto done;
2980 }
2981
39236c6e
A		 2982 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2983 if (error) {
2984 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2985 error = ENOBUFS;
2986 goto done;
2987 }
2988
2989 hdr.conn_id = ntohl(hdr.conn_id);
2990
2991 if (hdr.conn_id == 0) {
2992 switch (hdr.packet_type) {
0a7de745
A		 2993 case FLOW_DIVERT_PKT_GROUP_INIT:
2994 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2995 break;
2996 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2997 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2998 break;
2999 default:
3000 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
3001 break;
39236c6e
A		 3002 }
3003 goto done;
3004 }
3005
0a7de745 3006 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
39236c6e
A		 3007 if (fd_cb == NULL) {
3008 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
3009 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
3010 }
3011 goto done;
3012 }
3013
3014 switch (hdr.packet_type) {
0a7de745
A		 3015 case FLOW_DIVERT_PKT_CONNECT_RESULT:
3016 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
3017 break;
3018 case FLOW_DIVERT_PKT_CLOSE:
3019 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
3020 break;
3021 case FLOW_DIVERT_PKT_DATA:
f427ee49 3022 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
0a7de745
A		 3023 break;
3024 case FLOW_DIVERT_PKT_READ_NOTIFY:
3025 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
3026 break;
3027 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
3028 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
3029 break;
3030 default:
3031 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
3032 break;
39236c6e
A		 3033 }
3034
3035 FDRELEASE(fd_cb);
3036
3037done:
3e170ce0 3038 mbuf_freem(packet);
39236c6e
A		 3039 return error;
3040}
3041
3042static void
3043flow_divert_close_all(struct flow_divert_group *group)
3044{
0a7de745
A		 3045 struct flow_divert_pcb *fd_cb;
3046 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 3047
3048 SLIST_INIT(&tmp_list);
3049
3050 lck_rw_lock_exclusive(&group->lck);
3051
3052 MBUFQ_DRAIN(&group->send_queue);
3053
3054 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3055 FDRETAIN(fd_cb);
3056 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3057 }
3058
3059 lck_rw_done(&group->lck);
3060
3061 while (!SLIST_EMPTY(&tmp_list)) {
3062 fd_cb = SLIST_FIRST(&tmp_list);
3063 FDLOCK(fd_cb);
3064 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
3065 if (fd_cb->so != NULL) {
3066 socket_lock(fd_cb->so, 0);
3067 flow_divert_pcb_remove(fd_cb);
3068 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
3069 fd_cb->so->so_error = ECONNABORTED;
39037602 3070 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 3071 socket_unlock(fd_cb->so, 0);
3072 }
3073 FDUNLOCK(fd_cb);
3074 FDRELEASE(fd_cb);
3075 }
3076}
3077
3078void
3079flow_divert_detach(struct socket *so)
3080{
0a7de745 3081 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3082
3083 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3084
3085 so->so_flags &= ~SOF_FLOW_DIVERT;
3086 so->so_fd_pcb = NULL;
3087
3088 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
3089
3090 if (fd_cb->group != NULL) {
3091 /* Last-ditch effort to send any buffered data */
3092 flow_divert_send_buffered_data(fd_cb, TRUE);
3093
3e170ce0
A		 3094 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3095 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 3096 /* Remove from the group */
3097 flow_divert_pcb_remove(fd_cb);
3098 }
3099
3100 socket_unlock(so, 0);
3101 FDLOCK(fd_cb);
3102 fd_cb->so = NULL;
3103 FDUNLOCK(fd_cb);
3104 socket_lock(so, 0);
3105
0a7de745 3106 FDRELEASE(fd_cb); /* Release the socket's reference */
39236c6e
A		 3107}
3108
3109static int
3110flow_divert_close(struct socket *so)
3111{
0a7de745 3112 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3113
3114 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3115
3116 FDLOG0(LOG_INFO, fd_cb, "Closing");
3117
3e170ce0
A		 3118 if (SOCK_TYPE(so) == SOCK_STREAM) {
3119 soisdisconnecting(so);
3120 sbflush(&so->so_rcv);
3121 }
39236c6e
A		 3122
3123 flow_divert_send_buffered_data(fd_cb, TRUE);
3124 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3125 flow_divert_send_close_if_needed(fd_cb);
3126
3127 /* Remove from the group */
3128 flow_divert_pcb_remove(fd_cb);
3129
3130 return 0;
3131}
3132
3133static int
3e170ce0
A		 3134flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
3135 sae_connid_t cid __unused)
39236c6e 3136{
3e170ce0 3137 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
0a7de745 3138 return EINVAL;
39236c6e
A		 3139 }
3140
0a7de745 3141 return flow_divert_close(so);
39236c6e
A		 3142}
3143
3144static int
3145flow_divert_shutdown(struct socket *so)
3146{
0a7de745 3147 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3148
3149 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3150
3151 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
3152
3153 socantsendmore(so);
3154
3155 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
3156 flow_divert_send_close_if_needed(fd_cb);
3157
3158 return 0;
3159}
3160
3161static int
3162flow_divert_rcvd(struct socket *so, int flags __unused)
3163{
f427ee49
A		 3164 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3165 int space = sbspace(&so->so_rcv);
39236c6e
A		 3166
3167 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3168
f427ee49
A		 3169 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space);
3170 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) &&
3171 (space > 0) &&
3172 flow_divert_send_read_notification(fd_cb) == 0) {
3173 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification");
3174 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED;
39236c6e
A		 3175 }
3176
3177 return 0;
3178}
3179
3e170ce0
A		 3180static int
3181flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
3182{
3183 int error = 0;
3184 int port = 0;
3185
cb323159
A		 3186 if (!flow_divert_is_sockaddr_valid(toaddr)) {
3187 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
3188 error = EINVAL;
3189 goto done;
3190 }
3191
3e170ce0
A		 3192 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
3193 if (error) {
3194 goto done;
3195 }
3196
3197 if (toaddr->sa_family == AF_INET) {
3198 port = ntohs((satosin(toaddr))->sin_port);
f427ee49 3199 } else {
3e170ce0
A		 3200 port = ntohs((satosin6(toaddr))->sin6_port);
3201 }
3e170ce0
A		 3202
3203 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
3204 if (error) {
3205 goto done;
3206 }
3207
3208done:
3209 return error;
3210}
3211
3212struct sockaddr *
3213flow_divert_get_buffered_target_address(mbuf_t buffer)
3214{
3215 if (buffer != NULL && buffer->m_type == MT_SONAME) {
3216 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
3217 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
3218 return toaddr;
3219 }
3220 }
3221 return NULL;
3222}
3223
3224static boolean_t
3225flow_divert_is_sockaddr_valid(struct sockaddr *addr)
3226{
0a7de745
A		 3227 switch (addr->sa_family) {
3228 case AF_INET:
cb323159 3229 if (addr->sa_len < sizeof(struct sockaddr_in)) {
0a7de745
A		 3230 return FALSE;
3231 }
3232 break;
0a7de745 3233 case AF_INET6:
cb323159 3234 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3e170ce0 3235 return FALSE;
0a7de745
A		 3236 }
3237 break;
0a7de745
A		 3238 default:
3239 return FALSE;
3e170ce0
A		 3240 }
3241 return TRUE;
3242}
3243
39236c6e
A		 3244static errno_t
3245flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
0a7de745 3246 struct sockaddr **dup)
39236c6e 3247{
0a7de745
A		 3248 int error = 0;
3249 struct sockaddr *result;
3250 struct sockaddr_storage ss;
39236c6e
A		 3251
3252 if (addr != NULL) {
3253 result = addr;
3254 } else {
3255 memset(&ss, 0, sizeof(ss));
3256 ss.ss_family = family;
3257 if (ss.ss_family == AF_INET) {
3258 ss.ss_len = sizeof(struct sockaddr_in);
f427ee49 3259 } else if (ss.ss_family == AF_INET6) {
39236c6e 3260 ss.ss_len = sizeof(struct sockaddr_in6);
f427ee49 3261 } else {
39236c6e
A		 3262 error = EINVAL;
3263 }
3264 result = (struct sockaddr *)&ss;
3265 }
3266
3267 if (!error) {
3268 *dup = dup_sockaddr(result, 1);
3269 if (*dup == NULL) {
3270 error = ENOBUFS;
3271 }
3272 }
3273
3274 return error;
3275}
3276
3e170ce0
A		 3277static void
3278flow_divert_disconnect_socket(struct socket *so)
3279{
3280 soisdisconnected(so);
3281 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3282 struct inpcb *inp = NULL;
3283
3284 inp = sotoinpcb(so);
3285 if (inp != NULL) {
0a7de745 3286 if (SOCK_CHECK_DOM(so, PF_INET6)) {
3e170ce0 3287 in6_pcbdetach(inp);
f427ee49
A		 3288 } else {
3289 in_pcbdetach(inp);
3290 }
3e170ce0
A		 3291 }
3292 }
3293}
3294
39236c6e
A		 3295static errno_t
3296flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
3297{
0a7de745 3298 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3299
3300 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3301
3302 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
3303 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3304 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
3305 }
3306 }
3307
3308 if (SOCK_DOM(so) == PF_INET) {
3309 return g_tcp_protosw->pr_ctloutput(so, sopt);
f427ee49 3310 } else if (SOCK_DOM(so) == PF_INET6) {
39236c6e
A		 3311 return g_tcp6_protosw->pr_ctloutput(so, sopt);
3312 }
39236c6e
A		 3313 return 0;
3314}
3315
f427ee49
A		 3316static errno_t
3317flow_divert_connect_out_internal(struct socket *so, struct sockaddr *to, proc_t p, bool implicit)
39236c6e 3318{
0a7de745
A		 3319 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3320 int error = 0;
3321 struct inpcb *inp = sotoinpcb(so);
3322 struct sockaddr_in *sinp;
3323 mbuf_t connect_packet = NULL;
3324 int do_send = 1;
39236c6e
A		 3325
3326 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3327
3328 if (fd_cb->group == NULL) {
3329 error = ENETUNREACH;
3330 goto done;
3331 }
3332
3333 if (inp == NULL) {
3334 error = EINVAL;
3335 goto done;
3336 } else if (inp->inp_state == INPCB_STATE_DEAD) {
3337 if (so->so_error) {
3338 error = so->so_error;
3339 so->so_error = 0;
3340 } else {
3341 error = EINVAL;
3342 }
3343 goto done;
3344 }
3345
f427ee49 3346 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
39236c6e
A		 3347 error = EALREADY;
3348 goto done;
3349 }
3350
490019cf 3351 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 3352
490019cf 3353 if (fd_cb->connect_packet == NULL) {
f427ee49
A		 3354 struct sockaddr_in sin = {};
3355 struct ifnet *ifp = NULL;
3356
490019cf
A		 3357 if (to == NULL) {
3358 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
3359 error = EINVAL;
3360 goto done;
fe8ab488 3361 }
fe8ab488 3362
f427ee49
A		 3363 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0);
3364 if (fd_cb->original_remote_endpoint == NULL) {
3365 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint");
3366 error = ENOMEM;
3367 goto done;
3368 }
3369 fd_cb->original_vflag = inp->inp_vflag;
3370 fd_cb->original_last_outifp = inp->inp_last_outifp;
3371 fd_cb->original_last_outifp6 = inp->in6p_last_outifp;
3372
490019cf
A		 3373 sinp = (struct sockaddr_in *)(void *)to;
3374 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
3375 error = EAFNOSUPPORT;
3376 goto done;
fe8ab488
A		 3377 }
3378
f427ee49
A		 3379 if (to->sa_family == AF_INET6 && !(inp->inp_flags & IN6P_IPV6_V6ONLY)) {
3380 struct sockaddr_in6 sin6 = {};
3381 sin6.sin6_family = AF_INET6;
3382 sin6.sin6_len = sizeof(struct sockaddr_in6);
3383 sin6.sin6_port = satosin6(to)->sin6_port;
3384 sin6.sin6_addr = satosin6(to)->sin6_addr;
3385 if (IN6_IS_ADDR_V4MAPPED(&(sin6.sin6_addr))) {
3386 in6_sin6_2_sin(&sin, &sin6);
3387 to = (struct sockaddr *)&sin;
3388 }
3389 }
3390
3391 if (to->sa_family == AF_INET6) {
3392 inp->inp_vflag &= ~INP_IPV4;
3393 inp->inp_vflag |= INP_IPV6;
3394 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6);
3395 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6;
3396 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport;
3397 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp);
3398 if (error) {
3399 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error);
3400 error = 0;
3401 }
3402 if (ifp != NULL) {
3403 inp->in6p_last_outifp = ifp;
3404 ifnet_release(ifp);
3405 }
3406 } else if (to->sa_family == AF_INET) {
3407 inp->inp_vflag |= INP_IPV4;
3408 inp->inp_vflag &= ~INP_IPV6;
3409 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in);
3410 fd_cb->local_endpoint.sin.sin_family = AF_INET;
3411 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport;
3412 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0);
3413 if (error) {
3414 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error);
3415 error = 0;
3416 }
3417 if (ifp != NULL) {
3418 inp->inp_last_outifp = ifp;
3419 ifnet_release(ifp);
3420 }
3421 } else {
3422 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family);
3423 }
3424
3425 error = flow_divert_check_no_cellular(fd_cb) ||
3426 flow_divert_check_no_expensive(fd_cb) ||
3427 flow_divert_check_no_constrained(fd_cb);
3428 if (error) {
3429 goto done;
3430 }
3431
c3c9b80d
A		 3432 if (SOCK_TYPE(so) == SOCK_STREAM || /* TCP or */
3433 !implicit || /* connect() was called or */
3434 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) || /* local address is not un-specified */
3435 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
3436 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR;
3437 }
3438
490019cf
A		 3439 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
3440 if (error) {
3441 goto done;
fe8ab488
A		 3442 }
3443
f427ee49
A		 3444 if (!implicit || SOCK_TYPE(so) == SOCK_STREAM) {
3445 flow_divert_set_remote_endpoint(fd_cb, to);
c3c9b80d 3446 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
f427ee49
A		 3447 }
3448
3449 if (implicit) {
3450 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT;
3451 }
3452
490019cf
A		 3453 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
3454 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
3455 do_send = 0;
fe8ab488 3456 }
f427ee49
A		 3457
3458 fd_cb->connect_packet = connect_packet;
3459 connect_packet = NULL;
fe8ab488 3460 } else {
490019cf 3461 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
fe8ab488
A		 3462 }
3463
490019cf 3464 if (do_send) {
f427ee49 3465 error = flow_divert_send_connect_packet(fd_cb);
490019cf
A		 3466 if (error) {
3467 goto done;
3468 }
39236c6e 3469
490019cf 3470 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
39236c6e
A		 3471 }
3472
c3c9b80d 3473 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) {
f427ee49
A		 3474 soisconnected(so);
3475 } else {
3476 soisconnecting(so);
3477 }
39236c6e
A		 3478
3479done:
3480 return error;
3481}
3482
f427ee49
A		 3483errno_t
3484flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
3485{
a991bd8d
A		 3486#if CONTENT_FILTER
3487 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_flags & SOF_CONTENT_FILTER)) {
3488 int error = cfil_sock_attach(so, NULL, to, CFS_CONNECTION_DIR_OUT);
3489 if (error != 0) {
3490 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3491 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error);
3492 return error;
3493 }
3494 }
3495#endif /* CONTENT_FILTER */
3496
f427ee49
A		 3497 return flow_divert_connect_out_internal(so, to, p, false);
3498}
3499
39236c6e 3500static int
813fb2f6
A		 3501flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3502 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
39236c6e 3503{
39236c6e
A		 3504 struct inpcb *inp = sotoinpcb(so);
3505 int error;
3506
3507 if (inp == NULL) {
0a7de745 3508 return EINVAL;
39236c6e
A		 3509 }
3510
813fb2f6 3511 VERIFY(dst != NULL);
39236c6e 3512
813fb2f6 3513 error = flow_divert_connect_out(so, dst, p);
39236c6e 3514
490019cf
A		 3515 if (error != 0) {
3516 return error;
3517 }
3518
3519 /* if there is data, send it */
3520 if (auio != NULL) {
3521 user_ssize_t datalen = 0;
3522
3523 socket_unlock(so, 0);
3524
3525 VERIFY(bytes_written != NULL);
3526
3527 datalen = uio_resid(auio);
3528 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3529 socket_lock(so, 0);
3530
3531 if (error == 0 || error == EWOULDBLOCK) {
3532 *bytes_written = datalen - uio_resid(auio);
3533 }
3534
3535 /*
3536 * sosend returns EWOULDBLOCK if it's a non-blocking
3537 * socket or a timeout occured (this allows to return
3538 * the amount of queued data through sendit()).
3539 *
3540 * However, connectx() returns EINPROGRESS in case of a
3541 * blocking socket. So we change the return value here.
3542 */
3543 if (error == EWOULDBLOCK) {
3544 error = EINPROGRESS;
3545 }
3546 }
3547
39236c6e 3548 if (error == 0 && pcid != NULL) {
0a7de745 3549 *pcid = 1; /* there is only 1 connection for a TCP */
39236c6e
A		 3550 }
3551
0a7de745 3552 return error;
39236c6e
A		 3553}
3554
3555static int
813fb2f6
A		 3556flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3557 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3558 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3559 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 3560{
0a7de745 3561 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e
A		 3562}
3563
39236c6e 3564static int
813fb2f6
A		 3565flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3566 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3567 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3568 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3569{
0a7de745 3570 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e 3571}
39236c6e
A		 3572
3573static errno_t
490019cf 3574flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e 3575{
0a7de745
A		 3576 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3577 int error = 0;
39236c6e 3578 struct inpcb *inp;
bca245ac
A		 3579#if CONTENT_FILTER
3580 struct m_tag *cfil_tag = NULL;
3581#endif
39236c6e
A		 3582
3583 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3584
3585 inp = sotoinpcb(so);
3586 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3587 error = ECONNRESET;
3588 goto done;
3589 }
3590
3591 if (control && mbuf_len(control) > 0) {
3592 error = EINVAL;
3593 goto done;
3594 }
3595
3596 if (flags & MSG_OOB) {
3597 error = EINVAL;
3598 goto done; /* We don't support OOB data */
3599 }
0a7de745 3600
f427ee49
A		 3601#if CONTENT_FILTER
3602 /*
3603 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3604 * retrieve the CFIL saved remote address from the mbuf and use it.
3605 */
3606 if (to == NULL && so->so_cfil_db) {
3607 struct sockaddr *cfil_faddr = NULL;
3608 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3609 if (cfil_tag) {
3610 to = (struct sockaddr *)(void *)cfil_faddr;
3611 }
3612 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
39236c6e 3613 }
f427ee49 3614#endif
39236c6e
A		 3615
3616 /* Implicit connect */
3617 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3618 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
cb323159 3619
f427ee49 3620 error = flow_divert_connect_out_internal(so, to, p, true);
39236c6e
A		 3621 if (error) {
3622 goto done;
3623 }
f427ee49
A		 3624 } else {
3625 error = flow_divert_check_no_cellular(fd_cb) ||
3626 flow_divert_check_no_expensive(fd_cb) ||
3627 flow_divert_check_no_constrained(fd_cb);
3628 if (error) {
3629 goto done;
490019cf 3630 }
39236c6e
A		 3631 }
3632
3633 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3634
3635 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3636 error = flow_divert_send_app_data(fd_cb, data, to);
f427ee49
A		 3637
3638 data = NULL;
3639
39236c6e
A		 3640 if (error) {
3641 goto done;
3642 }
3643
39236c6e
A		 3644 if (flags & PRUS_EOF) {
3645 flow_divert_shutdown(so);
3646 }
3647
3648done:
3649 if (data) {
3e170ce0 3650 mbuf_freem(data);
39236c6e
A		 3651 }
3652 if (control) {
3653 mbuf_free(control);
3654 }
bca245ac
A		 3655#if CONTENT_FILTER
3656 if (cfil_tag) {
3657 m_tag_free(cfil_tag);
3658 }
3659#endif
3660
39236c6e
A		 3661 return error;
3662}
3663
490019cf
A		 3664static int
3665flow_divert_preconnect(struct socket *so)
3666{
490019cf 3667 int error = 0;
f427ee49 3668 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
490019cf 3669
f427ee49 3670 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
490019cf 3671
f427ee49
A		 3672 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3673 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3674 error = flow_divert_send_connect_packet(so->so_fd_pcb);
490019cf 3675 if (error) {
f427ee49 3676 return error;
490019cf
A		 3677 }
3678
3679 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3680 }
3681
5ba3f43e 3682 soclearfastopen(so);
490019cf
A		 3683
3684 return error;
3685}
3686
39236c6e
A		 3687static void
3688flow_divert_set_protosw(struct socket *so)
3689{
39236c6e
A		 3690 if (SOCK_DOM(so) == PF_INET) {
3691 so->so_proto = &g_flow_divert_in_protosw;
f427ee49 3692 } else {
39236c6e
A		 3693 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3694 }
39236c6e
A		 3695}
3696
3e170ce0
A		 3697static void
3698flow_divert_set_udp_protosw(struct socket *so)
3699{
0a7de745
A		 3700 if (SOCK_DOM(so) == PF_INET) {
3701 so->so_proto = &g_flow_divert_in_udp_protosw;
f427ee49 3702 } else {
0a7de745
A		 3703 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3704 }
39236c6e
A		 3705}
3706
3e170ce0
A		 3707errno_t
3708flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3709{
0a7de745 3710 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3e170ce0 3711 struct inpcb *inp;
0a7de745 3712 int error = 0;
3e170ce0
A		 3713
3714 inp = sotoinpcb(so);
3715 if (inp == NULL) {
0a7de745
A		 3716 return EINVAL;
3717 }
3718
3719 if (fd_cb == NULL) {
f427ee49
A		 3720 error = flow_divert_pcb_init(so);
3721 fd_cb = so->so_fd_pcb;
3722 if (error != 0 || fd_cb == NULL) {
0a7de745
A		 3723 goto done;
3724 }
3725 }
3726 return flow_divert_data_out(so, flags, data, to, control, p);
3e170ce0
A		 3727
3728done:
0a7de745
A		 3729 if (data) {
3730 mbuf_freem(data);
3731 }
3732 if (control) {
3733 mbuf_free(control);
3734 }
3e170ce0 3735
0a7de745 3736 return error;
3e170ce0
A		 3737}
3738
f427ee49
A		 3739static errno_t
3740flow_divert_pcb_init_internal(struct socket *so, uint32_t ctl_unit, uint32_t aggregate_unit)
39236c6e
A		 3741{
3742 errno_t error = 0;
3743 struct flow_divert_pcb *fd_cb;
f427ee49
A		 3744 uint32_t agg_unit = aggregate_unit;
3745 uint32_t group_unit = flow_divert_derive_kernel_control_unit(ctl_unit, &agg_unit);
3746
3747 if (group_unit == 0) {
3748 return EINVAL;
3749 }
39236c6e
A		 3750
3751 if (so->so_flags & SOF_FLOW_DIVERT) {
3752 return EALREADY;
3753 }
0a7de745 3754
39236c6e
A		 3755 fd_cb = flow_divert_pcb_create(so);
3756 if (fd_cb != NULL) {
f427ee49
A		 3757 so->so_fd_pcb = fd_cb;
3758 so->so_flags |= SOF_FLOW_DIVERT;
3759 fd_cb->control_group_unit = group_unit;
3760 fd_cb->policy_control_unit = ctl_unit;
3761 fd_cb->aggregate_unit = agg_unit;
3762
3763 error = flow_divert_pcb_insert(fd_cb, group_unit);
39236c6e
A		 3764 if (error) {
3765 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
f427ee49
A		 3766 so->so_fd_pcb = NULL;
3767 so->so_flags &= ~SOF_FLOW_DIVERT;
39236c6e
A		 3768 FDRELEASE(fd_cb);
3769 } else {
3e170ce0
A		 3770 if (SOCK_TYPE(so) == SOCK_STREAM) {
3771 flow_divert_set_protosw(so);
3772 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3773 flow_divert_set_udp_protosw(so);
3774 }
39236c6e
A		 3775
3776 FDLOG0(LOG_INFO, fd_cb, "Created");
3777 }
3778 } else {
3779 error = ENOMEM;
3780 }
3781
3782 return error;
3783}
3784
f427ee49
A		 3785errno_t
3786flow_divert_pcb_init(struct socket *so)
3787{
3788 struct inpcb *inp = sotoinpcb(so);
3789 uint32_t aggregate_units = 0;
3790 uint32_t ctl_unit = necp_socket_get_flow_divert_control_unit(inp, &aggregate_units);
3791 return flow_divert_pcb_init_internal(so, ctl_unit, aggregate_units);
3792}
3793
39236c6e
A		 3794errno_t
3795flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3796{
f427ee49
A		 3797 uint32_t ctl_unit = 0;
3798 uint32_t key_unit = 0;
3799 uint32_t aggregate_unit = 0;
3800 int error = 0;
3801 int hmac_error = 0;
3802 mbuf_t token = NULL;
39236c6e
A		 3803
3804 if (so->so_flags & SOF_FLOW_DIVERT) {
3805 error = EALREADY;
3806 goto done;
3807 }
3808
3809 if (g_init_result) {
3810 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3811 error = ENOPROTOOPT;
3812 goto done;
3813 }
3814
3e170ce0
A		 3815 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3816 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
f427ee49 3817 (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6)) {
39236c6e
A		 3818 error = EINVAL;
3819 goto done;
3820 } else {
3e170ce0
A		 3821 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3822 struct tcpcb *tp = sototcpcb(so);
3823 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3824 error = EINVAL;
3825 goto done;
3826 }
39236c6e
A		 3827 }
3828 }
3829
3830 error = soopt_getm(sopt, &token);
3831 if (error) {
5c9f4661 3832 token = NULL;
39236c6e
A		 3833 goto done;
3834 }
3835
3836 error = soopt_mcopyin(sopt, token);
3837 if (error) {
5c9f4661 3838 token = NULL;
39236c6e
A		 3839 goto done;
3840 }
3841
3842 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3843 if (!error) {
3844 key_unit = ntohl(key_unit);
4d15aeb1
A		 3845 if (key_unit >= GROUP_COUNT_MAX) {
3846 key_unit = 0;
3847 }
39236c6e
A		 3848 } else if (error != ENOENT) {
3849 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3850 goto done;
3851 } else {
3852 key_unit = 0;
3853 }
3854
3855 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3856 if (error) {
3857 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3858 goto done;
3859 }
3860
f427ee49
A		 3861 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_AGGREGATE_UNIT, sizeof(aggregate_unit), (void *)&aggregate_unit, NULL);
3862 if (error && error != ENOENT) {
3863 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the aggregate unit from the token: %d", error);
39236c6e
A		 3864 goto done;
3865 }
3866
f427ee49
A		 3867 /* A valid kernel control unit is required */
3868 ctl_unit = ntohl(ctl_unit);
3869 aggregate_unit = ntohl(aggregate_unit);
39236c6e 3870
f427ee49
A		 3871 if (ctl_unit > 0 && ctl_unit < GROUP_COUNT_MAX) {
3872 socket_unlock(so, 0);
3873 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3874 socket_lock(so, 0);
39236c6e 3875
f427ee49
A		 3876 if (hmac_error && hmac_error != ENOENT) {
3877 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3878 error = hmac_error;
3879 goto done;
3880 }
39236c6e
A		 3881 }
3882
f427ee49
A		 3883 error = flow_divert_pcb_init_internal(so, ctl_unit, aggregate_unit);
3884 if (error == 0) {
3885 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3886 int log_level = LOG_NOTICE;
39236c6e 3887
f427ee49
A		 3888 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
3889 if (error == 0) {
3890 fd_cb->log_level = (uint8_t)log_level;
39236c6e 3891 }
f427ee49
A		 3892 error = 0;
3893
3894 fd_cb->connect_token = token;
3895 token = NULL;
c3c9b80d
A		 3896
3897 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN;
39236c6e
A		 3898 }
3899
490019cf
A		 3900 if (hmac_error == 0) {
3901 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3902 if (fd_cb != NULL) {
3903 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3904 }
3905 }
3906
39236c6e
A		 3907done:
3908 if (token != NULL) {
3909 mbuf_freem(token);
3910 }
3911
3912 return error;
3913}
3914
3915errno_t
3916flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3917{
0a7de745
A		 3918 uint32_t ctl_unit;
3919 int error = 0;
3920 uint8_t hmac[SHA_DIGEST_LENGTH];
3921 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3922 mbuf_t token = NULL;
3923 struct flow_divert_group *control_group = NULL;
39236c6e
A		 3924
3925 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3926 error = EINVAL;
3927 goto done;
3928 }
3929
3930 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3931
3932 if (fd_cb->group == NULL) {
3933 error = EINVAL;
3934 goto done;
3935 }
3936
3937 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3938 if (error) {
3939 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3940 goto done;
3941 }
3942
3943 ctl_unit = htonl(fd_cb->group->ctl_unit);
3944
3945 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3946 if (error) {
3947 goto done;
3948 }
3949
3950 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3951 if (error) {
3952 goto done;
3953 }
3954
490019cf 3955 if (fd_cb->app_data != NULL) {
f427ee49 3956 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_data);
490019cf
A		 3957 if (error) {
3958 goto done;
3959 }
3960 }
3961
39236c6e
A		 3962 socket_unlock(so, 0);
3963 lck_rw_lock_shared(&g_flow_divert_group_lck);
3964
3965 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
0a7de745 3966 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
39236c6e
A		 3967 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3968 }
3969
3970 if (control_group != NULL) {
3971 lck_rw_lock_shared(&control_group->lck);
3972 ctl_unit = htonl(control_group->ctl_unit);
3973 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3974 if (!error) {
3975 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3976 }
3977 lck_rw_done(&control_group->lck);
3978 } else {
3979 error = ENOPROTOOPT;
3980 }
3981
3982 lck_rw_done(&g_flow_divert_group_lck);
3983 socket_lock(so, 0);
3984
3985 if (error) {
3986 goto done;
3987 }
3988
3989 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3990 if (error) {
3991 goto done;
3992 }
3993
39037602
A		 3994 if (sopt->sopt_val == USER_ADDR_NULL) {
3995 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3996 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3997 goto done;
3998 }
3999
39236c6e
A		 4000 error = soopt_mcopyout(sopt, token);
4001 if (error) {
0a7de745 4002 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
39236c6e
A		 4003 goto done;
4004 }
4005
4006done:
4007 if (token != NULL) {
4008 mbuf_freem(token);
4009 }
4010
4011 return error;
4012}
4013
4014static errno_t
4015flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
4016{
0a7de745
A		 4017 struct flow_divert_group *new_group = NULL;
4018 int error = 0;
39236c6e
A		 4019
4020 if (sac->sc_unit >= GROUP_COUNT_MAX) {
4021 error = EINVAL;
4022 goto done;
4023 }
4024
4025 *unitinfo = NULL;
4026
f427ee49 4027 new_group = zalloc_flags(flow_divert_group_zone, Z_WAITOK | Z_ZERO);
39236c6e
A		 4028 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4029 RB_INIT(&new_group->pcb_tree);
4030 new_group->ctl_unit = sac->sc_unit;
4031 MBUFQ_INIT(&new_group->send_queue);
39037602 4032 new_group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 4033
4034 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4035
4036 if (g_flow_divert_groups == NULL) {
4037 MALLOC(g_flow_divert_groups,
0a7de745
A		 4038 struct flow_divert_group **,
4039 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
4040 M_TEMP,
4041 M_WAITOK | M_ZERO);
39236c6e
A		 4042 }
4043
4044 if (g_flow_divert_groups == NULL) {
4045 error = ENOBUFS;
4046 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
4047 error = EALREADY;
4048 } else {
4049 g_flow_divert_groups[sac->sc_unit] = new_group;
4050 g_active_group_count++;
4051 }
4052
4053 lck_rw_done(&g_flow_divert_group_lck);
4054
39236c6e 4055done:
f427ee49
A		 4056 if (error == 0) {
4057 *unitinfo = new_group;
4058 } else if (new_group != NULL) {
4059 zfree(flow_divert_group_zone, new_group);
39236c6e
A		 4060 }
4061 return error;
4062}
4063
4064static errno_t
4065flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
4066{
0a7de745
A		 4067 struct flow_divert_group *group = NULL;
4068 errno_t error = 0;
39236c6e
A		 4069
4070 if (unit >= GROUP_COUNT_MAX) {
4071 return EINVAL;
4072 }
4073
f427ee49
A		 4074 if (unitinfo == NULL) {
4075 return 0;
4076 }
4077
39236c6e
A		 4078 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
4079
4080 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4081
4082 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
4083 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
0a7de745 4084 g_flow_divert_groups, g_active_group_count);
39236c6e
A		 4085 }
4086
4087 group = g_flow_divert_groups[unit];
4088
4089 if (group != (struct flow_divert_group *)unitinfo) {
4090 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
4091 }
4092
cb323159
A		 4093 g_flow_divert_groups[unit] = NULL;
4094 g_active_group_count--;
4095
4096 if (g_active_group_count == 0) {
4097 FREE(g_flow_divert_groups, M_TEMP);
4098 g_flow_divert_groups = NULL;
4099 }
4100
4101 lck_rw_done(&g_flow_divert_group_lck);
4102
39236c6e
A		 4103 if (group != NULL) {
4104 flow_divert_close_all(group);
cb323159
A		 4105
4106 lck_rw_lock_exclusive(&group->lck);
4107
39236c6e
A		 4108 if (group->token_key != NULL) {
4109 memset(group->token_key, 0, group->token_key_size);
4110 FREE(group->token_key, M_TEMP);
4111 group->token_key = NULL;
4112 group->token_key_size = 0;
4113 }
39037602
A		 4114
4115 /* Re-set the current trie */
4116 if (group->signing_id_trie.memory != NULL) {
4117 FREE(group->signing_id_trie.memory, M_TEMP);
4118 }
4119 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
4120 group->signing_id_trie.root = NULL_TRIE_IDX;
4121
cb323159
A		 4122 lck_rw_done(&group->lck);
4123
f427ee49 4124 zfree(flow_divert_group_zone, group);
39236c6e
A		 4125 } else {
4126 error = EINVAL;
4127 }
4128
39236c6e
A		 4129 return error;
4130}
4131
4132static errno_t
4133flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
4134{
4135 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
4136}
4137
4138static void
4139flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
4140{
0a7de745 4141 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
39236c6e
A		 4142
4143 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
0a7de745
A		 4144 struct flow_divert_pcb *fd_cb;
4145 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 4146
4147 lck_rw_lock_shared(&g_flow_divert_group_lck);
4148 lck_rw_lock_exclusive(&group->lck);
4149
4150 while (!MBUFQ_EMPTY(&group->send_queue)) {
4151 mbuf_t next_packet;
4152 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
4153 next_packet = MBUFQ_FIRST(&group->send_queue);
4154 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
4155 if (error) {
4156 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
4157 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
4158 lck_rw_done(&group->lck);
4159 lck_rw_done(&g_flow_divert_group_lck);
4160 return;
4161 }
4162 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
4163 }
4164
4165 SLIST_INIT(&tmp_list);
4166
4167 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
4168 FDRETAIN(fd_cb);
4169 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
4170 }
4171
4172 lck_rw_done(&group->lck);
4173
4174 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
4175 FDLOCK(fd_cb);
4176 if (fd_cb->so != NULL) {
4177 socket_lock(fd_cb->so, 0);
4178 if (fd_cb->group != NULL) {
4179 flow_divert_send_buffered_data(fd_cb, FALSE);
4180 }
4181 socket_unlock(fd_cb->so, 0);
4182 }
4183 FDUNLOCK(fd_cb);
4184 FDRELEASE(fd_cb);
4185 }
4186
4187 lck_rw_done(&g_flow_divert_group_lck);
4188 }
4189}
4190
4191static int
4192flow_divert_kctl_init(void)
4193{
0a7de745
A		 4194 struct kern_ctl_reg ctl_reg;
4195 int result;
39236c6e
A		 4196
4197 memset(&ctl_reg, 0, sizeof(ctl_reg));
4198
fe8ab488 4199 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
0a7de745 4200 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
39236c6e
A		 4201 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
4202 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
4203 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
4204
4205 ctl_reg.ctl_connect = flow_divert_kctl_connect;
4206 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
4207 ctl_reg.ctl_send = flow_divert_kctl_send;
4208 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
4209
4210 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
4211
4212 if (result) {
4213 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
4214 return result;
4215 }
4216
4217 return 0;
4218}
4219
4220void
4221flow_divert_init(void)
4222{
4223 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 4224 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 4225
4226 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4227
4228 VERIFY(g_tcp_protosw != NULL);
4229
4230 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4231 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4232
4233 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4234 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
39236c6e
A		 4235 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4236 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
39236c6e
A		 4237 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4238 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4239 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
490019cf 4240 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 4241
4242 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4243 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4244
4245 /*
4246 * Socket filters shouldn't attach/detach to/from this protosw
4247 * since pr_protosw is to be used instead, which points to the
4248 * real protocol; if they do, it is a bug and we should panic.
4249 */
4250 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4251 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4252 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4253 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4254
3e170ce0
A		 4255 /* UDP */
4256 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4257 VERIFY(g_udp_protosw != NULL);
4258
4259 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4260 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4261
4262 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4263 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3e170ce0
A		 4264 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4265 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3e170ce0
A		 4266 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4267 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4268 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3e170ce0
A		 4269 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4270 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 4271 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 4272
4273 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4274 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4275
4276 /*
0a7de745
A		 4277 * Socket filters shouldn't attach/detach to/from this protosw
4278 * since pr_protosw is to be used instead, which points to the
4279 * real protocol; if they do, it is a bug and we should panic.
4280 */
3e170ce0
A		 4281 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4282 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4283 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4284 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4285
39236c6e
A		 4286 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4287
4288 VERIFY(g_tcp6_protosw != NULL);
4289
4290 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4291 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4292
4293 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4294 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
39236c6e
A		 4295 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4296 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
39236c6e
A		 4297 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4298 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4299 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
490019cf 4300 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 4301
4302 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4303 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4304 /*
4305 * Socket filters shouldn't attach/detach to/from this protosw
4306 * since pr_protosw is to be used instead, which points to the
4307 * real protocol; if they do, it is a bug and we should panic.
4308 */
4309 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4310 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4311 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4312 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 4313
4314 /* UDP6 */
4315 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4316
4317 VERIFY(g_udp6_protosw != NULL);
4318
4319 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4320 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4321
4322 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4323 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3e170ce0
A		 4324 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4325 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3e170ce0
A		 4326 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4327 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4328 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3e170ce0
A		 4329 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4330 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 4331 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 4332
4333 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4334 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4335 /*
0a7de745
A		 4336 * Socket filters shouldn't attach/detach to/from this protosw
4337 * since pr_protosw is to be used instead, which points to the
4338 * real protocol; if they do, it is a bug and we should panic.
4339 */
3e170ce0
A		 4340 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4341 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4342 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4343 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
39236c6e
A		 4344
4345 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4346 if (flow_divert_grp_attr == NULL) {
4347 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4348 g_init_result = ENOMEM;
4349 goto done;
4350 }
4351
4352 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4353 if (flow_divert_mtx_grp == NULL) {
4354 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4355 g_init_result = ENOMEM;
4356 goto done;
4357 }
4358
4359 flow_divert_mtx_attr = lck_attr_alloc_init();
4360 if (flow_divert_mtx_attr == NULL) {
4361 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4362 g_init_result = ENOMEM;
4363 goto done;
4364 }
4365
4366 g_init_result = flow_divert_kctl_init();
4367 if (g_init_result) {
4368 goto done;
4369 }
4370
4371 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4372
39236c6e
A		 4373done:
4374 if (g_init_result != 0) {
4375 if (flow_divert_mtx_attr != NULL) {
4376 lck_attr_free(flow_divert_mtx_attr);
4377 flow_divert_mtx_attr = NULL;
4378 }
4379 if (flow_divert_mtx_grp != NULL) {
4380 lck_grp_free(flow_divert_mtx_grp);
4381 flow_divert_mtx_grp = NULL;
4382 }
4383 if (flow_divert_grp_attr != NULL) {
4384 lck_grp_attr_free(flow_divert_grp_attr);
4385 flow_divert_grp_attr = NULL;
4386 }
4387
4388 if (g_flow_divert_kctl_ref != NULL) {
4389 ctl_deregister(g_flow_divert_kctl_ref);
4390 g_flow_divert_kctl_ref = NULL;
4391 }
4392 }
4393}
mirror of XNU