]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-7195.81.3.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
f427ee49 2 * Copyright (c) 2012-2017, 2020 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
39236c6e
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
39236c6e
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
39236c6e
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
39236c6e
A		 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
cb323159
A		 48#include <kern/task.h>
49#include <mach/task_info.h>
39236c6e
A		 50#include <net/if_var.h>
51#include <net/route.h>
52#include <net/flowhash.h>
53#include <net/ntstat.h>
ea3f0419 54#include <net/content_filter.h>
f427ee49 55#include <net/necp.h>
39236c6e
A		 56#include <netinet/in.h>
57#include <netinet/in_var.h>
58#include <netinet/tcp.h>
59#include <netinet/tcp_var.h>
60#include <netinet/tcp_fsm.h>
61#include <netinet/flow_divert.h>
62#include <netinet/flow_divert_proto.h>
3e170ce0 63#include <netinet6/in6_pcb.h>
39236c6e 64#include <netinet6/ip6protosw.h>
39236c6e
A		 65#include <dev/random/randomdev.h>
66#include <libkern/crypto/sha1.h>
67#include <libkern/crypto/crypto_internal.h>
39037602 68#include <os/log.h>
cb323159
A		 69#include <corecrypto/cc.h>
70#if CONTENT_FILTER
71#include <net/content_filter.h>
72#endif /* CONTENT_FILTER */
39236c6e 73
0a7de745
A		 74#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75#define FLOW_DIVERT_READ_CLOSED 0x00000002
76#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
490019cf 79#define FLOW_DIVERT_HAS_HMAC 0x00000040
f427ee49
A		 80#define FLOW_DIVERT_NOTIFY_ON_RECEIVED 0x00000080
81#define FLOW_DIVERT_IMPLICIT_CONNECT 0x00000100
82#define FLOW_DIVERT_DID_SET_LOCAL_ADDR 0x00000200
39236c6e 83
39037602
A		 84#define FDLOG(level, pcb, format, ...) \
85 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
39236c6e 86
39037602
A		 87#define FDLOG0(level, pcb, msg) \
88 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
39236c6e 89
0a7de745
A		 90#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
91#define FDRELEASE(pcb) \
92 do { \
93 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
94 flow_divert_pcb_destroy(pcb); \
95 } \
39236c6e
A		 96 } while (0)
97
0a7de745
A		 98#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
99#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
39236c6e 100
0a7de745
A		 101#define FD_CTL_SENDBUFF_SIZE (128 * 1024)
102#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
39236c6e 103
0a7de745 104#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
39236c6e 105
f427ee49 106#define GROUP_COUNT_MAX 31
0a7de745
A		 107#define FLOW_DIVERT_MAX_NAME_SIZE 4096
108#define FLOW_DIVERT_MAX_KEY_SIZE 1024
109#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e 110
0a7de745 111struct flow_divert_trie_node {
39236c6e
A		 112 uint16_t start;
113 uint16_t length;
114 uint16_t child_map;
39236c6e
A		 115};
116
0a7de745
A		 117#define CHILD_MAP_SIZE 256
118#define NULL_TRIE_IDX 0xffff
119#define TRIE_NODE(t, i) ((t)->nodes[(i)])
120#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
121#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
39236c6e 122
0a7de745 123static struct flow_divert_pcb nil_pcb;
39236c6e
A		 124
125decl_lck_rw_data(static, g_flow_divert_group_lck);
0a7de745
A		 126static struct flow_divert_group **g_flow_divert_groups = NULL;
127static uint32_t g_active_group_count = 0;
39236c6e 128
0a7de745
A		 129static lck_grp_attr_t *flow_divert_grp_attr = NULL;
130static lck_attr_t *flow_divert_mtx_attr = NULL;
131static lck_grp_t *flow_divert_mtx_grp = NULL;
132static errno_t g_init_result = 0;
39236c6e 133
0a7de745 134static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
39236c6e 135
0a7de745
A		 136static struct protosw g_flow_divert_in_protosw;
137static struct pr_usrreqs g_flow_divert_in_usrreqs;
138static struct protosw g_flow_divert_in_udp_protosw;
139static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
0a7de745
A		 140static struct ip6protosw g_flow_divert_in6_protosw;
141static struct pr_usrreqs g_flow_divert_in6_usrreqs;
142static struct ip6protosw g_flow_divert_in6_udp_protosw;
143static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
39236c6e 144
0a7de745
A		 145static struct protosw *g_tcp_protosw = NULL;
146static struct ip6protosw *g_tcp6_protosw = NULL;
147static struct protosw *g_udp_protosw = NULL;
148static struct ip6protosw *g_udp6_protosw = NULL;
3e170ce0 149
f427ee49
A		 150ZONE_DECLARE(flow_divert_group_zone, "flow_divert_group",
151 sizeof(struct flow_divert_group), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
152ZONE_DECLARE(flow_divert_pcb_zone, "flow_divert_pcb",
153 sizeof(struct flow_divert_pcb), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
3e170ce0
A		 154
155static errno_t
f427ee49 156flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
3e170ce0
A		 157
158static boolean_t
159flow_divert_is_sockaddr_valid(struct sockaddr *addr);
160
161static int
162flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
163
164struct sockaddr *
165flow_divert_get_buffered_target_address(mbuf_t buffer);
166
3e170ce0
A		 167static void
168flow_divert_disconnect_socket(struct socket *so);
39236c6e 169
39037602
A		 170static inline uint8_t
171flow_divert_syslog_type_to_oslog_type(int syslog_type)
172{
173 switch (syslog_type) {
0a7de745
A		 174 case LOG_ERR: return OS_LOG_TYPE_ERROR;
175 case LOG_INFO: return OS_LOG_TYPE_INFO;
176 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
177 default: return OS_LOG_TYPE_DEFAULT;
39037602
A		 178 }
179}
180
39236c6e
A		 181static inline int
182flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
183{
184 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
185}
186
187RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
188RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
189
190static const char *
191flow_divert_packet_type2str(uint8_t packet_type)
192{
193 switch (packet_type) {
0a7de745
A		 194 case FLOW_DIVERT_PKT_CONNECT:
195 return "connect";
196 case FLOW_DIVERT_PKT_CONNECT_RESULT:
197 return "connect result";
198 case FLOW_DIVERT_PKT_DATA:
199 return "data";
200 case FLOW_DIVERT_PKT_CLOSE:
201 return "close";
202 case FLOW_DIVERT_PKT_READ_NOTIFY:
203 return "read notification";
204 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
205 return "properties update";
206 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
207 return "app map create";
208 default:
209 return "unknown";
39236c6e
A		 210 }
211}
212
213static struct flow_divert_pcb *
214flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
215{
0a7de745
A		 216 struct flow_divert_pcb key_item;
217 struct flow_divert_pcb *fd_cb = NULL;
39236c6e
A		 218
219 key_item.hash = hash;
220
221 lck_rw_lock_shared(&group->lck);
222 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
223 FDRETAIN(fd_cb);
224 lck_rw_done(&group->lck);
225
226 return fd_cb;
227}
228
229static errno_t
230flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
231{
0a7de745
A		 232 errno_t error = 0;
233 struct flow_divert_pcb *exist = NULL;
234 struct flow_divert_group *group;
235 static uint32_t g_nextkey = 1;
236 static uint32_t g_hash_seed = 0;
237 int try_count = 0;
39236c6e
A		 238
239 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
240 return EINVAL;
241 }
242
243 socket_unlock(fd_cb->so, 0);
244 lck_rw_lock_shared(&g_flow_divert_group_lck);
245
246 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
247 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
248 error = ENETUNREACH;
249 goto done;
250 }
251
252 group = g_flow_divert_groups[ctl_unit];
253 if (group == NULL) {
254 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
255 error = ENETUNREACH;
256 goto done;
257 }
258
259 socket_lock(fd_cb->so, 0);
260
261 do {
0a7de745
A		 262 uint32_t key[2];
263 uint32_t idx;
39236c6e
A		 264
265 key[0] = g_nextkey++;
266 key[1] = RandomULong();
267
268 if (g_hash_seed == 0) {
269 g_hash_seed = RandomULong();
270 }
271
272 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
273
274 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
275 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
276 if (curr_group != NULL && curr_group != group) {
277 lck_rw_lock_shared(&curr_group->lck);
278 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
279 lck_rw_done(&curr_group->lck);
280 if (exist != NULL) {
281 break;
282 }
283 }
284 }
285
286 if (exist == NULL) {
287 lck_rw_lock_exclusive(&group->lck);
288 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
289 lck_rw_done(&group->lck);
290 }
291 } while (exist != NULL && try_count++ < 3);
292
293 if (exist == NULL) {
294 fd_cb->group = group;
0a7de745 295 FDRETAIN(fd_cb); /* The group now has a reference */
39236c6e
A		 296 } else {
297 fd_cb->hash = 0;
3e170ce0 298 error = EEXIST;
39236c6e
A		 299 }
300
301 socket_unlock(fd_cb->so, 0);
302
303done:
304 lck_rw_done(&g_flow_divert_group_lck);
305 socket_lock(fd_cb->so, 0);
306
3e170ce0 307 return error;
39236c6e
A		 308}
309
310static struct flow_divert_pcb *
311flow_divert_pcb_create(socket_t so)
312{
f427ee49 313 struct flow_divert_pcb *new_pcb = NULL;
39236c6e 314
f427ee49 315 new_pcb = zalloc_flags(flow_divert_pcb_zone, Z_WAITOK | Z_ZERO);
39236c6e
A		 316 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
317 new_pcb->so = so;
318 new_pcb->log_level = nil_pcb.log_level;
319
0a7de745 320 FDRETAIN(new_pcb); /* Represents the socket's reference */
39236c6e
A		 321
322 return new_pcb;
323}
324
325static void
326flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
327{
f427ee49
A		 328 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u",
329 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
39236c6e 330
39236c6e
A		 331 if (fd_cb->connect_token != NULL) {
332 mbuf_freem(fd_cb->connect_token);
333 }
490019cf
A		 334 if (fd_cb->connect_packet != NULL) {
335 mbuf_freem(fd_cb->connect_packet);
336 }
337 if (fd_cb->app_data != NULL) {
338 FREE(fd_cb->app_data, M_TEMP);
339 }
f427ee49
A		 340 if (fd_cb->original_remote_endpoint != NULL) {
341 FREE(fd_cb->original_remote_endpoint, M_SONAME);
342 }
343 zfree(flow_divert_pcb_zone, fd_cb);
39236c6e
A		 344}
345
346static void
347flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
348{
349 if (fd_cb->group != NULL) {
350 struct flow_divert_group *group = fd_cb->group;
351 lck_rw_lock_exclusive(&group->lck);
352 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
353 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
354 fd_cb->group = NULL;
0a7de745 355 FDRELEASE(fd_cb); /* Release the group's reference */
39236c6e
A		 356 lck_rw_done(&group->lck);
357 }
358}
359
360static int
361flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
362{
0a7de745
A		 363 struct flow_divert_packet_header hdr;
364 int error = 0;
39236c6e
A		 365
366 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
367 if (error) {
368 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
369 return error;
370 }
371
372 hdr.packet_type = packet_type;
373 hdr.conn_id = htonl(fd_cb->hash);
374
375 /* Lay down the header */
376 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
377 if (error) {
378 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
379 mbuf_freem(*packet);
380 *packet = NULL;
381 return error;
382 }
383
384 return 0;
385}
386
387static int
3e170ce0 388flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 389{
0a7de745
A		 390 uint32_t net_length = htonl(length);
391 int error = 0;
39236c6e
A		 392
393 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
394 if (error) {
395 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
396 return error;
397 }
398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
400 if (error) {
39037602 401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
39236c6e
A		 402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
406 if (error) {
407 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
408 return error;
409 }
410
411 return error;
412}
413
414static int
415flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
416{
0a7de745
A		 417 size_t cursor = offset;
418 int error = 0;
419 uint32_t curr_length;
420 uint8_t curr_type;
39236c6e
A		 421
422 *err = 0;
423
424 do {
425 if (!next) {
426 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
427 if (error) {
428 *err = ENOENT;
429 return -1;
430 }
431 } else {
432 next = 0;
433 curr_type = FLOW_DIVERT_TLV_NIL;
434 }
435
436 if (curr_type != type) {
437 cursor += sizeof(curr_type);
438 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
439 if (error) {
440 *err = error;
441 return -1;
442 }
443
444 cursor += (sizeof(curr_length) + ntohl(curr_length));
445 }
446 } while (curr_type != type);
447
f427ee49 448 return (int)cursor;
39236c6e
A		 449}
450
451static int
3e170ce0 452flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 453{
0a7de745
A		 454 int error = 0;
455 uint32_t length;
456 int tlv_offset;
39236c6e
A		 457
458 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
459 if (tlv_offset < 0) {
460 return error;
461 }
462
463 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
464 if (error) {
465 return error;
466 }
467
468 length = ntohl(length);
469
cb323159
A		 470 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
471
472 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
473 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
474 return EINVAL;
475 }
476
39236c6e
A		 477 if (val_size != NULL) {
478 *val_size = length;
479 }
480
481 if (buff != NULL && buff_len > 0) {
cb323159 482 memset(buff, 0, buff_len);
39236c6e 483 size_t to_copy = (length < buff_len) ? length : buff_len;
cb323159 484 error = mbuf_copydata(packet, data_offset, to_copy, buff);
39236c6e
A		 485 if (error) {
486 return error;
487 }
488 }
489
490 return 0;
491}
492
493static int
494flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
495{
0a7de745 496 mbuf_t curr_mbuf = packet;
39236c6e
A		 497
498 if (g_crypto_funcs == NULL || group->token_key == NULL) {
499 return ENOPROTOOPT;
500 }
501
502 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
503 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
504
505 while (curr_mbuf != NULL) {
506 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
507 curr_mbuf = mbuf_next(curr_mbuf);
508 }
509
510 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
511
512 return 0;
513}
514
515static int
516flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
517{
0a7de745
A		 518 int error = 0;
519 struct flow_divert_group *group = NULL;
520 int hmac_offset;
521 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
522 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
523 mbuf_t tail;
39236c6e
A		 524
525 lck_rw_lock_shared(&g_flow_divert_group_lck);
526
527 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
528 group = g_flow_divert_groups[ctl_unit];
529 }
530
531 if (group == NULL) {
532 lck_rw_done(&g_flow_divert_group_lck);
533 return ENOPROTOOPT;
534 }
535
536 lck_rw_lock_shared(&group->lck);
537
538 if (group->token_key == NULL) {
539 error = ENOPROTOOPT;
540 goto done;
541 }
542
543 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
544 if (hmac_offset < 0) {
545 goto done;
546 }
547
548 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
549 if (error) {
550 goto done;
551 }
552
553 /* Chop off the HMAC TLV */
554 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
555 if (error) {
556 goto done;
557 }
558
559 mbuf_free(tail);
560
561 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
562 if (error) {
563 goto done;
564 }
565
cb323159 566 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
39236c6e
A		 567 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
568 error = EINVAL;
569 goto done;
570 }
571
572done:
573 lck_rw_done(&group->lck);
574 lck_rw_done(&g_flow_divert_group_lck);
575 return error;
576}
577
578static void
f427ee49 579flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send)
39236c6e
A		 580{
581 struct inpcb *inp = NULL;
582 struct ifnet *ifp = NULL;
583 Boolean cell = FALSE;
584 Boolean wifi = FALSE;
fe8ab488 585 Boolean wired = FALSE;
0a7de745 586
39236c6e
A		 587 inp = sotoinpcb(fd_cb->so);
588 if (inp == NULL) {
589 return;
590 }
591
f427ee49
A		 592 if (inp->inp_vflag & INP_IPV4) {
593 ifp = inp->inp_last_outifp;
594 } else if (inp->inp_vflag & INP_IPV6) {
595 ifp = inp->in6p_last_outifp;
596 }
39236c6e
A		 597 if (ifp != NULL) {
598 cell = IFNET_IS_CELLULAR(ifp);
599 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 600 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e 601 }
0a7de745 602
39236c6e 603 if (send) {
fe8ab488
A		 604 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
605 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 606 } else {
fe8ab488
A		 607 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
608 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e 609 }
5ba3f43e 610 inp_set_activity_bitmap(inp);
39236c6e
A		 611}
612
613static errno_t
614flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
615{
f427ee49
A		 616 struct inpcb *inp = sotoinpcb(fd_cb->so);
617 if (INP_NO_CELLULAR(inp)) {
618 struct ifnet *ifp = NULL;
619 if (inp->inp_vflag & INP_IPV4) {
620 ifp = inp->inp_last_outifp;
621 } else if (inp->inp_vflag & INP_IPV6) {
622 ifp = inp->in6p_last_outifp;
623 }
624 if (ifp != NULL && IFNET_IS_CELLULAR(ifp)) {
625 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied");
626 return EHOSTUNREACH;
627 }
0a7de745 628 }
fe8ab488
A		 629 return 0;
630}
631
632static errno_t
633flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
634{
f427ee49
A		 635 struct inpcb *inp = sotoinpcb(fd_cb->so);
636 if (INP_NO_EXPENSIVE(inp)) {
637 struct ifnet *ifp = NULL;
638 if (inp->inp_vflag & INP_IPV4) {
639 ifp = inp->inp_last_outifp;
640 } else if (inp->inp_vflag & INP_IPV6) {
641 ifp = inp->in6p_last_outifp;
642 }
643 if (ifp != NULL && IFNET_IS_EXPENSIVE(ifp)) {
644 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied");
645 return EHOSTUNREACH;
646 }
0a7de745 647 }
39236c6e
A		 648 return 0;
649}
650
cb323159
A		 651static errno_t
652flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
653{
f427ee49
A		 654 struct inpcb *inp = sotoinpcb(fd_cb->so);
655 if (INP_NO_CONSTRAINED(inp)) {
656 struct ifnet *ifp = NULL;
657 if (inp->inp_vflag & INP_IPV4) {
658 ifp = inp->inp_last_outifp;
659 } else if (inp->inp_vflag & INP_IPV6) {
660 ifp = inp->in6p_last_outifp;
661 }
662 if (ifp != NULL && IFNET_IS_CONSTRAINED(ifp)) {
663 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied");
664 return EHOSTUNREACH;
665 }
cb323159 666 }
cb323159
A		 667 return 0;
668}
669
39236c6e
A		 670static void
671flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
672{
673 if (how != SHUT_RD) {
674 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
675 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
676 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
677 /* If the tunnel is not accepting writes any more, then flush the send buffer */
678 sbflush(&fd_cb->so->so_snd);
679 }
680 }
681 if (how != SHUT_WR) {
682 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
683 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
684 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
685 }
686 }
687}
688
689static uint16_t
690trie_node_alloc(struct flow_divert_trie *trie)
691{
692 if (trie->nodes_free_next < trie->nodes_count) {
693 uint16_t node_idx = trie->nodes_free_next++;
694 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
695 return node_idx;
696 } else {
697 return NULL_TRIE_IDX;
698 }
699}
700
701static uint16_t
702trie_child_map_alloc(struct flow_divert_trie *trie)
703{
704 if (trie->child_maps_free_next < trie->child_maps_count) {
705 return trie->child_maps_free_next++;
706 } else {
707 return NULL_TRIE_IDX;
708 }
709}
710
711static uint16_t
712trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
713{
714 uint16_t start = trie->bytes_free_next;
715 if (start + bytes_size <= trie->bytes_count) {
716 if (start != bytes_idx) {
717 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
718 }
719 trie->bytes_free_next += bytes_size;
720 return start;
721 } else {
722 return NULL_TRIE_IDX;
723 }
724}
725
726static uint16_t
727flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
728{
729 uint16_t current = trie->root;
730 uint16_t child = trie->root;
f427ee49 731 uint16_t string_end = string_start + (uint16_t)string_len;
39236c6e 732 uint16_t string_idx = string_start;
f427ee49 733 uint16_t string_remainder = (uint16_t)string_len;
39236c6e
A		 734
735 while (child != NULL_TRIE_IDX) {
736 uint16_t parent = current;
737 uint16_t node_idx;
738 uint16_t current_end;
739
740 current = child;
741 child = NULL_TRIE_IDX;
742
743 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
744
745 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 746 node_idx < current_end &&
747 string_idx < string_end &&
748 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
749 node_idx++, string_idx++) {
750 ;
751 }
39236c6e
A		 752
753 string_remainder = string_end - string_idx;
754
755 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
756 /*
757 * We did not reach the end of the current node's string.
758 * We need to split the current node into two:
759 * 1. A new node that contains the prefix of the node that matches
760 * the prefix of the string being inserted.
761 * 2. The current node modified to point to the remainder
762 * of the current node's string.
763 */
764 uint16_t prefix = trie_node_alloc(trie);
765 if (prefix == NULL_TRIE_IDX) {
766 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
767 return NULL_TRIE_IDX;
768 }
769
770 /*
771 * Prefix points to the portion of the current nodes's string that has matched
772 * the input string thus far.
773 */
774 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
775 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
776
777 /*
778 * Prefix has the current node as the child corresponding to the first byte
779 * after the split.
780 */
781 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
782 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
783 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
784 return NULL_TRIE_IDX;
785 }
786 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
787
788 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
789 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
790
791 /* Current node is adjusted to point to the remainder */
792 TRIE_NODE(trie, current).start = node_idx;
793 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
794
795 /* We want to insert the new leaf (if any) as a child of the prefix */
796 current = prefix;
797 }
798
799 if (string_remainder > 0) {
800 /*
801 * We still have bytes in the string that have not been matched yet.
802 * If the current node has children, iterate to the child corresponding
803 * to the next byte in the string.
804 */
805 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
806 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
807 }
808 }
809 } /* while (child != NULL_TRIE_IDX) */
810
811 if (string_remainder > 0) {
812 /* Add a new leaf containing the remainder of the string */
813 uint16_t leaf = trie_node_alloc(trie);
814 if (leaf == NULL_TRIE_IDX) {
815 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
816 return NULL_TRIE_IDX;
817 }
818
819 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
820 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
821 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
822 return NULL_TRIE_IDX;
823 }
824 TRIE_NODE(trie, leaf).length = string_remainder;
825
826 /* Set the new leaf as the child of the current node */
827 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
828 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
829 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
830 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
831 return NULL_TRIE_IDX;
832 }
833 }
834 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
835 current = leaf;
836 } /* else duplicate or this string is a prefix of one of the existing strings */
837
838 return current;
839}
840
0a7de745 841#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 842static uint16_t
ea3f0419 843flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
39236c6e
A		 844{
845 uint16_t current = trie->root;
846 uint16_t string_idx = 0;
847
848 while (current != NULL_TRIE_IDX) {
849 uint16_t next = NULL_TRIE_IDX;
850 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
851 uint16_t node_idx;
852
853 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 854 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
855 node_idx++, string_idx++) {
856 ;
857 }
39236c6e
A		 858
859 if (node_idx == node_end) {
860 if (string_bytes[string_idx] == '\0') {
861 return current; /* Got an exact match */
3e170ce0 862 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
0a7de745 863 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
3e170ce0 864 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 865 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
866 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
867 }
868 }
869 current = next;
870 }
871
872 return NULL_TRIE_IDX;
873}
874
490019cf
A		 875struct uuid_search_info {
876 uuid_t target_uuid;
877 char *found_signing_id;
878 boolean_t found_multiple_signing_ids;
879 proc_t found_proc;
880};
881
882static int
883flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
884{
885 struct uuid_search_info *info = (struct uuid_search_info *)arg;
886 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
887
888 if (info->found_signing_id != NULL) {
889 if (!info->found_multiple_signing_ids) {
890 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
891 info->found_proc = p;
892 result = PROC_CLAIMED_DONE;
893 } else {
894 uuid_string_t uuid_str;
895 uuid_unparse(info->target_uuid, uuid_str);
896 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
897 }
898 FREE(info->found_signing_id, M_TEMP);
899 info->found_signing_id = NULL;
900 }
901
902 if (result == PROC_RETURNED_DONE) {
903 uuid_string_t uuid_str;
904 uuid_unparse(info->target_uuid, uuid_str);
905 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
906 }
907
908 return result;
909}
910
911static int
912flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
913{
914 struct uuid_search_info *info = (struct uuid_search_info *)arg;
915 int include = 0;
916
917 if (info->found_multiple_signing_ids) {
918 return include;
919 }
920
921 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
922 if (include) {
923 const char *signing_id = cs_identity_get(p);
924 if (signing_id != NULL) {
925 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
926 size_t signing_id_size = strlen(signing_id) + 1;
927 if (info->found_signing_id == NULL) {
928 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
929 memcpy(info->found_signing_id, signing_id, signing_id_size);
930 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
931 info->found_multiple_signing_ids = TRUE;
932 }
933 } else {
934 info->found_multiple_signing_ids = TRUE;
935 }
936 include = !info->found_multiple_signing_ids;
937 }
938
939 return include;
940}
941
942static proc_t
943flow_divert_find_proc_by_uuid(uuid_t uuid)
944{
945 struct uuid_search_info info;
946
947 if (LOG_INFO <= nil_pcb.log_level) {
948 uuid_string_t uuid_str;
949 uuid_unparse(uuid, uuid_str);
950 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
951 }
952
953 memset(&info, 0, sizeof(info));
954 info.found_proc = PROC_NULL;
955 uuid_copy(info.target_uuid, uuid);
956
957 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
958
959 return info.found_proc;
960}
961
39236c6e 962static int
ea3f0419 963flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
39236c6e 964{
ea3f0419 965 int error = 0;
f427ee49 966 uint8_t *cdhash = NULL;
ea3f0419
A		 967 audit_token_t audit_token = {};
968 const char *proc_cs_id = signing_id;
39236c6e 969
ea3f0419
A		 970 proc_lock(proc);
971
972 if (proc_cs_id == NULL) {
973 if (proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
974 proc_cs_id = cs_identity_get(proc);
975 } else {
976 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
490019cf 977 }
39236c6e
A		 978 }
979
ea3f0419
A		 980 if (is_effective) {
981 lck_rw_lock_shared(&fd_cb->group->lck);
982 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
983 if (proc_cs_id != NULL) {
984 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
985 if (result == NULL_TRIE_IDX) {
986 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
987 error = EPERM;
988 } else {
989 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
990 }
991 } else {
992 error = EPERM;
39236c6e 993 }
ea3f0419
A		 994 }
995 lck_rw_done(&fd_cb->group->lck);
996 }
997
998 if (error != 0) {
999 goto done;
1000 }
1001
1002 /*
1003 * If signing_id is not NULL then it came from the flow divert token and will be added
1004 * as part of the token, so there is no need to add it here.
1005 */
1006 if (signing_id == NULL && proc_cs_id != NULL) {
1007 error = flow_divert_packet_append_tlv(connect_packet,
1008 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
f427ee49 1009 (uint32_t)strlen(proc_cs_id),
ea3f0419
A		 1010 proc_cs_id);
1011 if (error != 0) {
1012 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1013 goto done;
1014 }
1015 }
1016
f427ee49
A		 1017 cdhash = cs_get_cdhash(proc);
1018 if (cdhash != NULL) {
ea3f0419
A		 1019 error = flow_divert_packet_append_tlv(connect_packet,
1020 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
f427ee49 1021 SHA1_RESULTLEN,
ea3f0419
A		 1022 cdhash);
1023 if (error) {
1024 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1025 goto done;
1026 }
1027 } else {
f427ee49 1028 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash");
ea3f0419
A		 1029 }
1030
1031 task_t task = proc_task(proc);
1032 if (task != TASK_NULL) {
1033 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1034 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1035 if (rc == KERN_SUCCESS) {
1036 int append_error = flow_divert_packet_append_tlv(connect_packet,
1037 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1038 sizeof(audit_token_t),
1039 &audit_token);
1040 if (append_error) {
1041 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1042 }
1043 }
1044 }
1045
1046done:
1047 proc_unlock(proc);
1048
1049 return error;
1050}
1051
1052static int
1053flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1054{
1055 int error = 0;
1056 proc_t effective_proc = PROC_NULL;
1057 proc_t responsible_proc = PROC_NULL;
1058 proc_t real_proc = proc_find(so->last_pid);
1059 bool release_real_proc = true;
1060
1061 proc_t src_proc = PROC_NULL;
1062 proc_t real_src_proc = PROC_NULL;
1063
1064 if (real_proc == PROC_NULL) {
1065 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1066 release_real_proc = false;
1067 real_proc = proc;
1068 if (real_proc == PROC_NULL) {
1069 real_proc = current_proc();
1070 }
1071 }
1072
1073 if (so->so_flags & SOF_DELEGATED) {
1074 if (real_proc->p_pid != so->e_pid) {
1075 effective_proc = proc_find(so->e_pid);
1076 } else if (uuid_compare(real_proc->p_uuid, so->e_uuid)) {
1077 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
39236c6e
A		 1078 }
1079 }
1080
ea3f0419
A		 1081#if defined(XNU_TARGET_OS_OSX)
1082 lck_rw_lock_shared(&fd_cb->group->lck);
1083 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1084 if (so->so_rpid > 0) {
1085 responsible_proc = proc_find(so->so_rpid);
1086 }
1087 }
1088 lck_rw_done(&fd_cb->group->lck);
1089#endif
1090
1091 real_src_proc = real_proc;
1092
1093 if (responsible_proc != PROC_NULL) {
1094 src_proc = responsible_proc;
1095 if (effective_proc != NULL) {
1096 real_src_proc = effective_proc;
1097 }
1098 } else if (effective_proc != PROC_NULL) {
1099 src_proc = effective_proc;
1100 } else {
1101 src_proc = real_proc;
1102 }
1103
1104 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1105 if (error != 0) {
1106 goto done;
1107 }
1108
1109 if (real_src_proc != NULL && real_src_proc != src_proc) {
1110 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1111 if (error != 0) {
1112 goto done;
1113 }
1114 }
1115
1116done:
1117 if (responsible_proc != PROC_NULL) {
1118 proc_rele(responsible_proc);
1119 }
1120
1121 if (effective_proc != PROC_NULL) {
1122 proc_rele(effective_proc);
1123 }
1124
1125 if (real_proc != PROC_NULL && release_real_proc) {
1126 proc_rele(real_proc);
1127 }
1128
1129 return error;
39236c6e
A		 1130}
1131
1132static int
1133flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1134{
0a7de745 1135 int error;
39236c6e
A		 1136
1137 if (fd_cb->group == NULL) {
1138 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 1139 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1140 return ECONNABORTED;
1141 }
1142
1143 lck_rw_lock_shared(&fd_cb->group->lck);
1144
1145 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1146 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1147 } else {
1148 error = ENOBUFS;
1149 }
1150
1151 if (error == ENOBUFS) {
1152 if (enqueue) {
1153 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1154 lck_rw_lock_exclusive(&fd_cb->group->lck);
1155 }
1156 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1157 error = 0;
1158 }
1159 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1160 }
1161
1162 lck_rw_done(&fd_cb->group->lck);
1163
1164 return error;
1165}
1166
1167static int
490019cf 1168flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 1169{
ea3f0419
A		 1170 int error = 0;
1171 int flow_type = 0;
0a7de745 1172 char *signing_id = NULL;
0a7de745 1173 mbuf_t connect_packet = NULL;
ea3f0419
A		 1174 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1175 const void *cfil_id = NULL;
1176 size_t cfil_id_size = 0;
1177 struct inpcb *inp = sotoinpcb(so);
f427ee49 1178 struct ifnet *ifp = NULL;
2a1bd2d3 1179 uint32_t flags = 0;
490019cf
A		 1180
1181 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1182 if (error) {
1183 goto done;
1184 }
1185
490019cf
A		 1186 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1187 uint32_t sid_size = 0;
1188 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1189 if (find_error == 0 && sid_size > 0) {
1190 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1191 if (signing_id != NULL) {
1192 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1193 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
490019cf
A		 1194 }
1195 }
1196 }
1197
1198 socket_unlock(so, 0);
39037602 1199
ea3f0419 1200 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
490019cf 1201
490019cf
A		 1202 socket_lock(so, 0);
1203
ea3f0419 1204 if (signing_id != NULL) {
490019cf
A		 1205 FREE(signing_id, M_TEMP);
1206 }
1207
1208 if (error) {
ea3f0419 1209 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
490019cf
A		 1210 goto done;
1211 }
39236c6e 1212
39236c6e 1213 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1214 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1215 sizeof(fd_cb->so->so_traffic_class),
1216 &fd_cb->so->so_traffic_class);
39236c6e
A		 1217 if (error) {
1218 goto done;
1219 }
1220
3e170ce0
A		 1221 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1222 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1223 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1224 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1225 } else {
1226 error = EINVAL;
1227 goto done;
1228 }
1229 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1230 FLOW_DIVERT_TLV_FLOW_TYPE,
1231 sizeof(flow_type),
1232 &flow_type);
3e170ce0
A		 1233
1234 if (error) {
1235 goto done;
1236 }
1237
39236c6e
A		 1238 if (fd_cb->connect_token != NULL) {
1239 unsigned int token_len = m_length(fd_cb->connect_token);
1240 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1241 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1242 fd_cb->connect_token = NULL;
1243 } else {
3e170ce0 1244 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1245 if (error) {
1246 goto done;
1247 }
39236c6e
A		 1248 }
1249
f427ee49
A		 1250 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1251 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
490019cf
A		 1252 if (error) {
1253 goto done;
1254 }
1255 }
3e170ce0 1256
f427ee49 1257 if (inp->inp_vflag & INP_IPV4) {
ea3f0419 1258 ifp = inp->inp_last_outifp;
f427ee49
A		 1259 } else if (inp->inp_vflag & INP_IPV6) {
1260 ifp = inp->in6p_last_outifp;
ea3f0419 1261 }
ea3f0419
A		 1262 if (ifp != NULL) {
1263 uint32_t flow_if_index = ifp->if_index;
1264 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1265 sizeof(flow_if_index), &flow_if_index);
1266 if (error) {
1267 goto done;
1268 }
1269 }
1270
490019cf 1271 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
2a1bd2d3
A		 1272 flags |= FLOW_DIVERT_TOKEN_FLAG_TFO;
1273 }
1274
1275 if ((inp->inp_flags & INP_BOUND_IF) ||
1276 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) ||
1277 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
1278 flags |= FLOW_DIVERT_TOKEN_FLAG_BOUND;
1279 }
1280
1281 if (flags != 0) {
490019cf
A		 1282 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1283 if (error) {
1284 goto done;
1285 }
39236c6e
A		 1286 }
1287
f427ee49
A		 1288 if (SOCK_TYPE(so) == SOCK_DGRAM) {
1289 cfil_sock_id = cfil_sock_id_from_datagram_socket(so, NULL, to);
1290 } else {
1291 cfil_sock_id = cfil_sock_id_from_socket(so);
1292 }
1293
ea3f0419
A		 1294 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1295 cfil_id = &cfil_sock_id;
1296 cfil_id_size = sizeof(cfil_sock_id);
1297 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1298 cfil_id = &inp->necp_client_uuid;
1299 cfil_id_size = sizeof(inp->necp_client_uuid);
1300 }
1301
1302 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
f427ee49 1303 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, (uint32_t)cfil_id_size, cfil_id);
ea3f0419
A		 1304 if (error) {
1305 goto done;
1306 }
1307 }
1308
39236c6e 1309done:
490019cf
A		 1310 if (!error) {
1311 *out_connect_packet = connect_packet;
1312 } else if (connect_packet != NULL) {
1313 mbuf_freem(connect_packet);
1314 }
1315
39236c6e
A		 1316 return error;
1317}
1318
f427ee49
A		 1319static int
1320flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb)
1321{
1322 int error = 0;
1323 mbuf_t connect_packet = fd_cb->connect_packet;
1324 mbuf_t saved_connect_packet = NULL;
1325
1326 if (connect_packet != NULL) {
1327 error = mbuf_copym(connect_packet, 0, mbuf_pkthdr_len(connect_packet), MBUF_DONTWAIT, &saved_connect_packet);
1328 if (error) {
1329 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet");
1330 goto done;
1331 }
1332
1333 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
1334 if (error) {
1335 goto done;
1336 }
1337
1338 fd_cb->connect_packet = saved_connect_packet;
1339 saved_connect_packet = NULL;
1340 } else {
1341 error = ENOENT;
1342 }
1343done:
1344 if (saved_connect_packet != NULL) {
1345 mbuf_freem(saved_connect_packet);
1346 }
1347
1348 return error;
1349}
1350
39236c6e
A		 1351static int
1352flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1353{
0a7de745
A		 1354 int error = 0;
1355 mbuf_t packet = NULL;
1356 int rbuff_space = 0;
39236c6e
A		 1357
1358 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1359 if (error) {
1360 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1361 goto done;
1362 }
1363
3e170ce0 1364 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1365 if (rbuff_space < 0) {
1366 rbuff_space = 0;
1367 }
1368 rbuff_space = htonl(rbuff_space);
1369 error = flow_divert_packet_append_tlv(packet,
0a7de745
A		 1370 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1371 sizeof(rbuff_space),
1372 &rbuff_space);
39236c6e
A		 1373 if (error) {
1374 goto done;
1375 }
1376
1377 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1378 if (error) {
1379 goto done;
1380 }
1381
1382done:
1383 if (error && packet != NULL) {
3e170ce0 1384 mbuf_freem(packet);
39236c6e
A		 1385 }
1386
1387 return error;
1388}
1389
1390static int
1391flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1392{
0a7de745
A		 1393 int error = 0;
1394 mbuf_t packet = NULL;
1395 uint32_t zero = 0;
39236c6e
A		 1396
1397 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1398 if (error) {
1399 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1400 goto done;
1401 }
1402
1403 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1404 if (error) {
1405 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1406 goto done;
1407 }
1408
1409 how = htonl(how);
1410 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1411 if (error) {
1412 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1413 goto done;
1414 }
1415
1416 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1417 if (error) {
1418 goto done;
1419 }
1420
1421done:
1422 if (error && packet != NULL) {
1423 mbuf_free(packet);
1424 }
1425
1426 return error;
1427}
1428
1429static int
1430flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1431{
0a7de745
A		 1432 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1433 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
39236c6e
A		 1434 return SHUT_RDWR;
1435 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1436 return SHUT_RD;
1437 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1438 return SHUT_WR;
1439 }
1440
1441 return -1;
1442}
1443
1444/*
1445 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1446 * writes. Returns FALSE otherwise.
1447 */
1448static void
1449flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1450{
0a7de745 1451 int how = -1;
39236c6e
A		 1452
1453 /* Do not send any close messages if there is still data in the send buffer */
1454 if (fd_cb->so->so_snd.sb_cc == 0) {
0a7de745 1455 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
39236c6e
A		 1456 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1457 how = SHUT_RD;
1458 }
0a7de745 1459 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
39236c6e
A		 1460 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1461 if (how == SHUT_RD) {
1462 how = SHUT_RDWR;
1463 } else {
1464 how = SHUT_WR;
1465 }
1466 }
1467 }
1468
1469 if (how != -1) {
1470 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1471 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1472 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1473 if (how != SHUT_RD) {
1474 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1475 }
1476 if (how != SHUT_WR) {
1477 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1478 }
1479 }
1480 }
1481
1482 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1483 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1484 }
1485}
1486
1487static errno_t
3e170ce0 1488flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e 1489{
eb6b6ca3
A		 1490 mbuf_t packet = NULL;
1491 mbuf_t last = NULL;
0a7de745 1492 int error = 0;
39236c6e
A		 1493
1494 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
eb6b6ca3 1495 if (error || packet == NULL) {
39236c6e 1496 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
eb6b6ca3 1497 goto done;
39236c6e
A		 1498 }
1499
3e170ce0
A		 1500 if (toaddr != NULL) {
1501 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1502 if (error) {
1503 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
eb6b6ca3 1504 goto done;
3e170ce0
A		 1505 }
1506 }
1507
f427ee49 1508 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
5ba3f43e
A		 1509 last = m_last(packet);
1510 mbuf_setnext(last, data);
f427ee49
A		 1511 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1512 } else {
1513 data_len = 0;
5ba3f43e 1514 }
39236c6e 1515 error = flow_divert_send_packet(fd_cb, packet, force);
eb6b6ca3 1516 if (error == 0 && data_len > 0) {
39236c6e
A		 1517 fd_cb->bytes_sent += data_len;
1518 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1519 }
1520
eb6b6ca3
A		 1521done:
1522 if (error) {
1523 if (last != NULL) {
1524 mbuf_setnext(last, NULL);
1525 }
1526 if (packet != NULL) {
1527 mbuf_freem(packet);
1528 }
1529 }
1530
39236c6e
A		 1531 return error;
1532}
1533
1534static void
1535flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1536{
0a7de745
A		 1537 size_t to_send;
1538 size_t sent = 0;
1539 int error = 0;
1540 mbuf_t buffer;
39236c6e
A		 1541
1542 to_send = fd_cb->so->so_snd.sb_cc;
1543 buffer = fd_cb->so->so_snd.sb_mb;
1544
1545 if (buffer == NULL && to_send > 0) {
1546 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1547 return;
1548 }
1549
1550 /* Ignore the send window if force is enabled */
1551 if (!force && (to_send > fd_cb->send_window)) {
1552 to_send = fd_cb->send_window;
1553 }
1554
3e170ce0
A		 1555 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1556 while (sent < to_send) {
0a7de745
A		 1557 mbuf_t data;
1558 size_t data_len;
39236c6e 1559
3e170ce0
A		 1560 data_len = to_send - sent;
1561 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1562 data_len = FLOW_DIVERT_CHUNK_SIZE;
1563 }
39236c6e 1564
3e170ce0
A		 1565 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1566 if (error) {
1567 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1568 break;
1569 }
39236c6e 1570
3e170ce0
A		 1571 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1572 if (error) {
f427ee49
A		 1573 if (data != NULL) {
1574 mbuf_freem(data);
1575 }
3e170ce0
A		 1576 break;
1577 }
39236c6e 1578
3e170ce0
A		 1579 sent += data_len;
1580 }
f427ee49 1581 sbdrop(&fd_cb->so->so_snd, (int)sent);
3e170ce0
A		 1582 sowwakeup(fd_cb->so);
1583 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1584 mbuf_t data;
1585 mbuf_t m;
1586 size_t data_len;
1587
0a7de745 1588 while (buffer) {
3e170ce0
A		 1589 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1590
1591 m = buffer;
1592 if (toaddr != NULL) {
1593 /* look for data in the chain */
1594 do {
1595 m = m->m_next;
1596 if (m != NULL && m->m_type == MT_DATA) {
1597 break;
1598 }
0a7de745 1599 } while (m);
3e170ce0
A		 1600 if (m == NULL) {
1601 /* unexpected */
1602 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1603 goto move_on;
1604 }
1605 }
1606 data_len = mbuf_pkthdr_len(m);
5ba3f43e
A		 1607 if (data_len > 0) {
1608 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1609 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1610 if (error) {
1611 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1612 break;
1613 }
1614 } else {
1615 data = NULL;
3e170ce0
A		 1616 }
1617 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1618 if (error) {
f427ee49
A		 1619 if (data != NULL) {
1620 mbuf_freem(data);
1621 }
3e170ce0
A		 1622 break;
1623 }
1624 sent += data_len;
1625move_on:
1626 buffer = buffer->m_nextpkt;
1627 (void) sbdroprecord(&(fd_cb->so->so_snd));
1628 }
39236c6e
A		 1629 }
1630
1631 if (sent > 0) {
1632 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1633 if (fd_cb->send_window >= sent) {
1634 fd_cb->send_window -= sent;
1635 } else {
1636 fd_cb->send_window = 0;
1637 }
39236c6e
A		 1638 }
1639}
1640
1641static int
3e170ce0 1642flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e 1643{
0a7de745
A		 1644 size_t to_send = mbuf_pkthdr_len(data);
1645 int error = 0;
39236c6e
A		 1646
1647 if (to_send > fd_cb->send_window) {
1648 to_send = fd_cb->send_window;
1649 }
1650
1651 if (fd_cb->so->so_snd.sb_cc > 0) {
0a7de745 1652 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
39236c6e
A		 1653 }
1654
3e170ce0 1655 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
0a7de745
A		 1656 size_t sent = 0;
1657 mbuf_t remaining_data = data;
1658 mbuf_t pkt_data = NULL;
490019cf 1659 while (sent < to_send && remaining_data != NULL) {
0a7de745 1660 size_t pkt_data_len;
3e170ce0
A		 1661
1662 pkt_data = remaining_data;
39236c6e 1663
3e170ce0
A		 1664 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1665 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1666 } else {
1667 pkt_data_len = to_send - sent;
1668 }
1669
1670 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1671 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1672 if (error) {
1673 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1674 pkt_data = NULL;
1675 break;
1676 }
1677 } else {
1678 remaining_data = NULL;
1679 }
1680
1681 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1682
39236c6e 1683 if (error) {
39236c6e
A		 1684 break;
1685 }
39236c6e 1686
3e170ce0
A		 1687 pkt_data = NULL;
1688 sent += pkt_data_len;
39236c6e
A		 1689 }
1690
3e170ce0 1691 fd_cb->send_window -= sent;
39236c6e 1692
3e170ce0 1693 error = 0;
39236c6e 1694
3e170ce0
A		 1695 if (pkt_data != NULL) {
1696 if (sbspace(&fd_cb->so->so_snd) > 0) {
1697 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1698 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
0a7de745 1699 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1700 }
1701 } else {
f427ee49 1702 mbuf_freem(pkt_data);
3e170ce0 1703 error = ENOBUFS;
39236c6e 1704 }
39236c6e 1705 }
39236c6e 1706
3e170ce0
A		 1707 if (remaining_data != NULL) {
1708 if (sbspace(&fd_cb->so->so_snd) > 0) {
1709 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1710 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
0a7de745 1711 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1712 }
1713 } else {
f427ee49 1714 mbuf_freem(remaining_data);
3e170ce0
A		 1715 error = ENOBUFS;
1716 }
1717 }
1718 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
5ba3f43e 1719 if (to_send || mbuf_pkthdr_len(data) == 0) {
3e170ce0
A		 1720 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1721 if (error) {
39037602 1722 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
f427ee49
A		 1723 if (data != NULL) {
1724 mbuf_freem(data);
1725 }
3e170ce0
A		 1726 } else {
1727 fd_cb->send_window -= to_send;
39236c6e
A		 1728 }
1729 } else {
3e170ce0
A		 1730 /* buffer it */
1731 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1732 if (toaddr != NULL) {
1733 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1734 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1735 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1736 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0 1737 }
bca245ac 1738 error = 0;
3e170ce0
A		 1739 } else {
1740 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1741 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1742 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1743 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0
A		 1744 }
1745 }
1746 } else {
f427ee49
A		 1747 if (data != NULL) {
1748 mbuf_freem(data);
1749 }
3e170ce0
A		 1750 error = ENOBUFS;
1751 }
39236c6e
A		 1752 }
1753 }
1754
1755 return error;
1756}
1757
1758static int
f427ee49 1759flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb)
39236c6e 1760{
f427ee49
A		 1761 int error = 0;
1762 mbuf_t packet = NULL;
39236c6e
A		 1763
1764 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1765 if (error) {
1766 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1767 goto done;
1768 }
1769
39236c6e
A		 1770 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1771 if (error) {
1772 goto done;
1773 }
1774
1775done:
1776 if (error && packet != NULL) {
1777 mbuf_free(packet);
1778 }
1779
1780 return error;
1781}
1782
1783static int
1784flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1785{
0a7de745
A		 1786 int error = 0;
1787 mbuf_t packet = NULL;
39236c6e
A		 1788
1789 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1790 if (error) {
1791 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1792 goto done;
1793 }
1794
1795 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1796 if (error) {
1797 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1798 goto done;
1799 }
1800
1801 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1802 if (error) {
1803 goto done;
1804 }
1805
1806done:
1807 if (error && packet != NULL) {
1808 mbuf_free(packet);
1809 }
1810
1811 return error;
1812}
1813
f427ee49
A		 1814static void
1815flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint, bool port_only)
1816{
1817 struct inpcb *inp = sotoinpcb(fd_cb->so);
1818
1819 if (local_endpoint->sa_family == AF_INET6) {
1820 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && !port_only) {
1821 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1822 inp->in6p_laddr = (satosin6(local_endpoint))->sin6_addr;
1823 }
1824 if (inp->inp_lport == 0) {
1825 inp->inp_lport = (satosin6(local_endpoint))->sin6_port;
1826 }
1827 } else if (local_endpoint->sa_family == AF_INET) {
1828 if (inp->inp_laddr.s_addr == INADDR_ANY && !port_only) {
1829 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1830 inp->inp_laddr = (satosin(local_endpoint))->sin_addr;
1831 }
1832 if (inp->inp_lport == 0) {
1833 inp->inp_lport = (satosin(local_endpoint))->sin_port;
1834 }
1835 }
1836}
1837
1838static void
1839flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint)
1840{
1841 struct inpcb *inp = sotoinpcb(fd_cb->so);
1842
1843 if (remote_endpoint->sa_family == AF_INET6) {
1844 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
1845 inp->in6p_faddr = (satosin6(remote_endpoint))->sin6_addr;
1846 }
1847 if (inp->inp_fport == 0) {
1848 inp->inp_fport = (satosin6(remote_endpoint))->sin6_port;
1849 }
1850 } else if (remote_endpoint->sa_family == AF_INET) {
1851 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1852 inp->inp_faddr = (satosin(remote_endpoint))->sin_addr;
1853 }
1854 if (inp->inp_fport == 0) {
1855 inp->inp_fport = (satosin(remote_endpoint))->sin_port;
1856 }
1857 }
1858}
1859
1860static uint32_t
1861flow_divert_derive_kernel_control_unit(uint32_t ctl_unit, uint32_t *aggregate_unit)
1862{
1863 if (aggregate_unit != NULL && *aggregate_unit != 0) {
1864 uint32_t counter;
1865 for (counter = 0; counter < (GROUP_COUNT_MAX - 1); counter++) {
1866 if ((*aggregate_unit) & (1 << counter)) {
1867 break;
1868 }
1869 }
1870 if (counter < (GROUP_COUNT_MAX - 1)) {
1871 *aggregate_unit &= ~(1 << counter);
1872 return counter + 1;
1873 } else {
1874 return ctl_unit;
1875 }
1876 } else {
1877 return ctl_unit;
1878 }
1879}
1880
1881static int
1882flow_divert_try_next(struct flow_divert_pcb *fd_cb)
1883{
1884 uint32_t current_ctl_unit = 0;
1885 uint32_t next_ctl_unit = 0;
1886 struct flow_divert_group *current_group = NULL;
1887 struct flow_divert_group *next_group = NULL;
1888 int error = 0;
1889
1890 next_ctl_unit = flow_divert_derive_kernel_control_unit(fd_cb->policy_control_unit, &(fd_cb->aggregate_unit));
1891 current_ctl_unit = fd_cb->control_group_unit;
1892
1893 if (current_ctl_unit == next_ctl_unit) {
1894 FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling flow divert");
1895 error = EALREADY;
1896 goto done;
1897 }
1898
1899 if (next_ctl_unit == 0 || next_ctl_unit >= GROUP_COUNT_MAX) {
1900 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert");
1901 error = ENOENT;
1902 goto done;
1903 }
1904
1905 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1906 FDLOG0(LOG_NOTICE, fd_cb, "No active groups, disabling flow divert");
1907 error = ENOENT;
1908 goto done;
1909 }
1910
1911 next_group = g_flow_divert_groups[next_ctl_unit];
1912 if (next_group == NULL) {
1913 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit);
1914 error = ENOENT;
1915 goto done;
1916 }
1917
1918 current_group = fd_cb->group;
1919
1920 lck_rw_lock_exclusive(&(current_group->lck));
1921 lck_rw_lock_exclusive(&(next_group->lck));
1922
1923 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", current_ctl_unit, next_ctl_unit);
1924
1925 RB_REMOVE(fd_pcb_tree, &(current_group->pcb_tree), fd_cb);
1926 if (RB_INSERT(fd_pcb_tree, &(next_group->pcb_tree), fd_cb) != NULL) {
1927 panic("group with unit %u already contains a connection with hash %u", next_ctl_unit, fd_cb->hash);
1928 }
1929
1930 fd_cb->group = next_group;
1931 fd_cb->control_group_unit = next_ctl_unit;
1932
1933 lck_rw_done(&(next_group->lck));
1934 lck_rw_done(&(current_group->lck));
1935
1936 error = flow_divert_send_connect_packet(fd_cb);
1937 if (error) {
1938 FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", next_ctl_unit);
1939 error = ENOENT;
1940 goto done;
1941 }
1942
1943done:
1944 return error;
1945}
1946
1947static void
1948flow_divert_disable(struct flow_divert_pcb *fd_cb)
1949{
1950 struct socket *so = NULL;
1951 mbuf_t buffer;
1952 int error = 0;
1953 proc_t last_proc = NULL;
1954 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint;
1955 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT);
1956 struct inpcb *inp = NULL;
1957
1958 so = fd_cb->so;
1959 if (so == NULL) {
1960 goto done;
1961 }
1962
1963 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert");
1964
1965 /* Restore the IP state */
1966 inp = sotoinpcb(so);
1967 inp->inp_vflag = fd_cb->original_vflag;
1968 inp->inp_faddr.s_addr = INADDR_ANY;
1969 inp->inp_fport = 0;
1970 memset(&(inp->in6p_faddr), 0, sizeof(inp->in6p_faddr));
1971 inp->in6p_fport = 0;
1972 /* If flow divert set the local address, clear it out */
1973 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) {
1974 inp->inp_laddr.s_addr = INADDR_ANY;
1975 memset(&(inp->in6p_laddr), 0, sizeof(inp->in6p_laddr));
1976 }
1977 inp->inp_last_outifp = fd_cb->original_last_outifp;
1978 inp->in6p_last_outifp = fd_cb->original_last_outifp6;
1979
1980 /* Dis-associate the socket */
1981 so->so_flags &= ~SOF_FLOW_DIVERT;
1982 so->so_flags1 |= SOF1_FLOW_DIVERT_SKIP;
1983 so->so_fd_pcb = NULL;
1984 fd_cb->so = NULL;
1985
1986 /* Remove from the group */
1987 flow_divert_pcb_remove(fd_cb);
1988
1989 FDRELEASE(fd_cb); /* Release the socket's reference */
1990
1991 /* Revert back to the original protocol */
1992 so->so_proto = pffindproto(SOCK_DOM(so), SOCK_PROTO(so), SOCK_TYPE(so));
1993
1994 last_proc = proc_find(so->last_pid);
1995
1996 if (do_connect) {
1997 /* Connect using the original protocol */
1998 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, remote_endpoint, (last_proc != NULL ? last_proc : current_proc()));
1999 if (error) {
2000 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error);
2001 goto done;
2002 }
2003 }
2004
2005 buffer = so->so_snd.sb_mb;
2006 if (buffer == NULL) {
2007 /* No buffered data, done */
2008 goto done;
2009 }
2010
2011 /* Send any buffered data using the original protocol */
2012 if (SOCK_TYPE(so) == SOCK_STREAM) {
2013 mbuf_t data_to_send = NULL;
2014 size_t data_len = so->so_snd.sb_cc;
2015
2016 error = mbuf_copym(buffer, 0, data_len, MBUF_DONTWAIT, &data_to_send);
2017 if (error) {
2018 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer");
2019 goto done;
2020 }
2021
2022 sbflush(&so->so_snd);
2023
2024 if (data_to_send->m_flags & M_PKTHDR) {
2025 mbuf_pkthdr_setlen(data_to_send, data_len);
2026 }
2027
2028 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2029 0,
2030 data_to_send,
2031 NULL,
2032 NULL,
2033 (last_proc != NULL ? last_proc : current_proc()));
2034
2035 if (error) {
2036 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2037 }
2038 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
2039 struct sockbuf *sb = &so->so_snd;
2040 MBUFQ_HEAD(send_queue_head) send_queue;
2041 MBUFQ_INIT(&send_queue);
2042
2043 /* Flush the send buffer, moving all records to a temporary queue */
2044 while (sb->sb_mb != NULL) {
2045 mbuf_t record = sb->sb_mb;
2046 mbuf_t m = record;
2047 sb->sb_mb = sb->sb_mb->m_nextpkt;
2048 while (m != NULL) {
2049 sbfree(sb, m);
2050 m = m->m_next;
2051 }
2052 record->m_nextpkt = NULL;
2053 MBUFQ_ENQUEUE(&send_queue, record);
2054 }
2055 SB_EMPTY_FIXUP(sb);
2056
2057 while (!MBUFQ_EMPTY(&send_queue)) {
2058 mbuf_t next_record = MBUFQ_FIRST(&send_queue);
2059 mbuf_t addr = NULL;
2060 mbuf_t control = NULL;
2061 mbuf_t last_control = NULL;
2062 mbuf_t data = NULL;
2063 mbuf_t m = next_record;
2064 struct sockaddr *to_endpoint = NULL;
2065
2066 MBUFQ_DEQUEUE(&send_queue, next_record);
2067
2068 while (m != NULL) {
2069 if (m->m_type == MT_SONAME) {
2070 addr = m;
2071 } else if (m->m_type == MT_CONTROL) {
2072 if (control == NULL) {
2073 control = m;
2074 }
2075 last_control = m;
2076 } else if (m->m_type == MT_DATA) {
2077 data = m;
2078 break;
2079 }
2080 m = m->m_next;
2081 }
2082
2083 if (addr != NULL) {
2084 to_endpoint = flow_divert_get_buffered_target_address(addr);
2085 if (to_endpoint == NULL) {
2086 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer");
2087 }
2088 }
2089
2090 if (data == NULL) {
2091 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data");
2092 mbuf_freem(next_record);
2093 continue;
2094 }
2095
2096 if (!(data->m_flags & M_PKTHDR)) {
2097 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header");
2098 mbuf_freem(next_record);
2099 continue;
2100 }
2101
2102 if (addr != NULL) {
2103 addr->m_next = NULL;
2104 }
2105
2106 if (last_control != NULL) {
2107 last_control->m_next = NULL;
2108 }
2109
2110 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2111 0,
2112 data,
2113 to_endpoint,
2114 control,
2115 (last_proc != NULL ? last_proc : current_proc()));
2116
2117 if (addr != NULL) {
2118 mbuf_freem(addr);
2119 }
2120
2121 if (error) {
2122 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2123 }
2124 }
2125 }
2126done:
2127 if (last_proc != NULL) {
2128 proc_rele(last_proc);
2129 }
2130
2131 if (error) {
2132 so->so_error = (uint16_t)error;
2133 flow_divert_disconnect_socket(so);
2134 }
2135}
2136
39236c6e
A		 2137static void
2138flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2139{
f427ee49 2140 uint32_t connect_error = 0;
0a7de745
A		 2141 uint32_t ctl_unit = 0;
2142 int error = 0;
2143 struct flow_divert_group *grp = NULL;
f427ee49
A		 2144 union sockaddr_in_4_6 local_endpoint = {};
2145 union sockaddr_in_4_6 remote_endpoint = {};
0a7de745 2146 int out_if_index = 0;
0a7de745
A		 2147 uint32_t send_window;
2148 uint32_t app_data_length = 0;
39236c6e 2149
f427ee49
A		 2150 memset(&local_endpoint, 0, sizeof(local_endpoint));
2151 memset(&remote_endpoint, 0, sizeof(remote_endpoint));
39236c6e
A		 2152
2153 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
2154 if (error) {
2155 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
2156 return;
2157 }
2158
f427ee49 2159 connect_error = ntohl(connect_error);
39236c6e
A		 2160 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
2161
2162 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
2163 if (error) {
2164 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
2165 return;
2166 }
2167
2168 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
2169 if (error) {
39037602 2170 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
39236c6e
A		 2171 }
2172
f427ee49 2173 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_endpoint), &(local_endpoint.sa), NULL);
39236c6e 2174 if (error) {
39037602 2175 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
39236c6e
A		 2176 }
2177
f427ee49 2178 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_endpoint), &(remote_endpoint.sa), NULL);
39236c6e 2179 if (error) {
39037602 2180 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
39236c6e
A		 2181 }
2182
2183 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2184 if (error) {
39037602
A		 2185 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
2186 }
2187
2188 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2189 if (error) {
2190 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
39236c6e
A		 2191 }
2192
39037602 2193 error = 0;
0a7de745 2194 ctl_unit = ntohl(ctl_unit);
39236c6e
A		 2195
2196 lck_rw_lock_shared(&g_flow_divert_group_lck);
2197
39037602
A		 2198 if (connect_error == 0 && ctl_unit > 0) {
2199 if (ctl_unit >= GROUP_COUNT_MAX) {
39236c6e
A		 2200 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
2201 error = EINVAL;
2202 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
2203 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
2204 error = EINVAL;
2205 } else {
2206 grp = g_flow_divert_groups[ctl_unit];
2207 if (grp == NULL) {
2208 error = ECONNRESET;
2209 }
2210 }
2211 }
2212
2213 FDLOCK(fd_cb);
2214 if (fd_cb->so != NULL) {
0a7de745
A		 2215 struct inpcb *inp = NULL;
2216 struct ifnet *ifp = NULL;
2217 struct flow_divert_group *old_group;
f427ee49 2218 struct socket *so = fd_cb->so;
39236c6e 2219
f427ee49 2220 socket_lock(so, 0);
39236c6e 2221
f427ee49
A		 2222 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_state & SS_ISCONNECTING)) {
2223 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result");
39236c6e
A		 2224 goto done;
2225 }
2226
f427ee49 2227 inp = sotoinpcb(so);
39236c6e
A		 2228
2229 if (connect_error || error) {
2230 goto set_socket_state;
2231 }
2232
f427ee49
A		 2233 if (flow_divert_is_sockaddr_valid(&(local_endpoint.sa))) {
2234 if (local_endpoint.sa.sa_family == AF_INET) {
2235 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2236 } else if (local_endpoint.sa.sa_family == AF_INET6) {
2237 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
cb323159 2238 }
f427ee49
A		 2239 fd_cb->local_endpoint = local_endpoint;
2240 flow_divert_set_local_endpoint(fd_cb, &(local_endpoint.sa), (SOCK_TYPE(so) == SOCK_DGRAM));
cb323159 2241 }
39236c6e 2242
f427ee49
A		 2243 if (flow_divert_is_sockaddr_valid(&(remote_endpoint.sa)) && SOCK_TYPE(so) == SOCK_STREAM) {
2244 if (remote_endpoint.sa.sa_family == AF_INET) {
2245 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2246 } else if (remote_endpoint.sa.sa_family == AF_INET6) {
2247 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
cb323159 2248 }
f427ee49 2249 flow_divert_set_remote_endpoint(fd_cb, &(remote_endpoint.sa));
39236c6e
A		 2250 }
2251
39037602 2252 if (app_data_length > 0) {
0a7de745 2253 uint8_t *app_data = NULL;
39037602
A		 2254 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2255 if (app_data != NULL) {
2256 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2257 if (error == 0) {
2258 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
2259 if (fd_cb->app_data != NULL) {
2260 FREE(fd_cb->app_data, M_TEMP);
2261 }
2262 fd_cb->app_data = app_data;
2263 fd_cb->app_data_length = app_data_length;
2264 } else {
2265 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
2266 FREE(app_data, M_TEMP);
2267 }
2268 } else {
2269 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
2270 }
2271 }
2272
39236c6e
A		 2273 ifnet_head_lock_shared();
2274 if (out_if_index > 0 && out_if_index <= if_index) {
2275 ifp = ifindex2ifnet[out_if_index];
2276 }
2277
2278 if (ifp != NULL) {
f427ee49
A		 2279 if (inp->inp_vflag & INP_IPV4) {
2280 inp->inp_last_outifp = ifp;
2281 } else if (inp->inp_vflag & INP_IPV6) {
2282 inp->in6p_last_outifp = ifp;
2283 }
39236c6e
A		 2284 } else {
2285 error = EINVAL;
2286 }
2287 ifnet_head_done();
2288
2289 if (error) {
2290 goto set_socket_state;
2291 }
2292
2293 if (fd_cb->group == NULL) {
2294 error = EINVAL;
2295 goto set_socket_state;
2296 }
2297
39037602
A		 2298 if (grp != NULL) {
2299 old_group = fd_cb->group;
39236c6e 2300
39037602
A		 2301 lck_rw_lock_exclusive(&old_group->lck);
2302 lck_rw_lock_exclusive(&grp->lck);
39236c6e 2303
39037602
A		 2304 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
2305 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
2306 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
2307 }
39236c6e 2308
39037602 2309 fd_cb->group = grp;
39236c6e 2310
39037602
A		 2311 lck_rw_done(&grp->lck);
2312 lck_rw_done(&old_group->lck);
2313 }
39236c6e
A		 2314
2315 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 2316
2317set_socket_state:
2318 if (!connect_error && !error) {
2319 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
2320 error = flow_divert_send_connect_result(fd_cb);
2321 }
2322
2323 if (connect_error || error) {
f427ee49
A		 2324 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) {
2325 error = flow_divert_try_next(fd_cb);
2326 if (error) {
2327 flow_divert_disable(fd_cb);
2328 }
2329 goto done;
2330 }
2331
2332 if (!connect_error) {
2333 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2334 so->so_error = (uint16_t)error;
2335 flow_divert_send_close_if_needed(fd_cb);
2336 } else {
39236c6e 2337 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
f427ee49 2338 so->so_error = (uint16_t)connect_error;
39236c6e 2339 }
f427ee49 2340 flow_divert_disconnect_socket(so);
39236c6e 2341 } else {
cb323159
A		 2342#if NECP
2343 /* Update NECP client with connected five-tuple */
2344 if (!uuid_is_null(inp->necp_client_uuid)) {
f427ee49
A		 2345 socket_unlock(so, 0);
2346 necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp);
2347 socket_lock(so, 0);
cb323159
A		 2348 }
2349#endif /* NECP */
2350
490019cf 2351 flow_divert_send_buffered_data(fd_cb, FALSE);
f427ee49
A		 2352 soisconnected(so);
2353 }
2354
2355 /* We don't need the connect packet any more */
2356 if (fd_cb->connect_packet != NULL) {
2357 mbuf_freem(fd_cb->connect_packet);
2358 fd_cb->connect_packet = NULL;
39236c6e
A		 2359 }
2360
f427ee49
A		 2361 /* We don't need the original remote endpoint any more */
2362 if (fd_cb->original_remote_endpoint != NULL) {
2363 FREE(fd_cb->original_remote_endpoint, M_SONAME);
2364 fd_cb->original_remote_endpoint = NULL;
2365 }
39236c6e 2366done:
f427ee49 2367 socket_unlock(so, 0);
39236c6e
A		 2368 }
2369 FDUNLOCK(fd_cb);
2370
2371 lck_rw_done(&g_flow_divert_group_lck);
2372}
2373
2374static void
2375flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2376{
0a7de745
A		 2377 uint32_t close_error;
2378 int error = 0;
2379 int how;
39236c6e
A		 2380
2381 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2382 if (error) {
2383 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2384 return;
2385 }
2386
2387 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2388 if (error) {
2389 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2390 return;
2391 }
2392
2393 how = ntohl(how);
2394
2395 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2396
2397 FDLOCK(fd_cb);
2398 if (fd_cb->so != NULL) {
2399 socket_lock(fd_cb->so, 0);
2400
f427ee49 2401 fd_cb->so->so_error = (uint16_t)ntohl(close_error);
39236c6e
A		 2402
2403 flow_divert_update_closed_state(fd_cb, how, TRUE);
0a7de745 2404
39236c6e
A		 2405 how = flow_divert_tunnel_how_closed(fd_cb);
2406 if (how == SHUT_RDWR) {
3e170ce0 2407 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2408 } else if (how == SHUT_RD) {
2409 socantrcvmore(fd_cb->so);
2410 } else if (how == SHUT_WR) {
2411 socantsendmore(fd_cb->so);
2412 }
2413
2414 socket_unlock(fd_cb->so, 0);
2415 }
2416 FDUNLOCK(fd_cb);
2417}
2418
3e170ce0 2419static mbuf_t
f427ee49 2420flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 2421{
cb323159 2422 struct inpcb *inp = sotoinpcb(fd_cb->so);
f427ee49
A		 2423 bool is_cfil_enabled = false;
2424#if CONTENT_FILTER
2425 /* Content Filter needs to see the local address */
2426 is_cfil_enabled = (inp->inp_socket && inp->inp_socket->so_cfil_db != NULL);
2427#endif
2428 if ((inp->inp_vflag & INP_IPV4) &&
2429 fd_cb->local_endpoint.sa.sa_family == AF_INET &&
2430 ((inp->inp_flags & INP_RECVDSTADDR) || is_cfil_enabled)) {
2431 return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2432 } else if ((inp->inp_vflag & INP_IPV6) &&
2433 fd_cb->local_endpoint.sa.sa_family == AF_INET6 &&
2434 ((inp->inp_flags & IN6P_PKTINFO) || is_cfil_enabled)) {
cb323159
A		 2435 struct in6_pktinfo pi6;
2436 memset(&pi6, 0, sizeof(pi6));
f427ee49 2437 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr;
cb323159
A		 2438
2439 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 2440 }
0a7de745 2441 return NULL;
3e170ce0 2442}
39236c6e 2443
f427ee49 2444static int
3e170ce0
A		 2445flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2446{
f427ee49
A		 2447 int error = 0;
2448
39236c6e
A		 2449 FDLOCK(fd_cb);
2450 if (fd_cb->so != NULL) {
0a7de745
A		 2451 mbuf_t data = NULL;
2452 size_t data_size;
3e170ce0
A		 2453 struct sockaddr_storage remote_address;
2454 boolean_t got_remote_sa = FALSE;
f427ee49
A		 2455 boolean_t appended = FALSE;
2456 boolean_t append_success = FALSE;
3e170ce0 2457
39236c6e 2458 socket_lock(fd_cb->so, 0);
3e170ce0 2459
f427ee49
A		 2460 if (sbspace(&fd_cb->so->so_rcv) == 0) {
2461 error = ENOBUFS;
2462 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED;
2463 FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some data");
2464 goto done;
2465 }
2466
3e170ce0
A		 2467 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2468 uint32_t val_size = 0;
2469
2470 /* check if we got remote address with data */
2471 memset(&remote_address, 0, sizeof(remote_address));
f427ee49 2472 error = flow_divert_packet_get_tlv(packet, (int)offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
3e170ce0
A		 2473 if (error || val_size > sizeof(remote_address)) {
2474 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2475 error = 0;
39236c6e 2476 } else {
bca245ac
A		 2477 if (remote_address.ss_len > sizeof(remote_address)) {
2478 remote_address.ss_len = sizeof(remote_address);
2479 }
3e170ce0
A		 2480 /* validate the address */
2481 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2482 got_remote_sa = TRUE;
f427ee49
A		 2483 } else {
2484 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid");
3e170ce0
A		 2485 }
2486 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2487 }
2488 }
2489
2490 data_size = (mbuf_pkthdr_len(packet) - offset);
2491
f427ee49
A		 2492 if (fd_cb->so->so_state & SS_CANTRCVMORE) {
2493 FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size);
2494 goto done;
2495 }
2496
2497 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) {
2498 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so));
2499 goto done;
2500 }
2501
3e170ce0
A		 2502 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2503
2504 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2505 if (error || data == NULL) {
2506 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
f427ee49
A		 2507 goto done;
2508 }
2509
2510 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2511 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0);
2512 append_success = TRUE;
3e170ce0 2513 } else {
f427ee49
A		 2514 struct sockaddr *append_sa = NULL;
2515 mbuf_t mctl;
3e170ce0 2516
f427ee49
A		 2517 if (got_remote_sa == TRUE) {
2518 error = flow_divert_dup_addr(remote_address.ss_family, (struct sockaddr *)&remote_address, &append_sa);
2519 } else {
2520 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) {
2521 error = in6_mapped_peeraddr(fd_cb->so, &append_sa);
2522 } else {
2523 error = in_getpeeraddr(fd_cb->so, &append_sa);
3e170ce0 2524 }
39236c6e 2525 }
f427ee49
A		 2526 if (error) {
2527 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2528 }
2529
2530 mctl = flow_divert_create_control_mbuf(fd_cb);
2531 int append_error = 0;
2532 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error) || append_error == EJUSTRETURN) {
2533 append_success = TRUE;
2534 appended = (append_error == 0);
2535 } else {
2536 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error);
2537 }
2538
2539 if (append_sa != NULL) {
2540 FREE(append_sa, M_SONAME);
2541 }
39236c6e 2542 }
f427ee49
A		 2543
2544 if (append_success) {
2545 fd_cb->bytes_received += data_size;
2546 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2547 }
2548
2549 if (appended) {
2550 sorwakeup(fd_cb->so);
2551 }
2552done:
39236c6e 2553 socket_unlock(fd_cb->so, 0);
39236c6e 2554 }
3e170ce0 2555 FDUNLOCK(fd_cb);
f427ee49
A		 2556
2557 return error;
39236c6e
A		 2558}
2559
2560static void
2561flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2562{
0a7de745
A		 2563 uint32_t read_count;
2564 int error = 0;
39236c6e
A		 2565
2566 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2567 if (error) {
2568 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2569 return;
2570 }
2571
3e170ce0 2572 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2573
2574 FDLOCK(fd_cb);
2575 if (fd_cb->so != NULL) {
2576 socket_lock(fd_cb->so, 0);
2577 fd_cb->send_window += ntohl(read_count);
2578 flow_divert_send_buffered_data(fd_cb, FALSE);
2579 socket_unlock(fd_cb->so, 0);
2580 }
2581 FDUNLOCK(fd_cb);
2582}
2583
2584static void
2585flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2586{
2587 int error = 0;
3e170ce0 2588 uint32_t key_size = 0;
39236c6e 2589 int log_level;
5ba3f43e 2590 uint32_t flags = 0;
39236c6e
A		 2591
2592 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2593 if (error) {
2594 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2595 return;
2596 }
2597
2598 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
39037602 2599 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
39236c6e
A		 2600 return;
2601 }
2602
2603 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2604 if (!error) {
f427ee49 2605 nil_pcb.log_level = (uint8_t)log_level;
39236c6e
A		 2606 }
2607
2608 lck_rw_lock_exclusive(&group->lck);
2609
cb323159
A		 2610 if (group->token_key != NULL) {
2611 FREE(group->token_key, M_TEMP);
2612 group->token_key = NULL;
2613 }
2614
39236c6e
A		 2615 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2616 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2617 if (error) {
2618 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2619 FREE(group->token_key, M_TEMP);
2620 group->token_key = NULL;
2621 lck_rw_done(&group->lck);
2622 return;
2623 }
2624
2625 group->token_key_size = key_size;
2626
5ba3f43e
A		 2627 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2628 if (!error) {
2629 group->flags = flags;
2630 }
2631
39236c6e
A		 2632 lck_rw_done(&group->lck);
2633}
2634
2635static void
2636flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2637{
0a7de745 2638 int error = 0;
0a7de745 2639 int out_if_index = 0;
0a7de745 2640 uint32_t app_data_length = 0;
39236c6e
A		 2641
2642 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2643
39236c6e
A		 2644 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2645 if (error) {
490019cf
A		 2646 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2647 }
2648
2649 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2650 if (error) {
2651 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2652 }
2653
2654 FDLOCK(fd_cb);
2655 if (fd_cb->so != NULL) {
39236c6e
A		 2656 socket_lock(fd_cb->so, 0);
2657
490019cf
A		 2658 if (out_if_index > 0) {
2659 struct inpcb *inp = NULL;
2660 struct ifnet *ifp = NULL;
2661
2662 inp = sotoinpcb(fd_cb->so);
2663
2664 ifnet_head_lock_shared();
2665 if (out_if_index <= if_index) {
2666 ifp = ifindex2ifnet[out_if_index];
2667 }
2668
2669 if (ifp != NULL) {
f427ee49
A		 2670 if (inp->inp_vflag & INP_IPV4) {
2671 inp->inp_last_outifp = ifp;
2672 } else if (inp->inp_vflag & INP_IPV6) {
2673 inp->in6p_last_outifp = ifp;
2674 }
490019cf
A		 2675 }
2676 ifnet_head_done();
39236c6e
A		 2677 }
2678
490019cf 2679 if (app_data_length > 0) {
0a7de745 2680 uint8_t *app_data = NULL;
490019cf
A		 2681 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2682 if (app_data != NULL) {
2683 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2684 if (error == 0) {
2685 if (fd_cb->app_data != NULL) {
2686 FREE(fd_cb->app_data, M_TEMP);
2687 }
2688 fd_cb->app_data = app_data;
2689 fd_cb->app_data_length = app_data_length;
2690 } else {
2691 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2692 FREE(app_data, M_TEMP);
2693 }
2694 } else {
2695 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2696 }
39236c6e 2697 }
39236c6e
A		 2698
2699 socket_unlock(fd_cb->so, 0);
2700 }
2701 FDUNLOCK(fd_cb);
2702}
2703
2704static void
39037602 2705flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
39236c6e
A		 2706{
2707 size_t bytes_mem_size;
2708 size_t child_maps_mem_size;
f427ee49
A		 2709 size_t nodes_mem_size;
2710 size_t trie_memory_size = 0;
39236c6e
A		 2711 int cursor;
2712 int error = 0;
2713 struct flow_divert_trie new_trie;
2714 int insert_error = 0;
c6bf4f31 2715 int prefix_count = -1;
39236c6e 2716 int signing_id_count = 0;
f427ee49
A		 2717 size_t bytes_count = 0;
2718 size_t nodes_count = 0;
2719 size_t maps_count = 0;
39236c6e 2720
39037602 2721 lck_rw_lock_exclusive(&group->lck);
39236c6e
A		 2722
2723 /* Re-set the current trie */
39037602
A		 2724 if (group->signing_id_trie.memory != NULL) {
2725 FREE(group->signing_id_trie.memory, M_TEMP);
39236c6e 2726 }
39037602
A		 2727 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2728 group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 2729
2730 memset(&new_trie, 0, sizeof(new_trie));
2731
2732 /* Get the number of shared prefixes in the new set of signing ID strings */
c6bf4f31 2733 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
39236c6e 2734
c6bf4f31
A		 2735 if (prefix_count < 0 || error) {
2736 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
39037602 2737 lck_rw_done(&group->lck);
ecc0ceb4
A		 2738 return;
2739 }
2740
39236c6e
A		 2741 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2742 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2743 cursor >= 0;
2744 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2745 uint32_t sid_size = 0;
c6bf4f31
A		 2746 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2747 if (error || sid_size == 0) {
2748 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
2749 signing_id_count = 0;
2750 break;
2751 }
f427ee49
A		 2752 if (os_add_overflow(bytes_count, sid_size, &bytes_count)) {
2753 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while incrementing number of bytes");
2754 signing_id_count = 0;
2755 break;
2756 }
39236c6e
A		 2757 signing_id_count++;
2758 }
2759
2760 if (signing_id_count == 0) {
39037602 2761 lck_rw_done(&group->lck);
f427ee49 2762 FDLOG0(LOG_NOTICE, &nil_pcb, "No signing identifiers");
39236c6e
A		 2763 return;
2764 }
2765
f427ee49
A		 2766 if (os_add3_overflow(prefix_count, signing_id_count, 1, &nodes_count)) { /* + 1 for the root node */
2767 lck_rw_done(&group->lck);
2768 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of nodes");
2769 return;
2770 }
2771
2772 if (os_add_overflow(prefix_count, 1, &maps_count)) { /* + 1 for the root node */
2773 lck_rw_done(&group->lck);
2774 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of maps");
2775 return;
2776 }
2777
2778 if (bytes_count > UINT16_MAX || nodes_count > UINT16_MAX || maps_count > UINT16_MAX) {
2779 lck_rw_done(&group->lck);
2780 FDLOG(LOG_NOTICE, &nil_pcb, "Invalid bytes count (%lu), nodes count (%lu) or maps count (%lu)", bytes_count, nodes_count, maps_count);
2781 return;
2782 }
39236c6e
A		 2783
2784 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
f427ee49 2785 nodes_count, maps_count, bytes_count);
39236c6e 2786
f427ee49
A		 2787 if (os_mul_overflow(sizeof(*new_trie.nodes), (size_t)nodes_count, &nodes_mem_size) ||
2788 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, (size_t)maps_count, &child_maps_mem_size) ||
2789 os_mul_overflow(sizeof(*new_trie.bytes), (size_t)bytes_count, &bytes_mem_size) ||
4ba76501
A		 2790 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2791 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2792 lck_rw_done(&group->lck);
2793 return;
2794 }
39236c6e 2795
ecc0ceb4 2796 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
39037602
A		 2797 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2798 lck_rw_done(&group->lck);
ecc0ceb4
A		 2799 return;
2800 }
2801
2802 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2803 if (new_trie.memory == NULL) {
2804 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
0a7de745 2805 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
39037602 2806 lck_rw_done(&group->lck);
39236c6e
A		 2807 return;
2808 }
2809
f427ee49
A		 2810 new_trie.bytes_count = (uint16_t)bytes_count;
2811 new_trie.nodes_count = (uint16_t)nodes_count;
2812 new_trie.child_maps_count = (uint16_t)maps_count;
2813
39236c6e
A		 2814 /* Initialize the free lists */
2815 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2816 new_trie.nodes_free_next = 0;
2817 memset(new_trie.nodes, 0, nodes_mem_size);
2818
2819 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2820 new_trie.child_maps_free_next = 0;
2821 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2822
2823 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2824 new_trie.bytes_free_next = 0;
c6bf4f31 2825 memset(new_trie.bytes, 0, bytes_mem_size);
39236c6e
A		 2826
2827 /* The root is an empty node */
2828 new_trie.root = trie_node_alloc(&new_trie);
2829
2830 /* Add each signing ID to the trie */
2831 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2832 cursor >= 0;
2833 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2834 uint32_t sid_size = 0;
c6bf4f31
A		 2835 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2836 if (error || sid_size == 0) {
2837 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
2838 insert_error = EINVAL;
2839 break;
2840 }
f427ee49 2841 if (sid_size <= UINT16_MAX && new_trie.bytes_free_next + (uint16_t)sid_size <= new_trie.bytes_count) {
39236c6e 2842 uint16_t new_node_idx;
c6bf4f31
A		 2843 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2844 if (error) {
2845 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
2846 insert_error = EINVAL;
2847 break;
2848 }
39236c6e 2849 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
39037602 2850 if (new_node_idx == NULL_TRIE_IDX) {
39236c6e
A		 2851 insert_error = EINVAL;
2852 break;
2853 }
2854 } else {
2855 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2856 insert_error = ENOBUFS;
2857 break;
2858 }
2859 }
2860
2861 if (!insert_error) {
39037602 2862 group->signing_id_trie = new_trie;
39236c6e
A		 2863 } else {
2864 FREE(new_trie.memory, M_TEMP);
2865 }
2866
39236c6e 2867 lck_rw_done(&group->lck);
39236c6e
A		 2868}
2869
2870static int
2871flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2872{
0a7de745
A		 2873 struct flow_divert_packet_header hdr;
2874 int error = 0;
2875 struct flow_divert_pcb *fd_cb;
39236c6e
A		 2876
2877 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2878 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2879 error = EINVAL;
2880 goto done;
2881 }
2882
3e170ce0 2883 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
39037602 2884 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
3e170ce0
A		 2885 error = EINVAL;
2886 goto done;
2887 }
2888
39236c6e
A		 2889 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2890 if (error) {
2891 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2892 error = ENOBUFS;
2893 goto done;
2894 }
2895
2896 hdr.conn_id = ntohl(hdr.conn_id);
2897
2898 if (hdr.conn_id == 0) {
2899 switch (hdr.packet_type) {
0a7de745
A		 2900 case FLOW_DIVERT_PKT_GROUP_INIT:
2901 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2902 break;
2903 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2904 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2905 break;
2906 default:
2907 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2908 break;
39236c6e
A		 2909 }
2910 goto done;
2911 }
2912
0a7de745 2913 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
39236c6e
A		 2914 if (fd_cb == NULL) {
2915 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2916 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2917 }
2918 goto done;
2919 }
2920
2921 switch (hdr.packet_type) {
0a7de745
A		 2922 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2923 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2924 break;
2925 case FLOW_DIVERT_PKT_CLOSE:
2926 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2927 break;
2928 case FLOW_DIVERT_PKT_DATA:
f427ee49 2929 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
0a7de745
A		 2930 break;
2931 case FLOW_DIVERT_PKT_READ_NOTIFY:
2932 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2933 break;
2934 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2935 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2936 break;
2937 default:
2938 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2939 break;
39236c6e
A		 2940 }
2941
2942 FDRELEASE(fd_cb);
2943
2944done:
3e170ce0 2945 mbuf_freem(packet);
39236c6e
A		 2946 return error;
2947}
2948
2949static void
2950flow_divert_close_all(struct flow_divert_group *group)
2951{
0a7de745
A		 2952 struct flow_divert_pcb *fd_cb;
2953 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 2954
2955 SLIST_INIT(&tmp_list);
2956
2957 lck_rw_lock_exclusive(&group->lck);
2958
2959 MBUFQ_DRAIN(&group->send_queue);
2960
2961 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2962 FDRETAIN(fd_cb);
2963 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2964 }
2965
2966 lck_rw_done(&group->lck);
2967
2968 while (!SLIST_EMPTY(&tmp_list)) {
2969 fd_cb = SLIST_FIRST(&tmp_list);
2970 FDLOCK(fd_cb);
2971 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2972 if (fd_cb->so != NULL) {
2973 socket_lock(fd_cb->so, 0);
2974 flow_divert_pcb_remove(fd_cb);
2975 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2976 fd_cb->so->so_error = ECONNABORTED;
39037602 2977 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2978 socket_unlock(fd_cb->so, 0);
2979 }
2980 FDUNLOCK(fd_cb);
2981 FDRELEASE(fd_cb);
2982 }
2983}
2984
2985void
2986flow_divert_detach(struct socket *so)
2987{
0a7de745 2988 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2989
2990 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2991
2992 so->so_flags &= ~SOF_FLOW_DIVERT;
2993 so->so_fd_pcb = NULL;
2994
2995 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2996
2997 if (fd_cb->group != NULL) {
2998 /* Last-ditch effort to send any buffered data */
2999 flow_divert_send_buffered_data(fd_cb, TRUE);
3000
3e170ce0
A		 3001 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3002 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 3003 /* Remove from the group */
3004 flow_divert_pcb_remove(fd_cb);
3005 }
3006
3007 socket_unlock(so, 0);
3008 FDLOCK(fd_cb);
3009 fd_cb->so = NULL;
3010 FDUNLOCK(fd_cb);
3011 socket_lock(so, 0);
3012
0a7de745 3013 FDRELEASE(fd_cb); /* Release the socket's reference */
39236c6e
A		 3014}
3015
3016static int
3017flow_divert_close(struct socket *so)
3018{
0a7de745 3019 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3020
3021 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3022
3023 FDLOG0(LOG_INFO, fd_cb, "Closing");
3024
3e170ce0
A		 3025 if (SOCK_TYPE(so) == SOCK_STREAM) {
3026 soisdisconnecting(so);
3027 sbflush(&so->so_rcv);
3028 }
39236c6e
A		 3029
3030 flow_divert_send_buffered_data(fd_cb, TRUE);
3031 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3032 flow_divert_send_close_if_needed(fd_cb);
3033
3034 /* Remove from the group */
3035 flow_divert_pcb_remove(fd_cb);
3036
3037 return 0;
3038}
3039
3040static int
3e170ce0
A		 3041flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
3042 sae_connid_t cid __unused)
39236c6e 3043{
3e170ce0 3044 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
0a7de745 3045 return EINVAL;
39236c6e
A		 3046 }
3047
0a7de745 3048 return flow_divert_close(so);
39236c6e
A		 3049}
3050
3051static int
3052flow_divert_shutdown(struct socket *so)
3053{
0a7de745 3054 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3055
3056 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3057
3058 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
3059
3060 socantsendmore(so);
3061
3062 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
3063 flow_divert_send_close_if_needed(fd_cb);
3064
3065 return 0;
3066}
3067
3068static int
3069flow_divert_rcvd(struct socket *so, int flags __unused)
3070{
f427ee49
A		 3071 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3072 int space = sbspace(&so->so_rcv);
39236c6e
A		 3073
3074 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3075
f427ee49
A		 3076 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space);
3077 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) &&
3078 (space > 0) &&
3079 flow_divert_send_read_notification(fd_cb) == 0) {
3080 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification");
3081 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED;
39236c6e
A		 3082 }
3083
3084 return 0;
3085}
3086
3e170ce0
A		 3087static int
3088flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
3089{
3090 int error = 0;
3091 int port = 0;
3092
cb323159
A		 3093 if (!flow_divert_is_sockaddr_valid(toaddr)) {
3094 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
3095 error = EINVAL;
3096 goto done;
3097 }
3098
3e170ce0
A		 3099 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
3100 if (error) {
3101 goto done;
3102 }
3103
3104 if (toaddr->sa_family == AF_INET) {
3105 port = ntohs((satosin(toaddr))->sin_port);
f427ee49 3106 } else {
3e170ce0
A		 3107 port = ntohs((satosin6(toaddr))->sin6_port);
3108 }
3e170ce0
A		 3109
3110 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
3111 if (error) {
3112 goto done;
3113 }
3114
3115done:
3116 return error;
3117}
3118
3119struct sockaddr *
3120flow_divert_get_buffered_target_address(mbuf_t buffer)
3121{
3122 if (buffer != NULL && buffer->m_type == MT_SONAME) {
3123 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
3124 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
3125 return toaddr;
3126 }
3127 }
3128 return NULL;
3129}
3130
3131static boolean_t
3132flow_divert_is_sockaddr_valid(struct sockaddr *addr)
3133{
0a7de745
A		 3134 switch (addr->sa_family) {
3135 case AF_INET:
cb323159 3136 if (addr->sa_len < sizeof(struct sockaddr_in)) {
0a7de745
A		 3137 return FALSE;
3138 }
3139 break;
0a7de745 3140 case AF_INET6:
cb323159 3141 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3e170ce0 3142 return FALSE;
0a7de745
A		 3143 }
3144 break;
0a7de745
A		 3145 default:
3146 return FALSE;
3e170ce0
A		 3147 }
3148 return TRUE;
3149}
3150
39236c6e
A		 3151static errno_t
3152flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
0a7de745 3153 struct sockaddr **dup)
39236c6e 3154{
0a7de745
A		 3155 int error = 0;
3156 struct sockaddr *result;
3157 struct sockaddr_storage ss;
39236c6e
A		 3158
3159 if (addr != NULL) {
3160 result = addr;
3161 } else {
3162 memset(&ss, 0, sizeof(ss));
3163 ss.ss_family = family;
3164 if (ss.ss_family == AF_INET) {
3165 ss.ss_len = sizeof(struct sockaddr_in);
f427ee49 3166 } else if (ss.ss_family == AF_INET6) {
39236c6e 3167 ss.ss_len = sizeof(struct sockaddr_in6);
f427ee49 3168 } else {
39236c6e
A		 3169 error = EINVAL;
3170 }
3171 result = (struct sockaddr *)&ss;
3172 }
3173
3174 if (!error) {
3175 *dup = dup_sockaddr(result, 1);
3176 if (*dup == NULL) {
3177 error = ENOBUFS;
3178 }
3179 }
3180
3181 return error;
3182}
3183
3e170ce0
A		 3184static void
3185flow_divert_disconnect_socket(struct socket *so)
3186{
3187 soisdisconnected(so);
3188 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3189 struct inpcb *inp = NULL;
3190
3191 inp = sotoinpcb(so);
3192 if (inp != NULL) {
0a7de745 3193 if (SOCK_CHECK_DOM(so, PF_INET6)) {
3e170ce0 3194 in6_pcbdetach(inp);
f427ee49
A		 3195 } else {
3196 in_pcbdetach(inp);
3197 }
3e170ce0
A		 3198 }
3199 }
3200}
3201
39236c6e
A		 3202static errno_t
3203flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
3204{
0a7de745 3205 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 3206
3207 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3208
3209 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
3210 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3211 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
3212 }
3213 }
3214
3215 if (SOCK_DOM(so) == PF_INET) {
3216 return g_tcp_protosw->pr_ctloutput(so, sopt);
f427ee49 3217 } else if (SOCK_DOM(so) == PF_INET6) {
39236c6e
A		 3218 return g_tcp6_protosw->pr_ctloutput(so, sopt);
3219 }
39236c6e
A		 3220 return 0;
3221}
3222
f427ee49
A		 3223static errno_t
3224flow_divert_connect_out_internal(struct socket *so, struct sockaddr *to, proc_t p, bool implicit)
39236c6e 3225{
0a7de745
A		 3226 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3227 int error = 0;
3228 struct inpcb *inp = sotoinpcb(so);
3229 struct sockaddr_in *sinp;
3230 mbuf_t connect_packet = NULL;
3231 int do_send = 1;
39236c6e
A		 3232
3233 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3234
3235 if (fd_cb->group == NULL) {
3236 error = ENETUNREACH;
3237 goto done;
3238 }
3239
3240 if (inp == NULL) {
3241 error = EINVAL;
3242 goto done;
3243 } else if (inp->inp_state == INPCB_STATE_DEAD) {
3244 if (so->so_error) {
3245 error = so->so_error;
3246 so->so_error = 0;
3247 } else {
3248 error = EINVAL;
3249 }
3250 goto done;
3251 }
3252
f427ee49 3253 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
39236c6e
A		 3254 error = EALREADY;
3255 goto done;
3256 }
3257
490019cf 3258 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 3259
490019cf 3260 if (fd_cb->connect_packet == NULL) {
f427ee49
A		 3261 struct sockaddr_in sin = {};
3262 struct ifnet *ifp = NULL;
3263
490019cf
A		 3264 if (to == NULL) {
3265 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
3266 error = EINVAL;
3267 goto done;
fe8ab488 3268 }
fe8ab488 3269
f427ee49
A		 3270 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0);
3271 if (fd_cb->original_remote_endpoint == NULL) {
3272 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint");
3273 error = ENOMEM;
3274 goto done;
3275 }
3276 fd_cb->original_vflag = inp->inp_vflag;
3277 fd_cb->original_last_outifp = inp->inp_last_outifp;
3278 fd_cb->original_last_outifp6 = inp->in6p_last_outifp;
3279
490019cf
A		 3280 sinp = (struct sockaddr_in *)(void *)to;
3281 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
3282 error = EAFNOSUPPORT;
3283 goto done;
fe8ab488
A		 3284 }
3285
f427ee49
A		 3286 if (to->sa_family == AF_INET6 && !(inp->inp_flags & IN6P_IPV6_V6ONLY)) {
3287 struct sockaddr_in6 sin6 = {};
3288 sin6.sin6_family = AF_INET6;
3289 sin6.sin6_len = sizeof(struct sockaddr_in6);
3290 sin6.sin6_port = satosin6(to)->sin6_port;
3291 sin6.sin6_addr = satosin6(to)->sin6_addr;
3292 if (IN6_IS_ADDR_V4MAPPED(&(sin6.sin6_addr))) {
3293 in6_sin6_2_sin(&sin, &sin6);
3294 to = (struct sockaddr *)&sin;
3295 }
3296 }
3297
3298 if (to->sa_family == AF_INET6) {
3299 inp->inp_vflag &= ~INP_IPV4;
3300 inp->inp_vflag |= INP_IPV6;
3301 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6);
3302 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6;
3303 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport;
3304 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp);
3305 if (error) {
3306 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error);
3307 error = 0;
3308 }
3309 if (ifp != NULL) {
3310 inp->in6p_last_outifp = ifp;
3311 ifnet_release(ifp);
3312 }
3313 } else if (to->sa_family == AF_INET) {
3314 inp->inp_vflag |= INP_IPV4;
3315 inp->inp_vflag &= ~INP_IPV6;
3316 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in);
3317 fd_cb->local_endpoint.sin.sin_family = AF_INET;
3318 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport;
3319 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0);
3320 if (error) {
3321 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error);
3322 error = 0;
3323 }
3324 if (ifp != NULL) {
3325 inp->inp_last_outifp = ifp;
3326 ifnet_release(ifp);
3327 }
3328 } else {
3329 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family);
3330 }
3331
3332 error = flow_divert_check_no_cellular(fd_cb) ||
3333 flow_divert_check_no_expensive(fd_cb) ||
3334 flow_divert_check_no_constrained(fd_cb);
3335 if (error) {
3336 goto done;
3337 }
3338
490019cf
A		 3339 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
3340 if (error) {
3341 goto done;
fe8ab488
A		 3342 }
3343
f427ee49
A		 3344 if (!implicit || SOCK_TYPE(so) == SOCK_STREAM) {
3345 flow_divert_set_remote_endpoint(fd_cb, to);
3346 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa), false);
3347 }
3348
3349 if (implicit) {
3350 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT;
3351 }
3352
490019cf
A		 3353 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
3354 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
3355 do_send = 0;
fe8ab488 3356 }
f427ee49
A		 3357
3358 fd_cb->connect_packet = connect_packet;
3359 connect_packet = NULL;
fe8ab488 3360 } else {
490019cf 3361 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
fe8ab488
A		 3362 }
3363
490019cf 3364 if (do_send) {
f427ee49 3365 error = flow_divert_send_connect_packet(fd_cb);
490019cf
A		 3366 if (error) {
3367 goto done;
3368 }
39236c6e 3369
490019cf 3370 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
39236c6e
A		 3371 }
3372
f427ee49
A		 3373 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3374 soisconnected(so);
3375 } else {
3376 soisconnecting(so);
3377 }
39236c6e
A		 3378
3379done:
3380 return error;
3381}
3382
f427ee49
A		 3383errno_t
3384flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
3385{
a991bd8d
A		 3386#if CONTENT_FILTER
3387 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_flags & SOF_CONTENT_FILTER)) {
3388 int error = cfil_sock_attach(so, NULL, to, CFS_CONNECTION_DIR_OUT);
3389 if (error != 0) {
3390 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3391 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error);
3392 return error;
3393 }
3394 }
3395#endif /* CONTENT_FILTER */
3396
f427ee49
A		 3397 return flow_divert_connect_out_internal(so, to, p, false);
3398}
3399
39236c6e 3400static int
813fb2f6
A		 3401flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3402 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
39236c6e 3403{
39236c6e
A		 3404 struct inpcb *inp = sotoinpcb(so);
3405 int error;
3406
3407 if (inp == NULL) {
0a7de745 3408 return EINVAL;
39236c6e
A		 3409 }
3410
813fb2f6 3411 VERIFY(dst != NULL);
39236c6e 3412
813fb2f6 3413 error = flow_divert_connect_out(so, dst, p);
39236c6e 3414
490019cf
A		 3415 if (error != 0) {
3416 return error;
3417 }
3418
3419 /* if there is data, send it */
3420 if (auio != NULL) {
3421 user_ssize_t datalen = 0;
3422
3423 socket_unlock(so, 0);
3424
3425 VERIFY(bytes_written != NULL);
3426
3427 datalen = uio_resid(auio);
3428 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3429 socket_lock(so, 0);
3430
3431 if (error == 0 || error == EWOULDBLOCK) {
3432 *bytes_written = datalen - uio_resid(auio);
3433 }
3434
3435 /*
3436 * sosend returns EWOULDBLOCK if it's a non-blocking
3437 * socket or a timeout occured (this allows to return
3438 * the amount of queued data through sendit()).
3439 *
3440 * However, connectx() returns EINPROGRESS in case of a
3441 * blocking socket. So we change the return value here.
3442 */
3443 if (error == EWOULDBLOCK) {
3444 error = EINPROGRESS;
3445 }
3446 }
3447
39236c6e 3448 if (error == 0 && pcid != NULL) {
0a7de745 3449 *pcid = 1; /* there is only 1 connection for a TCP */
39236c6e
A		 3450 }
3451
0a7de745 3452 return error;
39236c6e
A		 3453}
3454
3455static int
813fb2f6
A		 3456flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3457 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3458 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3459 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 3460{
0a7de745 3461 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e
A		 3462}
3463
39236c6e 3464static int
813fb2f6
A		 3465flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3466 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3467 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3468 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3469{
0a7de745 3470 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e 3471}
39236c6e
A		 3472
3473static errno_t
490019cf 3474flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e 3475{
0a7de745
A		 3476 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3477 int error = 0;
39236c6e 3478 struct inpcb *inp;
bca245ac
A		 3479#if CONTENT_FILTER
3480 struct m_tag *cfil_tag = NULL;
3481#endif
39236c6e
A		 3482
3483 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3484
3485 inp = sotoinpcb(so);
3486 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3487 error = ECONNRESET;
3488 goto done;
3489 }
3490
3491 if (control && mbuf_len(control) > 0) {
3492 error = EINVAL;
3493 goto done;
3494 }
3495
3496 if (flags & MSG_OOB) {
3497 error = EINVAL;
3498 goto done; /* We don't support OOB data */
3499 }
0a7de745 3500
f427ee49
A		 3501#if CONTENT_FILTER
3502 /*
3503 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3504 * retrieve the CFIL saved remote address from the mbuf and use it.
3505 */
3506 if (to == NULL && so->so_cfil_db) {
3507 struct sockaddr *cfil_faddr = NULL;
3508 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3509 if (cfil_tag) {
3510 to = (struct sockaddr *)(void *)cfil_faddr;
3511 }
3512 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
39236c6e 3513 }
f427ee49 3514#endif
39236c6e
A		 3515
3516 /* Implicit connect */
3517 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3518 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
cb323159 3519
f427ee49 3520 error = flow_divert_connect_out_internal(so, to, p, true);
39236c6e
A		 3521 if (error) {
3522 goto done;
3523 }
490019cf
A		 3524
3525 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3526 /* Open up the send window so that the data will get sent right away */
f427ee49
A		 3527 fd_cb->send_window = (uint32_t)mbuf_pkthdr_len(data);
3528 }
3529 } else {
3530 error = flow_divert_check_no_cellular(fd_cb) ||
3531 flow_divert_check_no_expensive(fd_cb) ||
3532 flow_divert_check_no_constrained(fd_cb);
3533 if (error) {
3534 goto done;
490019cf 3535 }
39236c6e
A		 3536 }
3537
3538 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3539
3540 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3541 error = flow_divert_send_app_data(fd_cb, data, to);
f427ee49
A		 3542
3543 data = NULL;
3544
39236c6e
A		 3545 if (error) {
3546 goto done;
3547 }
3548
39236c6e
A		 3549 if (flags & PRUS_EOF) {
3550 flow_divert_shutdown(so);
3551 }
3552
3553done:
3554 if (data) {
3e170ce0 3555 mbuf_freem(data);
39236c6e
A		 3556 }
3557 if (control) {
3558 mbuf_free(control);
3559 }
bca245ac
A		 3560#if CONTENT_FILTER
3561 if (cfil_tag) {
3562 m_tag_free(cfil_tag);
3563 }
3564#endif
3565
39236c6e
A		 3566 return error;
3567}
3568
490019cf
A		 3569static int
3570flow_divert_preconnect(struct socket *so)
3571{
490019cf 3572 int error = 0;
f427ee49 3573 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
490019cf 3574
f427ee49 3575 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
490019cf 3576
f427ee49
A		 3577 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3578 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3579 error = flow_divert_send_connect_packet(so->so_fd_pcb);
490019cf 3580 if (error) {
f427ee49 3581 return error;
490019cf
A		 3582 }
3583
3584 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3585 }
3586
5ba3f43e 3587 soclearfastopen(so);
490019cf
A		 3588
3589 return error;
3590}
3591
39236c6e
A		 3592static void
3593flow_divert_set_protosw(struct socket *so)
3594{
39236c6e
A		 3595 if (SOCK_DOM(so) == PF_INET) {
3596 so->so_proto = &g_flow_divert_in_protosw;
f427ee49 3597 } else {
39236c6e
A		 3598 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3599 }
39236c6e
A		 3600}
3601
3e170ce0
A		 3602static void
3603flow_divert_set_udp_protosw(struct socket *so)
3604{
0a7de745
A		 3605 if (SOCK_DOM(so) == PF_INET) {
3606 so->so_proto = &g_flow_divert_in_udp_protosw;
f427ee49 3607 } else {
0a7de745
A		 3608 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3609 }
39236c6e
A		 3610}
3611
3e170ce0
A		 3612errno_t
3613flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3614{
0a7de745 3615 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3e170ce0 3616 struct inpcb *inp;
0a7de745 3617 int error = 0;
3e170ce0
A		 3618
3619 inp = sotoinpcb(so);
3620 if (inp == NULL) {
0a7de745
A		 3621 return EINVAL;
3622 }
3623
3624 if (fd_cb == NULL) {
f427ee49
A		 3625 error = flow_divert_pcb_init(so);
3626 fd_cb = so->so_fd_pcb;
3627 if (error != 0 || fd_cb == NULL) {
0a7de745
A		 3628 goto done;
3629 }
3630 }
3631 return flow_divert_data_out(so, flags, data, to, control, p);
3e170ce0
A		 3632
3633done:
0a7de745
A		 3634 if (data) {
3635 mbuf_freem(data);
3636 }
3637 if (control) {
3638 mbuf_free(control);
3639 }
3e170ce0 3640
0a7de745 3641 return error;
3e170ce0
A		 3642}
3643
f427ee49
A		 3644static errno_t
3645flow_divert_pcb_init_internal(struct socket *so, uint32_t ctl_unit, uint32_t aggregate_unit)
39236c6e
A		 3646{
3647 errno_t error = 0;
3648 struct flow_divert_pcb *fd_cb;
f427ee49
A		 3649 uint32_t agg_unit = aggregate_unit;
3650 uint32_t group_unit = flow_divert_derive_kernel_control_unit(ctl_unit, &agg_unit);
3651
3652 if (group_unit == 0) {
3653 return EINVAL;
3654 }
39236c6e
A		 3655
3656 if (so->so_flags & SOF_FLOW_DIVERT) {
3657 return EALREADY;
3658 }
0a7de745 3659
39236c6e
A		 3660 fd_cb = flow_divert_pcb_create(so);
3661 if (fd_cb != NULL) {
f427ee49
A		 3662 so->so_fd_pcb = fd_cb;
3663 so->so_flags |= SOF_FLOW_DIVERT;
3664 fd_cb->control_group_unit = group_unit;
3665 fd_cb->policy_control_unit = ctl_unit;
3666 fd_cb->aggregate_unit = agg_unit;
3667
3668 error = flow_divert_pcb_insert(fd_cb, group_unit);
39236c6e
A		 3669 if (error) {
3670 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
f427ee49
A		 3671 so->so_fd_pcb = NULL;
3672 so->so_flags &= ~SOF_FLOW_DIVERT;
39236c6e
A		 3673 FDRELEASE(fd_cb);
3674 } else {
3e170ce0
A		 3675 if (SOCK_TYPE(so) == SOCK_STREAM) {
3676 flow_divert_set_protosw(so);
3677 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3678 flow_divert_set_udp_protosw(so);
3679 }
39236c6e
A		 3680
3681 FDLOG0(LOG_INFO, fd_cb, "Created");
3682 }
3683 } else {
3684 error = ENOMEM;
3685 }
3686
3687 return error;
3688}
3689
f427ee49
A		 3690errno_t
3691flow_divert_pcb_init(struct socket *so)
3692{
3693 struct inpcb *inp = sotoinpcb(so);
3694 uint32_t aggregate_units = 0;
3695 uint32_t ctl_unit = necp_socket_get_flow_divert_control_unit(inp, &aggregate_units);
3696 return flow_divert_pcb_init_internal(so, ctl_unit, aggregate_units);
3697}
3698
39236c6e
A		 3699errno_t
3700flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3701{
f427ee49
A		 3702 uint32_t ctl_unit = 0;
3703 uint32_t key_unit = 0;
3704 uint32_t aggregate_unit = 0;
3705 int error = 0;
3706 int hmac_error = 0;
3707 mbuf_t token = NULL;
39236c6e
A		 3708
3709 if (so->so_flags & SOF_FLOW_DIVERT) {
3710 error = EALREADY;
3711 goto done;
3712 }
3713
3714 if (g_init_result) {
3715 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3716 error = ENOPROTOOPT;
3717 goto done;
3718 }
3719
3e170ce0
A		 3720 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3721 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
f427ee49 3722 (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6)) {
39236c6e
A		 3723 error = EINVAL;
3724 goto done;
3725 } else {
3e170ce0
A		 3726 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3727 struct tcpcb *tp = sototcpcb(so);
3728 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3729 error = EINVAL;
3730 goto done;
3731 }
39236c6e
A		 3732 }
3733 }
3734
3735 error = soopt_getm(sopt, &token);
3736 if (error) {
5c9f4661 3737 token = NULL;
39236c6e
A		 3738 goto done;
3739 }
3740
3741 error = soopt_mcopyin(sopt, token);
3742 if (error) {
5c9f4661 3743 token = NULL;
39236c6e
A		 3744 goto done;
3745 }
3746
3747 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3748 if (!error) {
3749 key_unit = ntohl(key_unit);
4d15aeb1
A		 3750 if (key_unit >= GROUP_COUNT_MAX) {
3751 key_unit = 0;
3752 }
39236c6e
A		 3753 } else if (error != ENOENT) {
3754 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3755 goto done;
3756 } else {
3757 key_unit = 0;
3758 }
3759
3760 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3761 if (error) {
3762 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3763 goto done;
3764 }
3765
f427ee49
A		 3766 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_AGGREGATE_UNIT, sizeof(aggregate_unit), (void *)&aggregate_unit, NULL);
3767 if (error && error != ENOENT) {
3768 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the aggregate unit from the token: %d", error);
39236c6e
A		 3769 goto done;
3770 }
3771
f427ee49
A		 3772 /* A valid kernel control unit is required */
3773 ctl_unit = ntohl(ctl_unit);
3774 aggregate_unit = ntohl(aggregate_unit);
39236c6e 3775
f427ee49
A		 3776 if (ctl_unit > 0 && ctl_unit < GROUP_COUNT_MAX) {
3777 socket_unlock(so, 0);
3778 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3779 socket_lock(so, 0);
39236c6e 3780
f427ee49
A		 3781 if (hmac_error && hmac_error != ENOENT) {
3782 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3783 error = hmac_error;
3784 goto done;
3785 }
39236c6e
A		 3786 }
3787
f427ee49
A		 3788 error = flow_divert_pcb_init_internal(so, ctl_unit, aggregate_unit);
3789 if (error == 0) {
3790 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3791 int log_level = LOG_NOTICE;
39236c6e 3792
f427ee49
A		 3793 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
3794 if (error == 0) {
3795 fd_cb->log_level = (uint8_t)log_level;
39236c6e 3796 }
f427ee49
A		 3797 error = 0;
3798
3799 fd_cb->connect_token = token;
3800 token = NULL;
39236c6e
A		 3801 }
3802
490019cf
A		 3803 if (hmac_error == 0) {
3804 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3805 if (fd_cb != NULL) {
3806 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3807 }
3808 }
3809
39236c6e
A		 3810done:
3811 if (token != NULL) {
3812 mbuf_freem(token);
3813 }
3814
3815 return error;
3816}
3817
3818errno_t
3819flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3820{
0a7de745
A		 3821 uint32_t ctl_unit;
3822 int error = 0;
3823 uint8_t hmac[SHA_DIGEST_LENGTH];
3824 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3825 mbuf_t token = NULL;
3826 struct flow_divert_group *control_group = NULL;
39236c6e
A		 3827
3828 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3829 error = EINVAL;
3830 goto done;
3831 }
3832
3833 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3834
3835 if (fd_cb->group == NULL) {
3836 error = EINVAL;
3837 goto done;
3838 }
3839
3840 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3841 if (error) {
3842 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3843 goto done;
3844 }
3845
3846 ctl_unit = htonl(fd_cb->group->ctl_unit);
3847
3848 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3849 if (error) {
3850 goto done;
3851 }
3852
3853 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3854 if (error) {
3855 goto done;
3856 }
3857
490019cf 3858 if (fd_cb->app_data != NULL) {
f427ee49 3859 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_data);
490019cf
A		 3860 if (error) {
3861 goto done;
3862 }
3863 }
3864
39236c6e
A		 3865 socket_unlock(so, 0);
3866 lck_rw_lock_shared(&g_flow_divert_group_lck);
3867
3868 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
0a7de745 3869 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
39236c6e
A		 3870 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3871 }
3872
3873 if (control_group != NULL) {
3874 lck_rw_lock_shared(&control_group->lck);
3875 ctl_unit = htonl(control_group->ctl_unit);
3876 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3877 if (!error) {
3878 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3879 }
3880 lck_rw_done(&control_group->lck);
3881 } else {
3882 error = ENOPROTOOPT;
3883 }
3884
3885 lck_rw_done(&g_flow_divert_group_lck);
3886 socket_lock(so, 0);
3887
3888 if (error) {
3889 goto done;
3890 }
3891
3892 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3893 if (error) {
3894 goto done;
3895 }
3896
39037602
A		 3897 if (sopt->sopt_val == USER_ADDR_NULL) {
3898 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3899 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3900 goto done;
3901 }
3902
39236c6e
A		 3903 error = soopt_mcopyout(sopt, token);
3904 if (error) {
0a7de745 3905 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
39236c6e
A		 3906 goto done;
3907 }
3908
3909done:
3910 if (token != NULL) {
3911 mbuf_freem(token);
3912 }
3913
3914 return error;
3915}
3916
3917static errno_t
3918flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3919{
0a7de745
A		 3920 struct flow_divert_group *new_group = NULL;
3921 int error = 0;
39236c6e
A		 3922
3923 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3924 error = EINVAL;
3925 goto done;
3926 }
3927
3928 *unitinfo = NULL;
3929
f427ee49 3930 new_group = zalloc_flags(flow_divert_group_zone, Z_WAITOK | Z_ZERO);
39236c6e
A		 3931 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3932 RB_INIT(&new_group->pcb_tree);
3933 new_group->ctl_unit = sac->sc_unit;
3934 MBUFQ_INIT(&new_group->send_queue);
39037602 3935 new_group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 3936
3937 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3938
3939 if (g_flow_divert_groups == NULL) {
3940 MALLOC(g_flow_divert_groups,
0a7de745
A		 3941 struct flow_divert_group **,
3942 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3943 M_TEMP,
3944 M_WAITOK | M_ZERO);
39236c6e
A		 3945 }
3946
3947 if (g_flow_divert_groups == NULL) {
3948 error = ENOBUFS;
3949 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3950 error = EALREADY;
3951 } else {
3952 g_flow_divert_groups[sac->sc_unit] = new_group;
3953 g_active_group_count++;
3954 }
3955
3956 lck_rw_done(&g_flow_divert_group_lck);
3957
39236c6e 3958done:
f427ee49
A		 3959 if (error == 0) {
3960 *unitinfo = new_group;
3961 } else if (new_group != NULL) {
3962 zfree(flow_divert_group_zone, new_group);
39236c6e
A		 3963 }
3964 return error;
3965}
3966
3967static errno_t
3968flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3969{
0a7de745
A		 3970 struct flow_divert_group *group = NULL;
3971 errno_t error = 0;
39236c6e
A		 3972
3973 if (unit >= GROUP_COUNT_MAX) {
3974 return EINVAL;
3975 }
3976
f427ee49
A		 3977 if (unitinfo == NULL) {
3978 return 0;
3979 }
3980
39236c6e
A		 3981 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3982
3983 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3984
3985 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3986 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
0a7de745 3987 g_flow_divert_groups, g_active_group_count);
39236c6e
A		 3988 }
3989
3990 group = g_flow_divert_groups[unit];
3991
3992 if (group != (struct flow_divert_group *)unitinfo) {
3993 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3994 }
3995
cb323159
A		 3996 g_flow_divert_groups[unit] = NULL;
3997 g_active_group_count--;
3998
3999 if (g_active_group_count == 0) {
4000 FREE(g_flow_divert_groups, M_TEMP);
4001 g_flow_divert_groups = NULL;
4002 }
4003
4004 lck_rw_done(&g_flow_divert_group_lck);
4005
39236c6e
A		 4006 if (group != NULL) {
4007 flow_divert_close_all(group);
cb323159
A		 4008
4009 lck_rw_lock_exclusive(&group->lck);
4010
39236c6e
A		 4011 if (group->token_key != NULL) {
4012 memset(group->token_key, 0, group->token_key_size);
4013 FREE(group->token_key, M_TEMP);
4014 group->token_key = NULL;
4015 group->token_key_size = 0;
4016 }
39037602
A		 4017
4018 /* Re-set the current trie */
4019 if (group->signing_id_trie.memory != NULL) {
4020 FREE(group->signing_id_trie.memory, M_TEMP);
4021 }
4022 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
4023 group->signing_id_trie.root = NULL_TRIE_IDX;
4024
cb323159
A		 4025 lck_rw_done(&group->lck);
4026
f427ee49 4027 zfree(flow_divert_group_zone, group);
39236c6e
A		 4028 } else {
4029 error = EINVAL;
4030 }
4031
39236c6e
A		 4032 return error;
4033}
4034
4035static errno_t
4036flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
4037{
4038 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
4039}
4040
4041static void
4042flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
4043{
0a7de745 4044 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
39236c6e
A		 4045
4046 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
0a7de745
A		 4047 struct flow_divert_pcb *fd_cb;
4048 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 4049
4050 lck_rw_lock_shared(&g_flow_divert_group_lck);
4051 lck_rw_lock_exclusive(&group->lck);
4052
4053 while (!MBUFQ_EMPTY(&group->send_queue)) {
4054 mbuf_t next_packet;
4055 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
4056 next_packet = MBUFQ_FIRST(&group->send_queue);
4057 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
4058 if (error) {
4059 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
4060 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
4061 lck_rw_done(&group->lck);
4062 lck_rw_done(&g_flow_divert_group_lck);
4063 return;
4064 }
4065 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
4066 }
4067
4068 SLIST_INIT(&tmp_list);
4069
4070 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
4071 FDRETAIN(fd_cb);
4072 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
4073 }
4074
4075 lck_rw_done(&group->lck);
4076
4077 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
4078 FDLOCK(fd_cb);
4079 if (fd_cb->so != NULL) {
4080 socket_lock(fd_cb->so, 0);
4081 if (fd_cb->group != NULL) {
4082 flow_divert_send_buffered_data(fd_cb, FALSE);
4083 }
4084 socket_unlock(fd_cb->so, 0);
4085 }
4086 FDUNLOCK(fd_cb);
4087 FDRELEASE(fd_cb);
4088 }
4089
4090 lck_rw_done(&g_flow_divert_group_lck);
4091 }
4092}
4093
4094static int
4095flow_divert_kctl_init(void)
4096{
0a7de745
A		 4097 struct kern_ctl_reg ctl_reg;
4098 int result;
39236c6e
A		 4099
4100 memset(&ctl_reg, 0, sizeof(ctl_reg));
4101
fe8ab488 4102 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
0a7de745 4103 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
39236c6e
A		 4104 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
4105 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
4106 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
4107
4108 ctl_reg.ctl_connect = flow_divert_kctl_connect;
4109 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
4110 ctl_reg.ctl_send = flow_divert_kctl_send;
4111 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
4112
4113 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
4114
4115 if (result) {
4116 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
4117 return result;
4118 }
4119
4120 return 0;
4121}
4122
4123void
4124flow_divert_init(void)
4125{
4126 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 4127 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 4128
4129 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4130
4131 VERIFY(g_tcp_protosw != NULL);
4132
4133 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4134 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4135
4136 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4137 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
39236c6e
A		 4138 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4139 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
39236c6e
A		 4140 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4141 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4142 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
490019cf 4143 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 4144
4145 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4146 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4147
4148 /*
4149 * Socket filters shouldn't attach/detach to/from this protosw
4150 * since pr_protosw is to be used instead, which points to the
4151 * real protocol; if they do, it is a bug and we should panic.
4152 */
4153 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4154 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4155 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4156 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4157
3e170ce0
A		 4158 /* UDP */
4159 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4160 VERIFY(g_udp_protosw != NULL);
4161
4162 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4163 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4164
4165 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4166 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3e170ce0
A		 4167 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4168 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3e170ce0
A		 4169 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4170 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4171 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3e170ce0
A		 4172 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4173 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 4174 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 4175
4176 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4177 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4178
4179 /*
0a7de745
A		 4180 * Socket filters shouldn't attach/detach to/from this protosw
4181 * since pr_protosw is to be used instead, which points to the
4182 * real protocol; if they do, it is a bug and we should panic.
4183 */
3e170ce0
A		 4184 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4185 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4186 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4187 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4188
39236c6e
A		 4189 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4190
4191 VERIFY(g_tcp6_protosw != NULL);
4192
4193 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4194 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4195
4196 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4197 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
39236c6e
A		 4198 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4199 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
39236c6e
A		 4200 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4201 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4202 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
490019cf 4203 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 4204
4205 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4206 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4207 /*
4208 * Socket filters shouldn't attach/detach to/from this protosw
4209 * since pr_protosw is to be used instead, which points to the
4210 * real protocol; if they do, it is a bug and we should panic.
4211 */
4212 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4213 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4214 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4215 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 4216
4217 /* UDP6 */
4218 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4219
4220 VERIFY(g_udp6_protosw != NULL);
4221
4222 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4223 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4224
4225 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4226 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3e170ce0
A		 4227 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4228 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3e170ce0
A		 4229 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4230 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4231 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3e170ce0
A		 4232 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4233 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 4234 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 4235
4236 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4237 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4238 /*
0a7de745
A		 4239 * Socket filters shouldn't attach/detach to/from this protosw
4240 * since pr_protosw is to be used instead, which points to the
4241 * real protocol; if they do, it is a bug and we should panic.
4242 */
3e170ce0
A		 4243 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4244 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4245 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4246 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
39236c6e
A		 4247
4248 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4249 if (flow_divert_grp_attr == NULL) {
4250 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4251 g_init_result = ENOMEM;
4252 goto done;
4253 }
4254
4255 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4256 if (flow_divert_mtx_grp == NULL) {
4257 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4258 g_init_result = ENOMEM;
4259 goto done;
4260 }
4261
4262 flow_divert_mtx_attr = lck_attr_alloc_init();
4263 if (flow_divert_mtx_attr == NULL) {
4264 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4265 g_init_result = ENOMEM;
4266 goto done;
4267 }
4268
4269 g_init_result = flow_divert_kctl_init();
4270 if (g_init_result) {
4271 goto done;
4272 }
4273
4274 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4275
39236c6e
A		 4276done:
4277 if (g_init_result != 0) {
4278 if (flow_divert_mtx_attr != NULL) {
4279 lck_attr_free(flow_divert_mtx_attr);
4280 flow_divert_mtx_attr = NULL;
4281 }
4282 if (flow_divert_mtx_grp != NULL) {
4283 lck_grp_free(flow_divert_mtx_grp);
4284 flow_divert_mtx_grp = NULL;
4285 }
4286 if (flow_divert_grp_attr != NULL) {
4287 lck_grp_attr_free(flow_divert_grp_attr);
4288 flow_divert_grp_attr = NULL;
4289 }
4290
4291 if (g_flow_divert_kctl_ref != NULL) {
4292 ctl_deregister(g_flow_divert_kctl_ref);
4293 g_flow_divert_kctl_ref = NULL;
4294 }
4295 }
4296}
mirror of XNU