]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.101.1.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2017, 2020, 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <kern/task.h>
49 #include <mach/task_info.h>
50 #include <net/if_var.h>
51 #include <net/route.h>
52 #include <net/flowhash.h>
53 #include <net/ntstat.h>
54 #include <net/content_filter.h>
55 #include <net/necp.h>
56 #include <netinet/in.h>
57 #include <netinet/in_var.h>
58 #include <netinet/tcp.h>
59 #include <netinet/tcp_var.h>
60 #include <netinet/tcp_fsm.h>
61 #include <netinet/flow_divert.h>
62 #include <netinet/flow_divert_proto.h>
63 #include <netinet6/in6_pcb.h>
64 #include <netinet6/ip6protosw.h>
65 #include <dev/random/randomdev.h>
66 #include <libkern/crypto/sha1.h>
67 #include <libkern/crypto/crypto_internal.h>
68 #include <os/log.h>
69 #include <corecrypto/cc.h>
70 #if CONTENT_FILTER
71 #include <net/content_filter.h>
72 #endif /* CONTENT_FILTER */
73
74 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75 #define FLOW_DIVERT_READ_CLOSED 0x00000002
76 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
79 #define FLOW_DIVERT_HAS_HMAC 0x00000040
80 #define FLOW_DIVERT_NOTIFY_ON_RECEIVED 0x00000080
81 #define FLOW_DIVERT_IMPLICIT_CONNECT 0x00000100
82 #define FLOW_DIVERT_DID_SET_LOCAL_ADDR 0x00000200
83 #define FLOW_DIVERT_HAS_TOKEN 0x00000400
84 #define FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR 0x00000800
85
86 #define FDLOG(level, pcb, format, ...) \
87 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
88
89 #define FDLOG0(level, pcb, msg) \
90 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
91
92 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
93 #define FDRELEASE(pcb) \
94 do { \
95 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
96 flow_divert_pcb_destroy(pcb); \
97 } \
98 } while (0)
99
100 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
101 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
102
103 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
104 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
105
106 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
107
108 #define GROUP_COUNT_MAX 31
109 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
110 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
111 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
112
113 struct flow_divert_trie_node {
114 uint16_t start;
115 uint16_t length;
116 uint16_t child_map;
117 };
118
119 #define CHILD_MAP_SIZE 256
120 #define NULL_TRIE_IDX 0xffff
121 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
122 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
123 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
124
125 static struct flow_divert_pcb nil_pcb;
126
127 decl_lck_rw_data(static, g_flow_divert_group_lck);
128 static struct flow_divert_group **g_flow_divert_groups = NULL;
129 static uint32_t g_active_group_count = 0;
130
131 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
132 static lck_attr_t *flow_divert_mtx_attr = NULL;
133 static lck_grp_t *flow_divert_mtx_grp = NULL;
134 static errno_t g_init_result = 0;
135
136 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
137
138 static struct protosw g_flow_divert_in_protosw;
139 static struct pr_usrreqs g_flow_divert_in_usrreqs;
140 static struct protosw g_flow_divert_in_udp_protosw;
141 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
142 static struct ip6protosw g_flow_divert_in6_protosw;
143 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
144 static struct ip6protosw g_flow_divert_in6_udp_protosw;
145 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
146
147 static struct protosw *g_tcp_protosw = NULL;
148 static struct ip6protosw *g_tcp6_protosw = NULL;
149 static struct protosw *g_udp_protosw = NULL;
150 static struct ip6protosw *g_udp6_protosw = NULL;
151
152 ZONE_DECLARE(flow_divert_group_zone, "flow_divert_group",
153 sizeof(struct flow_divert_group), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
154 ZONE_DECLARE(flow_divert_pcb_zone, "flow_divert_pcb",
155 sizeof(struct flow_divert_pcb), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
156
157 static errno_t
158 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
159
160 static boolean_t
161 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
162
163 static int
164 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
165
166 struct sockaddr *
167 flow_divert_get_buffered_target_address(mbuf_t buffer);
168
169 static void
170 flow_divert_disconnect_socket(struct socket *so);
171
172 static inline uint8_t
173 flow_divert_syslog_type_to_oslog_type(int syslog_type)
174 {
175 switch (syslog_type) {
176 case LOG_ERR: return OS_LOG_TYPE_ERROR;
177 case LOG_INFO: return OS_LOG_TYPE_INFO;
178 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
179 default: return OS_LOG_TYPE_DEFAULT;
180 }
181 }
182
183 static inline int
184 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
185 {
186 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
187 }
188
189 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
190 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
191
192 static const char *
193 flow_divert_packet_type2str(uint8_t packet_type)
194 {
195 switch (packet_type) {
196 case FLOW_DIVERT_PKT_CONNECT:
197 return "connect";
198 case FLOW_DIVERT_PKT_CONNECT_RESULT:
199 return "connect result";
200 case FLOW_DIVERT_PKT_DATA:
201 return "data";
202 case FLOW_DIVERT_PKT_CLOSE:
203 return "close";
204 case FLOW_DIVERT_PKT_READ_NOTIFY:
205 return "read notification";
206 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
207 return "properties update";
208 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
209 return "app map create";
210 default:
211 return "unknown";
212 }
213 }
214
215 static struct flow_divert_pcb *
216 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
217 {
218 struct flow_divert_pcb key_item;
219 struct flow_divert_pcb *fd_cb = NULL;
220
221 key_item.hash = hash;
222
223 lck_rw_lock_shared(&group->lck);
224 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
225 FDRETAIN(fd_cb);
226 lck_rw_done(&group->lck);
227
228 return fd_cb;
229 }
230
231 static errno_t
232 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
233 {
234 errno_t error = 0;
235 struct flow_divert_pcb *exist = NULL;
236 struct flow_divert_group *group;
237 static uint32_t g_nextkey = 1;
238 static uint32_t g_hash_seed = 0;
239 int try_count = 0;
240
241 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
242 return EINVAL;
243 }
244
245 socket_unlock(fd_cb->so, 0);
246 lck_rw_lock_shared(&g_flow_divert_group_lck);
247
248 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
249 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
250 error = ENETUNREACH;
251 goto done;
252 }
253
254 group = g_flow_divert_groups[ctl_unit];
255 if (group == NULL) {
256 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
257 error = ENETUNREACH;
258 goto done;
259 }
260
261 socket_lock(fd_cb->so, 0);
262
263 do {
264 uint32_t key[2];
265 uint32_t idx;
266
267 key[0] = g_nextkey++;
268 key[1] = RandomULong();
269
270 if (g_hash_seed == 0) {
271 g_hash_seed = RandomULong();
272 }
273
274 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
275
276 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
277 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
278 if (curr_group != NULL && curr_group != group) {
279 lck_rw_lock_shared(&curr_group->lck);
280 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
281 lck_rw_done(&curr_group->lck);
282 if (exist != NULL) {
283 break;
284 }
285 }
286 }
287
288 if (exist == NULL) {
289 lck_rw_lock_exclusive(&group->lck);
290 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
291 lck_rw_done(&group->lck);
292 }
293 } while (exist != NULL && try_count++ < 3);
294
295 if (exist == NULL) {
296 fd_cb->group = group;
297 FDRETAIN(fd_cb); /* The group now has a reference */
298 } else {
299 fd_cb->hash = 0;
300 error = EEXIST;
301 }
302
303 socket_unlock(fd_cb->so, 0);
304
305 done:
306 lck_rw_done(&g_flow_divert_group_lck);
307 socket_lock(fd_cb->so, 0);
308
309 return error;
310 }
311
312 static struct flow_divert_pcb *
313 flow_divert_pcb_create(socket_t so)
314 {
315 struct flow_divert_pcb *new_pcb = NULL;
316
317 new_pcb = zalloc_flags(flow_divert_pcb_zone, Z_WAITOK | Z_ZERO);
318 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
319 new_pcb->so = so;
320 new_pcb->log_level = nil_pcb.log_level;
321
322 FDRETAIN(new_pcb); /* Represents the socket's reference */
323
324 return new_pcb;
325 }
326
327 static void
328 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
329 {
330 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u",
331 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
332
333 if (fd_cb->connect_token != NULL) {
334 mbuf_freem(fd_cb->connect_token);
335 }
336 if (fd_cb->connect_packet != NULL) {
337 mbuf_freem(fd_cb->connect_packet);
338 }
339 if (fd_cb->app_data != NULL) {
340 FREE(fd_cb->app_data, M_TEMP);
341 }
342 if (fd_cb->original_remote_endpoint != NULL) {
343 FREE(fd_cb->original_remote_endpoint, M_SONAME);
344 }
345 zfree(flow_divert_pcb_zone, fd_cb);
346 }
347
348 static void
349 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
350 {
351 if (fd_cb->group != NULL) {
352 struct flow_divert_group *group = fd_cb->group;
353 lck_rw_lock_exclusive(&group->lck);
354 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
355 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
356 fd_cb->group = NULL;
357 FDRELEASE(fd_cb); /* Release the group's reference */
358 lck_rw_done(&group->lck);
359 }
360 }
361
362 static int
363 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
364 {
365 struct flow_divert_packet_header hdr;
366 int error = 0;
367
368 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
369 if (error) {
370 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
371 return error;
372 }
373
374 hdr.packet_type = packet_type;
375 hdr.conn_id = htonl(fd_cb->hash);
376
377 /* Lay down the header */
378 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
379 if (error) {
380 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
381 mbuf_freem(*packet);
382 *packet = NULL;
383 return error;
384 }
385
386 return 0;
387 }
388
389 static int
390 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
391 {
392 uint32_t net_length = htonl(length);
393 int error = 0;
394
395 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
396 if (error) {
397 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
398 return error;
399 }
400
401 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
402 if (error) {
403 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
404 return error;
405 }
406
407 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
408 if (error) {
409 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
410 return error;
411 }
412
413 return error;
414 }
415
416 static int
417 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
418 {
419 size_t cursor = offset;
420 int error = 0;
421 uint32_t curr_length;
422 uint8_t curr_type;
423
424 *err = 0;
425
426 do {
427 if (!next) {
428 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
429 if (error) {
430 *err = ENOENT;
431 return -1;
432 }
433 } else {
434 next = 0;
435 curr_type = FLOW_DIVERT_TLV_NIL;
436 }
437
438 if (curr_type != type) {
439 cursor += sizeof(curr_type);
440 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
441 if (error) {
442 *err = error;
443 return -1;
444 }
445
446 cursor += (sizeof(curr_length) + ntohl(curr_length));
447 }
448 } while (curr_type != type);
449
450 return (int)cursor;
451 }
452
453 static int
454 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
455 {
456 int error = 0;
457 uint32_t length;
458 int tlv_offset;
459
460 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
461 if (tlv_offset < 0) {
462 return error;
463 }
464
465 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
466 if (error) {
467 return error;
468 }
469
470 length = ntohl(length);
471
472 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
473
474 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
475 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
476 return EINVAL;
477 }
478
479 if (val_size != NULL) {
480 *val_size = length;
481 }
482
483 if (buff != NULL && buff_len > 0) {
484 memset(buff, 0, buff_len);
485 size_t to_copy = (length < buff_len) ? length : buff_len;
486 error = mbuf_copydata(packet, data_offset, to_copy, buff);
487 if (error) {
488 return error;
489 }
490 }
491
492 return 0;
493 }
494
495 static int
496 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
497 {
498 mbuf_t curr_mbuf = packet;
499
500 if (g_crypto_funcs == NULL || group->token_key == NULL) {
501 return ENOPROTOOPT;
502 }
503
504 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
505 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
506
507 while (curr_mbuf != NULL) {
508 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
509 curr_mbuf = mbuf_next(curr_mbuf);
510 }
511
512 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
513
514 return 0;
515 }
516
517 static int
518 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
519 {
520 int error = 0;
521 struct flow_divert_group *group = NULL;
522 int hmac_offset;
523 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
524 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
525 mbuf_t tail;
526
527 lck_rw_lock_shared(&g_flow_divert_group_lck);
528
529 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
530 group = g_flow_divert_groups[ctl_unit];
531 }
532
533 if (group == NULL) {
534 lck_rw_done(&g_flow_divert_group_lck);
535 return ENOPROTOOPT;
536 }
537
538 lck_rw_lock_shared(&group->lck);
539
540 if (group->token_key == NULL) {
541 error = ENOPROTOOPT;
542 goto done;
543 }
544
545 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
546 if (hmac_offset < 0) {
547 goto done;
548 }
549
550 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
551 if (error) {
552 goto done;
553 }
554
555 /* Chop off the HMAC TLV */
556 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
557 if (error) {
558 goto done;
559 }
560
561 mbuf_free(tail);
562
563 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
564 if (error) {
565 goto done;
566 }
567
568 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
569 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
570 error = EINVAL;
571 goto done;
572 }
573
574 done:
575 lck_rw_done(&group->lck);
576 lck_rw_done(&g_flow_divert_group_lck);
577 return error;
578 }
579
580 static void
581 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send)
582 {
583 struct inpcb *inp = NULL;
584 struct ifnet *ifp = NULL;
585 Boolean cell = FALSE;
586 Boolean wifi = FALSE;
587 Boolean wired = FALSE;
588
589 inp = sotoinpcb(fd_cb->so);
590 if (inp == NULL) {
591 return;
592 }
593
594 if (inp->inp_vflag & INP_IPV4) {
595 ifp = inp->inp_last_outifp;
596 } else if (inp->inp_vflag & INP_IPV6) {
597 ifp = inp->in6p_last_outifp;
598 }
599 if (ifp != NULL) {
600 cell = IFNET_IS_CELLULAR(ifp);
601 wifi = (!cell && IFNET_IS_WIFI(ifp));
602 wired = (!wifi && IFNET_IS_WIRED(ifp));
603 }
604
605 if (send) {
606 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
607 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
608 } else {
609 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
610 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
611 }
612 inp_set_activity_bitmap(inp);
613 }
614
615 static errno_t
616 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
617 {
618 struct inpcb *inp = sotoinpcb(fd_cb->so);
619 if (INP_NO_CELLULAR(inp)) {
620 struct ifnet *ifp = NULL;
621 if (inp->inp_vflag & INP_IPV4) {
622 ifp = inp->inp_last_outifp;
623 } else if (inp->inp_vflag & INP_IPV6) {
624 ifp = inp->in6p_last_outifp;
625 }
626 if (ifp != NULL && IFNET_IS_CELLULAR(ifp)) {
627 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied");
628 return EHOSTUNREACH;
629 }
630 }
631 return 0;
632 }
633
634 static errno_t
635 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
636 {
637 struct inpcb *inp = sotoinpcb(fd_cb->so);
638 if (INP_NO_EXPENSIVE(inp)) {
639 struct ifnet *ifp = NULL;
640 if (inp->inp_vflag & INP_IPV4) {
641 ifp = inp->inp_last_outifp;
642 } else if (inp->inp_vflag & INP_IPV6) {
643 ifp = inp->in6p_last_outifp;
644 }
645 if (ifp != NULL && IFNET_IS_EXPENSIVE(ifp)) {
646 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied");
647 return EHOSTUNREACH;
648 }
649 }
650 return 0;
651 }
652
653 static errno_t
654 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
655 {
656 struct inpcb *inp = sotoinpcb(fd_cb->so);
657 if (INP_NO_CONSTRAINED(inp)) {
658 struct ifnet *ifp = NULL;
659 if (inp->inp_vflag & INP_IPV4) {
660 ifp = inp->inp_last_outifp;
661 } else if (inp->inp_vflag & INP_IPV6) {
662 ifp = inp->in6p_last_outifp;
663 }
664 if (ifp != NULL && IFNET_IS_CONSTRAINED(ifp)) {
665 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied");
666 return EHOSTUNREACH;
667 }
668 }
669 return 0;
670 }
671
672 static void
673 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
674 {
675 if (how != SHUT_RD) {
676 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
677 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
678 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
679 /* If the tunnel is not accepting writes any more, then flush the send buffer */
680 sbflush(&fd_cb->so->so_snd);
681 }
682 }
683 if (how != SHUT_WR) {
684 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
685 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
686 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
687 }
688 }
689 }
690
691 static uint16_t
692 trie_node_alloc(struct flow_divert_trie *trie)
693 {
694 if (trie->nodes_free_next < trie->nodes_count) {
695 uint16_t node_idx = trie->nodes_free_next++;
696 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
697 return node_idx;
698 } else {
699 return NULL_TRIE_IDX;
700 }
701 }
702
703 static uint16_t
704 trie_child_map_alloc(struct flow_divert_trie *trie)
705 {
706 if (trie->child_maps_free_next < trie->child_maps_count) {
707 return trie->child_maps_free_next++;
708 } else {
709 return NULL_TRIE_IDX;
710 }
711 }
712
713 static uint16_t
714 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
715 {
716 uint16_t start = trie->bytes_free_next;
717 if (start + bytes_size <= trie->bytes_count) {
718 if (start != bytes_idx) {
719 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
720 }
721 trie->bytes_free_next += bytes_size;
722 return start;
723 } else {
724 return NULL_TRIE_IDX;
725 }
726 }
727
728 static uint16_t
729 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
730 {
731 uint16_t current = trie->root;
732 uint16_t child = trie->root;
733 uint16_t string_end = string_start + (uint16_t)string_len;
734 uint16_t string_idx = string_start;
735 uint16_t string_remainder = (uint16_t)string_len;
736
737 while (child != NULL_TRIE_IDX) {
738 uint16_t parent = current;
739 uint16_t node_idx;
740 uint16_t current_end;
741
742 current = child;
743 child = NULL_TRIE_IDX;
744
745 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
746
747 for (node_idx = TRIE_NODE(trie, current).start;
748 node_idx < current_end &&
749 string_idx < string_end &&
750 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
751 node_idx++, string_idx++) {
752 ;
753 }
754
755 string_remainder = string_end - string_idx;
756
757 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
758 /*
759 * We did not reach the end of the current node's string.
760 * We need to split the current node into two:
761 * 1. A new node that contains the prefix of the node that matches
762 * the prefix of the string being inserted.
763 * 2. The current node modified to point to the remainder
764 * of the current node's string.
765 */
766 uint16_t prefix = trie_node_alloc(trie);
767 if (prefix == NULL_TRIE_IDX) {
768 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
769 return NULL_TRIE_IDX;
770 }
771
772 /*
773 * Prefix points to the portion of the current nodes's string that has matched
774 * the input string thus far.
775 */
776 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
777 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
778
779 /*
780 * Prefix has the current node as the child corresponding to the first byte
781 * after the split.
782 */
783 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
784 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
785 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
786 return NULL_TRIE_IDX;
787 }
788 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
789
790 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
791 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
792
793 /* Current node is adjusted to point to the remainder */
794 TRIE_NODE(trie, current).start = node_idx;
795 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
796
797 /* We want to insert the new leaf (if any) as a child of the prefix */
798 current = prefix;
799 }
800
801 if (string_remainder > 0) {
802 /*
803 * We still have bytes in the string that have not been matched yet.
804 * If the current node has children, iterate to the child corresponding
805 * to the next byte in the string.
806 */
807 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
808 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
809 }
810 }
811 } /* while (child != NULL_TRIE_IDX) */
812
813 if (string_remainder > 0) {
814 /* Add a new leaf containing the remainder of the string */
815 uint16_t leaf = trie_node_alloc(trie);
816 if (leaf == NULL_TRIE_IDX) {
817 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
818 return NULL_TRIE_IDX;
819 }
820
821 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
822 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
823 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
824 return NULL_TRIE_IDX;
825 }
826 TRIE_NODE(trie, leaf).length = string_remainder;
827
828 /* Set the new leaf as the child of the current node */
829 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
830 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
831 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
832 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
833 return NULL_TRIE_IDX;
834 }
835 }
836 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
837 current = leaf;
838 } /* else duplicate or this string is a prefix of one of the existing strings */
839
840 return current;
841 }
842
843 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
844 static uint16_t
845 flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
846 {
847 uint16_t current = trie->root;
848 uint16_t string_idx = 0;
849
850 while (current != NULL_TRIE_IDX) {
851 uint16_t next = NULL_TRIE_IDX;
852 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
853 uint16_t node_idx;
854
855 for (node_idx = TRIE_NODE(trie, current).start;
856 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
857 node_idx++, string_idx++) {
858 ;
859 }
860
861 if (node_idx == node_end) {
862 if (string_bytes[string_idx] == '\0') {
863 return current; /* Got an exact match */
864 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
865 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
866 return current; /* Got an apple webclip id prefix match */
867 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
868 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
869 }
870 }
871 current = next;
872 }
873
874 return NULL_TRIE_IDX;
875 }
876
877 struct uuid_search_info {
878 uuid_t target_uuid;
879 char *found_signing_id;
880 boolean_t found_multiple_signing_ids;
881 proc_t found_proc;
882 };
883
884 static int
885 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
886 {
887 struct uuid_search_info *info = (struct uuid_search_info *)arg;
888 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
889
890 if (info->found_signing_id != NULL) {
891 if (!info->found_multiple_signing_ids) {
892 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
893 info->found_proc = p;
894 result = PROC_CLAIMED_DONE;
895 } else {
896 uuid_string_t uuid_str;
897 uuid_unparse(info->target_uuid, uuid_str);
898 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
899 }
900 FREE(info->found_signing_id, M_TEMP);
901 info->found_signing_id = NULL;
902 }
903
904 if (result == PROC_RETURNED_DONE) {
905 uuid_string_t uuid_str;
906 uuid_unparse(info->target_uuid, uuid_str);
907 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
908 }
909
910 return result;
911 }
912
913 static int
914 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
915 {
916 struct uuid_search_info *info = (struct uuid_search_info *)arg;
917 int include = 0;
918
919 if (info->found_multiple_signing_ids) {
920 return include;
921 }
922
923 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
924 if (include) {
925 const char *signing_id = cs_identity_get(p);
926 if (signing_id != NULL) {
927 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
928 size_t signing_id_size = strlen(signing_id) + 1;
929 if (info->found_signing_id == NULL) {
930 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
931 memcpy(info->found_signing_id, signing_id, signing_id_size);
932 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
933 info->found_multiple_signing_ids = TRUE;
934 }
935 } else {
936 info->found_multiple_signing_ids = TRUE;
937 }
938 include = !info->found_multiple_signing_ids;
939 }
940
941 return include;
942 }
943
944 static proc_t
945 flow_divert_find_proc_by_uuid(uuid_t uuid)
946 {
947 struct uuid_search_info info;
948
949 if (LOG_INFO <= nil_pcb.log_level) {
950 uuid_string_t uuid_str;
951 uuid_unparse(uuid, uuid_str);
952 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
953 }
954
955 memset(&info, 0, sizeof(info));
956 info.found_proc = PROC_NULL;
957 uuid_copy(info.target_uuid, uuid);
958
959 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
960
961 return info.found_proc;
962 }
963
964 static int
965 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
966 {
967 int error = 0;
968 uint8_t *cdhash = NULL;
969 audit_token_t audit_token = {};
970 const char *proc_cs_id = signing_id;
971
972 proc_lock(proc);
973
974 if (proc_cs_id == NULL) {
975 if (proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
976 proc_cs_id = cs_identity_get(proc);
977 } else {
978 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
979 }
980 }
981
982 if (is_effective) {
983 lck_rw_lock_shared(&fd_cb->group->lck);
984 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
985 if (proc_cs_id != NULL) {
986 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
987 if (result == NULL_TRIE_IDX) {
988 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
989 error = EPERM;
990 } else {
991 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
992 }
993 } else {
994 error = EPERM;
995 }
996 }
997 lck_rw_done(&fd_cb->group->lck);
998 }
999
1000 if (error != 0) {
1001 goto done;
1002 }
1003
1004 /*
1005 * If signing_id is not NULL then it came from the flow divert token and will be added
1006 * as part of the token, so there is no need to add it here.
1007 */
1008 if (signing_id == NULL && proc_cs_id != NULL) {
1009 error = flow_divert_packet_append_tlv(connect_packet,
1010 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
1011 (uint32_t)strlen(proc_cs_id),
1012 proc_cs_id);
1013 if (error != 0) {
1014 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1015 goto done;
1016 }
1017 }
1018
1019 cdhash = cs_get_cdhash(proc);
1020 if (cdhash != NULL) {
1021 error = flow_divert_packet_append_tlv(connect_packet,
1022 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
1023 SHA1_RESULTLEN,
1024 cdhash);
1025 if (error) {
1026 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1027 goto done;
1028 }
1029 } else {
1030 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash");
1031 }
1032
1033 task_t task = proc_task(proc);
1034 if (task != TASK_NULL) {
1035 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1036 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1037 if (rc == KERN_SUCCESS) {
1038 int append_error = flow_divert_packet_append_tlv(connect_packet,
1039 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1040 sizeof(audit_token_t),
1041 &audit_token);
1042 if (append_error) {
1043 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1044 }
1045 }
1046 }
1047
1048 done:
1049 proc_unlock(proc);
1050
1051 return error;
1052 }
1053
1054 static int
1055 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1056 {
1057 int error = 0;
1058 proc_t effective_proc = PROC_NULL;
1059 proc_t responsible_proc = PROC_NULL;
1060 proc_t real_proc = proc_find(so->last_pid);
1061 bool release_real_proc = true;
1062
1063 proc_t src_proc = PROC_NULL;
1064 proc_t real_src_proc = PROC_NULL;
1065
1066 if (real_proc == PROC_NULL) {
1067 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1068 release_real_proc = false;
1069 real_proc = proc;
1070 if (real_proc == PROC_NULL) {
1071 real_proc = current_proc();
1072 }
1073 }
1074
1075 if (so->so_flags & SOF_DELEGATED) {
1076 if (real_proc->p_pid != so->e_pid) {
1077 effective_proc = proc_find(so->e_pid);
1078 } else if (uuid_compare(real_proc->p_uuid, so->e_uuid)) {
1079 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
1080 }
1081 }
1082
1083 #if defined(XNU_TARGET_OS_OSX)
1084 lck_rw_lock_shared(&fd_cb->group->lck);
1085 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1086 if (so->so_rpid > 0) {
1087 responsible_proc = proc_find(so->so_rpid);
1088 }
1089 }
1090 lck_rw_done(&fd_cb->group->lck);
1091 #endif
1092
1093 real_src_proc = real_proc;
1094
1095 if (responsible_proc != PROC_NULL) {
1096 src_proc = responsible_proc;
1097 if (effective_proc != NULL) {
1098 real_src_proc = effective_proc;
1099 }
1100 } else if (effective_proc != PROC_NULL) {
1101 src_proc = effective_proc;
1102 } else {
1103 src_proc = real_proc;
1104 }
1105
1106 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1107 if (error != 0) {
1108 goto done;
1109 }
1110
1111 if (real_src_proc != NULL && real_src_proc != src_proc) {
1112 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1113 if (error != 0) {
1114 goto done;
1115 }
1116 }
1117
1118 done:
1119 if (responsible_proc != PROC_NULL) {
1120 proc_rele(responsible_proc);
1121 }
1122
1123 if (effective_proc != PROC_NULL) {
1124 proc_rele(effective_proc);
1125 }
1126
1127 if (real_proc != PROC_NULL && release_real_proc) {
1128 proc_rele(real_proc);
1129 }
1130
1131 return error;
1132 }
1133
1134 static int
1135 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1136 {
1137 int error;
1138
1139 if (fd_cb->group == NULL) {
1140 fd_cb->so->so_error = ECONNABORTED;
1141 flow_divert_disconnect_socket(fd_cb->so);
1142 return ECONNABORTED;
1143 }
1144
1145 lck_rw_lock_shared(&fd_cb->group->lck);
1146
1147 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1148 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1149 } else {
1150 error = ENOBUFS;
1151 }
1152
1153 if (error == ENOBUFS) {
1154 if (enqueue) {
1155 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1156 lck_rw_lock_exclusive(&fd_cb->group->lck);
1157 }
1158 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1159 error = 0;
1160 }
1161 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1162 }
1163
1164 lck_rw_done(&fd_cb->group->lck);
1165
1166 return error;
1167 }
1168
1169 static int
1170 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
1171 {
1172 int error = 0;
1173 int flow_type = 0;
1174 char *signing_id = NULL;
1175 mbuf_t connect_packet = NULL;
1176 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1177 const void *cfil_id = NULL;
1178 size_t cfil_id_size = 0;
1179 struct inpcb *inp = sotoinpcb(so);
1180 struct ifnet *ifp = NULL;
1181 uint32_t flags = 0;
1182
1183 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1184 if (error) {
1185 goto done;
1186 }
1187
1188 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1189 uint32_t sid_size = 0;
1190 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1191 if (find_error == 0 && sid_size > 0) {
1192 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1193 if (signing_id != NULL) {
1194 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1195 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1196 }
1197 }
1198 }
1199
1200 socket_unlock(so, 0);
1201
1202 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
1203
1204 socket_lock(so, 0);
1205
1206 if (signing_id != NULL) {
1207 FREE(signing_id, M_TEMP);
1208 }
1209
1210 if (error) {
1211 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
1212 goto done;
1213 }
1214
1215 error = flow_divert_packet_append_tlv(connect_packet,
1216 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1217 sizeof(fd_cb->so->so_traffic_class),
1218 &fd_cb->so->so_traffic_class);
1219 if (error) {
1220 goto done;
1221 }
1222
1223 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1224 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1225 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1226 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1227 } else {
1228 error = EINVAL;
1229 goto done;
1230 }
1231 error = flow_divert_packet_append_tlv(connect_packet,
1232 FLOW_DIVERT_TLV_FLOW_TYPE,
1233 sizeof(flow_type),
1234 &flow_type);
1235
1236 if (error) {
1237 goto done;
1238 }
1239
1240 if (fd_cb->connect_token != NULL) {
1241 unsigned int token_len = m_length(fd_cb->connect_token);
1242 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1243 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1244 fd_cb->connect_token = NULL;
1245 } else {
1246 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1247 if (error) {
1248 goto done;
1249 }
1250 }
1251
1252 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1253 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1254 if (error) {
1255 goto done;
1256 }
1257 }
1258
1259 if (inp->inp_vflag & INP_IPV4) {
1260 ifp = inp->inp_last_outifp;
1261 } else if (inp->inp_vflag & INP_IPV6) {
1262 ifp = inp->in6p_last_outifp;
1263 }
1264 if (ifp != NULL) {
1265 uint32_t flow_if_index = ifp->if_index;
1266 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1267 sizeof(flow_if_index), &flow_if_index);
1268 if (error) {
1269 goto done;
1270 }
1271 }
1272
1273 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1274 flags |= FLOW_DIVERT_TOKEN_FLAG_TFO;
1275 }
1276
1277 if ((inp->inp_flags & INP_BOUND_IF) ||
1278 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) ||
1279 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
1280 flags |= FLOW_DIVERT_TOKEN_FLAG_BOUND;
1281 }
1282
1283 if (flags != 0) {
1284 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1285 if (error) {
1286 goto done;
1287 }
1288 }
1289
1290 if (SOCK_TYPE(so) == SOCK_DGRAM) {
1291 cfil_sock_id = cfil_sock_id_from_datagram_socket(so, NULL, to);
1292 } else {
1293 cfil_sock_id = cfil_sock_id_from_socket(so);
1294 }
1295
1296 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1297 cfil_id = &cfil_sock_id;
1298 cfil_id_size = sizeof(cfil_sock_id);
1299 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1300 cfil_id = &inp->necp_client_uuid;
1301 cfil_id_size = sizeof(inp->necp_client_uuid);
1302 }
1303
1304 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
1305 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, (uint32_t)cfil_id_size, cfil_id);
1306 if (error) {
1307 goto done;
1308 }
1309 }
1310
1311 done:
1312 if (!error) {
1313 *out_connect_packet = connect_packet;
1314 } else if (connect_packet != NULL) {
1315 mbuf_freem(connect_packet);
1316 }
1317
1318 return error;
1319 }
1320
1321 static int
1322 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb)
1323 {
1324 int error = 0;
1325 mbuf_t connect_packet = fd_cb->connect_packet;
1326 mbuf_t saved_connect_packet = NULL;
1327
1328 if (connect_packet != NULL) {
1329 error = mbuf_copym(connect_packet, 0, mbuf_pkthdr_len(connect_packet), MBUF_DONTWAIT, &saved_connect_packet);
1330 if (error) {
1331 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet");
1332 goto done;
1333 }
1334
1335 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
1336 if (error) {
1337 goto done;
1338 }
1339
1340 fd_cb->connect_packet = saved_connect_packet;
1341 saved_connect_packet = NULL;
1342 } else {
1343 error = ENOENT;
1344 }
1345 done:
1346 if (saved_connect_packet != NULL) {
1347 mbuf_freem(saved_connect_packet);
1348 }
1349
1350 return error;
1351 }
1352
1353 static int
1354 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1355 {
1356 int error = 0;
1357 mbuf_t packet = NULL;
1358 int rbuff_space = 0;
1359
1360 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1361 if (error) {
1362 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1363 goto done;
1364 }
1365
1366 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1367 if (rbuff_space < 0) {
1368 rbuff_space = 0;
1369 }
1370 rbuff_space = htonl(rbuff_space);
1371 error = flow_divert_packet_append_tlv(packet,
1372 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1373 sizeof(rbuff_space),
1374 &rbuff_space);
1375 if (error) {
1376 goto done;
1377 }
1378
1379 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1380 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1381 if (error) {
1382 goto done;
1383 }
1384 }
1385
1386 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1387 if (error) {
1388 goto done;
1389 }
1390
1391 done:
1392 if (error && packet != NULL) {
1393 mbuf_freem(packet);
1394 }
1395
1396 return error;
1397 }
1398
1399 static int
1400 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1401 {
1402 int error = 0;
1403 mbuf_t packet = NULL;
1404 uint32_t zero = 0;
1405
1406 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1407 if (error) {
1408 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1409 goto done;
1410 }
1411
1412 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1413 if (error) {
1414 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1415 goto done;
1416 }
1417
1418 how = htonl(how);
1419 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1420 if (error) {
1421 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1422 goto done;
1423 }
1424
1425 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1426 if (error) {
1427 goto done;
1428 }
1429
1430 done:
1431 if (error && packet != NULL) {
1432 mbuf_free(packet);
1433 }
1434
1435 return error;
1436 }
1437
1438 static int
1439 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1440 {
1441 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1442 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1443 return SHUT_RDWR;
1444 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1445 return SHUT_RD;
1446 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1447 return SHUT_WR;
1448 }
1449
1450 return -1;
1451 }
1452
1453 /*
1454 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1455 * writes. Returns FALSE otherwise.
1456 */
1457 static void
1458 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1459 {
1460 int how = -1;
1461
1462 /* Do not send any close messages if there is still data in the send buffer */
1463 if (fd_cb->so->so_snd.sb_cc == 0) {
1464 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1465 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1466 how = SHUT_RD;
1467 }
1468 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1469 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1470 if (how == SHUT_RD) {
1471 how = SHUT_RDWR;
1472 } else {
1473 how = SHUT_WR;
1474 }
1475 }
1476 }
1477
1478 if (how != -1) {
1479 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1480 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1481 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1482 if (how != SHUT_RD) {
1483 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1484 }
1485 if (how != SHUT_WR) {
1486 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1487 }
1488 }
1489 }
1490
1491 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1492 flow_divert_disconnect_socket(fd_cb->so);
1493 }
1494 }
1495
1496 static errno_t
1497 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1498 {
1499 mbuf_t packet = NULL;
1500 mbuf_t last = NULL;
1501 int error = 0;
1502
1503 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1504 if (error || packet == NULL) {
1505 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1506 goto done;
1507 }
1508
1509 if (toaddr != NULL) {
1510 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1511 if (error) {
1512 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1513 goto done;
1514 }
1515 }
1516
1517 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
1518 last = m_last(packet);
1519 mbuf_setnext(last, data);
1520 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1521 } else {
1522 data_len = 0;
1523 }
1524 error = flow_divert_send_packet(fd_cb, packet, force);
1525 if (error == 0 && data_len > 0) {
1526 fd_cb->bytes_sent += data_len;
1527 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1528 }
1529
1530 done:
1531 if (error) {
1532 if (last != NULL) {
1533 mbuf_setnext(last, NULL);
1534 }
1535 if (packet != NULL) {
1536 mbuf_freem(packet);
1537 }
1538 }
1539
1540 return error;
1541 }
1542
1543 static void
1544 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1545 {
1546 size_t to_send;
1547 size_t sent = 0;
1548 int error = 0;
1549 mbuf_t buffer;
1550
1551 to_send = fd_cb->so->so_snd.sb_cc;
1552 buffer = fd_cb->so->so_snd.sb_mb;
1553
1554 if (buffer == NULL && to_send > 0) {
1555 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1556 return;
1557 }
1558
1559 /* Ignore the send window if force is enabled */
1560 if (!force && (to_send > fd_cb->send_window)) {
1561 to_send = fd_cb->send_window;
1562 }
1563
1564 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1565 while (sent < to_send) {
1566 mbuf_t data;
1567 size_t data_len;
1568
1569 data_len = to_send - sent;
1570 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1571 data_len = FLOW_DIVERT_CHUNK_SIZE;
1572 }
1573
1574 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1575 if (error) {
1576 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1577 break;
1578 }
1579
1580 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1581 if (error) {
1582 if (data != NULL) {
1583 mbuf_freem(data);
1584 }
1585 break;
1586 }
1587
1588 sent += data_len;
1589 }
1590 sbdrop(&fd_cb->so->so_snd, (int)sent);
1591 sowwakeup(fd_cb->so);
1592 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1593 mbuf_t data;
1594 mbuf_t m;
1595 size_t data_len;
1596
1597 while (buffer) {
1598 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1599
1600 m = buffer;
1601 if (toaddr != NULL) {
1602 /* look for data in the chain */
1603 do {
1604 m = m->m_next;
1605 if (m != NULL && m->m_type == MT_DATA) {
1606 break;
1607 }
1608 } while (m);
1609 if (m == NULL) {
1610 /* unexpected */
1611 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1612 goto move_on;
1613 }
1614 }
1615 data_len = mbuf_pkthdr_len(m);
1616 if (data_len > 0) {
1617 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1618 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1619 if (error) {
1620 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1621 break;
1622 }
1623 } else {
1624 data = NULL;
1625 }
1626 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1627 if (error) {
1628 if (data != NULL) {
1629 mbuf_freem(data);
1630 }
1631 break;
1632 }
1633 sent += data_len;
1634 move_on:
1635 buffer = buffer->m_nextpkt;
1636 (void) sbdroprecord(&(fd_cb->so->so_snd));
1637 }
1638 }
1639
1640 if (sent > 0) {
1641 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1642 if (fd_cb->send_window >= sent) {
1643 fd_cb->send_window -= sent;
1644 } else {
1645 fd_cb->send_window = 0;
1646 }
1647 }
1648 }
1649
1650 static int
1651 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1652 {
1653 size_t to_send = mbuf_pkthdr_len(data);
1654 int error = 0;
1655
1656 if (to_send > fd_cb->send_window) {
1657 to_send = fd_cb->send_window;
1658 }
1659
1660 if (fd_cb->so->so_snd.sb_cc > 0) {
1661 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1662 }
1663
1664 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1665 size_t sent = 0;
1666 mbuf_t remaining_data = data;
1667 mbuf_t pkt_data = NULL;
1668 while (sent < to_send && remaining_data != NULL) {
1669 size_t pkt_data_len;
1670
1671 pkt_data = remaining_data;
1672
1673 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1674 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1675 } else {
1676 pkt_data_len = to_send - sent;
1677 }
1678
1679 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1680 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1681 if (error) {
1682 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1683 pkt_data = NULL;
1684 break;
1685 }
1686 } else {
1687 remaining_data = NULL;
1688 }
1689
1690 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1691
1692 if (error) {
1693 break;
1694 }
1695
1696 pkt_data = NULL;
1697 sent += pkt_data_len;
1698 }
1699
1700 fd_cb->send_window -= sent;
1701
1702 error = 0;
1703
1704 if (pkt_data != NULL) {
1705 if (sbspace(&fd_cb->so->so_snd) > 0) {
1706 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1707 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1708 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1709 }
1710 } else {
1711 mbuf_freem(pkt_data);
1712 error = ENOBUFS;
1713 }
1714 }
1715
1716 if (remaining_data != NULL) {
1717 if (sbspace(&fd_cb->so->so_snd) > 0) {
1718 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1719 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1720 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1721 }
1722 } else {
1723 mbuf_freem(remaining_data);
1724 error = ENOBUFS;
1725 }
1726 }
1727 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1728 if (to_send || mbuf_pkthdr_len(data) == 0) {
1729 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1730 if (error) {
1731 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1732 if (data != NULL) {
1733 mbuf_freem(data);
1734 }
1735 } else {
1736 fd_cb->send_window -= to_send;
1737 }
1738 } else {
1739 /* buffer it */
1740 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1741 if (toaddr != NULL) {
1742 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1743 FDLOG(LOG_ERR, fd_cb,
1744 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1745 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1746 }
1747 error = 0;
1748 } else {
1749 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1750 FDLOG(LOG_ERR, fd_cb,
1751 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1752 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1753 }
1754 }
1755 } else {
1756 if (data != NULL) {
1757 mbuf_freem(data);
1758 }
1759 error = ENOBUFS;
1760 }
1761 }
1762 }
1763
1764 return error;
1765 }
1766
1767 static int
1768 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb)
1769 {
1770 int error = 0;
1771 mbuf_t packet = NULL;
1772
1773 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1774 if (error) {
1775 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1776 goto done;
1777 }
1778
1779 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1780 if (error) {
1781 goto done;
1782 }
1783
1784 done:
1785 if (error && packet != NULL) {
1786 mbuf_free(packet);
1787 }
1788
1789 return error;
1790 }
1791
1792 static int
1793 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1794 {
1795 int error = 0;
1796 mbuf_t packet = NULL;
1797
1798 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1799 if (error) {
1800 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1801 goto done;
1802 }
1803
1804 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1805 if (error) {
1806 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1807 goto done;
1808 }
1809
1810 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1811 if (error) {
1812 goto done;
1813 }
1814
1815 done:
1816 if (error && packet != NULL) {
1817 mbuf_free(packet);
1818 }
1819
1820 return error;
1821 }
1822
1823 static void
1824 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint)
1825 {
1826 struct inpcb *inp = sotoinpcb(fd_cb->so);
1827
1828 if (local_endpoint->sa_family == AF_INET6) {
1829 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
1830 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1831 inp->in6p_laddr = (satosin6(local_endpoint))->sin6_addr;
1832 }
1833 if (inp->inp_lport == 0) {
1834 inp->inp_lport = (satosin6(local_endpoint))->sin6_port;
1835 }
1836 } else if (local_endpoint->sa_family == AF_INET) {
1837 if (inp->inp_laddr.s_addr == INADDR_ANY && (fd_cb->flags & FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR)) {
1838 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1839 inp->inp_laddr = (satosin(local_endpoint))->sin_addr;
1840 }
1841 if (inp->inp_lport == 0) {
1842 inp->inp_lport = (satosin(local_endpoint))->sin_port;
1843 }
1844 }
1845 }
1846
1847 static void
1848 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint)
1849 {
1850 struct inpcb *inp = sotoinpcb(fd_cb->so);
1851
1852 if (remote_endpoint->sa_family == AF_INET6) {
1853 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
1854 inp->in6p_faddr = (satosin6(remote_endpoint))->sin6_addr;
1855 }
1856 if (inp->inp_fport == 0) {
1857 inp->inp_fport = (satosin6(remote_endpoint))->sin6_port;
1858 }
1859 } else if (remote_endpoint->sa_family == AF_INET) {
1860 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1861 inp->inp_faddr = (satosin(remote_endpoint))->sin_addr;
1862 }
1863 if (inp->inp_fport == 0) {
1864 inp->inp_fport = (satosin(remote_endpoint))->sin_port;
1865 }
1866 }
1867 }
1868
1869 static uint32_t
1870 flow_divert_derive_kernel_control_unit(uint32_t ctl_unit, uint32_t *aggregate_unit)
1871 {
1872 if (aggregate_unit != NULL && *aggregate_unit != 0) {
1873 uint32_t counter;
1874 for (counter = 0; counter < (GROUP_COUNT_MAX - 1); counter++) {
1875 if ((*aggregate_unit) & (1 << counter)) {
1876 break;
1877 }
1878 }
1879 if (counter < (GROUP_COUNT_MAX - 1)) {
1880 *aggregate_unit &= ~(1 << counter);
1881 return counter + 1;
1882 } else {
1883 return ctl_unit;
1884 }
1885 } else {
1886 return ctl_unit;
1887 }
1888 }
1889
1890 static int
1891 flow_divert_try_next(struct flow_divert_pcb *fd_cb)
1892 {
1893 uint32_t current_ctl_unit = 0;
1894 uint32_t next_ctl_unit = 0;
1895 struct flow_divert_group *current_group = NULL;
1896 struct flow_divert_group *next_group = NULL;
1897 int error = 0;
1898
1899 next_ctl_unit = flow_divert_derive_kernel_control_unit(fd_cb->policy_control_unit, &(fd_cb->aggregate_unit));
1900 current_ctl_unit = fd_cb->control_group_unit;
1901
1902 if (current_ctl_unit == next_ctl_unit) {
1903 FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling flow divert");
1904 error = EALREADY;
1905 goto done;
1906 }
1907
1908 if (next_ctl_unit == 0 || next_ctl_unit >= GROUP_COUNT_MAX) {
1909 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert");
1910 error = ENOENT;
1911 goto done;
1912 }
1913
1914 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1915 FDLOG0(LOG_NOTICE, fd_cb, "No active groups, disabling flow divert");
1916 error = ENOENT;
1917 goto done;
1918 }
1919
1920 next_group = g_flow_divert_groups[next_ctl_unit];
1921 if (next_group == NULL) {
1922 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit);
1923 error = ENOENT;
1924 goto done;
1925 }
1926
1927 current_group = fd_cb->group;
1928
1929 lck_rw_lock_exclusive(&(current_group->lck));
1930 lck_rw_lock_exclusive(&(next_group->lck));
1931
1932 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", current_ctl_unit, next_ctl_unit);
1933
1934 RB_REMOVE(fd_pcb_tree, &(current_group->pcb_tree), fd_cb);
1935 if (RB_INSERT(fd_pcb_tree, &(next_group->pcb_tree), fd_cb) != NULL) {
1936 panic("group with unit %u already contains a connection with hash %u", next_ctl_unit, fd_cb->hash);
1937 }
1938
1939 fd_cb->group = next_group;
1940 fd_cb->control_group_unit = next_ctl_unit;
1941
1942 lck_rw_done(&(next_group->lck));
1943 lck_rw_done(&(current_group->lck));
1944
1945 error = flow_divert_send_connect_packet(fd_cb);
1946 if (error) {
1947 FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", next_ctl_unit);
1948 error = ENOENT;
1949 goto done;
1950 }
1951
1952 done:
1953 return error;
1954 }
1955
1956 static void
1957 flow_divert_disable(struct flow_divert_pcb *fd_cb)
1958 {
1959 struct socket *so = NULL;
1960 mbuf_t buffer;
1961 int error = 0;
1962 proc_t last_proc = NULL;
1963 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint;
1964 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT);
1965 struct inpcb *inp = NULL;
1966
1967 so = fd_cb->so;
1968 if (so == NULL) {
1969 goto done;
1970 }
1971
1972 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert");
1973
1974 /* Restore the IP state */
1975 inp = sotoinpcb(so);
1976 inp->inp_vflag = fd_cb->original_vflag;
1977 inp->inp_faddr.s_addr = INADDR_ANY;
1978 inp->inp_fport = 0;
1979 memset(&(inp->in6p_faddr), 0, sizeof(inp->in6p_faddr));
1980 inp->in6p_fport = 0;
1981 /* If flow divert set the local address, clear it out */
1982 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) {
1983 inp->inp_laddr.s_addr = INADDR_ANY;
1984 memset(&(inp->in6p_laddr), 0, sizeof(inp->in6p_laddr));
1985 }
1986 inp->inp_last_outifp = fd_cb->original_last_outifp;
1987 inp->in6p_last_outifp = fd_cb->original_last_outifp6;
1988
1989 /* Dis-associate the socket */
1990 so->so_flags &= ~SOF_FLOW_DIVERT;
1991 so->so_flags1 |= SOF1_FLOW_DIVERT_SKIP;
1992 so->so_fd_pcb = NULL;
1993 fd_cb->so = NULL;
1994
1995 /* Remove from the group */
1996 flow_divert_pcb_remove(fd_cb);
1997
1998 FDRELEASE(fd_cb); /* Release the socket's reference */
1999
2000 /* Revert back to the original protocol */
2001 so->so_proto = pffindproto(SOCK_DOM(so), SOCK_PROTO(so), SOCK_TYPE(so));
2002
2003 last_proc = proc_find(so->last_pid);
2004
2005 if (do_connect) {
2006 /* Connect using the original protocol */
2007 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, remote_endpoint, (last_proc != NULL ? last_proc : current_proc()));
2008 if (error) {
2009 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error);
2010 goto done;
2011 }
2012 }
2013
2014 buffer = so->so_snd.sb_mb;
2015 if (buffer == NULL) {
2016 /* No buffered data, done */
2017 goto done;
2018 }
2019
2020 /* Send any buffered data using the original protocol */
2021 if (SOCK_TYPE(so) == SOCK_STREAM) {
2022 mbuf_t data_to_send = NULL;
2023 size_t data_len = so->so_snd.sb_cc;
2024
2025 error = mbuf_copym(buffer, 0, data_len, MBUF_DONTWAIT, &data_to_send);
2026 if (error) {
2027 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer");
2028 goto done;
2029 }
2030
2031 sbflush(&so->so_snd);
2032
2033 if (data_to_send->m_flags & M_PKTHDR) {
2034 mbuf_pkthdr_setlen(data_to_send, data_len);
2035 }
2036
2037 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2038 0,
2039 data_to_send,
2040 NULL,
2041 NULL,
2042 (last_proc != NULL ? last_proc : current_proc()));
2043
2044 if (error && error != EWOULDBLOCK) {
2045 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2046 } else {
2047 error = 0;
2048 }
2049 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
2050 struct sockbuf *sb = &so->so_snd;
2051 MBUFQ_HEAD(send_queue_head) send_queue;
2052 MBUFQ_INIT(&send_queue);
2053
2054 /* Flush the send buffer, moving all records to a temporary queue */
2055 while (sb->sb_mb != NULL) {
2056 mbuf_t record = sb->sb_mb;
2057 mbuf_t m = record;
2058 sb->sb_mb = sb->sb_mb->m_nextpkt;
2059 while (m != NULL) {
2060 sbfree(sb, m);
2061 m = m->m_next;
2062 }
2063 record->m_nextpkt = NULL;
2064 MBUFQ_ENQUEUE(&send_queue, record);
2065 }
2066 SB_EMPTY_FIXUP(sb);
2067
2068 while (!MBUFQ_EMPTY(&send_queue)) {
2069 mbuf_t next_record = MBUFQ_FIRST(&send_queue);
2070 mbuf_t addr = NULL;
2071 mbuf_t control = NULL;
2072 mbuf_t last_control = NULL;
2073 mbuf_t data = NULL;
2074 mbuf_t m = next_record;
2075 struct sockaddr *to_endpoint = NULL;
2076
2077 MBUFQ_DEQUEUE(&send_queue, next_record);
2078
2079 while (m != NULL) {
2080 if (m->m_type == MT_SONAME) {
2081 addr = m;
2082 } else if (m->m_type == MT_CONTROL) {
2083 if (control == NULL) {
2084 control = m;
2085 }
2086 last_control = m;
2087 } else if (m->m_type == MT_DATA) {
2088 data = m;
2089 break;
2090 }
2091 m = m->m_next;
2092 }
2093
2094 if (addr != NULL) {
2095 to_endpoint = flow_divert_get_buffered_target_address(addr);
2096 if (to_endpoint == NULL) {
2097 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer");
2098 }
2099 }
2100
2101 if (data == NULL) {
2102 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data");
2103 mbuf_freem(next_record);
2104 continue;
2105 }
2106
2107 if (!(data->m_flags & M_PKTHDR)) {
2108 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header");
2109 mbuf_freem(next_record);
2110 continue;
2111 }
2112
2113 if (addr != NULL) {
2114 addr->m_next = NULL;
2115 }
2116
2117 if (last_control != NULL) {
2118 last_control->m_next = NULL;
2119 }
2120
2121 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2122 0,
2123 data,
2124 to_endpoint,
2125 control,
2126 (last_proc != NULL ? last_proc : current_proc()));
2127
2128 if (addr != NULL) {
2129 mbuf_freem(addr);
2130 }
2131
2132 if (error) {
2133 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2134 }
2135 }
2136 }
2137 done:
2138 if (last_proc != NULL) {
2139 proc_rele(last_proc);
2140 }
2141
2142 if (error) {
2143 so->so_error = (uint16_t)error;
2144 flow_divert_disconnect_socket(so);
2145 }
2146 }
2147
2148 static void
2149 flow_divert_scope(struct flow_divert_pcb *fd_cb, int out_if_index, bool derive_new_address)
2150 {
2151 struct socket *so = NULL;
2152 struct inpcb *inp = NULL;
2153 struct ifnet *current_ifp = NULL;
2154 struct ifnet *new_ifp = NULL;
2155 int error = 0;
2156
2157 so = fd_cb->so;
2158 if (so == NULL) {
2159 return;
2160 }
2161
2162 inp = sotoinpcb(so);
2163
2164 if (out_if_index <= 0) {
2165 return;
2166 }
2167
2168 if (inp->inp_vflag & INP_IPV6) {
2169 current_ifp = inp->in6p_last_outifp;
2170 } else {
2171 current_ifp = inp->inp_last_outifp;
2172 }
2173
2174 if (current_ifp != NULL) {
2175 if (current_ifp->if_index == out_if_index) {
2176 /* No change */
2177 return;
2178 }
2179
2180 /* Scope the socket to the given interface */
2181 error = inp_bindif(inp, out_if_index, &new_ifp);
2182 if (error != 0) {
2183 FDLOG(LOG_ERR, fd_cb, "failed to scope to %d because inp_bindif returned %d", out_if_index, error);
2184 return;
2185 }
2186
2187 if (derive_new_address && fd_cb->original_remote_endpoint != NULL) {
2188 /* Get the appropriate address for the given interface */
2189 if (inp->inp_vflag & INP_IPV6) {
2190 inp->in6p_laddr = sa6_any.sin6_addr;
2191 error = in6_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin6.sin6_addr), NULL);
2192 } else {
2193 inp->inp_laddr.s_addr = INADDR_ANY;
2194 error = in_pcbladdr(inp, fd_cb->original_remote_endpoint, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, NULL, 0);
2195 }
2196
2197 if (error != 0) {
2198 FDLOG(LOG_WARNING, fd_cb, "failed to derive a new local address from %d because in_pcbladdr returned %d", out_if_index, error);
2199 }
2200 }
2201 } else {
2202 ifnet_head_lock_shared();
2203 if (out_if_index <= if_index) {
2204 new_ifp = ifindex2ifnet[out_if_index];
2205 }
2206 ifnet_head_done();
2207 }
2208
2209 /* Update the "last interface" of the socket */
2210 if (new_ifp != NULL) {
2211 if (inp->inp_vflag & INP_IPV6) {
2212 inp->in6p_last_outifp = new_ifp;
2213 } else {
2214 inp->inp_last_outifp = new_ifp;
2215 }
2216
2217 }
2218 }
2219
2220 static void
2221 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2222 {
2223 uint32_t connect_error = 0;
2224 uint32_t ctl_unit = 0;
2225 int error = 0;
2226 struct flow_divert_group *grp = NULL;
2227 union sockaddr_in_4_6 local_endpoint = {};
2228 union sockaddr_in_4_6 remote_endpoint = {};
2229 int out_if_index = 0;
2230 uint32_t send_window;
2231 uint32_t app_data_length = 0;
2232
2233 memset(&local_endpoint, 0, sizeof(local_endpoint));
2234 memset(&remote_endpoint, 0, sizeof(remote_endpoint));
2235
2236 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
2237 if (error) {
2238 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
2239 return;
2240 }
2241
2242 connect_error = ntohl(connect_error);
2243 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
2244
2245 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
2246 if (error) {
2247 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
2248 return;
2249 }
2250
2251 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
2252 if (error) {
2253 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
2254 }
2255
2256 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_endpoint), &(local_endpoint.sa), NULL);
2257 if (error) {
2258 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
2259 }
2260
2261 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_endpoint), &(remote_endpoint.sa), NULL);
2262 if (error) {
2263 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2264 }
2265
2266 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2267 if (error) {
2268 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
2269 }
2270
2271 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2272 if (error) {
2273 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
2274 }
2275
2276 error = 0;
2277 ctl_unit = ntohl(ctl_unit);
2278
2279 lck_rw_lock_shared(&g_flow_divert_group_lck);
2280
2281 if (connect_error == 0 && ctl_unit > 0) {
2282 if (ctl_unit >= GROUP_COUNT_MAX) {
2283 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
2284 error = EINVAL;
2285 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
2286 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
2287 error = EINVAL;
2288 } else {
2289 grp = g_flow_divert_groups[ctl_unit];
2290 if (grp == NULL) {
2291 error = ECONNRESET;
2292 }
2293 }
2294 }
2295
2296 FDLOCK(fd_cb);
2297 if (fd_cb->so != NULL) {
2298 struct inpcb *inp = NULL;
2299 struct flow_divert_group *old_group;
2300 struct socket *so = fd_cb->so;
2301 bool local_address_is_valid = false;
2302
2303 socket_lock(so, 0);
2304
2305 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
2306 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring connect result");
2307 goto done;
2308 }
2309
2310 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_state & SS_ISCONNECTING)) {
2311 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result");
2312 goto done;
2313 }
2314
2315 inp = sotoinpcb(so);
2316
2317 if (connect_error || error) {
2318 goto set_socket_state;
2319 }
2320
2321 if (flow_divert_is_sockaddr_valid(&(local_endpoint.sa))) {
2322 if (local_endpoint.sa.sa_family == AF_INET) {
2323 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2324 if ((inp->inp_vflag & INP_IPV4) && local_endpoint.sin.sin_addr.s_addr != INADDR_ANY) {
2325 local_address_is_valid = true;
2326 fd_cb->local_endpoint = local_endpoint;
2327 inp->inp_laddr.s_addr = INADDR_ANY;
2328 } else {
2329 fd_cb->local_endpoint.sin.sin_port = local_endpoint.sin.sin_port;
2330 }
2331 } else if (local_endpoint.sa.sa_family == AF_INET6) {
2332 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2333 if ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&local_endpoint.sin6.sin6_addr)) {
2334 local_address_is_valid = true;
2335 fd_cb->local_endpoint = local_endpoint;
2336 inp->in6p_laddr = sa6_any.sin6_addr;
2337 } else {
2338 fd_cb->local_endpoint.sin6.sin6_port = local_endpoint.sin6.sin6_port;
2339 }
2340 }
2341 }
2342
2343 flow_divert_scope(fd_cb, out_if_index, !local_address_is_valid);
2344 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
2345
2346 if (flow_divert_is_sockaddr_valid(&(remote_endpoint.sa)) && SOCK_TYPE(so) == SOCK_STREAM) {
2347 if (remote_endpoint.sa.sa_family == AF_INET) {
2348 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2349 } else if (remote_endpoint.sa.sa_family == AF_INET6) {
2350 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2351 }
2352 flow_divert_set_remote_endpoint(fd_cb, &(remote_endpoint.sa));
2353 }
2354
2355 if (app_data_length > 0) {
2356 uint8_t *app_data = NULL;
2357 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2358 if (app_data != NULL) {
2359 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2360 if (error == 0) {
2361 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
2362 if (fd_cb->app_data != NULL) {
2363 FREE(fd_cb->app_data, M_TEMP);
2364 }
2365 fd_cb->app_data = app_data;
2366 fd_cb->app_data_length = app_data_length;
2367 } else {
2368 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
2369 FREE(app_data, M_TEMP);
2370 }
2371 } else {
2372 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
2373 }
2374 }
2375
2376 if (error) {
2377 goto set_socket_state;
2378 }
2379
2380 if (fd_cb->group == NULL) {
2381 error = EINVAL;
2382 goto set_socket_state;
2383 }
2384
2385 if (grp != NULL) {
2386 old_group = fd_cb->group;
2387
2388 lck_rw_lock_exclusive(&old_group->lck);
2389 lck_rw_lock_exclusive(&grp->lck);
2390
2391 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
2392 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
2393 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
2394 }
2395
2396 fd_cb->group = grp;
2397
2398 lck_rw_done(&grp->lck);
2399 lck_rw_done(&old_group->lck);
2400 }
2401
2402 fd_cb->send_window = ntohl(send_window);
2403
2404 set_socket_state:
2405 if (!connect_error && !error) {
2406 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
2407 error = flow_divert_send_connect_result(fd_cb);
2408 }
2409
2410 if (connect_error || error) {
2411 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) {
2412 error = flow_divert_try_next(fd_cb);
2413 if (error) {
2414 flow_divert_disable(fd_cb);
2415 }
2416 goto done;
2417 }
2418
2419 if (!connect_error) {
2420 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2421 so->so_error = (uint16_t)error;
2422 flow_divert_send_close_if_needed(fd_cb);
2423 } else {
2424 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2425 so->so_error = (uint16_t)connect_error;
2426 }
2427 flow_divert_disconnect_socket(so);
2428 } else {
2429 #if NECP
2430 /* Update NECP client with connected five-tuple */
2431 if (!uuid_is_null(inp->necp_client_uuid)) {
2432 socket_unlock(so, 0);
2433 necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp);
2434 socket_lock(so, 0);
2435 }
2436 #endif /* NECP */
2437
2438 flow_divert_send_buffered_data(fd_cb, FALSE);
2439 soisconnected(so);
2440 }
2441
2442 /* We don't need the connect packet any more */
2443 if (fd_cb->connect_packet != NULL) {
2444 mbuf_freem(fd_cb->connect_packet);
2445 fd_cb->connect_packet = NULL;
2446 }
2447
2448 /* We don't need the original remote endpoint any more */
2449 if (fd_cb->original_remote_endpoint != NULL) {
2450 FREE(fd_cb->original_remote_endpoint, M_SONAME);
2451 fd_cb->original_remote_endpoint = NULL;
2452 }
2453 done:
2454 socket_unlock(so, 0);
2455 }
2456 FDUNLOCK(fd_cb);
2457
2458 lck_rw_done(&g_flow_divert_group_lck);
2459 }
2460
2461 static void
2462 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2463 {
2464 uint32_t close_error;
2465 int error = 0;
2466 int how;
2467
2468 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2469 if (error) {
2470 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2471 return;
2472 }
2473
2474 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2475 if (error) {
2476 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2477 return;
2478 }
2479
2480 how = ntohl(how);
2481
2482 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2483
2484 FDLOCK(fd_cb);
2485 if (fd_cb->so != NULL) {
2486 socket_lock(fd_cb->so, 0);
2487
2488 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2489 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring close from provider");
2490 goto done;
2491 }
2492
2493 fd_cb->so->so_error = (uint16_t)ntohl(close_error);
2494
2495 flow_divert_update_closed_state(fd_cb, how, TRUE);
2496
2497 how = flow_divert_tunnel_how_closed(fd_cb);
2498 if (how == SHUT_RDWR) {
2499 flow_divert_disconnect_socket(fd_cb->so);
2500 } else if (how == SHUT_RD) {
2501 socantrcvmore(fd_cb->so);
2502 } else if (how == SHUT_WR) {
2503 socantsendmore(fd_cb->so);
2504 }
2505 done:
2506 socket_unlock(fd_cb->so, 0);
2507 }
2508 FDUNLOCK(fd_cb);
2509 }
2510
2511 static mbuf_t
2512 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb)
2513 {
2514 struct inpcb *inp = sotoinpcb(fd_cb->so);
2515 bool is_cfil_enabled = false;
2516 #if CONTENT_FILTER
2517 /* Content Filter needs to see the local address */
2518 is_cfil_enabled = (inp->inp_socket && inp->inp_socket->so_cfil_db != NULL);
2519 #endif
2520 if ((inp->inp_vflag & INP_IPV4) &&
2521 fd_cb->local_endpoint.sa.sa_family == AF_INET &&
2522 ((inp->inp_flags & INP_RECVDSTADDR) || is_cfil_enabled)) {
2523 return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2524 } else if ((inp->inp_vflag & INP_IPV6) &&
2525 fd_cb->local_endpoint.sa.sa_family == AF_INET6 &&
2526 ((inp->inp_flags & IN6P_PKTINFO) || is_cfil_enabled)) {
2527 struct in6_pktinfo pi6;
2528 memset(&pi6, 0, sizeof(pi6));
2529 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr;
2530
2531 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
2532 }
2533 return NULL;
2534 }
2535
2536 static int
2537 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2538 {
2539 int error = 0;
2540
2541 FDLOCK(fd_cb);
2542 if (fd_cb->so != NULL) {
2543 mbuf_t data = NULL;
2544 size_t data_size;
2545 struct sockaddr_storage remote_address;
2546 boolean_t got_remote_sa = FALSE;
2547 boolean_t appended = FALSE;
2548 boolean_t append_success = FALSE;
2549
2550 socket_lock(fd_cb->so, 0);
2551
2552 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2553 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring inbound data");
2554 goto done;
2555 }
2556
2557 if (sbspace(&fd_cb->so->so_rcv) == 0) {
2558 error = ENOBUFS;
2559 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED;
2560 FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some data");
2561 goto done;
2562 }
2563
2564 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2565 uint32_t val_size = 0;
2566
2567 /* check if we got remote address with data */
2568 memset(&remote_address, 0, sizeof(remote_address));
2569 error = flow_divert_packet_get_tlv(packet, (int)offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2570 if (error || val_size > sizeof(remote_address)) {
2571 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2572 error = 0;
2573 } else {
2574 if (remote_address.ss_len > sizeof(remote_address)) {
2575 remote_address.ss_len = sizeof(remote_address);
2576 }
2577 /* validate the address */
2578 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2579 got_remote_sa = TRUE;
2580 } else {
2581 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid");
2582 }
2583 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2584 }
2585 }
2586
2587 data_size = (mbuf_pkthdr_len(packet) - offset);
2588
2589 if (fd_cb->so->so_state & SS_CANTRCVMORE) {
2590 FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size);
2591 goto done;
2592 }
2593
2594 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) {
2595 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so));
2596 goto done;
2597 }
2598
2599 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2600
2601 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2602 if (error || data == NULL) {
2603 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2604 goto done;
2605 }
2606
2607 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2608 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0);
2609 append_success = TRUE;
2610 } else {
2611 struct sockaddr *append_sa = NULL;
2612 mbuf_t mctl;
2613
2614 if (got_remote_sa == TRUE) {
2615 error = flow_divert_dup_addr(remote_address.ss_family, (struct sockaddr *)&remote_address, &append_sa);
2616 } else {
2617 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) {
2618 error = in6_mapped_peeraddr(fd_cb->so, &append_sa);
2619 } else {
2620 error = in_getpeeraddr(fd_cb->so, &append_sa);
2621 }
2622 }
2623 if (error) {
2624 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2625 }
2626
2627 mctl = flow_divert_create_control_mbuf(fd_cb);
2628 int append_error = 0;
2629 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error) || append_error == EJUSTRETURN) {
2630 append_success = TRUE;
2631 appended = (append_error == 0);
2632 } else {
2633 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error);
2634 }
2635
2636 if (append_sa != NULL) {
2637 FREE(append_sa, M_SONAME);
2638 }
2639 }
2640
2641 if (append_success) {
2642 fd_cb->bytes_received += data_size;
2643 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2644 }
2645
2646 if (appended) {
2647 sorwakeup(fd_cb->so);
2648 }
2649 done:
2650 socket_unlock(fd_cb->so, 0);
2651 }
2652 FDUNLOCK(fd_cb);
2653
2654 return error;
2655 }
2656
2657 static void
2658 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2659 {
2660 uint32_t read_count;
2661 int error = 0;
2662
2663 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2664 if (error) {
2665 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2666 return;
2667 }
2668
2669 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2670
2671 FDLOCK(fd_cb);
2672 if (fd_cb->so != NULL) {
2673 socket_lock(fd_cb->so, 0);
2674
2675 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2676 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring read notification");
2677 goto done;
2678 }
2679
2680 fd_cb->send_window += ntohl(read_count);
2681 flow_divert_send_buffered_data(fd_cb, FALSE);
2682 done:
2683 socket_unlock(fd_cb->so, 0);
2684 }
2685 FDUNLOCK(fd_cb);
2686 }
2687
2688 static void
2689 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2690 {
2691 int error = 0;
2692 uint32_t key_size = 0;
2693 int log_level;
2694 uint32_t flags = 0;
2695
2696 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2697 if (error) {
2698 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2699 return;
2700 }
2701
2702 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2703 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2704 return;
2705 }
2706
2707 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2708 if (!error) {
2709 nil_pcb.log_level = (uint8_t)log_level;
2710 }
2711
2712 lck_rw_lock_exclusive(&group->lck);
2713
2714 if (group->token_key != NULL) {
2715 FREE(group->token_key, M_TEMP);
2716 group->token_key = NULL;
2717 }
2718
2719 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2720 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2721 if (error) {
2722 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2723 FREE(group->token_key, M_TEMP);
2724 group->token_key = NULL;
2725 lck_rw_done(&group->lck);
2726 return;
2727 }
2728
2729 group->token_key_size = key_size;
2730
2731 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2732 if (!error) {
2733 group->flags = flags;
2734 }
2735
2736 lck_rw_done(&group->lck);
2737 }
2738
2739 static void
2740 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2741 {
2742 int error = 0;
2743 int out_if_index = 0;
2744 uint32_t app_data_length = 0;
2745
2746 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2747
2748 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2749 if (error) {
2750 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2751 }
2752
2753 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2754 if (error) {
2755 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2756 }
2757
2758 FDLOCK(fd_cb);
2759 if (fd_cb->so != NULL) {
2760 socket_lock(fd_cb->so, 0);
2761
2762 if (!(fd_cb->so->so_flags & SOF_FLOW_DIVERT)) {
2763 FDLOG0(LOG_NOTICE, fd_cb, "socket is not attached any more, ignoring properties update");
2764 goto done;
2765 }
2766
2767 if (out_if_index > 0) {
2768 flow_divert_scope(fd_cb, out_if_index, true);
2769 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
2770 }
2771
2772 if (app_data_length > 0) {
2773 uint8_t *app_data = NULL;
2774 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2775 if (app_data != NULL) {
2776 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2777 if (error == 0) {
2778 if (fd_cb->app_data != NULL) {
2779 FREE(fd_cb->app_data, M_TEMP);
2780 }
2781 fd_cb->app_data = app_data;
2782 fd_cb->app_data_length = app_data_length;
2783 } else {
2784 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2785 FREE(app_data, M_TEMP);
2786 }
2787 } else {
2788 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2789 }
2790 }
2791 done:
2792 socket_unlock(fd_cb->so, 0);
2793 }
2794 FDUNLOCK(fd_cb);
2795 }
2796
2797 static void
2798 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2799 {
2800 size_t bytes_mem_size;
2801 size_t child_maps_mem_size;
2802 size_t nodes_mem_size;
2803 size_t trie_memory_size = 0;
2804 int cursor;
2805 int error = 0;
2806 struct flow_divert_trie new_trie;
2807 int insert_error = 0;
2808 int prefix_count = -1;
2809 int signing_id_count = 0;
2810 size_t bytes_count = 0;
2811 size_t nodes_count = 0;
2812 size_t maps_count = 0;
2813
2814 lck_rw_lock_exclusive(&group->lck);
2815
2816 /* Re-set the current trie */
2817 if (group->signing_id_trie.memory != NULL) {
2818 FREE(group->signing_id_trie.memory, M_TEMP);
2819 }
2820 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2821 group->signing_id_trie.root = NULL_TRIE_IDX;
2822
2823 memset(&new_trie, 0, sizeof(new_trie));
2824
2825 /* Get the number of shared prefixes in the new set of signing ID strings */
2826 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2827
2828 if (prefix_count < 0 || error) {
2829 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
2830 lck_rw_done(&group->lck);
2831 return;
2832 }
2833
2834 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2835 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2836 cursor >= 0;
2837 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2838 uint32_t sid_size = 0;
2839 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2840 if (error || sid_size == 0) {
2841 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
2842 signing_id_count = 0;
2843 break;
2844 }
2845 if (os_add_overflow(bytes_count, sid_size, &bytes_count)) {
2846 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while incrementing number of bytes");
2847 signing_id_count = 0;
2848 break;
2849 }
2850 signing_id_count++;
2851 }
2852
2853 if (signing_id_count == 0) {
2854 lck_rw_done(&group->lck);
2855 FDLOG0(LOG_NOTICE, &nil_pcb, "No signing identifiers");
2856 return;
2857 }
2858
2859 if (os_add3_overflow(prefix_count, signing_id_count, 1, &nodes_count)) { /* + 1 for the root node */
2860 lck_rw_done(&group->lck);
2861 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of nodes");
2862 return;
2863 }
2864
2865 if (os_add_overflow(prefix_count, 1, &maps_count)) { /* + 1 for the root node */
2866 lck_rw_done(&group->lck);
2867 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of maps");
2868 return;
2869 }
2870
2871 if (bytes_count > UINT16_MAX || nodes_count > UINT16_MAX || maps_count > UINT16_MAX) {
2872 lck_rw_done(&group->lck);
2873 FDLOG(LOG_NOTICE, &nil_pcb, "Invalid bytes count (%lu), nodes count (%lu) or maps count (%lu)", bytes_count, nodes_count, maps_count);
2874 return;
2875 }
2876
2877 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2878 nodes_count, maps_count, bytes_count);
2879
2880 if (os_mul_overflow(sizeof(*new_trie.nodes), (size_t)nodes_count, &nodes_mem_size) ||
2881 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, (size_t)maps_count, &child_maps_mem_size) ||
2882 os_mul_overflow(sizeof(*new_trie.bytes), (size_t)bytes_count, &bytes_mem_size) ||
2883 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2884 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2885 lck_rw_done(&group->lck);
2886 return;
2887 }
2888
2889 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2890 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2891 lck_rw_done(&group->lck);
2892 return;
2893 }
2894
2895 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2896 if (new_trie.memory == NULL) {
2897 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2898 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2899 lck_rw_done(&group->lck);
2900 return;
2901 }
2902
2903 new_trie.bytes_count = (uint16_t)bytes_count;
2904 new_trie.nodes_count = (uint16_t)nodes_count;
2905 new_trie.child_maps_count = (uint16_t)maps_count;
2906
2907 /* Initialize the free lists */
2908 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2909 new_trie.nodes_free_next = 0;
2910 memset(new_trie.nodes, 0, nodes_mem_size);
2911
2912 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2913 new_trie.child_maps_free_next = 0;
2914 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2915
2916 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2917 new_trie.bytes_free_next = 0;
2918 memset(new_trie.bytes, 0, bytes_mem_size);
2919
2920 /* The root is an empty node */
2921 new_trie.root = trie_node_alloc(&new_trie);
2922
2923 /* Add each signing ID to the trie */
2924 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2925 cursor >= 0;
2926 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2927 uint32_t sid_size = 0;
2928 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2929 if (error || sid_size == 0) {
2930 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
2931 insert_error = EINVAL;
2932 break;
2933 }
2934 if (sid_size <= UINT16_MAX && new_trie.bytes_free_next + (uint16_t)sid_size <= new_trie.bytes_count) {
2935 uint16_t new_node_idx;
2936 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2937 if (error) {
2938 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
2939 insert_error = EINVAL;
2940 break;
2941 }
2942 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2943 if (new_node_idx == NULL_TRIE_IDX) {
2944 insert_error = EINVAL;
2945 break;
2946 }
2947 } else {
2948 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2949 insert_error = ENOBUFS;
2950 break;
2951 }
2952 }
2953
2954 if (!insert_error) {
2955 group->signing_id_trie = new_trie;
2956 } else {
2957 FREE(new_trie.memory, M_TEMP);
2958 }
2959
2960 lck_rw_done(&group->lck);
2961 }
2962
2963 static int
2964 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2965 {
2966 struct flow_divert_packet_header hdr;
2967 int error = 0;
2968 struct flow_divert_pcb *fd_cb;
2969
2970 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2971 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2972 error = EINVAL;
2973 goto done;
2974 }
2975
2976 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2977 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2978 error = EINVAL;
2979 goto done;
2980 }
2981
2982 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2983 if (error) {
2984 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2985 error = ENOBUFS;
2986 goto done;
2987 }
2988
2989 hdr.conn_id = ntohl(hdr.conn_id);
2990
2991 if (hdr.conn_id == 0) {
2992 switch (hdr.packet_type) {
2993 case FLOW_DIVERT_PKT_GROUP_INIT:
2994 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2995 break;
2996 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2997 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2998 break;
2999 default:
3000 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
3001 break;
3002 }
3003 goto done;
3004 }
3005
3006 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
3007 if (fd_cb == NULL) {
3008 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
3009 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
3010 }
3011 goto done;
3012 }
3013
3014 switch (hdr.packet_type) {
3015 case FLOW_DIVERT_PKT_CONNECT_RESULT:
3016 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
3017 break;
3018 case FLOW_DIVERT_PKT_CLOSE:
3019 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
3020 break;
3021 case FLOW_DIVERT_PKT_DATA:
3022 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
3023 break;
3024 case FLOW_DIVERT_PKT_READ_NOTIFY:
3025 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
3026 break;
3027 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
3028 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
3029 break;
3030 default:
3031 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
3032 break;
3033 }
3034
3035 FDRELEASE(fd_cb);
3036
3037 done:
3038 mbuf_freem(packet);
3039 return error;
3040 }
3041
3042 static void
3043 flow_divert_close_all(struct flow_divert_group *group)
3044 {
3045 struct flow_divert_pcb *fd_cb;
3046 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3047
3048 SLIST_INIT(&tmp_list);
3049
3050 lck_rw_lock_exclusive(&group->lck);
3051
3052 MBUFQ_DRAIN(&group->send_queue);
3053
3054 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3055 FDRETAIN(fd_cb);
3056 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3057 }
3058
3059 lck_rw_done(&group->lck);
3060
3061 while (!SLIST_EMPTY(&tmp_list)) {
3062 fd_cb = SLIST_FIRST(&tmp_list);
3063 FDLOCK(fd_cb);
3064 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
3065 if (fd_cb->so != NULL) {
3066 socket_lock(fd_cb->so, 0);
3067 flow_divert_pcb_remove(fd_cb);
3068 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
3069 fd_cb->so->so_error = ECONNABORTED;
3070 flow_divert_disconnect_socket(fd_cb->so);
3071 socket_unlock(fd_cb->so, 0);
3072 }
3073 FDUNLOCK(fd_cb);
3074 FDRELEASE(fd_cb);
3075 }
3076 }
3077
3078 void
3079 flow_divert_detach(struct socket *so)
3080 {
3081 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3082
3083 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3084
3085 so->so_flags &= ~SOF_FLOW_DIVERT;
3086 so->so_fd_pcb = NULL;
3087
3088 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
3089
3090 if (fd_cb->group != NULL) {
3091 /* Last-ditch effort to send any buffered data */
3092 flow_divert_send_buffered_data(fd_cb, TRUE);
3093
3094 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3095 flow_divert_send_close_if_needed(fd_cb);
3096 /* Remove from the group */
3097 flow_divert_pcb_remove(fd_cb);
3098 }
3099
3100 socket_unlock(so, 0);
3101 FDLOCK(fd_cb);
3102 fd_cb->so = NULL;
3103 FDUNLOCK(fd_cb);
3104 socket_lock(so, 0);
3105
3106 FDRELEASE(fd_cb); /* Release the socket's reference */
3107 }
3108
3109 static int
3110 flow_divert_close(struct socket *so)
3111 {
3112 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3113
3114 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3115
3116 FDLOG0(LOG_INFO, fd_cb, "Closing");
3117
3118 if (SOCK_TYPE(so) == SOCK_STREAM) {
3119 soisdisconnecting(so);
3120 sbflush(&so->so_rcv);
3121 }
3122
3123 flow_divert_send_buffered_data(fd_cb, TRUE);
3124 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3125 flow_divert_send_close_if_needed(fd_cb);
3126
3127 /* Remove from the group */
3128 flow_divert_pcb_remove(fd_cb);
3129
3130 return 0;
3131 }
3132
3133 static int
3134 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
3135 sae_connid_t cid __unused)
3136 {
3137 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
3138 return EINVAL;
3139 }
3140
3141 return flow_divert_close(so);
3142 }
3143
3144 static int
3145 flow_divert_shutdown(struct socket *so)
3146 {
3147 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3148
3149 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3150
3151 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
3152
3153 socantsendmore(so);
3154
3155 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
3156 flow_divert_send_close_if_needed(fd_cb);
3157
3158 return 0;
3159 }
3160
3161 static int
3162 flow_divert_rcvd(struct socket *so, int flags __unused)
3163 {
3164 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3165 int space = sbspace(&so->so_rcv);
3166
3167 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3168
3169 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space);
3170 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) &&
3171 (space > 0) &&
3172 flow_divert_send_read_notification(fd_cb) == 0) {
3173 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification");
3174 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED;
3175 }
3176
3177 return 0;
3178 }
3179
3180 static int
3181 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
3182 {
3183 int error = 0;
3184 int port = 0;
3185
3186 if (!flow_divert_is_sockaddr_valid(toaddr)) {
3187 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
3188 error = EINVAL;
3189 goto done;
3190 }
3191
3192 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
3193 if (error) {
3194 goto done;
3195 }
3196
3197 if (toaddr->sa_family == AF_INET) {
3198 port = ntohs((satosin(toaddr))->sin_port);
3199 } else {
3200 port = ntohs((satosin6(toaddr))->sin6_port);
3201 }
3202
3203 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
3204 if (error) {
3205 goto done;
3206 }
3207
3208 done:
3209 return error;
3210 }
3211
3212 struct sockaddr *
3213 flow_divert_get_buffered_target_address(mbuf_t buffer)
3214 {
3215 if (buffer != NULL && buffer->m_type == MT_SONAME) {
3216 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
3217 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
3218 return toaddr;
3219 }
3220 }
3221 return NULL;
3222 }
3223
3224 static boolean_t
3225 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
3226 {
3227 switch (addr->sa_family) {
3228 case AF_INET:
3229 if (addr->sa_len < sizeof(struct sockaddr_in)) {
3230 return FALSE;
3231 }
3232 break;
3233 case AF_INET6:
3234 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3235 return FALSE;
3236 }
3237 break;
3238 default:
3239 return FALSE;
3240 }
3241 return TRUE;
3242 }
3243
3244 static errno_t
3245 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
3246 struct sockaddr **dup)
3247 {
3248 int error = 0;
3249 struct sockaddr *result;
3250 struct sockaddr_storage ss;
3251
3252 if (addr != NULL) {
3253 result = addr;
3254 } else {
3255 memset(&ss, 0, sizeof(ss));
3256 ss.ss_family = family;
3257 if (ss.ss_family == AF_INET) {
3258 ss.ss_len = sizeof(struct sockaddr_in);
3259 } else if (ss.ss_family == AF_INET6) {
3260 ss.ss_len = sizeof(struct sockaddr_in6);
3261 } else {
3262 error = EINVAL;
3263 }
3264 result = (struct sockaddr *)&ss;
3265 }
3266
3267 if (!error) {
3268 *dup = dup_sockaddr(result, 1);
3269 if (*dup == NULL) {
3270 error = ENOBUFS;
3271 }
3272 }
3273
3274 return error;
3275 }
3276
3277 static void
3278 flow_divert_disconnect_socket(struct socket *so)
3279 {
3280 soisdisconnected(so);
3281 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3282 struct inpcb *inp = NULL;
3283
3284 inp = sotoinpcb(so);
3285 if (inp != NULL) {
3286 if (SOCK_CHECK_DOM(so, PF_INET6)) {
3287 in6_pcbdetach(inp);
3288 } else {
3289 in_pcbdetach(inp);
3290 }
3291 }
3292 }
3293 }
3294
3295 static errno_t
3296 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
3297 {
3298 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3299
3300 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3301
3302 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
3303 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3304 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
3305 }
3306 }
3307
3308 if (SOCK_DOM(so) == PF_INET) {
3309 return g_tcp_protosw->pr_ctloutput(so, sopt);
3310 } else if (SOCK_DOM(so) == PF_INET6) {
3311 return g_tcp6_protosw->pr_ctloutput(so, sopt);
3312 }
3313 return 0;
3314 }
3315
3316 static errno_t
3317 flow_divert_connect_out_internal(struct socket *so, struct sockaddr *to, proc_t p, bool implicit)
3318 {
3319 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3320 int error = 0;
3321 struct inpcb *inp = sotoinpcb(so);
3322 struct sockaddr_in *sinp;
3323 mbuf_t connect_packet = NULL;
3324 int do_send = 1;
3325
3326 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3327
3328 if (fd_cb->group == NULL) {
3329 error = ENETUNREACH;
3330 goto done;
3331 }
3332
3333 if (inp == NULL) {
3334 error = EINVAL;
3335 goto done;
3336 } else if (inp->inp_state == INPCB_STATE_DEAD) {
3337 if (so->so_error) {
3338 error = so->so_error;
3339 so->so_error = 0;
3340 } else {
3341 error = EINVAL;
3342 }
3343 goto done;
3344 }
3345
3346 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3347 error = EALREADY;
3348 goto done;
3349 }
3350
3351 FDLOG0(LOG_INFO, fd_cb, "Connecting");
3352
3353 if (fd_cb->connect_packet == NULL) {
3354 struct sockaddr_in sin = {};
3355 struct ifnet *ifp = NULL;
3356
3357 if (to == NULL) {
3358 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
3359 error = EINVAL;
3360 goto done;
3361 }
3362
3363 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0);
3364 if (fd_cb->original_remote_endpoint == NULL) {
3365 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint");
3366 error = ENOMEM;
3367 goto done;
3368 }
3369 fd_cb->original_vflag = inp->inp_vflag;
3370 fd_cb->original_last_outifp = inp->inp_last_outifp;
3371 fd_cb->original_last_outifp6 = inp->in6p_last_outifp;
3372
3373 sinp = (struct sockaddr_in *)(void *)to;
3374 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
3375 error = EAFNOSUPPORT;
3376 goto done;
3377 }
3378
3379 if (to->sa_family == AF_INET6 && !(inp->inp_flags & IN6P_IPV6_V6ONLY)) {
3380 struct sockaddr_in6 sin6 = {};
3381 sin6.sin6_family = AF_INET6;
3382 sin6.sin6_len = sizeof(struct sockaddr_in6);
3383 sin6.sin6_port = satosin6(to)->sin6_port;
3384 sin6.sin6_addr = satosin6(to)->sin6_addr;
3385 if (IN6_IS_ADDR_V4MAPPED(&(sin6.sin6_addr))) {
3386 in6_sin6_2_sin(&sin, &sin6);
3387 to = (struct sockaddr *)&sin;
3388 }
3389 }
3390
3391 if (to->sa_family == AF_INET6) {
3392 inp->inp_vflag &= ~INP_IPV4;
3393 inp->inp_vflag |= INP_IPV6;
3394 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6);
3395 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6;
3396 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport;
3397 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp);
3398 if (error) {
3399 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error);
3400 error = 0;
3401 }
3402 if (ifp != NULL) {
3403 inp->in6p_last_outifp = ifp;
3404 ifnet_release(ifp);
3405 }
3406 } else if (to->sa_family == AF_INET) {
3407 inp->inp_vflag |= INP_IPV4;
3408 inp->inp_vflag &= ~INP_IPV6;
3409 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in);
3410 fd_cb->local_endpoint.sin.sin_family = AF_INET;
3411 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport;
3412 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0);
3413 if (error) {
3414 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error);
3415 error = 0;
3416 }
3417 if (ifp != NULL) {
3418 inp->inp_last_outifp = ifp;
3419 ifnet_release(ifp);
3420 }
3421 } else {
3422 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family);
3423 }
3424
3425 error = flow_divert_check_no_cellular(fd_cb) ||
3426 flow_divert_check_no_expensive(fd_cb) ||
3427 flow_divert_check_no_constrained(fd_cb);
3428 if (error) {
3429 goto done;
3430 }
3431
3432 if (SOCK_TYPE(so) == SOCK_STREAM || /* TCP or */
3433 !implicit || /* connect() was called or */
3434 ((inp->inp_vflag & INP_IPV6) && !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) || /* local address is not un-specified */
3435 ((inp->inp_vflag & INP_IPV4) && inp->inp_laddr.s_addr != INADDR_ANY)) {
3436 fd_cb->flags |= FLOW_DIVERT_SHOULD_SET_LOCAL_ADDR;
3437 }
3438
3439 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
3440 if (error) {
3441 goto done;
3442 }
3443
3444 if (!implicit || SOCK_TYPE(so) == SOCK_STREAM) {
3445 flow_divert_set_remote_endpoint(fd_cb, to);
3446 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa));
3447 }
3448
3449 if (implicit) {
3450 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT;
3451 }
3452
3453 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
3454 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
3455 do_send = 0;
3456 }
3457
3458 fd_cb->connect_packet = connect_packet;
3459 connect_packet = NULL;
3460 } else {
3461 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
3462 }
3463
3464 if (do_send) {
3465 error = flow_divert_send_connect_packet(fd_cb);
3466 if (error) {
3467 goto done;
3468 }
3469
3470 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3471 }
3472
3473 if (SOCK_TYPE(so) == SOCK_DGRAM && !(fd_cb->flags & FLOW_DIVERT_HAS_TOKEN)) {
3474 soisconnected(so);
3475 } else {
3476 soisconnecting(so);
3477 }
3478
3479 done:
3480 return error;
3481 }
3482
3483 errno_t
3484 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
3485 {
3486 #if CONTENT_FILTER
3487 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_flags & SOF_CONTENT_FILTER)) {
3488 int error = cfil_sock_attach(so, NULL, to, CFS_CONNECTION_DIR_OUT);
3489 if (error != 0) {
3490 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3491 FDLOG(LOG_ERR, fd_cb, "Failed to attach cfil: %d", error);
3492 return error;
3493 }
3494 }
3495 #endif /* CONTENT_FILTER */
3496
3497 return flow_divert_connect_out_internal(so, to, p, false);
3498 }
3499
3500 static int
3501 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3502 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
3503 {
3504 struct inpcb *inp = sotoinpcb(so);
3505 int error;
3506
3507 if (inp == NULL) {
3508 return EINVAL;
3509 }
3510
3511 VERIFY(dst != NULL);
3512
3513 error = flow_divert_connect_out(so, dst, p);
3514
3515 if (error != 0) {
3516 return error;
3517 }
3518
3519 /* if there is data, send it */
3520 if (auio != NULL) {
3521 user_ssize_t datalen = 0;
3522
3523 socket_unlock(so, 0);
3524
3525 VERIFY(bytes_written != NULL);
3526
3527 datalen = uio_resid(auio);
3528 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3529 socket_lock(so, 0);
3530
3531 if (error == 0 || error == EWOULDBLOCK) {
3532 *bytes_written = datalen - uio_resid(auio);
3533 }
3534
3535 /*
3536 * sosend returns EWOULDBLOCK if it's a non-blocking
3537 * socket or a timeout occured (this allows to return
3538 * the amount of queued data through sendit()).
3539 *
3540 * However, connectx() returns EINPROGRESS in case of a
3541 * blocking socket. So we change the return value here.
3542 */
3543 if (error == EWOULDBLOCK) {
3544 error = EINPROGRESS;
3545 }
3546 }
3547
3548 if (error == 0 && pcid != NULL) {
3549 *pcid = 1; /* there is only 1 connection for a TCP */
3550 }
3551
3552 return error;
3553 }
3554
3555 static int
3556 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3557 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3558 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3559 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3560 {
3561 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3562 }
3563
3564 static int
3565 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3566 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3567 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3568 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3569 {
3570 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3571 }
3572
3573 static errno_t
3574 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3575 {
3576 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3577 int error = 0;
3578 struct inpcb *inp;
3579 #if CONTENT_FILTER
3580 struct m_tag *cfil_tag = NULL;
3581 #endif
3582
3583 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3584
3585 inp = sotoinpcb(so);
3586 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3587 error = ECONNRESET;
3588 goto done;
3589 }
3590
3591 if (control && mbuf_len(control) > 0) {
3592 error = EINVAL;
3593 goto done;
3594 }
3595
3596 if (flags & MSG_OOB) {
3597 error = EINVAL;
3598 goto done; /* We don't support OOB data */
3599 }
3600
3601 #if CONTENT_FILTER
3602 /*
3603 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3604 * retrieve the CFIL saved remote address from the mbuf and use it.
3605 */
3606 if (to == NULL && so->so_cfil_db) {
3607 struct sockaddr *cfil_faddr = NULL;
3608 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3609 if (cfil_tag) {
3610 to = (struct sockaddr *)(void *)cfil_faddr;
3611 }
3612 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3613 }
3614 #endif
3615
3616 /* Implicit connect */
3617 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3618 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3619
3620 error = flow_divert_connect_out_internal(so, to, p, true);
3621 if (error) {
3622 goto done;
3623 }
3624 } else {
3625 error = flow_divert_check_no_cellular(fd_cb) ||
3626 flow_divert_check_no_expensive(fd_cb) ||
3627 flow_divert_check_no_constrained(fd_cb);
3628 if (error) {
3629 goto done;
3630 }
3631 }
3632
3633 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3634
3635 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3636 error = flow_divert_send_app_data(fd_cb, data, to);
3637
3638 data = NULL;
3639
3640 if (error) {
3641 goto done;
3642 }
3643
3644 if (flags & PRUS_EOF) {
3645 flow_divert_shutdown(so);
3646 }
3647
3648 done:
3649 if (data) {
3650 mbuf_freem(data);
3651 }
3652 if (control) {
3653 mbuf_free(control);
3654 }
3655 #if CONTENT_FILTER
3656 if (cfil_tag) {
3657 m_tag_free(cfil_tag);
3658 }
3659 #endif
3660
3661 return error;
3662 }
3663
3664 static int
3665 flow_divert_preconnect(struct socket *so)
3666 {
3667 int error = 0;
3668 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3669
3670 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3671
3672 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3673 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3674 error = flow_divert_send_connect_packet(so->so_fd_pcb);
3675 if (error) {
3676 return error;
3677 }
3678
3679 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3680 }
3681
3682 soclearfastopen(so);
3683
3684 return error;
3685 }
3686
3687 static void
3688 flow_divert_set_protosw(struct socket *so)
3689 {
3690 if (SOCK_DOM(so) == PF_INET) {
3691 so->so_proto = &g_flow_divert_in_protosw;
3692 } else {
3693 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3694 }
3695 }
3696
3697 static void
3698 flow_divert_set_udp_protosw(struct socket *so)
3699 {
3700 if (SOCK_DOM(so) == PF_INET) {
3701 so->so_proto = &g_flow_divert_in_udp_protosw;
3702 } else {
3703 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3704 }
3705 }
3706
3707 errno_t
3708 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3709 {
3710 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3711 struct inpcb *inp;
3712 int error = 0;
3713
3714 inp = sotoinpcb(so);
3715 if (inp == NULL) {
3716 return EINVAL;
3717 }
3718
3719 if (fd_cb == NULL) {
3720 error = flow_divert_pcb_init(so);
3721 fd_cb = so->so_fd_pcb;
3722 if (error != 0 || fd_cb == NULL) {
3723 goto done;
3724 }
3725 }
3726 return flow_divert_data_out(so, flags, data, to, control, p);
3727
3728 done:
3729 if (data) {
3730 mbuf_freem(data);
3731 }
3732 if (control) {
3733 mbuf_free(control);
3734 }
3735
3736 return error;
3737 }
3738
3739 static errno_t
3740 flow_divert_pcb_init_internal(struct socket *so, uint32_t ctl_unit, uint32_t aggregate_unit)
3741 {
3742 errno_t error = 0;
3743 struct flow_divert_pcb *fd_cb;
3744 uint32_t agg_unit = aggregate_unit;
3745 uint32_t group_unit = flow_divert_derive_kernel_control_unit(ctl_unit, &agg_unit);
3746
3747 if (group_unit == 0) {
3748 return EINVAL;
3749 }
3750
3751 if (so->so_flags & SOF_FLOW_DIVERT) {
3752 return EALREADY;
3753 }
3754
3755 fd_cb = flow_divert_pcb_create(so);
3756 if (fd_cb != NULL) {
3757 so->so_fd_pcb = fd_cb;
3758 so->so_flags |= SOF_FLOW_DIVERT;
3759 fd_cb->control_group_unit = group_unit;
3760 fd_cb->policy_control_unit = ctl_unit;
3761 fd_cb->aggregate_unit = agg_unit;
3762
3763 error = flow_divert_pcb_insert(fd_cb, group_unit);
3764 if (error) {
3765 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3766 so->so_fd_pcb = NULL;
3767 so->so_flags &= ~SOF_FLOW_DIVERT;
3768 FDRELEASE(fd_cb);
3769 } else {
3770 if (SOCK_TYPE(so) == SOCK_STREAM) {
3771 flow_divert_set_protosw(so);
3772 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3773 flow_divert_set_udp_protosw(so);
3774 }
3775
3776 FDLOG0(LOG_INFO, fd_cb, "Created");
3777 }
3778 } else {
3779 error = ENOMEM;
3780 }
3781
3782 return error;
3783 }
3784
3785 errno_t
3786 flow_divert_pcb_init(struct socket *so)
3787 {
3788 struct inpcb *inp = sotoinpcb(so);
3789 uint32_t aggregate_units = 0;
3790 uint32_t ctl_unit = necp_socket_get_flow_divert_control_unit(inp, &aggregate_units);
3791 return flow_divert_pcb_init_internal(so, ctl_unit, aggregate_units);
3792 }
3793
3794 errno_t
3795 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3796 {
3797 uint32_t ctl_unit = 0;
3798 uint32_t key_unit = 0;
3799 uint32_t aggregate_unit = 0;
3800 int error = 0;
3801 int hmac_error = 0;
3802 mbuf_t token = NULL;
3803
3804 if (so->so_flags & SOF_FLOW_DIVERT) {
3805 error = EALREADY;
3806 goto done;
3807 }
3808
3809 if (g_init_result) {
3810 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3811 error = ENOPROTOOPT;
3812 goto done;
3813 }
3814
3815 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3816 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3817 (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6)) {
3818 error = EINVAL;
3819 goto done;
3820 } else {
3821 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3822 struct tcpcb *tp = sototcpcb(so);
3823 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3824 error = EINVAL;
3825 goto done;
3826 }
3827 }
3828 }
3829
3830 error = soopt_getm(sopt, &token);
3831 if (error) {
3832 token = NULL;
3833 goto done;
3834 }
3835
3836 error = soopt_mcopyin(sopt, token);
3837 if (error) {
3838 token = NULL;
3839 goto done;
3840 }
3841
3842 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3843 if (!error) {
3844 key_unit = ntohl(key_unit);
3845 if (key_unit >= GROUP_COUNT_MAX) {
3846 key_unit = 0;
3847 }
3848 } else if (error != ENOENT) {
3849 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3850 goto done;
3851 } else {
3852 key_unit = 0;
3853 }
3854
3855 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3856 if (error) {
3857 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3858 goto done;
3859 }
3860
3861 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_AGGREGATE_UNIT, sizeof(aggregate_unit), (void *)&aggregate_unit, NULL);
3862 if (error && error != ENOENT) {
3863 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the aggregate unit from the token: %d", error);
3864 goto done;
3865 }
3866
3867 /* A valid kernel control unit is required */
3868 ctl_unit = ntohl(ctl_unit);
3869 aggregate_unit = ntohl(aggregate_unit);
3870
3871 if (ctl_unit > 0 && ctl_unit < GROUP_COUNT_MAX) {
3872 socket_unlock(so, 0);
3873 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3874 socket_lock(so, 0);
3875
3876 if (hmac_error && hmac_error != ENOENT) {
3877 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3878 error = hmac_error;
3879 goto done;
3880 }
3881 }
3882
3883 error = flow_divert_pcb_init_internal(so, ctl_unit, aggregate_unit);
3884 if (error == 0) {
3885 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3886 int log_level = LOG_NOTICE;
3887
3888 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
3889 if (error == 0) {
3890 fd_cb->log_level = (uint8_t)log_level;
3891 }
3892 error = 0;
3893
3894 fd_cb->connect_token = token;
3895 token = NULL;
3896
3897 fd_cb->flags |= FLOW_DIVERT_HAS_TOKEN;
3898 }
3899
3900 if (hmac_error == 0) {
3901 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3902 if (fd_cb != NULL) {
3903 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3904 }
3905 }
3906
3907 done:
3908 if (token != NULL) {
3909 mbuf_freem(token);
3910 }
3911
3912 return error;
3913 }
3914
3915 errno_t
3916 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3917 {
3918 uint32_t ctl_unit;
3919 int error = 0;
3920 uint8_t hmac[SHA_DIGEST_LENGTH];
3921 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3922 mbuf_t token = NULL;
3923 struct flow_divert_group *control_group = NULL;
3924
3925 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3926 error = EINVAL;
3927 goto done;
3928 }
3929
3930 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3931
3932 if (fd_cb->group == NULL) {
3933 error = EINVAL;
3934 goto done;
3935 }
3936
3937 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3938 if (error) {
3939 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3940 goto done;
3941 }
3942
3943 ctl_unit = htonl(fd_cb->group->ctl_unit);
3944
3945 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3946 if (error) {
3947 goto done;
3948 }
3949
3950 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3951 if (error) {
3952 goto done;
3953 }
3954
3955 if (fd_cb->app_data != NULL) {
3956 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_data);
3957 if (error) {
3958 goto done;
3959 }
3960 }
3961
3962 socket_unlock(so, 0);
3963 lck_rw_lock_shared(&g_flow_divert_group_lck);
3964
3965 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3966 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
3967 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3968 }
3969
3970 if (control_group != NULL) {
3971 lck_rw_lock_shared(&control_group->lck);
3972 ctl_unit = htonl(control_group->ctl_unit);
3973 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3974 if (!error) {
3975 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3976 }
3977 lck_rw_done(&control_group->lck);
3978 } else {
3979 error = ENOPROTOOPT;
3980 }
3981
3982 lck_rw_done(&g_flow_divert_group_lck);
3983 socket_lock(so, 0);
3984
3985 if (error) {
3986 goto done;
3987 }
3988
3989 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3990 if (error) {
3991 goto done;
3992 }
3993
3994 if (sopt->sopt_val == USER_ADDR_NULL) {
3995 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3996 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3997 goto done;
3998 }
3999
4000 error = soopt_mcopyout(sopt, token);
4001 if (error) {
4002 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
4003 goto done;
4004 }
4005
4006 done:
4007 if (token != NULL) {
4008 mbuf_freem(token);
4009 }
4010
4011 return error;
4012 }
4013
4014 static errno_t
4015 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
4016 {
4017 struct flow_divert_group *new_group = NULL;
4018 int error = 0;
4019
4020 if (sac->sc_unit >= GROUP_COUNT_MAX) {
4021 error = EINVAL;
4022 goto done;
4023 }
4024
4025 *unitinfo = NULL;
4026
4027 new_group = zalloc_flags(flow_divert_group_zone, Z_WAITOK | Z_ZERO);
4028 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4029 RB_INIT(&new_group->pcb_tree);
4030 new_group->ctl_unit = sac->sc_unit;
4031 MBUFQ_INIT(&new_group->send_queue);
4032 new_group->signing_id_trie.root = NULL_TRIE_IDX;
4033
4034 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4035
4036 if (g_flow_divert_groups == NULL) {
4037 MALLOC(g_flow_divert_groups,
4038 struct flow_divert_group **,
4039 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
4040 M_TEMP,
4041 M_WAITOK | M_ZERO);
4042 }
4043
4044 if (g_flow_divert_groups == NULL) {
4045 error = ENOBUFS;
4046 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
4047 error = EALREADY;
4048 } else {
4049 g_flow_divert_groups[sac->sc_unit] = new_group;
4050 g_active_group_count++;
4051 }
4052
4053 lck_rw_done(&g_flow_divert_group_lck);
4054
4055 done:
4056 if (error == 0) {
4057 *unitinfo = new_group;
4058 } else if (new_group != NULL) {
4059 zfree(flow_divert_group_zone, new_group);
4060 }
4061 return error;
4062 }
4063
4064 static errno_t
4065 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
4066 {
4067 struct flow_divert_group *group = NULL;
4068 errno_t error = 0;
4069
4070 if (unit >= GROUP_COUNT_MAX) {
4071 return EINVAL;
4072 }
4073
4074 if (unitinfo == NULL) {
4075 return 0;
4076 }
4077
4078 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
4079
4080 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
4081
4082 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
4083 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
4084 g_flow_divert_groups, g_active_group_count);
4085 }
4086
4087 group = g_flow_divert_groups[unit];
4088
4089 if (group != (struct flow_divert_group *)unitinfo) {
4090 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
4091 }
4092
4093 g_flow_divert_groups[unit] = NULL;
4094 g_active_group_count--;
4095
4096 if (g_active_group_count == 0) {
4097 FREE(g_flow_divert_groups, M_TEMP);
4098 g_flow_divert_groups = NULL;
4099 }
4100
4101 lck_rw_done(&g_flow_divert_group_lck);
4102
4103 if (group != NULL) {
4104 flow_divert_close_all(group);
4105
4106 lck_rw_lock_exclusive(&group->lck);
4107
4108 if (group->token_key != NULL) {
4109 memset(group->token_key, 0, group->token_key_size);
4110 FREE(group->token_key, M_TEMP);
4111 group->token_key = NULL;
4112 group->token_key_size = 0;
4113 }
4114
4115 /* Re-set the current trie */
4116 if (group->signing_id_trie.memory != NULL) {
4117 FREE(group->signing_id_trie.memory, M_TEMP);
4118 }
4119 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
4120 group->signing_id_trie.root = NULL_TRIE_IDX;
4121
4122 lck_rw_done(&group->lck);
4123
4124 zfree(flow_divert_group_zone, group);
4125 } else {
4126 error = EINVAL;
4127 }
4128
4129 return error;
4130 }
4131
4132 static errno_t
4133 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
4134 {
4135 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
4136 }
4137
4138 static void
4139 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
4140 {
4141 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
4142
4143 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
4144 struct flow_divert_pcb *fd_cb;
4145 SLIST_HEAD(, flow_divert_pcb) tmp_list;
4146
4147 lck_rw_lock_shared(&g_flow_divert_group_lck);
4148 lck_rw_lock_exclusive(&group->lck);
4149
4150 while (!MBUFQ_EMPTY(&group->send_queue)) {
4151 mbuf_t next_packet;
4152 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
4153 next_packet = MBUFQ_FIRST(&group->send_queue);
4154 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
4155 if (error) {
4156 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
4157 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
4158 lck_rw_done(&group->lck);
4159 lck_rw_done(&g_flow_divert_group_lck);
4160 return;
4161 }
4162 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
4163 }
4164
4165 SLIST_INIT(&tmp_list);
4166
4167 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
4168 FDRETAIN(fd_cb);
4169 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
4170 }
4171
4172 lck_rw_done(&group->lck);
4173
4174 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
4175 FDLOCK(fd_cb);
4176 if (fd_cb->so != NULL) {
4177 socket_lock(fd_cb->so, 0);
4178 if (fd_cb->group != NULL) {
4179 flow_divert_send_buffered_data(fd_cb, FALSE);
4180 }
4181 socket_unlock(fd_cb->so, 0);
4182 }
4183 FDUNLOCK(fd_cb);
4184 FDRELEASE(fd_cb);
4185 }
4186
4187 lck_rw_done(&g_flow_divert_group_lck);
4188 }
4189 }
4190
4191 static int
4192 flow_divert_kctl_init(void)
4193 {
4194 struct kern_ctl_reg ctl_reg;
4195 int result;
4196
4197 memset(&ctl_reg, 0, sizeof(ctl_reg));
4198
4199 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
4200 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
4201 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
4202 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
4203 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
4204
4205 ctl_reg.ctl_connect = flow_divert_kctl_connect;
4206 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
4207 ctl_reg.ctl_send = flow_divert_kctl_send;
4208 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
4209
4210 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
4211
4212 if (result) {
4213 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
4214 return result;
4215 }
4216
4217 return 0;
4218 }
4219
4220 void
4221 flow_divert_init(void)
4222 {
4223 memset(&nil_pcb, 0, sizeof(nil_pcb));
4224 nil_pcb.log_level = LOG_NOTICE;
4225
4226 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4227
4228 VERIFY(g_tcp_protosw != NULL);
4229
4230 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4231 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4232
4233 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4234 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
4235 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4236 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4237 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4238 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4239 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
4240 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
4241
4242 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4243 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4244
4245 /*
4246 * Socket filters shouldn't attach/detach to/from this protosw
4247 * since pr_protosw is to be used instead, which points to the
4248 * real protocol; if they do, it is a bug and we should panic.
4249 */
4250 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4251 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4252 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4253 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4254
4255 /* UDP */
4256 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4257 VERIFY(g_udp_protosw != NULL);
4258
4259 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4260 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4261
4262 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4263 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
4264 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4265 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4266 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4267 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4268 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4269 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4270 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4271 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4272
4273 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4274 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4275
4276 /*
4277 * Socket filters shouldn't attach/detach to/from this protosw
4278 * since pr_protosw is to be used instead, which points to the
4279 * real protocol; if they do, it is a bug and we should panic.
4280 */
4281 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4282 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4283 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4284 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4285
4286 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4287
4288 VERIFY(g_tcp6_protosw != NULL);
4289
4290 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4291 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4292
4293 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4294 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
4295 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4296 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4297 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4298 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4299 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
4300 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
4301
4302 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4303 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4304 /*
4305 * Socket filters shouldn't attach/detach to/from this protosw
4306 * since pr_protosw is to be used instead, which points to the
4307 * real protocol; if they do, it is a bug and we should panic.
4308 */
4309 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4310 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4311 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4312 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4313
4314 /* UDP6 */
4315 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4316
4317 VERIFY(g_udp6_protosw != NULL);
4318
4319 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4320 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4321
4322 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4323 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4324 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4325 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4326 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4327 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4328 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4329 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4330 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4331 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4332
4333 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4334 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4335 /*
4336 * Socket filters shouldn't attach/detach to/from this protosw
4337 * since pr_protosw is to be used instead, which points to the
4338 * real protocol; if they do, it is a bug and we should panic.
4339 */
4340 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4341 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4342 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4343 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4344
4345 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4346 if (flow_divert_grp_attr == NULL) {
4347 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4348 g_init_result = ENOMEM;
4349 goto done;
4350 }
4351
4352 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4353 if (flow_divert_mtx_grp == NULL) {
4354 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4355 g_init_result = ENOMEM;
4356 goto done;
4357 }
4358
4359 flow_divert_mtx_attr = lck_attr_alloc_init();
4360 if (flow_divert_mtx_attr == NULL) {
4361 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4362 g_init_result = ENOMEM;
4363 goto done;
4364 }
4365
4366 g_init_result = flow_divert_kctl_init();
4367 if (g_init_result) {
4368 goto done;
4369 }
4370
4371 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4372
4373 done:
4374 if (g_init_result != 0) {
4375 if (flow_divert_mtx_attr != NULL) {
4376 lck_attr_free(flow_divert_mtx_attr);
4377 flow_divert_mtx_attr = NULL;
4378 }
4379 if (flow_divert_mtx_grp != NULL) {
4380 lck_grp_free(flow_divert_mtx_grp);
4381 flow_divert_mtx_grp = NULL;
4382 }
4383 if (flow_divert_grp_attr != NULL) {
4384 lck_grp_attr_free(flow_divert_grp_attr);
4385 flow_divert_grp_attr = NULL;
4386 }
4387
4388 if (g_flow_divert_kctl_ref != NULL) {
4389 ctl_deregister(g_flow_divert_kctl_ref);
4390 g_flow_divert_kctl_ref = NULL;
4391 }
4392 }
4393 }
mirror of XNU