]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:


8a2b4c4f86b737afc8c4f1fe436438e5c41af05e
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2017, 2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <kern/task.h>
49 #include <mach/task_info.h>
50 #include <net/if_var.h>
51 #include <net/route.h>
52 #include <net/flowhash.h>
53 #include <net/ntstat.h>
54 #include <net/content_filter.h>
55 #include <net/necp.h>
56 #include <netinet/in.h>
57 #include <netinet/in_var.h>
58 #include <netinet/tcp.h>
59 #include <netinet/tcp_var.h>
60 #include <netinet/tcp_fsm.h>
61 #include <netinet/flow_divert.h>
62 #include <netinet/flow_divert_proto.h>
63 #include <netinet6/in6_pcb.h>
64 #include <netinet6/ip6protosw.h>
65 #include <dev/random/randomdev.h>
66 #include <libkern/crypto/sha1.h>
67 #include <libkern/crypto/crypto_internal.h>
68 #include <os/log.h>
69 #include <corecrypto/cc.h>
70 #if CONTENT_FILTER
71 #include <net/content_filter.h>
72 #endif /* CONTENT_FILTER */
73
74 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75 #define FLOW_DIVERT_READ_CLOSED 0x00000002
76 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
79 #define FLOW_DIVERT_HAS_HMAC 0x00000040
80 #define FLOW_DIVERT_NOTIFY_ON_RECEIVED 0x00000080
81 #define FLOW_DIVERT_IMPLICIT_CONNECT 0x00000100
82 #define FLOW_DIVERT_DID_SET_LOCAL_ADDR 0x00000200
83
84 #define FDLOG(level, pcb, format, ...) \
85 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
86
87 #define FDLOG0(level, pcb, msg) \
88 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
89
90 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
91 #define FDRELEASE(pcb) \
92 do { \
93 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
94 flow_divert_pcb_destroy(pcb); \
95 } \
96 } while (0)
97
98 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
99 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
100
101 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
102 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
103
104 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
105
106 #define GROUP_COUNT_MAX 31
107 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
108 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
109 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
110
111 struct flow_divert_trie_node {
112 uint16_t start;
113 uint16_t length;
114 uint16_t child_map;
115 };
116
117 #define CHILD_MAP_SIZE 256
118 #define NULL_TRIE_IDX 0xffff
119 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
120 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
121 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
122
123 static struct flow_divert_pcb nil_pcb;
124
125 decl_lck_rw_data(static, g_flow_divert_group_lck);
126 static struct flow_divert_group **g_flow_divert_groups = NULL;
127 static uint32_t g_active_group_count = 0;
128
129 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
130 static lck_attr_t *flow_divert_mtx_attr = NULL;
131 static lck_grp_t *flow_divert_mtx_grp = NULL;
132 static errno_t g_init_result = 0;
133
134 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
135
136 static struct protosw g_flow_divert_in_protosw;
137 static struct pr_usrreqs g_flow_divert_in_usrreqs;
138 static struct protosw g_flow_divert_in_udp_protosw;
139 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
140 static struct ip6protosw g_flow_divert_in6_protosw;
141 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
142 static struct ip6protosw g_flow_divert_in6_udp_protosw;
143 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
144
145 static struct protosw *g_tcp_protosw = NULL;
146 static struct ip6protosw *g_tcp6_protosw = NULL;
147 static struct protosw *g_udp_protosw = NULL;
148 static struct ip6protosw *g_udp6_protosw = NULL;
149
150 ZONE_DECLARE(flow_divert_group_zone, "flow_divert_group",
151 sizeof(struct flow_divert_group), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
152 ZONE_DECLARE(flow_divert_pcb_zone, "flow_divert_pcb",
153 sizeof(struct flow_divert_pcb), ZC_ZFREE_CLEARMEM | ZC_NOENCRYPT);
154
155 static errno_t
156 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
157
158 static boolean_t
159 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
160
161 static int
162 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
163
164 struct sockaddr *
165 flow_divert_get_buffered_target_address(mbuf_t buffer);
166
167 static void
168 flow_divert_disconnect_socket(struct socket *so);
169
170 static inline uint8_t
171 flow_divert_syslog_type_to_oslog_type(int syslog_type)
172 {
173 switch (syslog_type) {
174 case LOG_ERR: return OS_LOG_TYPE_ERROR;
175 case LOG_INFO: return OS_LOG_TYPE_INFO;
176 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
177 default: return OS_LOG_TYPE_DEFAULT;
178 }
179 }
180
181 static inline int
182 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
183 {
184 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
185 }
186
187 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
188 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
189
190 static const char *
191 flow_divert_packet_type2str(uint8_t packet_type)
192 {
193 switch (packet_type) {
194 case FLOW_DIVERT_PKT_CONNECT:
195 return "connect";
196 case FLOW_DIVERT_PKT_CONNECT_RESULT:
197 return "connect result";
198 case FLOW_DIVERT_PKT_DATA:
199 return "data";
200 case FLOW_DIVERT_PKT_CLOSE:
201 return "close";
202 case FLOW_DIVERT_PKT_READ_NOTIFY:
203 return "read notification";
204 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
205 return "properties update";
206 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
207 return "app map create";
208 default:
209 return "unknown";
210 }
211 }
212
213 static struct flow_divert_pcb *
214 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
215 {
216 struct flow_divert_pcb key_item;
217 struct flow_divert_pcb *fd_cb = NULL;
218
219 key_item.hash = hash;
220
221 lck_rw_lock_shared(&group->lck);
222 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
223 FDRETAIN(fd_cb);
224 lck_rw_done(&group->lck);
225
226 return fd_cb;
227 }
228
229 static errno_t
230 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
231 {
232 errno_t error = 0;
233 struct flow_divert_pcb *exist = NULL;
234 struct flow_divert_group *group;
235 static uint32_t g_nextkey = 1;
236 static uint32_t g_hash_seed = 0;
237 int try_count = 0;
238
239 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
240 return EINVAL;
241 }
242
243 socket_unlock(fd_cb->so, 0);
244 lck_rw_lock_shared(&g_flow_divert_group_lck);
245
246 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
247 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
248 error = ENETUNREACH;
249 goto done;
250 }
251
252 group = g_flow_divert_groups[ctl_unit];
253 if (group == NULL) {
254 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
255 error = ENETUNREACH;
256 goto done;
257 }
258
259 socket_lock(fd_cb->so, 0);
260
261 do {
262 uint32_t key[2];
263 uint32_t idx;
264
265 key[0] = g_nextkey++;
266 key[1] = RandomULong();
267
268 if (g_hash_seed == 0) {
269 g_hash_seed = RandomULong();
270 }
271
272 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
273
274 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
275 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
276 if (curr_group != NULL && curr_group != group) {
277 lck_rw_lock_shared(&curr_group->lck);
278 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
279 lck_rw_done(&curr_group->lck);
280 if (exist != NULL) {
281 break;
282 }
283 }
284 }
285
286 if (exist == NULL) {
287 lck_rw_lock_exclusive(&group->lck);
288 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
289 lck_rw_done(&group->lck);
290 }
291 } while (exist != NULL && try_count++ < 3);
292
293 if (exist == NULL) {
294 fd_cb->group = group;
295 FDRETAIN(fd_cb); /* The group now has a reference */
296 } else {
297 fd_cb->hash = 0;
298 error = EEXIST;
299 }
300
301 socket_unlock(fd_cb->so, 0);
302
303 done:
304 lck_rw_done(&g_flow_divert_group_lck);
305 socket_lock(fd_cb->so, 0);
306
307 return error;
308 }
309
310 static struct flow_divert_pcb *
311 flow_divert_pcb_create(socket_t so)
312 {
313 struct flow_divert_pcb *new_pcb = NULL;
314
315 new_pcb = zalloc_flags(flow_divert_pcb_zone, Z_WAITOK | Z_ZERO);
316 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
317 new_pcb->so = so;
318 new_pcb->log_level = nil_pcb.log_level;
319
320 FDRETAIN(new_pcb); /* Represents the socket's reference */
321
322 return new_pcb;
323 }
324
325 static void
326 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
327 {
328 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, tunnel tx %u, tunnel rx %u",
329 fd_cb->bytes_written_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
330
331 if (fd_cb->connect_token != NULL) {
332 mbuf_freem(fd_cb->connect_token);
333 }
334 if (fd_cb->connect_packet != NULL) {
335 mbuf_freem(fd_cb->connect_packet);
336 }
337 if (fd_cb->app_data != NULL) {
338 FREE(fd_cb->app_data, M_TEMP);
339 }
340 if (fd_cb->original_remote_endpoint != NULL) {
341 FREE(fd_cb->original_remote_endpoint, M_SONAME);
342 }
343 zfree(flow_divert_pcb_zone, fd_cb);
344 }
345
346 static void
347 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
348 {
349 if (fd_cb->group != NULL) {
350 struct flow_divert_group *group = fd_cb->group;
351 lck_rw_lock_exclusive(&group->lck);
352 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
353 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
354 fd_cb->group = NULL;
355 FDRELEASE(fd_cb); /* Release the group's reference */
356 lck_rw_done(&group->lck);
357 }
358 }
359
360 static int
361 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
362 {
363 struct flow_divert_packet_header hdr;
364 int error = 0;
365
366 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
367 if (error) {
368 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
369 return error;
370 }
371
372 hdr.packet_type = packet_type;
373 hdr.conn_id = htonl(fd_cb->hash);
374
375 /* Lay down the header */
376 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
377 if (error) {
378 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
379 mbuf_freem(*packet);
380 *packet = NULL;
381 return error;
382 }
383
384 return 0;
385 }
386
387 static int
388 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
389 {
390 uint32_t net_length = htonl(length);
391 int error = 0;
392
393 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
394 if (error) {
395 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
396 return error;
397 }
398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
400 if (error) {
401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
406 if (error) {
407 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
408 return error;
409 }
410
411 return error;
412 }
413
414 static int
415 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
416 {
417 size_t cursor = offset;
418 int error = 0;
419 uint32_t curr_length;
420 uint8_t curr_type;
421
422 *err = 0;
423
424 do {
425 if (!next) {
426 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
427 if (error) {
428 *err = ENOENT;
429 return -1;
430 }
431 } else {
432 next = 0;
433 curr_type = FLOW_DIVERT_TLV_NIL;
434 }
435
436 if (curr_type != type) {
437 cursor += sizeof(curr_type);
438 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
439 if (error) {
440 *err = error;
441 return -1;
442 }
443
444 cursor += (sizeof(curr_length) + ntohl(curr_length));
445 }
446 } while (curr_type != type);
447
448 return (int)cursor;
449 }
450
451 static int
452 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
453 {
454 int error = 0;
455 uint32_t length;
456 int tlv_offset;
457
458 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
459 if (tlv_offset < 0) {
460 return error;
461 }
462
463 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
464 if (error) {
465 return error;
466 }
467
468 length = ntohl(length);
469
470 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
471
472 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
473 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
474 return EINVAL;
475 }
476
477 if (val_size != NULL) {
478 *val_size = length;
479 }
480
481 if (buff != NULL && buff_len > 0) {
482 memset(buff, 0, buff_len);
483 size_t to_copy = (length < buff_len) ? length : buff_len;
484 error = mbuf_copydata(packet, data_offset, to_copy, buff);
485 if (error) {
486 return error;
487 }
488 }
489
490 return 0;
491 }
492
493 static int
494 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
495 {
496 mbuf_t curr_mbuf = packet;
497
498 if (g_crypto_funcs == NULL || group->token_key == NULL) {
499 return ENOPROTOOPT;
500 }
501
502 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
503 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
504
505 while (curr_mbuf != NULL) {
506 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
507 curr_mbuf = mbuf_next(curr_mbuf);
508 }
509
510 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
511
512 return 0;
513 }
514
515 static int
516 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
517 {
518 int error = 0;
519 struct flow_divert_group *group = NULL;
520 int hmac_offset;
521 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
522 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
523 mbuf_t tail;
524
525 lck_rw_lock_shared(&g_flow_divert_group_lck);
526
527 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
528 group = g_flow_divert_groups[ctl_unit];
529 }
530
531 if (group == NULL) {
532 lck_rw_done(&g_flow_divert_group_lck);
533 return ENOPROTOOPT;
534 }
535
536 lck_rw_lock_shared(&group->lck);
537
538 if (group->token_key == NULL) {
539 error = ENOPROTOOPT;
540 goto done;
541 }
542
543 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
544 if (hmac_offset < 0) {
545 goto done;
546 }
547
548 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
549 if (error) {
550 goto done;
551 }
552
553 /* Chop off the HMAC TLV */
554 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
555 if (error) {
556 goto done;
557 }
558
559 mbuf_free(tail);
560
561 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
562 if (error) {
563 goto done;
564 }
565
566 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
567 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
568 error = EINVAL;
569 goto done;
570 }
571
572 done:
573 lck_rw_done(&group->lck);
574 lck_rw_done(&g_flow_divert_group_lck);
575 return error;
576 }
577
578 static void
579 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, size_t data_len, Boolean send)
580 {
581 struct inpcb *inp = NULL;
582 struct ifnet *ifp = NULL;
583 Boolean cell = FALSE;
584 Boolean wifi = FALSE;
585 Boolean wired = FALSE;
586
587 inp = sotoinpcb(fd_cb->so);
588 if (inp == NULL) {
589 return;
590 }
591
592 if (inp->inp_vflag & INP_IPV4) {
593 ifp = inp->inp_last_outifp;
594 } else if (inp->inp_vflag & INP_IPV6) {
595 ifp = inp->in6p_last_outifp;
596 }
597 if (ifp != NULL) {
598 cell = IFNET_IS_CELLULAR(ifp);
599 wifi = (!cell && IFNET_IS_WIFI(ifp));
600 wired = (!wifi && IFNET_IS_WIRED(ifp));
601 }
602
603 if (send) {
604 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
605 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
606 } else {
607 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
608 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
609 }
610 inp_set_activity_bitmap(inp);
611 }
612
613 static errno_t
614 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
615 {
616 struct inpcb *inp = sotoinpcb(fd_cb->so);
617 if (INP_NO_CELLULAR(inp)) {
618 struct ifnet *ifp = NULL;
619 if (inp->inp_vflag & INP_IPV4) {
620 ifp = inp->inp_last_outifp;
621 } else if (inp->inp_vflag & INP_IPV6) {
622 ifp = inp->in6p_last_outifp;
623 }
624 if (ifp != NULL && IFNET_IS_CELLULAR(ifp)) {
625 FDLOG0(LOG_ERR, fd_cb, "Cellular is denied");
626 return EHOSTUNREACH;
627 }
628 }
629 return 0;
630 }
631
632 static errno_t
633 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
634 {
635 struct inpcb *inp = sotoinpcb(fd_cb->so);
636 if (INP_NO_EXPENSIVE(inp)) {
637 struct ifnet *ifp = NULL;
638 if (inp->inp_vflag & INP_IPV4) {
639 ifp = inp->inp_last_outifp;
640 } else if (inp->inp_vflag & INP_IPV6) {
641 ifp = inp->in6p_last_outifp;
642 }
643 if (ifp != NULL && IFNET_IS_EXPENSIVE(ifp)) {
644 FDLOG0(LOG_ERR, fd_cb, "Expensive is denied");
645 return EHOSTUNREACH;
646 }
647 }
648 return 0;
649 }
650
651 static errno_t
652 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
653 {
654 struct inpcb *inp = sotoinpcb(fd_cb->so);
655 if (INP_NO_CONSTRAINED(inp)) {
656 struct ifnet *ifp = NULL;
657 if (inp->inp_vflag & INP_IPV4) {
658 ifp = inp->inp_last_outifp;
659 } else if (inp->inp_vflag & INP_IPV6) {
660 ifp = inp->in6p_last_outifp;
661 }
662 if (ifp != NULL && IFNET_IS_CONSTRAINED(ifp)) {
663 FDLOG0(LOG_ERR, fd_cb, "Constrained is denied");
664 return EHOSTUNREACH;
665 }
666 }
667 return 0;
668 }
669
670 static void
671 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
672 {
673 if (how != SHUT_RD) {
674 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
675 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
676 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
677 /* If the tunnel is not accepting writes any more, then flush the send buffer */
678 sbflush(&fd_cb->so->so_snd);
679 }
680 }
681 if (how != SHUT_WR) {
682 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
683 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
684 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
685 }
686 }
687 }
688
689 static uint16_t
690 trie_node_alloc(struct flow_divert_trie *trie)
691 {
692 if (trie->nodes_free_next < trie->nodes_count) {
693 uint16_t node_idx = trie->nodes_free_next++;
694 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
695 return node_idx;
696 } else {
697 return NULL_TRIE_IDX;
698 }
699 }
700
701 static uint16_t
702 trie_child_map_alloc(struct flow_divert_trie *trie)
703 {
704 if (trie->child_maps_free_next < trie->child_maps_count) {
705 return trie->child_maps_free_next++;
706 } else {
707 return NULL_TRIE_IDX;
708 }
709 }
710
711 static uint16_t
712 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
713 {
714 uint16_t start = trie->bytes_free_next;
715 if (start + bytes_size <= trie->bytes_count) {
716 if (start != bytes_idx) {
717 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
718 }
719 trie->bytes_free_next += bytes_size;
720 return start;
721 } else {
722 return NULL_TRIE_IDX;
723 }
724 }
725
726 static uint16_t
727 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
728 {
729 uint16_t current = trie->root;
730 uint16_t child = trie->root;
731 uint16_t string_end = string_start + (uint16_t)string_len;
732 uint16_t string_idx = string_start;
733 uint16_t string_remainder = (uint16_t)string_len;
734
735 while (child != NULL_TRIE_IDX) {
736 uint16_t parent = current;
737 uint16_t node_idx;
738 uint16_t current_end;
739
740 current = child;
741 child = NULL_TRIE_IDX;
742
743 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
744
745 for (node_idx = TRIE_NODE(trie, current).start;
746 node_idx < current_end &&
747 string_idx < string_end &&
748 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
749 node_idx++, string_idx++) {
750 ;
751 }
752
753 string_remainder = string_end - string_idx;
754
755 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
756 /*
757 * We did not reach the end of the current node's string.
758 * We need to split the current node into two:
759 * 1. A new node that contains the prefix of the node that matches
760 * the prefix of the string being inserted.
761 * 2. The current node modified to point to the remainder
762 * of the current node's string.
763 */
764 uint16_t prefix = trie_node_alloc(trie);
765 if (prefix == NULL_TRIE_IDX) {
766 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
767 return NULL_TRIE_IDX;
768 }
769
770 /*
771 * Prefix points to the portion of the current nodes's string that has matched
772 * the input string thus far.
773 */
774 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
775 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
776
777 /*
778 * Prefix has the current node as the child corresponding to the first byte
779 * after the split.
780 */
781 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
782 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
783 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
784 return NULL_TRIE_IDX;
785 }
786 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
787
788 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
789 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
790
791 /* Current node is adjusted to point to the remainder */
792 TRIE_NODE(trie, current).start = node_idx;
793 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
794
795 /* We want to insert the new leaf (if any) as a child of the prefix */
796 current = prefix;
797 }
798
799 if (string_remainder > 0) {
800 /*
801 * We still have bytes in the string that have not been matched yet.
802 * If the current node has children, iterate to the child corresponding
803 * to the next byte in the string.
804 */
805 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
806 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
807 }
808 }
809 } /* while (child != NULL_TRIE_IDX) */
810
811 if (string_remainder > 0) {
812 /* Add a new leaf containing the remainder of the string */
813 uint16_t leaf = trie_node_alloc(trie);
814 if (leaf == NULL_TRIE_IDX) {
815 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
816 return NULL_TRIE_IDX;
817 }
818
819 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
820 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
821 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
822 return NULL_TRIE_IDX;
823 }
824 TRIE_NODE(trie, leaf).length = string_remainder;
825
826 /* Set the new leaf as the child of the current node */
827 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
828 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
829 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
830 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
831 return NULL_TRIE_IDX;
832 }
833 }
834 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
835 current = leaf;
836 } /* else duplicate or this string is a prefix of one of the existing strings */
837
838 return current;
839 }
840
841 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
842 static uint16_t
843 flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
844 {
845 uint16_t current = trie->root;
846 uint16_t string_idx = 0;
847
848 while (current != NULL_TRIE_IDX) {
849 uint16_t next = NULL_TRIE_IDX;
850 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
851 uint16_t node_idx;
852
853 for (node_idx = TRIE_NODE(trie, current).start;
854 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
855 node_idx++, string_idx++) {
856 ;
857 }
858
859 if (node_idx == node_end) {
860 if (string_bytes[string_idx] == '\0') {
861 return current; /* Got an exact match */
862 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
863 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
864 return current; /* Got an apple webclip id prefix match */
865 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
866 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
867 }
868 }
869 current = next;
870 }
871
872 return NULL_TRIE_IDX;
873 }
874
875 struct uuid_search_info {
876 uuid_t target_uuid;
877 char *found_signing_id;
878 boolean_t found_multiple_signing_ids;
879 proc_t found_proc;
880 };
881
882 static int
883 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
884 {
885 struct uuid_search_info *info = (struct uuid_search_info *)arg;
886 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
887
888 if (info->found_signing_id != NULL) {
889 if (!info->found_multiple_signing_ids) {
890 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
891 info->found_proc = p;
892 result = PROC_CLAIMED_DONE;
893 } else {
894 uuid_string_t uuid_str;
895 uuid_unparse(info->target_uuid, uuid_str);
896 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
897 }
898 FREE(info->found_signing_id, M_TEMP);
899 info->found_signing_id = NULL;
900 }
901
902 if (result == PROC_RETURNED_DONE) {
903 uuid_string_t uuid_str;
904 uuid_unparse(info->target_uuid, uuid_str);
905 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
906 }
907
908 return result;
909 }
910
911 static int
912 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
913 {
914 struct uuid_search_info *info = (struct uuid_search_info *)arg;
915 int include = 0;
916
917 if (info->found_multiple_signing_ids) {
918 return include;
919 }
920
921 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
922 if (include) {
923 const char *signing_id = cs_identity_get(p);
924 if (signing_id != NULL) {
925 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
926 size_t signing_id_size = strlen(signing_id) + 1;
927 if (info->found_signing_id == NULL) {
928 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
929 memcpy(info->found_signing_id, signing_id, signing_id_size);
930 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
931 info->found_multiple_signing_ids = TRUE;
932 }
933 } else {
934 info->found_multiple_signing_ids = TRUE;
935 }
936 include = !info->found_multiple_signing_ids;
937 }
938
939 return include;
940 }
941
942 static proc_t
943 flow_divert_find_proc_by_uuid(uuid_t uuid)
944 {
945 struct uuid_search_info info;
946
947 if (LOG_INFO <= nil_pcb.log_level) {
948 uuid_string_t uuid_str;
949 uuid_unparse(uuid, uuid_str);
950 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
951 }
952
953 memset(&info, 0, sizeof(info));
954 info.found_proc = PROC_NULL;
955 uuid_copy(info.target_uuid, uuid);
956
957 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
958
959 return info.found_proc;
960 }
961
962 static int
963 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
964 {
965 int error = 0;
966 uint8_t *cdhash = NULL;
967 audit_token_t audit_token = {};
968 const char *proc_cs_id = signing_id;
969
970 proc_lock(proc);
971
972 if (proc_cs_id == NULL) {
973 if (proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
974 proc_cs_id = cs_identity_get(proc);
975 } else {
976 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
977 }
978 }
979
980 if (is_effective) {
981 lck_rw_lock_shared(&fd_cb->group->lck);
982 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
983 if (proc_cs_id != NULL) {
984 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
985 if (result == NULL_TRIE_IDX) {
986 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
987 error = EPERM;
988 } else {
989 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
990 }
991 } else {
992 error = EPERM;
993 }
994 }
995 lck_rw_done(&fd_cb->group->lck);
996 }
997
998 if (error != 0) {
999 goto done;
1000 }
1001
1002 /*
1003 * If signing_id is not NULL then it came from the flow divert token and will be added
1004 * as part of the token, so there is no need to add it here.
1005 */
1006 if (signing_id == NULL && proc_cs_id != NULL) {
1007 error = flow_divert_packet_append_tlv(connect_packet,
1008 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
1009 (uint32_t)strlen(proc_cs_id),
1010 proc_cs_id);
1011 if (error != 0) {
1012 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1013 goto done;
1014 }
1015 }
1016
1017 cdhash = cs_get_cdhash(proc);
1018 if (cdhash != NULL) {
1019 error = flow_divert_packet_append_tlv(connect_packet,
1020 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
1021 SHA1_RESULTLEN,
1022 cdhash);
1023 if (error) {
1024 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1025 goto done;
1026 }
1027 } else {
1028 FDLOG0(LOG_ERR, fd_cb, "failed to get the cdhash");
1029 }
1030
1031 task_t task = proc_task(proc);
1032 if (task != TASK_NULL) {
1033 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1034 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1035 if (rc == KERN_SUCCESS) {
1036 int append_error = flow_divert_packet_append_tlv(connect_packet,
1037 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1038 sizeof(audit_token_t),
1039 &audit_token);
1040 if (append_error) {
1041 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1042 }
1043 }
1044 }
1045
1046 done:
1047 proc_unlock(proc);
1048
1049 return error;
1050 }
1051
1052 static int
1053 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1054 {
1055 int error = 0;
1056 proc_t effective_proc = PROC_NULL;
1057 proc_t responsible_proc = PROC_NULL;
1058 proc_t real_proc = proc_find(so->last_pid);
1059 bool release_real_proc = true;
1060
1061 proc_t src_proc = PROC_NULL;
1062 proc_t real_src_proc = PROC_NULL;
1063
1064 if (real_proc == PROC_NULL) {
1065 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1066 release_real_proc = false;
1067 real_proc = proc;
1068 if (real_proc == PROC_NULL) {
1069 real_proc = current_proc();
1070 }
1071 }
1072
1073 if (so->so_flags & SOF_DELEGATED) {
1074 if (real_proc->p_pid != so->e_pid) {
1075 effective_proc = proc_find(so->e_pid);
1076 } else if (uuid_compare(real_proc->p_uuid, so->e_uuid)) {
1077 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
1078 }
1079 }
1080
1081 #if defined(XNU_TARGET_OS_OSX)
1082 lck_rw_lock_shared(&fd_cb->group->lck);
1083 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1084 if (so->so_rpid > 0) {
1085 responsible_proc = proc_find(so->so_rpid);
1086 }
1087 }
1088 lck_rw_done(&fd_cb->group->lck);
1089 #endif
1090
1091 real_src_proc = real_proc;
1092
1093 if (responsible_proc != PROC_NULL) {
1094 src_proc = responsible_proc;
1095 if (effective_proc != NULL) {
1096 real_src_proc = effective_proc;
1097 }
1098 } else if (effective_proc != PROC_NULL) {
1099 src_proc = effective_proc;
1100 } else {
1101 src_proc = real_proc;
1102 }
1103
1104 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1105 if (error != 0) {
1106 goto done;
1107 }
1108
1109 if (real_src_proc != NULL && real_src_proc != src_proc) {
1110 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1111 if (error != 0) {
1112 goto done;
1113 }
1114 }
1115
1116 done:
1117 if (responsible_proc != PROC_NULL) {
1118 proc_rele(responsible_proc);
1119 }
1120
1121 if (effective_proc != PROC_NULL) {
1122 proc_rele(effective_proc);
1123 }
1124
1125 if (real_proc != PROC_NULL && release_real_proc) {
1126 proc_rele(real_proc);
1127 }
1128
1129 return error;
1130 }
1131
1132 static int
1133 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1134 {
1135 int error;
1136
1137 if (fd_cb->group == NULL) {
1138 fd_cb->so->so_error = ECONNABORTED;
1139 flow_divert_disconnect_socket(fd_cb->so);
1140 return ECONNABORTED;
1141 }
1142
1143 lck_rw_lock_shared(&fd_cb->group->lck);
1144
1145 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1146 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1147 } else {
1148 error = ENOBUFS;
1149 }
1150
1151 if (error == ENOBUFS) {
1152 if (enqueue) {
1153 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1154 lck_rw_lock_exclusive(&fd_cb->group->lck);
1155 }
1156 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1157 error = 0;
1158 }
1159 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1160 }
1161
1162 lck_rw_done(&fd_cb->group->lck);
1163
1164 return error;
1165 }
1166
1167 static int
1168 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
1169 {
1170 int error = 0;
1171 int flow_type = 0;
1172 char *signing_id = NULL;
1173 mbuf_t connect_packet = NULL;
1174 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1175 const void *cfil_id = NULL;
1176 size_t cfil_id_size = 0;
1177 struct inpcb *inp = sotoinpcb(so);
1178 struct ifnet *ifp = NULL;
1179
1180 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1181 if (error) {
1182 goto done;
1183 }
1184
1185 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1186 uint32_t sid_size = 0;
1187 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1188 if (find_error == 0 && sid_size > 0) {
1189 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1190 if (signing_id != NULL) {
1191 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1192 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1193 }
1194 }
1195 }
1196
1197 socket_unlock(so, 0);
1198
1199 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
1200
1201 socket_lock(so, 0);
1202
1203 if (signing_id != NULL) {
1204 FREE(signing_id, M_TEMP);
1205 }
1206
1207 if (error) {
1208 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
1209 goto done;
1210 }
1211
1212 error = flow_divert_packet_append_tlv(connect_packet,
1213 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1214 sizeof(fd_cb->so->so_traffic_class),
1215 &fd_cb->so->so_traffic_class);
1216 if (error) {
1217 goto done;
1218 }
1219
1220 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1221 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1222 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1223 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1224 } else {
1225 error = EINVAL;
1226 goto done;
1227 }
1228 error = flow_divert_packet_append_tlv(connect_packet,
1229 FLOW_DIVERT_TLV_FLOW_TYPE,
1230 sizeof(flow_type),
1231 &flow_type);
1232
1233 if (error) {
1234 goto done;
1235 }
1236
1237 if (fd_cb->connect_token != NULL) {
1238 unsigned int token_len = m_length(fd_cb->connect_token);
1239 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1240 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1241 fd_cb->connect_token = NULL;
1242 } else {
1243 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1244 if (error) {
1245 goto done;
1246 }
1247 }
1248
1249 if (fd_cb->local_endpoint.sa.sa_family == AF_INET || fd_cb->local_endpoint.sa.sa_family == AF_INET6) {
1250 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR, fd_cb->local_endpoint.sa.sa_len, &(fd_cb->local_endpoint.sa));
1251 if (error) {
1252 goto done;
1253 }
1254 }
1255
1256 if (inp->inp_vflag & INP_IPV4) {
1257 ifp = inp->inp_last_outifp;
1258 } else if (inp->inp_vflag & INP_IPV6) {
1259 ifp = inp->in6p_last_outifp;
1260 }
1261 if (ifp != NULL) {
1262 uint32_t flow_if_index = ifp->if_index;
1263 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1264 sizeof(flow_if_index), &flow_if_index);
1265 if (error) {
1266 goto done;
1267 }
1268 }
1269
1270 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1271 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1272 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1273 if (error) {
1274 goto done;
1275 }
1276 }
1277
1278 if (SOCK_TYPE(so) == SOCK_DGRAM) {
1279 cfil_sock_id = cfil_sock_id_from_datagram_socket(so, NULL, to);
1280 } else {
1281 cfil_sock_id = cfil_sock_id_from_socket(so);
1282 }
1283
1284 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1285 cfil_id = &cfil_sock_id;
1286 cfil_id_size = sizeof(cfil_sock_id);
1287 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1288 cfil_id = &inp->necp_client_uuid;
1289 cfil_id_size = sizeof(inp->necp_client_uuid);
1290 }
1291
1292 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
1293 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, (uint32_t)cfil_id_size, cfil_id);
1294 if (error) {
1295 goto done;
1296 }
1297 }
1298
1299 done:
1300 if (!error) {
1301 *out_connect_packet = connect_packet;
1302 } else if (connect_packet != NULL) {
1303 mbuf_freem(connect_packet);
1304 }
1305
1306 return error;
1307 }
1308
1309 static int
1310 flow_divert_send_connect_packet(struct flow_divert_pcb *fd_cb)
1311 {
1312 int error = 0;
1313 mbuf_t connect_packet = fd_cb->connect_packet;
1314 mbuf_t saved_connect_packet = NULL;
1315
1316 if (connect_packet != NULL) {
1317 error = mbuf_copym(connect_packet, 0, mbuf_pkthdr_len(connect_packet), MBUF_DONTWAIT, &saved_connect_packet);
1318 if (error) {
1319 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the connect packet");
1320 goto done;
1321 }
1322
1323 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
1324 if (error) {
1325 goto done;
1326 }
1327
1328 fd_cb->connect_packet = saved_connect_packet;
1329 saved_connect_packet = NULL;
1330 } else {
1331 error = ENOENT;
1332 }
1333 done:
1334 if (saved_connect_packet != NULL) {
1335 mbuf_freem(saved_connect_packet);
1336 }
1337
1338 return error;
1339 }
1340
1341 static int
1342 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1343 {
1344 int error = 0;
1345 mbuf_t packet = NULL;
1346 int rbuff_space = 0;
1347
1348 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1349 if (error) {
1350 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1351 goto done;
1352 }
1353
1354 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1355 if (rbuff_space < 0) {
1356 rbuff_space = 0;
1357 }
1358 rbuff_space = htonl(rbuff_space);
1359 error = flow_divert_packet_append_tlv(packet,
1360 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1361 sizeof(rbuff_space),
1362 &rbuff_space);
1363 if (error) {
1364 goto done;
1365 }
1366
1367 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1368 if (error) {
1369 goto done;
1370 }
1371
1372 done:
1373 if (error && packet != NULL) {
1374 mbuf_freem(packet);
1375 }
1376
1377 return error;
1378 }
1379
1380 static int
1381 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1382 {
1383 int error = 0;
1384 mbuf_t packet = NULL;
1385 uint32_t zero = 0;
1386
1387 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1388 if (error) {
1389 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1390 goto done;
1391 }
1392
1393 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1394 if (error) {
1395 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1396 goto done;
1397 }
1398
1399 how = htonl(how);
1400 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1401 if (error) {
1402 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1403 goto done;
1404 }
1405
1406 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1407 if (error) {
1408 goto done;
1409 }
1410
1411 done:
1412 if (error && packet != NULL) {
1413 mbuf_free(packet);
1414 }
1415
1416 return error;
1417 }
1418
1419 static int
1420 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1421 {
1422 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1423 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1424 return SHUT_RDWR;
1425 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1426 return SHUT_RD;
1427 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1428 return SHUT_WR;
1429 }
1430
1431 return -1;
1432 }
1433
1434 /*
1435 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1436 * writes. Returns FALSE otherwise.
1437 */
1438 static void
1439 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1440 {
1441 int how = -1;
1442
1443 /* Do not send any close messages if there is still data in the send buffer */
1444 if (fd_cb->so->so_snd.sb_cc == 0) {
1445 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1446 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1447 how = SHUT_RD;
1448 }
1449 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1450 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1451 if (how == SHUT_RD) {
1452 how = SHUT_RDWR;
1453 } else {
1454 how = SHUT_WR;
1455 }
1456 }
1457 }
1458
1459 if (how != -1) {
1460 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1461 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1462 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1463 if (how != SHUT_RD) {
1464 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1465 }
1466 if (how != SHUT_WR) {
1467 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1468 }
1469 }
1470 }
1471
1472 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1473 flow_divert_disconnect_socket(fd_cb->so);
1474 }
1475 }
1476
1477 static errno_t
1478 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1479 {
1480 mbuf_t packet = NULL;
1481 mbuf_t last = NULL;
1482 int error = 0;
1483
1484 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1485 if (error || packet == NULL) {
1486 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1487 goto done;
1488 }
1489
1490 if (toaddr != NULL) {
1491 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1492 if (error) {
1493 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1494 goto done;
1495 }
1496 }
1497
1498 if (data_len > 0 && data_len <= INT_MAX && data != NULL) {
1499 last = m_last(packet);
1500 mbuf_setnext(last, data);
1501 mbuf_pkthdr_adjustlen(packet, (int)data_len);
1502 } else {
1503 data_len = 0;
1504 }
1505 error = flow_divert_send_packet(fd_cb, packet, force);
1506 if (error == 0 && data_len > 0) {
1507 fd_cb->bytes_sent += data_len;
1508 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1509 }
1510
1511 done:
1512 if (error) {
1513 if (last != NULL) {
1514 mbuf_setnext(last, NULL);
1515 }
1516 if (packet != NULL) {
1517 mbuf_freem(packet);
1518 }
1519 }
1520
1521 return error;
1522 }
1523
1524 static void
1525 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1526 {
1527 size_t to_send;
1528 size_t sent = 0;
1529 int error = 0;
1530 mbuf_t buffer;
1531
1532 to_send = fd_cb->so->so_snd.sb_cc;
1533 buffer = fd_cb->so->so_snd.sb_mb;
1534
1535 if (buffer == NULL && to_send > 0) {
1536 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1537 return;
1538 }
1539
1540 /* Ignore the send window if force is enabled */
1541 if (!force && (to_send > fd_cb->send_window)) {
1542 to_send = fd_cb->send_window;
1543 }
1544
1545 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1546 while (sent < to_send) {
1547 mbuf_t data;
1548 size_t data_len;
1549
1550 data_len = to_send - sent;
1551 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1552 data_len = FLOW_DIVERT_CHUNK_SIZE;
1553 }
1554
1555 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1556 if (error) {
1557 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1558 break;
1559 }
1560
1561 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1562 if (error) {
1563 if (data != NULL) {
1564 mbuf_freem(data);
1565 }
1566 break;
1567 }
1568
1569 sent += data_len;
1570 }
1571 sbdrop(&fd_cb->so->so_snd, (int)sent);
1572 sowwakeup(fd_cb->so);
1573 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1574 mbuf_t data;
1575 mbuf_t m;
1576 size_t data_len;
1577
1578 while (buffer) {
1579 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1580
1581 m = buffer;
1582 if (toaddr != NULL) {
1583 /* look for data in the chain */
1584 do {
1585 m = m->m_next;
1586 if (m != NULL && m->m_type == MT_DATA) {
1587 break;
1588 }
1589 } while (m);
1590 if (m == NULL) {
1591 /* unexpected */
1592 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1593 goto move_on;
1594 }
1595 }
1596 data_len = mbuf_pkthdr_len(m);
1597 if (data_len > 0) {
1598 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1599 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1600 if (error) {
1601 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1602 break;
1603 }
1604 } else {
1605 data = NULL;
1606 }
1607 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1608 if (error) {
1609 if (data != NULL) {
1610 mbuf_freem(data);
1611 }
1612 break;
1613 }
1614 sent += data_len;
1615 move_on:
1616 buffer = buffer->m_nextpkt;
1617 (void) sbdroprecord(&(fd_cb->so->so_snd));
1618 }
1619 }
1620
1621 if (sent > 0) {
1622 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1623 if (fd_cb->send_window >= sent) {
1624 fd_cb->send_window -= sent;
1625 } else {
1626 fd_cb->send_window = 0;
1627 }
1628 }
1629 }
1630
1631 static int
1632 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1633 {
1634 size_t to_send = mbuf_pkthdr_len(data);
1635 int error = 0;
1636
1637 if (to_send > fd_cb->send_window) {
1638 to_send = fd_cb->send_window;
1639 }
1640
1641 if (fd_cb->so->so_snd.sb_cc > 0) {
1642 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1643 }
1644
1645 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1646 size_t sent = 0;
1647 mbuf_t remaining_data = data;
1648 mbuf_t pkt_data = NULL;
1649 while (sent < to_send && remaining_data != NULL) {
1650 size_t pkt_data_len;
1651
1652 pkt_data = remaining_data;
1653
1654 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1655 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1656 } else {
1657 pkt_data_len = to_send - sent;
1658 }
1659
1660 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1661 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1662 if (error) {
1663 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1664 pkt_data = NULL;
1665 break;
1666 }
1667 } else {
1668 remaining_data = NULL;
1669 }
1670
1671 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1672
1673 if (error) {
1674 break;
1675 }
1676
1677 pkt_data = NULL;
1678 sent += pkt_data_len;
1679 }
1680
1681 fd_cb->send_window -= sent;
1682
1683 error = 0;
1684
1685 if (pkt_data != NULL) {
1686 if (sbspace(&fd_cb->so->so_snd) > 0) {
1687 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1688 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1689 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1690 }
1691 } else {
1692 mbuf_freem(pkt_data);
1693 error = ENOBUFS;
1694 }
1695 }
1696
1697 if (remaining_data != NULL) {
1698 if (sbspace(&fd_cb->so->so_snd) > 0) {
1699 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1700 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1701 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1702 }
1703 } else {
1704 mbuf_freem(remaining_data);
1705 error = ENOBUFS;
1706 }
1707 }
1708 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1709 if (to_send || mbuf_pkthdr_len(data) == 0) {
1710 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1711 if (error) {
1712 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1713 if (data != NULL) {
1714 mbuf_freem(data);
1715 }
1716 } else {
1717 fd_cb->send_window -= to_send;
1718 }
1719 } else {
1720 /* buffer it */
1721 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1722 if (toaddr != NULL) {
1723 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1724 FDLOG(LOG_ERR, fd_cb,
1725 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1726 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1727 }
1728 error = 0;
1729 } else {
1730 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1731 FDLOG(LOG_ERR, fd_cb,
1732 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1733 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1734 }
1735 }
1736 } else {
1737 if (data != NULL) {
1738 mbuf_freem(data);
1739 }
1740 error = ENOBUFS;
1741 }
1742 }
1743 }
1744
1745 return error;
1746 }
1747
1748 static int
1749 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb)
1750 {
1751 int error = 0;
1752 mbuf_t packet = NULL;
1753
1754 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1755 if (error) {
1756 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1757 goto done;
1758 }
1759
1760 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1761 if (error) {
1762 goto done;
1763 }
1764
1765 done:
1766 if (error && packet != NULL) {
1767 mbuf_free(packet);
1768 }
1769
1770 return error;
1771 }
1772
1773 static int
1774 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1775 {
1776 int error = 0;
1777 mbuf_t packet = NULL;
1778
1779 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1780 if (error) {
1781 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1782 goto done;
1783 }
1784
1785 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1786 if (error) {
1787 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1788 goto done;
1789 }
1790
1791 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1792 if (error) {
1793 goto done;
1794 }
1795
1796 done:
1797 if (error && packet != NULL) {
1798 mbuf_free(packet);
1799 }
1800
1801 return error;
1802 }
1803
1804 static void
1805 flow_divert_set_local_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *local_endpoint, bool port_only)
1806 {
1807 struct inpcb *inp = sotoinpcb(fd_cb->so);
1808
1809 if (local_endpoint->sa_family == AF_INET6) {
1810 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) && !port_only) {
1811 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1812 inp->in6p_laddr = (satosin6(local_endpoint))->sin6_addr;
1813 }
1814 if (inp->inp_lport == 0) {
1815 inp->inp_lport = (satosin6(local_endpoint))->sin6_port;
1816 }
1817 } else if (local_endpoint->sa_family == AF_INET) {
1818 if (inp->inp_laddr.s_addr == INADDR_ANY && !port_only) {
1819 fd_cb->flags |= FLOW_DIVERT_DID_SET_LOCAL_ADDR;
1820 inp->inp_laddr = (satosin(local_endpoint))->sin_addr;
1821 }
1822 if (inp->inp_lport == 0) {
1823 inp->inp_lport = (satosin(local_endpoint))->sin_port;
1824 }
1825 }
1826 }
1827
1828 static void
1829 flow_divert_set_remote_endpoint(struct flow_divert_pcb *fd_cb, struct sockaddr *remote_endpoint)
1830 {
1831 struct inpcb *inp = sotoinpcb(fd_cb->so);
1832
1833 if (remote_endpoint->sa_family == AF_INET6) {
1834 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
1835 inp->in6p_faddr = (satosin6(remote_endpoint))->sin6_addr;
1836 }
1837 if (inp->inp_fport == 0) {
1838 inp->inp_fport = (satosin6(remote_endpoint))->sin6_port;
1839 }
1840 } else if (remote_endpoint->sa_family == AF_INET) {
1841 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1842 inp->inp_faddr = (satosin(remote_endpoint))->sin_addr;
1843 }
1844 if (inp->inp_fport == 0) {
1845 inp->inp_fport = (satosin(remote_endpoint))->sin_port;
1846 }
1847 }
1848 }
1849
1850 static uint32_t
1851 flow_divert_derive_kernel_control_unit(uint32_t ctl_unit, uint32_t *aggregate_unit)
1852 {
1853 if (aggregate_unit != NULL && *aggregate_unit != 0) {
1854 uint32_t counter;
1855 for (counter = 0; counter < (GROUP_COUNT_MAX - 1); counter++) {
1856 if ((*aggregate_unit) & (1 << counter)) {
1857 break;
1858 }
1859 }
1860 if (counter < (GROUP_COUNT_MAX - 1)) {
1861 *aggregate_unit &= ~(1 << counter);
1862 return counter + 1;
1863 } else {
1864 return ctl_unit;
1865 }
1866 } else {
1867 return ctl_unit;
1868 }
1869 }
1870
1871 static int
1872 flow_divert_try_next(struct flow_divert_pcb *fd_cb)
1873 {
1874 uint32_t current_ctl_unit = 0;
1875 uint32_t next_ctl_unit = 0;
1876 struct flow_divert_group *current_group = NULL;
1877 struct flow_divert_group *next_group = NULL;
1878 int error = 0;
1879
1880 next_ctl_unit = flow_divert_derive_kernel_control_unit(fd_cb->policy_control_unit, &(fd_cb->aggregate_unit));
1881 current_ctl_unit = fd_cb->control_group_unit;
1882
1883 if (current_ctl_unit == next_ctl_unit) {
1884 FDLOG0(LOG_NOTICE, fd_cb, "Next control unit is the same as the current control unit, disabling flow divert");
1885 error = EALREADY;
1886 goto done;
1887 }
1888
1889 if (next_ctl_unit == 0 || next_ctl_unit >= GROUP_COUNT_MAX) {
1890 FDLOG0(LOG_NOTICE, fd_cb, "No more valid control units, disabling flow divert");
1891 error = ENOENT;
1892 goto done;
1893 }
1894
1895 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1896 FDLOG0(LOG_NOTICE, fd_cb, "No active groups, disabling flow divert");
1897 error = ENOENT;
1898 goto done;
1899 }
1900
1901 next_group = g_flow_divert_groups[next_ctl_unit];
1902 if (next_group == NULL) {
1903 FDLOG(LOG_NOTICE, fd_cb, "Group for control unit %u does not exist", next_ctl_unit);
1904 error = ENOENT;
1905 goto done;
1906 }
1907
1908 current_group = fd_cb->group;
1909
1910 lck_rw_lock_exclusive(&(current_group->lck));
1911 lck_rw_lock_exclusive(&(next_group->lck));
1912
1913 FDLOG(LOG_NOTICE, fd_cb, "Moving from %u to %u", current_ctl_unit, next_ctl_unit);
1914
1915 RB_REMOVE(fd_pcb_tree, &(current_group->pcb_tree), fd_cb);
1916 if (RB_INSERT(fd_pcb_tree, &(next_group->pcb_tree), fd_cb) != NULL) {
1917 panic("group with unit %u already contains a connection with hash %u", next_ctl_unit, fd_cb->hash);
1918 }
1919
1920 fd_cb->group = next_group;
1921 fd_cb->control_group_unit = next_ctl_unit;
1922
1923 lck_rw_done(&(next_group->lck));
1924 lck_rw_done(&(current_group->lck));
1925
1926 error = flow_divert_send_connect_packet(fd_cb);
1927 if (error) {
1928 FDLOG(LOG_NOTICE, fd_cb, "Failed to send the connect packet to %u, disabling flow divert", next_ctl_unit);
1929 error = ENOENT;
1930 goto done;
1931 }
1932
1933 done:
1934 return error;
1935 }
1936
1937 static void
1938 flow_divert_disable(struct flow_divert_pcb *fd_cb)
1939 {
1940 struct socket *so = NULL;
1941 mbuf_t buffer;
1942 int error = 0;
1943 proc_t last_proc = NULL;
1944 struct sockaddr *remote_endpoint = fd_cb->original_remote_endpoint;
1945 bool do_connect = !(fd_cb->flags & FLOW_DIVERT_IMPLICIT_CONNECT);
1946 struct inpcb *inp = NULL;
1947
1948 so = fd_cb->so;
1949 if (so == NULL) {
1950 goto done;
1951 }
1952
1953 FDLOG0(LOG_NOTICE, fd_cb, "Skipped all flow divert services, disabling flow divert");
1954
1955 /* Restore the IP state */
1956 inp = sotoinpcb(so);
1957 inp->inp_vflag = fd_cb->original_vflag;
1958 inp->inp_faddr.s_addr = INADDR_ANY;
1959 inp->inp_fport = 0;
1960 memset(&(inp->in6p_faddr), 0, sizeof(inp->in6p_faddr));
1961 inp->in6p_fport = 0;
1962 /* If flow divert set the local address, clear it out */
1963 if (fd_cb->flags & FLOW_DIVERT_DID_SET_LOCAL_ADDR) {
1964 inp->inp_laddr.s_addr = INADDR_ANY;
1965 memset(&(inp->in6p_laddr), 0, sizeof(inp->in6p_laddr));
1966 }
1967 inp->inp_last_outifp = fd_cb->original_last_outifp;
1968 inp->in6p_last_outifp = fd_cb->original_last_outifp6;
1969
1970 /* Dis-associate the socket */
1971 so->so_flags &= ~SOF_FLOW_DIVERT;
1972 so->so_flags1 |= SOF1_FLOW_DIVERT_SKIP;
1973 so->so_fd_pcb = NULL;
1974 fd_cb->so = NULL;
1975
1976 /* Remove from the group */
1977 flow_divert_pcb_remove(fd_cb);
1978
1979 FDRELEASE(fd_cb); /* Release the socket's reference */
1980
1981 /* Revert back to the original protocol */
1982 so->so_proto = pffindproto(SOCK_DOM(so), SOCK_PROTO(so), SOCK_TYPE(so));
1983
1984 last_proc = proc_find(so->last_pid);
1985
1986 if (do_connect) {
1987 /* Connect using the original protocol */
1988 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, remote_endpoint, (last_proc != NULL ? last_proc : current_proc()));
1989 if (error) {
1990 FDLOG(LOG_ERR, fd_cb, "Failed to connect using the socket's original protocol: %d", error);
1991 goto done;
1992 }
1993 }
1994
1995 buffer = so->so_snd.sb_mb;
1996 if (buffer == NULL) {
1997 /* No buffered data, done */
1998 goto done;
1999 }
2000
2001 /* Send any buffered data using the original protocol */
2002 if (SOCK_TYPE(so) == SOCK_STREAM) {
2003 mbuf_t data_to_send = NULL;
2004 size_t data_len = so->so_snd.sb_cc;
2005
2006 error = mbuf_copym(buffer, 0, data_len, MBUF_DONTWAIT, &data_to_send);
2007 if (error) {
2008 FDLOG0(LOG_ERR, fd_cb, "Failed to copy the mbuf chain in the socket's send buffer");
2009 goto done;
2010 }
2011
2012 sbflush(&so->so_snd);
2013
2014 if (data_to_send->m_flags & M_PKTHDR) {
2015 mbuf_pkthdr_setlen(data_to_send, data_len);
2016 }
2017
2018 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2019 0,
2020 data_to_send,
2021 NULL,
2022 NULL,
2023 (last_proc != NULL ? last_proc : current_proc()));
2024
2025 if (error) {
2026 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2027 }
2028 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
2029 struct sockbuf *sb = &so->so_snd;
2030 MBUFQ_HEAD(send_queue_head) send_queue;
2031 MBUFQ_INIT(&send_queue);
2032
2033 /* Flush the send buffer, moving all records to a temporary queue */
2034 while (sb->sb_mb != NULL) {
2035 mbuf_t record = sb->sb_mb;
2036 mbuf_t m = record;
2037 sb->sb_mb = sb->sb_mb->m_nextpkt;
2038 while (m != NULL) {
2039 sbfree(sb, m);
2040 m = m->m_next;
2041 }
2042 record->m_nextpkt = NULL;
2043 MBUFQ_ENQUEUE(&send_queue, record);
2044 }
2045 SB_EMPTY_FIXUP(sb);
2046
2047 while (!MBUFQ_EMPTY(&send_queue)) {
2048 mbuf_t next_record = MBUFQ_FIRST(&send_queue);
2049 mbuf_t addr = NULL;
2050 mbuf_t control = NULL;
2051 mbuf_t last_control = NULL;
2052 mbuf_t data = NULL;
2053 mbuf_t m = next_record;
2054 struct sockaddr *to_endpoint = NULL;
2055
2056 MBUFQ_DEQUEUE(&send_queue, next_record);
2057
2058 while (m != NULL) {
2059 if (m->m_type == MT_SONAME) {
2060 addr = m;
2061 } else if (m->m_type == MT_CONTROL) {
2062 if (control == NULL) {
2063 control = m;
2064 }
2065 last_control = m;
2066 } else if (m->m_type == MT_DATA) {
2067 data = m;
2068 break;
2069 }
2070 m = m->m_next;
2071 }
2072
2073 if (addr != NULL) {
2074 to_endpoint = flow_divert_get_buffered_target_address(addr);
2075 if (to_endpoint == NULL) {
2076 FDLOG0(LOG_NOTICE, fd_cb, "Failed to get the remote address from the buffer");
2077 }
2078 }
2079
2080 if (data == NULL) {
2081 FDLOG0(LOG_ERR, fd_cb, "Buffered record does not contain any data");
2082 mbuf_freem(next_record);
2083 continue;
2084 }
2085
2086 if (!(data->m_flags & M_PKTHDR)) {
2087 FDLOG0(LOG_ERR, fd_cb, "Buffered data does not have a packet header");
2088 mbuf_freem(next_record);
2089 continue;
2090 }
2091
2092 if (addr != NULL) {
2093 addr->m_next = NULL;
2094 }
2095
2096 if (last_control != NULL) {
2097 last_control->m_next = NULL;
2098 }
2099
2100 error = (*so->so_proto->pr_usrreqs->pru_send)(so,
2101 0,
2102 data,
2103 to_endpoint,
2104 control,
2105 (last_proc != NULL ? last_proc : current_proc()));
2106
2107 if (addr != NULL) {
2108 mbuf_freem(addr);
2109 }
2110
2111 if (error) {
2112 FDLOG(LOG_ERR, fd_cb, "Failed to send queued data using the socket's original protocol: %d", error);
2113 }
2114 }
2115 }
2116 done:
2117 if (last_proc != NULL) {
2118 proc_rele(last_proc);
2119 }
2120
2121 if (error) {
2122 so->so_error = (uint16_t)error;
2123 flow_divert_disconnect_socket(so);
2124 }
2125 }
2126
2127 static void
2128 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2129 {
2130 uint32_t connect_error = 0;
2131 uint32_t ctl_unit = 0;
2132 int error = 0;
2133 struct flow_divert_group *grp = NULL;
2134 union sockaddr_in_4_6 local_endpoint = {};
2135 union sockaddr_in_4_6 remote_endpoint = {};
2136 int out_if_index = 0;
2137 uint32_t send_window;
2138 uint32_t app_data_length = 0;
2139
2140 memset(&local_endpoint, 0, sizeof(local_endpoint));
2141 memset(&remote_endpoint, 0, sizeof(remote_endpoint));
2142
2143 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
2144 if (error) {
2145 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
2146 return;
2147 }
2148
2149 connect_error = ntohl(connect_error);
2150 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
2151
2152 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
2153 if (error) {
2154 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
2155 return;
2156 }
2157
2158 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
2159 if (error) {
2160 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
2161 }
2162
2163 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_endpoint), &(local_endpoint.sa), NULL);
2164 if (error) {
2165 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
2166 }
2167
2168 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_endpoint), &(remote_endpoint.sa), NULL);
2169 if (error) {
2170 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2171 }
2172
2173 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2174 if (error) {
2175 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
2176 }
2177
2178 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2179 if (error) {
2180 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
2181 }
2182
2183 error = 0;
2184 ctl_unit = ntohl(ctl_unit);
2185
2186 lck_rw_lock_shared(&g_flow_divert_group_lck);
2187
2188 if (connect_error == 0 && ctl_unit > 0) {
2189 if (ctl_unit >= GROUP_COUNT_MAX) {
2190 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
2191 error = EINVAL;
2192 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
2193 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
2194 error = EINVAL;
2195 } else {
2196 grp = g_flow_divert_groups[ctl_unit];
2197 if (grp == NULL) {
2198 error = ECONNRESET;
2199 }
2200 }
2201 }
2202
2203 FDLOCK(fd_cb);
2204 if (fd_cb->so != NULL) {
2205 struct inpcb *inp = NULL;
2206 struct ifnet *ifp = NULL;
2207 struct flow_divert_group *old_group;
2208 struct socket *so = fd_cb->so;
2209
2210 socket_lock(so, 0);
2211
2212 if (SOCK_TYPE(so) == SOCK_STREAM && !(so->so_state & SS_ISCONNECTING)) {
2213 FDLOG0(LOG_ERR, fd_cb, "TCP socket is not in the connecting state, ignoring connect result");
2214 goto done;
2215 }
2216
2217 inp = sotoinpcb(so);
2218
2219 if (connect_error || error) {
2220 goto set_socket_state;
2221 }
2222
2223 if (flow_divert_is_sockaddr_valid(&(local_endpoint.sa))) {
2224 if (local_endpoint.sa.sa_family == AF_INET) {
2225 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2226 } else if (local_endpoint.sa.sa_family == AF_INET6) {
2227 local_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2228 }
2229 fd_cb->local_endpoint = local_endpoint;
2230 flow_divert_set_local_endpoint(fd_cb, &(local_endpoint.sa), (SOCK_TYPE(so) == SOCK_DGRAM));
2231 }
2232
2233 if (flow_divert_is_sockaddr_valid(&(remote_endpoint.sa)) && SOCK_TYPE(so) == SOCK_STREAM) {
2234 if (remote_endpoint.sa.sa_family == AF_INET) {
2235 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in);
2236 } else if (remote_endpoint.sa.sa_family == AF_INET6) {
2237 remote_endpoint.sa.sa_len = sizeof(struct sockaddr_in6);
2238 }
2239 flow_divert_set_remote_endpoint(fd_cb, &(remote_endpoint.sa));
2240 }
2241
2242 if (app_data_length > 0) {
2243 uint8_t *app_data = NULL;
2244 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2245 if (app_data != NULL) {
2246 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2247 if (error == 0) {
2248 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
2249 if (fd_cb->app_data != NULL) {
2250 FREE(fd_cb->app_data, M_TEMP);
2251 }
2252 fd_cb->app_data = app_data;
2253 fd_cb->app_data_length = app_data_length;
2254 } else {
2255 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
2256 FREE(app_data, M_TEMP);
2257 }
2258 } else {
2259 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
2260 }
2261 }
2262
2263 ifnet_head_lock_shared();
2264 if (out_if_index > 0 && out_if_index <= if_index) {
2265 ifp = ifindex2ifnet[out_if_index];
2266 }
2267
2268 if (ifp != NULL) {
2269 if (inp->inp_vflag & INP_IPV4) {
2270 inp->inp_last_outifp = ifp;
2271 } else if (inp->inp_vflag & INP_IPV6) {
2272 inp->in6p_last_outifp = ifp;
2273 }
2274 } else {
2275 error = EINVAL;
2276 }
2277 ifnet_head_done();
2278
2279 if (error) {
2280 goto set_socket_state;
2281 }
2282
2283 if (fd_cb->group == NULL) {
2284 error = EINVAL;
2285 goto set_socket_state;
2286 }
2287
2288 if (grp != NULL) {
2289 old_group = fd_cb->group;
2290
2291 lck_rw_lock_exclusive(&old_group->lck);
2292 lck_rw_lock_exclusive(&grp->lck);
2293
2294 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
2295 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
2296 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
2297 }
2298
2299 fd_cb->group = grp;
2300
2301 lck_rw_done(&grp->lck);
2302 lck_rw_done(&old_group->lck);
2303 }
2304
2305 fd_cb->send_window = ntohl(send_window);
2306
2307 set_socket_state:
2308 if (!connect_error && !error) {
2309 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
2310 error = flow_divert_send_connect_result(fd_cb);
2311 }
2312
2313 if (connect_error || error) {
2314 if (connect_error && fd_cb->control_group_unit != fd_cb->policy_control_unit) {
2315 error = flow_divert_try_next(fd_cb);
2316 if (error) {
2317 flow_divert_disable(fd_cb);
2318 }
2319 goto done;
2320 }
2321
2322 if (!connect_error) {
2323 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2324 so->so_error = (uint16_t)error;
2325 flow_divert_send_close_if_needed(fd_cb);
2326 } else {
2327 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2328 so->so_error = (uint16_t)connect_error;
2329 }
2330 flow_divert_disconnect_socket(so);
2331 } else {
2332 #if NECP
2333 /* Update NECP client with connected five-tuple */
2334 if (!uuid_is_null(inp->necp_client_uuid)) {
2335 socket_unlock(so, 0);
2336 necp_client_assign_from_socket(so->last_pid, inp->necp_client_uuid, inp);
2337 socket_lock(so, 0);
2338 }
2339 #endif /* NECP */
2340
2341 flow_divert_send_buffered_data(fd_cb, FALSE);
2342 soisconnected(so);
2343 }
2344
2345 /* We don't need the connect packet any more */
2346 if (fd_cb->connect_packet != NULL) {
2347 mbuf_freem(fd_cb->connect_packet);
2348 fd_cb->connect_packet = NULL;
2349 }
2350
2351 /* We don't need the original remote endpoint any more */
2352 if (fd_cb->original_remote_endpoint != NULL) {
2353 FREE(fd_cb->original_remote_endpoint, M_SONAME);
2354 fd_cb->original_remote_endpoint = NULL;
2355 }
2356 done:
2357 socket_unlock(so, 0);
2358 }
2359 FDUNLOCK(fd_cb);
2360
2361 lck_rw_done(&g_flow_divert_group_lck);
2362 }
2363
2364 static void
2365 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2366 {
2367 uint32_t close_error;
2368 int error = 0;
2369 int how;
2370
2371 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2372 if (error) {
2373 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2374 return;
2375 }
2376
2377 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2378 if (error) {
2379 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2380 return;
2381 }
2382
2383 how = ntohl(how);
2384
2385 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2386
2387 FDLOCK(fd_cb);
2388 if (fd_cb->so != NULL) {
2389 socket_lock(fd_cb->so, 0);
2390
2391 fd_cb->so->so_error = (uint16_t)ntohl(close_error);
2392
2393 flow_divert_update_closed_state(fd_cb, how, TRUE);
2394
2395 how = flow_divert_tunnel_how_closed(fd_cb);
2396 if (how == SHUT_RDWR) {
2397 flow_divert_disconnect_socket(fd_cb->so);
2398 } else if (how == SHUT_RD) {
2399 socantrcvmore(fd_cb->so);
2400 } else if (how == SHUT_WR) {
2401 socantsendmore(fd_cb->so);
2402 }
2403
2404 socket_unlock(fd_cb->so, 0);
2405 }
2406 FDUNLOCK(fd_cb);
2407 }
2408
2409 static mbuf_t
2410 flow_divert_create_control_mbuf(struct flow_divert_pcb *fd_cb)
2411 {
2412 struct inpcb *inp = sotoinpcb(fd_cb->so);
2413 bool is_cfil_enabled = false;
2414 #if CONTENT_FILTER
2415 /* Content Filter needs to see the local address */
2416 is_cfil_enabled = (inp->inp_socket && inp->inp_socket->so_cfil_db != NULL);
2417 #endif
2418 if ((inp->inp_vflag & INP_IPV4) &&
2419 fd_cb->local_endpoint.sa.sa_family == AF_INET &&
2420 ((inp->inp_flags & INP_RECVDSTADDR) || is_cfil_enabled)) {
2421 return sbcreatecontrol((caddr_t)&(fd_cb->local_endpoint.sin.sin_addr), sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
2422 } else if ((inp->inp_vflag & INP_IPV6) &&
2423 fd_cb->local_endpoint.sa.sa_family == AF_INET6 &&
2424 ((inp->inp_flags & IN6P_PKTINFO) || is_cfil_enabled)) {
2425 struct in6_pktinfo pi6;
2426 memset(&pi6, 0, sizeof(pi6));
2427 pi6.ipi6_addr = fd_cb->local_endpoint.sin6.sin6_addr;
2428
2429 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
2430 }
2431 return NULL;
2432 }
2433
2434 static int
2435 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2436 {
2437 int error = 0;
2438
2439 FDLOCK(fd_cb);
2440 if (fd_cb->so != NULL) {
2441 mbuf_t data = NULL;
2442 size_t data_size;
2443 struct sockaddr_storage remote_address;
2444 boolean_t got_remote_sa = FALSE;
2445 boolean_t appended = FALSE;
2446 boolean_t append_success = FALSE;
2447
2448 socket_lock(fd_cb->so, 0);
2449
2450 if (sbspace(&fd_cb->so->so_rcv) == 0) {
2451 error = ENOBUFS;
2452 fd_cb->flags |= FLOW_DIVERT_NOTIFY_ON_RECEIVED;
2453 FDLOG0(LOG_INFO, fd_cb, "Receive buffer is full, will send read notification when app reads some data");
2454 goto done;
2455 }
2456
2457 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2458 uint32_t val_size = 0;
2459
2460 /* check if we got remote address with data */
2461 memset(&remote_address, 0, sizeof(remote_address));
2462 error = flow_divert_packet_get_tlv(packet, (int)offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2463 if (error || val_size > sizeof(remote_address)) {
2464 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2465 error = 0;
2466 } else {
2467 if (remote_address.ss_len > sizeof(remote_address)) {
2468 remote_address.ss_len = sizeof(remote_address);
2469 }
2470 /* validate the address */
2471 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2472 got_remote_sa = TRUE;
2473 } else {
2474 FDLOG0(LOG_INFO, fd_cb, "Remote address is invalid");
2475 }
2476 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2477 }
2478 }
2479
2480 data_size = (mbuf_pkthdr_len(packet) - offset);
2481
2482 if (fd_cb->so->so_state & SS_CANTRCVMORE) {
2483 FDLOG(LOG_NOTICE, fd_cb, "app cannot receive any more data, dropping %lu bytes of data", data_size);
2484 goto done;
2485 }
2486
2487 if (SOCK_TYPE(fd_cb->so) != SOCK_STREAM && SOCK_TYPE(fd_cb->so) != SOCK_DGRAM) {
2488 FDLOG(LOG_ERR, fd_cb, "socket has an unsupported type: %d", SOCK_TYPE(fd_cb->so));
2489 goto done;
2490 }
2491
2492 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2493
2494 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2495 if (error || data == NULL) {
2496 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2497 goto done;
2498 }
2499
2500 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2501 appended = (sbappendstream(&fd_cb->so->so_rcv, data) != 0);
2502 append_success = TRUE;
2503 } else {
2504 struct sockaddr *append_sa = NULL;
2505 mbuf_t mctl;
2506
2507 if (got_remote_sa == TRUE) {
2508 error = flow_divert_dup_addr(remote_address.ss_family, (struct sockaddr *)&remote_address, &append_sa);
2509 } else {
2510 if (fd_cb->so->so_proto->pr_domain->dom_family == AF_INET6) {
2511 error = in6_mapped_peeraddr(fd_cb->so, &append_sa);
2512 } else {
2513 error = in_getpeeraddr(fd_cb->so, &append_sa);
2514 }
2515 }
2516 if (error) {
2517 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2518 }
2519
2520 mctl = flow_divert_create_control_mbuf(fd_cb);
2521 int append_error = 0;
2522 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error) || append_error == EJUSTRETURN) {
2523 append_success = TRUE;
2524 appended = (append_error == 0);
2525 } else {
2526 FDLOG(LOG_ERR, fd_cb, "failed to append %lu bytes of data: %d", data_size, append_error);
2527 }
2528
2529 if (append_sa != NULL) {
2530 FREE(append_sa, M_SONAME);
2531 }
2532 }
2533
2534 if (append_success) {
2535 fd_cb->bytes_received += data_size;
2536 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2537 }
2538
2539 if (appended) {
2540 sorwakeup(fd_cb->so);
2541 }
2542 done:
2543 socket_unlock(fd_cb->so, 0);
2544 }
2545 FDUNLOCK(fd_cb);
2546
2547 return error;
2548 }
2549
2550 static void
2551 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2552 {
2553 uint32_t read_count;
2554 int error = 0;
2555
2556 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2557 if (error) {
2558 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2559 return;
2560 }
2561
2562 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2563
2564 FDLOCK(fd_cb);
2565 if (fd_cb->so != NULL) {
2566 socket_lock(fd_cb->so, 0);
2567 fd_cb->send_window += ntohl(read_count);
2568 flow_divert_send_buffered_data(fd_cb, FALSE);
2569 socket_unlock(fd_cb->so, 0);
2570 }
2571 FDUNLOCK(fd_cb);
2572 }
2573
2574 static void
2575 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2576 {
2577 int error = 0;
2578 uint32_t key_size = 0;
2579 int log_level;
2580 uint32_t flags = 0;
2581
2582 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2583 if (error) {
2584 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2585 return;
2586 }
2587
2588 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2589 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2590 return;
2591 }
2592
2593 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2594 if (!error) {
2595 nil_pcb.log_level = (uint8_t)log_level;
2596 }
2597
2598 lck_rw_lock_exclusive(&group->lck);
2599
2600 if (group->token_key != NULL) {
2601 FREE(group->token_key, M_TEMP);
2602 group->token_key = NULL;
2603 }
2604
2605 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2606 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2607 if (error) {
2608 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2609 FREE(group->token_key, M_TEMP);
2610 group->token_key = NULL;
2611 lck_rw_done(&group->lck);
2612 return;
2613 }
2614
2615 group->token_key_size = key_size;
2616
2617 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2618 if (!error) {
2619 group->flags = flags;
2620 }
2621
2622 lck_rw_done(&group->lck);
2623 }
2624
2625 static void
2626 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2627 {
2628 int error = 0;
2629 int out_if_index = 0;
2630 uint32_t app_data_length = 0;
2631
2632 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2633
2634 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2635 if (error) {
2636 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2637 }
2638
2639 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2640 if (error) {
2641 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2642 }
2643
2644 FDLOCK(fd_cb);
2645 if (fd_cb->so != NULL) {
2646 socket_lock(fd_cb->so, 0);
2647
2648 if (out_if_index > 0) {
2649 struct inpcb *inp = NULL;
2650 struct ifnet *ifp = NULL;
2651
2652 inp = sotoinpcb(fd_cb->so);
2653
2654 ifnet_head_lock_shared();
2655 if (out_if_index <= if_index) {
2656 ifp = ifindex2ifnet[out_if_index];
2657 }
2658
2659 if (ifp != NULL) {
2660 if (inp->inp_vflag & INP_IPV4) {
2661 inp->inp_last_outifp = ifp;
2662 } else if (inp->inp_vflag & INP_IPV6) {
2663 inp->in6p_last_outifp = ifp;
2664 }
2665 }
2666 ifnet_head_done();
2667 }
2668
2669 if (app_data_length > 0) {
2670 uint8_t *app_data = NULL;
2671 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2672 if (app_data != NULL) {
2673 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2674 if (error == 0) {
2675 if (fd_cb->app_data != NULL) {
2676 FREE(fd_cb->app_data, M_TEMP);
2677 }
2678 fd_cb->app_data = app_data;
2679 fd_cb->app_data_length = app_data_length;
2680 } else {
2681 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2682 FREE(app_data, M_TEMP);
2683 }
2684 } else {
2685 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2686 }
2687 }
2688
2689 socket_unlock(fd_cb->so, 0);
2690 }
2691 FDUNLOCK(fd_cb);
2692 }
2693
2694 static void
2695 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2696 {
2697 size_t bytes_mem_size;
2698 size_t child_maps_mem_size;
2699 size_t nodes_mem_size;
2700 size_t trie_memory_size = 0;
2701 int cursor;
2702 int error = 0;
2703 struct flow_divert_trie new_trie;
2704 int insert_error = 0;
2705 int prefix_count = -1;
2706 int signing_id_count = 0;
2707 size_t bytes_count = 0;
2708 size_t nodes_count = 0;
2709 size_t maps_count = 0;
2710
2711 lck_rw_lock_exclusive(&group->lck);
2712
2713 /* Re-set the current trie */
2714 if (group->signing_id_trie.memory != NULL) {
2715 FREE(group->signing_id_trie.memory, M_TEMP);
2716 }
2717 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2718 group->signing_id_trie.root = NULL_TRIE_IDX;
2719
2720 memset(&new_trie, 0, sizeof(new_trie));
2721
2722 /* Get the number of shared prefixes in the new set of signing ID strings */
2723 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2724
2725 if (prefix_count < 0 || error) {
2726 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
2727 lck_rw_done(&group->lck);
2728 return;
2729 }
2730
2731 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2732 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2733 cursor >= 0;
2734 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2735 uint32_t sid_size = 0;
2736 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2737 if (error || sid_size == 0) {
2738 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
2739 signing_id_count = 0;
2740 break;
2741 }
2742 if (os_add_overflow(bytes_count, sid_size, &bytes_count)) {
2743 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while incrementing number of bytes");
2744 signing_id_count = 0;
2745 break;
2746 }
2747 signing_id_count++;
2748 }
2749
2750 if (signing_id_count == 0) {
2751 lck_rw_done(&group->lck);
2752 FDLOG0(LOG_NOTICE, &nil_pcb, "No signing identifiers");
2753 return;
2754 }
2755
2756 if (os_add3_overflow(prefix_count, signing_id_count, 1, &nodes_count)) { /* + 1 for the root node */
2757 lck_rw_done(&group->lck);
2758 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of nodes");
2759 return;
2760 }
2761
2762 if (os_add_overflow(prefix_count, 1, &maps_count)) { /* + 1 for the root node */
2763 lck_rw_done(&group->lck);
2764 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing the number of maps");
2765 return;
2766 }
2767
2768 if (bytes_count > UINT16_MAX || nodes_count > UINT16_MAX || maps_count > UINT16_MAX) {
2769 lck_rw_done(&group->lck);
2770 FDLOG(LOG_NOTICE, &nil_pcb, "Invalid bytes count (%lu), nodes count (%lu) or maps count (%lu)", bytes_count, nodes_count, maps_count);
2771 return;
2772 }
2773
2774 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2775 nodes_count, maps_count, bytes_count);
2776
2777 if (os_mul_overflow(sizeof(*new_trie.nodes), (size_t)nodes_count, &nodes_mem_size) ||
2778 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, (size_t)maps_count, &child_maps_mem_size) ||
2779 os_mul_overflow(sizeof(*new_trie.bytes), (size_t)bytes_count, &bytes_mem_size) ||
2780 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2781 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2782 lck_rw_done(&group->lck);
2783 return;
2784 }
2785
2786 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2787 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2788 lck_rw_done(&group->lck);
2789 return;
2790 }
2791
2792 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2793 if (new_trie.memory == NULL) {
2794 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2795 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2796 lck_rw_done(&group->lck);
2797 return;
2798 }
2799
2800 new_trie.bytes_count = (uint16_t)bytes_count;
2801 new_trie.nodes_count = (uint16_t)nodes_count;
2802 new_trie.child_maps_count = (uint16_t)maps_count;
2803
2804 /* Initialize the free lists */
2805 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2806 new_trie.nodes_free_next = 0;
2807 memset(new_trie.nodes, 0, nodes_mem_size);
2808
2809 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2810 new_trie.child_maps_free_next = 0;
2811 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2812
2813 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2814 new_trie.bytes_free_next = 0;
2815 memset(new_trie.bytes, 0, bytes_mem_size);
2816
2817 /* The root is an empty node */
2818 new_trie.root = trie_node_alloc(&new_trie);
2819
2820 /* Add each signing ID to the trie */
2821 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2822 cursor >= 0;
2823 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2824 uint32_t sid_size = 0;
2825 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2826 if (error || sid_size == 0) {
2827 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
2828 insert_error = EINVAL;
2829 break;
2830 }
2831 if (sid_size <= UINT16_MAX && new_trie.bytes_free_next + (uint16_t)sid_size <= new_trie.bytes_count) {
2832 uint16_t new_node_idx;
2833 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2834 if (error) {
2835 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
2836 insert_error = EINVAL;
2837 break;
2838 }
2839 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2840 if (new_node_idx == NULL_TRIE_IDX) {
2841 insert_error = EINVAL;
2842 break;
2843 }
2844 } else {
2845 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2846 insert_error = ENOBUFS;
2847 break;
2848 }
2849 }
2850
2851 if (!insert_error) {
2852 group->signing_id_trie = new_trie;
2853 } else {
2854 FREE(new_trie.memory, M_TEMP);
2855 }
2856
2857 lck_rw_done(&group->lck);
2858 }
2859
2860 static int
2861 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2862 {
2863 struct flow_divert_packet_header hdr;
2864 int error = 0;
2865 struct flow_divert_pcb *fd_cb;
2866
2867 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2868 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2869 error = EINVAL;
2870 goto done;
2871 }
2872
2873 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2874 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2875 error = EINVAL;
2876 goto done;
2877 }
2878
2879 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2880 if (error) {
2881 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2882 error = ENOBUFS;
2883 goto done;
2884 }
2885
2886 hdr.conn_id = ntohl(hdr.conn_id);
2887
2888 if (hdr.conn_id == 0) {
2889 switch (hdr.packet_type) {
2890 case FLOW_DIVERT_PKT_GROUP_INIT:
2891 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2892 break;
2893 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2894 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2895 break;
2896 default:
2897 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2898 break;
2899 }
2900 goto done;
2901 }
2902
2903 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2904 if (fd_cb == NULL) {
2905 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2906 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2907 }
2908 goto done;
2909 }
2910
2911 switch (hdr.packet_type) {
2912 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2913 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2914 break;
2915 case FLOW_DIVERT_PKT_CLOSE:
2916 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2917 break;
2918 case FLOW_DIVERT_PKT_DATA:
2919 error = flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2920 break;
2921 case FLOW_DIVERT_PKT_READ_NOTIFY:
2922 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2923 break;
2924 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2925 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2926 break;
2927 default:
2928 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2929 break;
2930 }
2931
2932 FDRELEASE(fd_cb);
2933
2934 done:
2935 mbuf_freem(packet);
2936 return error;
2937 }
2938
2939 static void
2940 flow_divert_close_all(struct flow_divert_group *group)
2941 {
2942 struct flow_divert_pcb *fd_cb;
2943 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2944
2945 SLIST_INIT(&tmp_list);
2946
2947 lck_rw_lock_exclusive(&group->lck);
2948
2949 MBUFQ_DRAIN(&group->send_queue);
2950
2951 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2952 FDRETAIN(fd_cb);
2953 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2954 }
2955
2956 lck_rw_done(&group->lck);
2957
2958 while (!SLIST_EMPTY(&tmp_list)) {
2959 fd_cb = SLIST_FIRST(&tmp_list);
2960 FDLOCK(fd_cb);
2961 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2962 if (fd_cb->so != NULL) {
2963 socket_lock(fd_cb->so, 0);
2964 flow_divert_pcb_remove(fd_cb);
2965 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2966 fd_cb->so->so_error = ECONNABORTED;
2967 flow_divert_disconnect_socket(fd_cb->so);
2968 socket_unlock(fd_cb->so, 0);
2969 }
2970 FDUNLOCK(fd_cb);
2971 FDRELEASE(fd_cb);
2972 }
2973 }
2974
2975 void
2976 flow_divert_detach(struct socket *so)
2977 {
2978 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2979
2980 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2981
2982 so->so_flags &= ~SOF_FLOW_DIVERT;
2983 so->so_fd_pcb = NULL;
2984
2985 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2986
2987 if (fd_cb->group != NULL) {
2988 /* Last-ditch effort to send any buffered data */
2989 flow_divert_send_buffered_data(fd_cb, TRUE);
2990
2991 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2992 flow_divert_send_close_if_needed(fd_cb);
2993 /* Remove from the group */
2994 flow_divert_pcb_remove(fd_cb);
2995 }
2996
2997 socket_unlock(so, 0);
2998 FDLOCK(fd_cb);
2999 fd_cb->so = NULL;
3000 FDUNLOCK(fd_cb);
3001 socket_lock(so, 0);
3002
3003 FDRELEASE(fd_cb); /* Release the socket's reference */
3004 }
3005
3006 static int
3007 flow_divert_close(struct socket *so)
3008 {
3009 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3010
3011 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3012
3013 FDLOG0(LOG_INFO, fd_cb, "Closing");
3014
3015 if (SOCK_TYPE(so) == SOCK_STREAM) {
3016 soisdisconnecting(so);
3017 sbflush(&so->so_rcv);
3018 }
3019
3020 flow_divert_send_buffered_data(fd_cb, TRUE);
3021 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
3022 flow_divert_send_close_if_needed(fd_cb);
3023
3024 /* Remove from the group */
3025 flow_divert_pcb_remove(fd_cb);
3026
3027 return 0;
3028 }
3029
3030 static int
3031 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
3032 sae_connid_t cid __unused)
3033 {
3034 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
3035 return EINVAL;
3036 }
3037
3038 return flow_divert_close(so);
3039 }
3040
3041 static int
3042 flow_divert_shutdown(struct socket *so)
3043 {
3044 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3045
3046 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3047
3048 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
3049
3050 socantsendmore(so);
3051
3052 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
3053 flow_divert_send_close_if_needed(fd_cb);
3054
3055 return 0;
3056 }
3057
3058 static int
3059 flow_divert_rcvd(struct socket *so, int flags __unused)
3060 {
3061 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3062 int space = sbspace(&so->so_rcv);
3063
3064 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3065
3066 FDLOG(LOG_DEBUG, fd_cb, "app read bytes, space = %d", space);
3067 if ((fd_cb->flags & FLOW_DIVERT_NOTIFY_ON_RECEIVED) &&
3068 (space > 0) &&
3069 flow_divert_send_read_notification(fd_cb) == 0) {
3070 FDLOG0(LOG_INFO, fd_cb, "Sent a read notification");
3071 fd_cb->flags &= ~FLOW_DIVERT_NOTIFY_ON_RECEIVED;
3072 }
3073
3074 return 0;
3075 }
3076
3077 static int
3078 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
3079 {
3080 int error = 0;
3081 int port = 0;
3082
3083 if (!flow_divert_is_sockaddr_valid(toaddr)) {
3084 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
3085 error = EINVAL;
3086 goto done;
3087 }
3088
3089 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
3090 if (error) {
3091 goto done;
3092 }
3093
3094 if (toaddr->sa_family == AF_INET) {
3095 port = ntohs((satosin(toaddr))->sin_port);
3096 } else {
3097 port = ntohs((satosin6(toaddr))->sin6_port);
3098 }
3099
3100 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
3101 if (error) {
3102 goto done;
3103 }
3104
3105 done:
3106 return error;
3107 }
3108
3109 struct sockaddr *
3110 flow_divert_get_buffered_target_address(mbuf_t buffer)
3111 {
3112 if (buffer != NULL && buffer->m_type == MT_SONAME) {
3113 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
3114 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
3115 return toaddr;
3116 }
3117 }
3118 return NULL;
3119 }
3120
3121 static boolean_t
3122 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
3123 {
3124 switch (addr->sa_family) {
3125 case AF_INET:
3126 if (addr->sa_len < sizeof(struct sockaddr_in)) {
3127 return FALSE;
3128 }
3129 break;
3130 case AF_INET6:
3131 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3132 return FALSE;
3133 }
3134 break;
3135 default:
3136 return FALSE;
3137 }
3138 return TRUE;
3139 }
3140
3141 static errno_t
3142 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
3143 struct sockaddr **dup)
3144 {
3145 int error = 0;
3146 struct sockaddr *result;
3147 struct sockaddr_storage ss;
3148
3149 if (addr != NULL) {
3150 result = addr;
3151 } else {
3152 memset(&ss, 0, sizeof(ss));
3153 ss.ss_family = family;
3154 if (ss.ss_family == AF_INET) {
3155 ss.ss_len = sizeof(struct sockaddr_in);
3156 } else if (ss.ss_family == AF_INET6) {
3157 ss.ss_len = sizeof(struct sockaddr_in6);
3158 } else {
3159 error = EINVAL;
3160 }
3161 result = (struct sockaddr *)&ss;
3162 }
3163
3164 if (!error) {
3165 *dup = dup_sockaddr(result, 1);
3166 if (*dup == NULL) {
3167 error = ENOBUFS;
3168 }
3169 }
3170
3171 return error;
3172 }
3173
3174 static void
3175 flow_divert_disconnect_socket(struct socket *so)
3176 {
3177 soisdisconnected(so);
3178 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3179 struct inpcb *inp = NULL;
3180
3181 inp = sotoinpcb(so);
3182 if (inp != NULL) {
3183 if (SOCK_CHECK_DOM(so, PF_INET6)) {
3184 in6_pcbdetach(inp);
3185 } else {
3186 in_pcbdetach(inp);
3187 }
3188 }
3189 }
3190 }
3191
3192 static errno_t
3193 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
3194 {
3195 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3196
3197 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3198
3199 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
3200 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3201 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
3202 }
3203 }
3204
3205 if (SOCK_DOM(so) == PF_INET) {
3206 return g_tcp_protosw->pr_ctloutput(so, sopt);
3207 } else if (SOCK_DOM(so) == PF_INET6) {
3208 return g_tcp6_protosw->pr_ctloutput(so, sopt);
3209 }
3210 return 0;
3211 }
3212
3213 static errno_t
3214 flow_divert_connect_out_internal(struct socket *so, struct sockaddr *to, proc_t p, bool implicit)
3215 {
3216 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3217 int error = 0;
3218 struct inpcb *inp = sotoinpcb(so);
3219 struct sockaddr_in *sinp;
3220 mbuf_t connect_packet = NULL;
3221 int do_send = 1;
3222
3223 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3224
3225 if (fd_cb->group == NULL) {
3226 error = ENETUNREACH;
3227 goto done;
3228 }
3229
3230 if (inp == NULL) {
3231 error = EINVAL;
3232 goto done;
3233 } else if (inp->inp_state == INPCB_STATE_DEAD) {
3234 if (so->so_error) {
3235 error = so->so_error;
3236 so->so_error = 0;
3237 } else {
3238 error = EINVAL;
3239 }
3240 goto done;
3241 }
3242
3243 if (fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
3244 error = EALREADY;
3245 goto done;
3246 }
3247
3248 FDLOG0(LOG_INFO, fd_cb, "Connecting");
3249
3250 if (fd_cb->connect_packet == NULL) {
3251 struct sockaddr_in sin = {};
3252 struct ifnet *ifp = NULL;
3253
3254 if (to == NULL) {
3255 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
3256 error = EINVAL;
3257 goto done;
3258 }
3259
3260 fd_cb->original_remote_endpoint = dup_sockaddr(to, 0);
3261 if (fd_cb->original_remote_endpoint == NULL) {
3262 FDLOG0(LOG_ERR, fd_cb, "Failed to dup the remote endpoint");
3263 error = ENOMEM;
3264 goto done;
3265 }
3266 fd_cb->original_vflag = inp->inp_vflag;
3267 fd_cb->original_last_outifp = inp->inp_last_outifp;
3268 fd_cb->original_last_outifp6 = inp->in6p_last_outifp;
3269
3270 sinp = (struct sockaddr_in *)(void *)to;
3271 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
3272 error = EAFNOSUPPORT;
3273 goto done;
3274 }
3275
3276 if (to->sa_family == AF_INET6 && !(inp->inp_flags & IN6P_IPV6_V6ONLY)) {
3277 struct sockaddr_in6 sin6 = {};
3278 sin6.sin6_family = AF_INET6;
3279 sin6.sin6_len = sizeof(struct sockaddr_in6);
3280 sin6.sin6_port = satosin6(to)->sin6_port;
3281 sin6.sin6_addr = satosin6(to)->sin6_addr;
3282 if (IN6_IS_ADDR_V4MAPPED(&(sin6.sin6_addr))) {
3283 in6_sin6_2_sin(&sin, &sin6);
3284 to = (struct sockaddr *)&sin;
3285 }
3286 }
3287
3288 if (to->sa_family == AF_INET6) {
3289 inp->inp_vflag &= ~INP_IPV4;
3290 inp->inp_vflag |= INP_IPV6;
3291 fd_cb->local_endpoint.sin6.sin6_len = sizeof(struct sockaddr_in6);
3292 fd_cb->local_endpoint.sin6.sin6_family = AF_INET6;
3293 fd_cb->local_endpoint.sin6.sin6_port = inp->inp_lport;
3294 error = in6_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin6.sin6_addr), &ifp);
3295 if (error) {
3296 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv6 address: %d", error);
3297 error = 0;
3298 }
3299 if (ifp != NULL) {
3300 inp->in6p_last_outifp = ifp;
3301 ifnet_release(ifp);
3302 }
3303 } else if (to->sa_family == AF_INET) {
3304 inp->inp_vflag |= INP_IPV4;
3305 inp->inp_vflag &= ~INP_IPV6;
3306 fd_cb->local_endpoint.sin.sin_len = sizeof(struct sockaddr_in);
3307 fd_cb->local_endpoint.sin.sin_family = AF_INET;
3308 fd_cb->local_endpoint.sin.sin_port = inp->inp_lport;
3309 error = in_pcbladdr(inp, to, &(fd_cb->local_endpoint.sin.sin_addr), IFSCOPE_NONE, &ifp, 0);
3310 if (error) {
3311 FDLOG(LOG_WARNING, fd_cb, "failed to get a local IPv4 address: %d", error);
3312 error = 0;
3313 }
3314 if (ifp != NULL) {
3315 inp->inp_last_outifp = ifp;
3316 ifnet_release(ifp);
3317 }
3318 } else {
3319 FDLOG(LOG_WARNING, fd_cb, "target address has an unsupported family: %d", to->sa_family);
3320 }
3321
3322 error = flow_divert_check_no_cellular(fd_cb) ||
3323 flow_divert_check_no_expensive(fd_cb) ||
3324 flow_divert_check_no_constrained(fd_cb);
3325 if (error) {
3326 goto done;
3327 }
3328
3329 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
3330 if (error) {
3331 goto done;
3332 }
3333
3334 if (!implicit || SOCK_TYPE(so) == SOCK_STREAM) {
3335 flow_divert_set_remote_endpoint(fd_cb, to);
3336 flow_divert_set_local_endpoint(fd_cb, &(fd_cb->local_endpoint.sa), false);
3337 }
3338
3339 if (implicit) {
3340 fd_cb->flags |= FLOW_DIVERT_IMPLICIT_CONNECT;
3341 }
3342
3343 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
3344 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
3345 do_send = 0;
3346 }
3347
3348 fd_cb->connect_packet = connect_packet;
3349 connect_packet = NULL;
3350 } else {
3351 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
3352 }
3353
3354 if (do_send) {
3355 error = flow_divert_send_connect_packet(fd_cb);
3356 if (error) {
3357 goto done;
3358 }
3359
3360 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3361 }
3362
3363 if (SOCK_TYPE(so) == SOCK_DGRAM) {
3364 soisconnected(so);
3365 } else {
3366 soisconnecting(so);
3367 }
3368
3369 done:
3370 return error;
3371 }
3372
3373 errno_t
3374 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
3375 {
3376 return flow_divert_connect_out_internal(so, to, p, false);
3377 }
3378
3379 static int
3380 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3381 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
3382 {
3383 struct inpcb *inp = sotoinpcb(so);
3384 int error;
3385
3386 if (inp == NULL) {
3387 return EINVAL;
3388 }
3389
3390 VERIFY(dst != NULL);
3391
3392 error = flow_divert_connect_out(so, dst, p);
3393
3394 if (error != 0) {
3395 return error;
3396 }
3397
3398 /* if there is data, send it */
3399 if (auio != NULL) {
3400 user_ssize_t datalen = 0;
3401
3402 socket_unlock(so, 0);
3403
3404 VERIFY(bytes_written != NULL);
3405
3406 datalen = uio_resid(auio);
3407 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3408 socket_lock(so, 0);
3409
3410 if (error == 0 || error == EWOULDBLOCK) {
3411 *bytes_written = datalen - uio_resid(auio);
3412 }
3413
3414 /*
3415 * sosend returns EWOULDBLOCK if it's a non-blocking
3416 * socket or a timeout occured (this allows to return
3417 * the amount of queued data through sendit()).
3418 *
3419 * However, connectx() returns EINPROGRESS in case of a
3420 * blocking socket. So we change the return value here.
3421 */
3422 if (error == EWOULDBLOCK) {
3423 error = EINPROGRESS;
3424 }
3425 }
3426
3427 if (error == 0 && pcid != NULL) {
3428 *pcid = 1; /* there is only 1 connection for a TCP */
3429 }
3430
3431 return error;
3432 }
3433
3434 static int
3435 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3436 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3437 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3438 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3439 {
3440 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3441 }
3442
3443 static int
3444 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3445 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3446 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3447 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3448 {
3449 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3450 }
3451
3452 static errno_t
3453 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3454 {
3455 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3456 int error = 0;
3457 struct inpcb *inp;
3458 #if CONTENT_FILTER
3459 struct m_tag *cfil_tag = NULL;
3460 #endif
3461
3462 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3463
3464 inp = sotoinpcb(so);
3465 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3466 error = ECONNRESET;
3467 goto done;
3468 }
3469
3470 if (control && mbuf_len(control) > 0) {
3471 error = EINVAL;
3472 goto done;
3473 }
3474
3475 if (flags & MSG_OOB) {
3476 error = EINVAL;
3477 goto done; /* We don't support OOB data */
3478 }
3479
3480 #if CONTENT_FILTER
3481 /*
3482 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3483 * retrieve the CFIL saved remote address from the mbuf and use it.
3484 */
3485 if (to == NULL && so->so_cfil_db) {
3486 struct sockaddr *cfil_faddr = NULL;
3487 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3488 if (cfil_tag) {
3489 to = (struct sockaddr *)(void *)cfil_faddr;
3490 }
3491 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3492 }
3493 #endif
3494
3495 /* Implicit connect */
3496 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3497 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3498
3499 error = flow_divert_connect_out_internal(so, to, p, true);
3500 if (error) {
3501 goto done;
3502 }
3503
3504 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3505 /* Open up the send window so that the data will get sent right away */
3506 fd_cb->send_window = (uint32_t)mbuf_pkthdr_len(data);
3507 }
3508 } else {
3509 error = flow_divert_check_no_cellular(fd_cb) ||
3510 flow_divert_check_no_expensive(fd_cb) ||
3511 flow_divert_check_no_constrained(fd_cb);
3512 if (error) {
3513 goto done;
3514 }
3515 }
3516
3517 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3518
3519 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3520 error = flow_divert_send_app_data(fd_cb, data, to);
3521
3522 data = NULL;
3523
3524 if (error) {
3525 goto done;
3526 }
3527
3528 if (flags & PRUS_EOF) {
3529 flow_divert_shutdown(so);
3530 }
3531
3532 done:
3533 if (data) {
3534 mbuf_freem(data);
3535 }
3536 if (control) {
3537 mbuf_free(control);
3538 }
3539 #if CONTENT_FILTER
3540 if (cfil_tag) {
3541 m_tag_free(cfil_tag);
3542 }
3543 #endif
3544
3545 return error;
3546 }
3547
3548 static int
3549 flow_divert_preconnect(struct socket *so)
3550 {
3551 int error = 0;
3552 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3553
3554 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3555
3556 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3557 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3558 error = flow_divert_send_connect_packet(so->so_fd_pcb);
3559 if (error) {
3560 return error;
3561 }
3562
3563 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3564 }
3565
3566 soclearfastopen(so);
3567
3568 return error;
3569 }
3570
3571 static void
3572 flow_divert_set_protosw(struct socket *so)
3573 {
3574 if (SOCK_DOM(so) == PF_INET) {
3575 so->so_proto = &g_flow_divert_in_protosw;
3576 } else {
3577 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3578 }
3579 }
3580
3581 static void
3582 flow_divert_set_udp_protosw(struct socket *so)
3583 {
3584 if (SOCK_DOM(so) == PF_INET) {
3585 so->so_proto = &g_flow_divert_in_udp_protosw;
3586 } else {
3587 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3588 }
3589 }
3590
3591 errno_t
3592 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3593 {
3594 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3595 struct inpcb *inp;
3596 int error = 0;
3597
3598 inp = sotoinpcb(so);
3599 if (inp == NULL) {
3600 return EINVAL;
3601 }
3602
3603 if (fd_cb == NULL) {
3604 error = flow_divert_pcb_init(so);
3605 fd_cb = so->so_fd_pcb;
3606 if (error != 0 || fd_cb == NULL) {
3607 goto done;
3608 }
3609 }
3610 return flow_divert_data_out(so, flags, data, to, control, p);
3611
3612 done:
3613 if (data) {
3614 mbuf_freem(data);
3615 }
3616 if (control) {
3617 mbuf_free(control);
3618 }
3619
3620 return error;
3621 }
3622
3623 static errno_t
3624 flow_divert_pcb_init_internal(struct socket *so, uint32_t ctl_unit, uint32_t aggregate_unit)
3625 {
3626 errno_t error = 0;
3627 struct flow_divert_pcb *fd_cb;
3628 uint32_t agg_unit = aggregate_unit;
3629 uint32_t group_unit = flow_divert_derive_kernel_control_unit(ctl_unit, &agg_unit);
3630
3631 if (group_unit == 0) {
3632 return EINVAL;
3633 }
3634
3635 if (so->so_flags & SOF_FLOW_DIVERT) {
3636 return EALREADY;
3637 }
3638
3639 fd_cb = flow_divert_pcb_create(so);
3640 if (fd_cb != NULL) {
3641 so->so_fd_pcb = fd_cb;
3642 so->so_flags |= SOF_FLOW_DIVERT;
3643 fd_cb->control_group_unit = group_unit;
3644 fd_cb->policy_control_unit = ctl_unit;
3645 fd_cb->aggregate_unit = agg_unit;
3646
3647 error = flow_divert_pcb_insert(fd_cb, group_unit);
3648 if (error) {
3649 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3650 so->so_fd_pcb = NULL;
3651 so->so_flags &= ~SOF_FLOW_DIVERT;
3652 FDRELEASE(fd_cb);
3653 } else {
3654 if (SOCK_TYPE(so) == SOCK_STREAM) {
3655 flow_divert_set_protosw(so);
3656 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3657 flow_divert_set_udp_protosw(so);
3658 }
3659
3660 FDLOG0(LOG_INFO, fd_cb, "Created");
3661 }
3662 } else {
3663 error = ENOMEM;
3664 }
3665
3666 return error;
3667 }
3668
3669 errno_t
3670 flow_divert_pcb_init(struct socket *so)
3671 {
3672 struct inpcb *inp = sotoinpcb(so);
3673 uint32_t aggregate_units = 0;
3674 uint32_t ctl_unit = necp_socket_get_flow_divert_control_unit(inp, &aggregate_units);
3675 return flow_divert_pcb_init_internal(so, ctl_unit, aggregate_units);
3676 }
3677
3678 errno_t
3679 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3680 {
3681 uint32_t ctl_unit = 0;
3682 uint32_t key_unit = 0;
3683 uint32_t aggregate_unit = 0;
3684 int error = 0;
3685 int hmac_error = 0;
3686 mbuf_t token = NULL;
3687
3688 if (so->so_flags & SOF_FLOW_DIVERT) {
3689 error = EALREADY;
3690 goto done;
3691 }
3692
3693 if (g_init_result) {
3694 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3695 error = ENOPROTOOPT;
3696 goto done;
3697 }
3698
3699 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3700 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3701 (SOCK_DOM(so) != PF_INET && SOCK_DOM(so) != PF_INET6)) {
3702 error = EINVAL;
3703 goto done;
3704 } else {
3705 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3706 struct tcpcb *tp = sototcpcb(so);
3707 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3708 error = EINVAL;
3709 goto done;
3710 }
3711 }
3712 }
3713
3714 error = soopt_getm(sopt, &token);
3715 if (error) {
3716 token = NULL;
3717 goto done;
3718 }
3719
3720 error = soopt_mcopyin(sopt, token);
3721 if (error) {
3722 token = NULL;
3723 goto done;
3724 }
3725
3726 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3727 if (!error) {
3728 key_unit = ntohl(key_unit);
3729 if (key_unit >= GROUP_COUNT_MAX) {
3730 key_unit = 0;
3731 }
3732 } else if (error != ENOENT) {
3733 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3734 goto done;
3735 } else {
3736 key_unit = 0;
3737 }
3738
3739 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3740 if (error) {
3741 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3742 goto done;
3743 }
3744
3745 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_AGGREGATE_UNIT, sizeof(aggregate_unit), (void *)&aggregate_unit, NULL);
3746 if (error && error != ENOENT) {
3747 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the aggregate unit from the token: %d", error);
3748 goto done;
3749 }
3750
3751 /* A valid kernel control unit is required */
3752 ctl_unit = ntohl(ctl_unit);
3753 aggregate_unit = ntohl(aggregate_unit);
3754
3755 if (ctl_unit > 0 && ctl_unit < GROUP_COUNT_MAX) {
3756 socket_unlock(so, 0);
3757 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3758 socket_lock(so, 0);
3759
3760 if (hmac_error && hmac_error != ENOENT) {
3761 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3762 error = hmac_error;
3763 goto done;
3764 }
3765 }
3766
3767 error = flow_divert_pcb_init_internal(so, ctl_unit, aggregate_unit);
3768 if (error == 0) {
3769 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3770 int log_level = LOG_NOTICE;
3771
3772 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
3773 if (error == 0) {
3774 fd_cb->log_level = (uint8_t)log_level;
3775 }
3776 error = 0;
3777
3778 fd_cb->connect_token = token;
3779 token = NULL;
3780 }
3781
3782 if (hmac_error == 0) {
3783 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3784 if (fd_cb != NULL) {
3785 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3786 }
3787 }
3788
3789 done:
3790 if (token != NULL) {
3791 mbuf_freem(token);
3792 }
3793
3794 return error;
3795 }
3796
3797 errno_t
3798 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3799 {
3800 uint32_t ctl_unit;
3801 int error = 0;
3802 uint8_t hmac[SHA_DIGEST_LENGTH];
3803 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3804 mbuf_t token = NULL;
3805 struct flow_divert_group *control_group = NULL;
3806
3807 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3808 error = EINVAL;
3809 goto done;
3810 }
3811
3812 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3813
3814 if (fd_cb->group == NULL) {
3815 error = EINVAL;
3816 goto done;
3817 }
3818
3819 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3820 if (error) {
3821 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3822 goto done;
3823 }
3824
3825 ctl_unit = htonl(fd_cb->group->ctl_unit);
3826
3827 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3828 if (error) {
3829 goto done;
3830 }
3831
3832 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3833 if (error) {
3834 goto done;
3835 }
3836
3837 if (fd_cb->app_data != NULL) {
3838 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, (uint32_t)fd_cb->app_data_length, fd_cb->app_data);
3839 if (error) {
3840 goto done;
3841 }
3842 }
3843
3844 socket_unlock(so, 0);
3845 lck_rw_lock_shared(&g_flow_divert_group_lck);
3846
3847 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3848 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
3849 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3850 }
3851
3852 if (control_group != NULL) {
3853 lck_rw_lock_shared(&control_group->lck);
3854 ctl_unit = htonl(control_group->ctl_unit);
3855 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3856 if (!error) {
3857 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3858 }
3859 lck_rw_done(&control_group->lck);
3860 } else {
3861 error = ENOPROTOOPT;
3862 }
3863
3864 lck_rw_done(&g_flow_divert_group_lck);
3865 socket_lock(so, 0);
3866
3867 if (error) {
3868 goto done;
3869 }
3870
3871 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3872 if (error) {
3873 goto done;
3874 }
3875
3876 if (sopt->sopt_val == USER_ADDR_NULL) {
3877 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3878 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3879 goto done;
3880 }
3881
3882 error = soopt_mcopyout(sopt, token);
3883 if (error) {
3884 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3885 goto done;
3886 }
3887
3888 done:
3889 if (token != NULL) {
3890 mbuf_freem(token);
3891 }
3892
3893 return error;
3894 }
3895
3896 static errno_t
3897 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3898 {
3899 struct flow_divert_group *new_group = NULL;
3900 int error = 0;
3901
3902 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3903 error = EINVAL;
3904 goto done;
3905 }
3906
3907 *unitinfo = NULL;
3908
3909 new_group = zalloc_flags(flow_divert_group_zone, Z_WAITOK | Z_ZERO);
3910 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3911 RB_INIT(&new_group->pcb_tree);
3912 new_group->ctl_unit = sac->sc_unit;
3913 MBUFQ_INIT(&new_group->send_queue);
3914 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3915
3916 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3917
3918 if (g_flow_divert_groups == NULL) {
3919 MALLOC(g_flow_divert_groups,
3920 struct flow_divert_group **,
3921 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3922 M_TEMP,
3923 M_WAITOK | M_ZERO);
3924 }
3925
3926 if (g_flow_divert_groups == NULL) {
3927 error = ENOBUFS;
3928 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3929 error = EALREADY;
3930 } else {
3931 g_flow_divert_groups[sac->sc_unit] = new_group;
3932 g_active_group_count++;
3933 }
3934
3935 lck_rw_done(&g_flow_divert_group_lck);
3936
3937 done:
3938 if (error == 0) {
3939 *unitinfo = new_group;
3940 } else if (new_group != NULL) {
3941 zfree(flow_divert_group_zone, new_group);
3942 }
3943 return error;
3944 }
3945
3946 static errno_t
3947 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3948 {
3949 struct flow_divert_group *group = NULL;
3950 errno_t error = 0;
3951
3952 if (unit >= GROUP_COUNT_MAX) {
3953 return EINVAL;
3954 }
3955
3956 if (unitinfo == NULL) {
3957 return 0;
3958 }
3959
3960 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3961
3962 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3963
3964 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3965 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3966 g_flow_divert_groups, g_active_group_count);
3967 }
3968
3969 group = g_flow_divert_groups[unit];
3970
3971 if (group != (struct flow_divert_group *)unitinfo) {
3972 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3973 }
3974
3975 g_flow_divert_groups[unit] = NULL;
3976 g_active_group_count--;
3977
3978 if (g_active_group_count == 0) {
3979 FREE(g_flow_divert_groups, M_TEMP);
3980 g_flow_divert_groups = NULL;
3981 }
3982
3983 lck_rw_done(&g_flow_divert_group_lck);
3984
3985 if (group != NULL) {
3986 flow_divert_close_all(group);
3987
3988 lck_rw_lock_exclusive(&group->lck);
3989
3990 if (group->token_key != NULL) {
3991 memset(group->token_key, 0, group->token_key_size);
3992 FREE(group->token_key, M_TEMP);
3993 group->token_key = NULL;
3994 group->token_key_size = 0;
3995 }
3996
3997 /* Re-set the current trie */
3998 if (group->signing_id_trie.memory != NULL) {
3999 FREE(group->signing_id_trie.memory, M_TEMP);
4000 }
4001 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
4002 group->signing_id_trie.root = NULL_TRIE_IDX;
4003
4004 lck_rw_done(&group->lck);
4005
4006 zfree(flow_divert_group_zone, group);
4007 } else {
4008 error = EINVAL;
4009 }
4010
4011 return error;
4012 }
4013
4014 static errno_t
4015 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
4016 {
4017 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
4018 }
4019
4020 static void
4021 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
4022 {
4023 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
4024
4025 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
4026 struct flow_divert_pcb *fd_cb;
4027 SLIST_HEAD(, flow_divert_pcb) tmp_list;
4028
4029 lck_rw_lock_shared(&g_flow_divert_group_lck);
4030 lck_rw_lock_exclusive(&group->lck);
4031
4032 while (!MBUFQ_EMPTY(&group->send_queue)) {
4033 mbuf_t next_packet;
4034 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
4035 next_packet = MBUFQ_FIRST(&group->send_queue);
4036 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
4037 if (error) {
4038 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
4039 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
4040 lck_rw_done(&group->lck);
4041 lck_rw_done(&g_flow_divert_group_lck);
4042 return;
4043 }
4044 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
4045 }
4046
4047 SLIST_INIT(&tmp_list);
4048
4049 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
4050 FDRETAIN(fd_cb);
4051 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
4052 }
4053
4054 lck_rw_done(&group->lck);
4055
4056 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
4057 FDLOCK(fd_cb);
4058 if (fd_cb->so != NULL) {
4059 socket_lock(fd_cb->so, 0);
4060 if (fd_cb->group != NULL) {
4061 flow_divert_send_buffered_data(fd_cb, FALSE);
4062 }
4063 socket_unlock(fd_cb->so, 0);
4064 }
4065 FDUNLOCK(fd_cb);
4066 FDRELEASE(fd_cb);
4067 }
4068
4069 lck_rw_done(&g_flow_divert_group_lck);
4070 }
4071 }
4072
4073 static int
4074 flow_divert_kctl_init(void)
4075 {
4076 struct kern_ctl_reg ctl_reg;
4077 int result;
4078
4079 memset(&ctl_reg, 0, sizeof(ctl_reg));
4080
4081 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
4082 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
4083 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
4084 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
4085 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
4086
4087 ctl_reg.ctl_connect = flow_divert_kctl_connect;
4088 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
4089 ctl_reg.ctl_send = flow_divert_kctl_send;
4090 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
4091
4092 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
4093
4094 if (result) {
4095 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
4096 return result;
4097 }
4098
4099 return 0;
4100 }
4101
4102 void
4103 flow_divert_init(void)
4104 {
4105 memset(&nil_pcb, 0, sizeof(nil_pcb));
4106 nil_pcb.log_level = LOG_NOTICE;
4107
4108 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4109
4110 VERIFY(g_tcp_protosw != NULL);
4111
4112 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4113 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4114
4115 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4116 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
4117 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4118 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4119 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4120 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4121 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
4122 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
4123
4124 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4125 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4126
4127 /*
4128 * Socket filters shouldn't attach/detach to/from this protosw
4129 * since pr_protosw is to be used instead, which points to the
4130 * real protocol; if they do, it is a bug and we should panic.
4131 */
4132 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4133 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4134 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4135 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4136
4137 /* UDP */
4138 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4139 VERIFY(g_udp_protosw != NULL);
4140
4141 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4142 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4143
4144 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4145 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
4146 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4147 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4148 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4149 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4150 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4151 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4152 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4153 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4154
4155 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4156 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4157
4158 /*
4159 * Socket filters shouldn't attach/detach to/from this protosw
4160 * since pr_protosw is to be used instead, which points to the
4161 * real protocol; if they do, it is a bug and we should panic.
4162 */
4163 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4164 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4165 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4166 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4167
4168 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4169
4170 VERIFY(g_tcp6_protosw != NULL);
4171
4172 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4173 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4174
4175 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4176 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
4177 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4178 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4179 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4180 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4181 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
4182 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
4183
4184 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4185 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4186 /*
4187 * Socket filters shouldn't attach/detach to/from this protosw
4188 * since pr_protosw is to be used instead, which points to the
4189 * real protocol; if they do, it is a bug and we should panic.
4190 */
4191 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4192 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4193 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4194 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4195
4196 /* UDP6 */
4197 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4198
4199 VERIFY(g_udp6_protosw != NULL);
4200
4201 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4202 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4203
4204 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4205 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4206 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4207 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4208 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4209 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4210 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4211 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4212 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4213 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4214
4215 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4216 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4217 /*
4218 * Socket filters shouldn't attach/detach to/from this protosw
4219 * since pr_protosw is to be used instead, which points to the
4220 * real protocol; if they do, it is a bug and we should panic.
4221 */
4222 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4223 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4224 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4225 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4226
4227 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4228 if (flow_divert_grp_attr == NULL) {
4229 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4230 g_init_result = ENOMEM;
4231 goto done;
4232 }
4233
4234 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4235 if (flow_divert_mtx_grp == NULL) {
4236 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4237 g_init_result = ENOMEM;
4238 goto done;
4239 }
4240
4241 flow_divert_mtx_attr = lck_attr_alloc_init();
4242 if (flow_divert_mtx_attr == NULL) {
4243 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4244 g_init_result = ENOMEM;
4245 goto done;
4246 }
4247
4248 g_init_result = flow_divert_kctl_init();
4249 if (g_init_result) {
4250 goto done;
4251 }
4252
4253 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4254
4255 done:
4256 if (g_init_result != 0) {
4257 if (flow_divert_mtx_attr != NULL) {
4258 lck_attr_free(flow_divert_mtx_attr);
4259 flow_divert_mtx_attr = NULL;
4260 }
4261 if (flow_divert_mtx_grp != NULL) {
4262 lck_grp_free(flow_divert_mtx_grp);
4263 flow_divert_mtx_grp = NULL;
4264 }
4265 if (flow_divert_grp_attr != NULL) {
4266 lck_grp_attr_free(flow_divert_grp_attr);
4267 flow_divert_grp_attr = NULL;
4268 }
4269
4270 if (g_flow_divert_kctl_ref != NULL) {
4271 ctl_deregister(g_flow_divert_kctl_ref);
4272 g_flow_divert_kctl_ref = NULL;
4273 }
4274 }
4275 }
mirror of XNU