]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:


84f3f4f483e4b76050665e4f5feed971ef7a0722
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <kern/task.h>
49 #include <mach/task_info.h>
50 #include <net/if_var.h>
51 #include <net/route.h>
52 #include <net/flowhash.h>
53 #include <net/ntstat.h>
54 #include <netinet/in.h>
55 #include <netinet/in_var.h>
56 #include <netinet/tcp.h>
57 #include <netinet/tcp_var.h>
58 #include <netinet/tcp_fsm.h>
59 #include <netinet/flow_divert.h>
60 #include <netinet/flow_divert_proto.h>
61 #if INET6
62 #include <netinet6/in6_pcb.h>
63 #include <netinet6/ip6protosw.h>
64 #endif /* INET6 */
65 #include <dev/random/randomdev.h>
66 #include <libkern/crypto/sha1.h>
67 #include <libkern/crypto/crypto_internal.h>
68 #include <os/log.h>
69 #include <corecrypto/cc.h>
70 #if CONTENT_FILTER
71 #include <net/content_filter.h>
72 #endif /* CONTENT_FILTER */
73
74 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75 #define FLOW_DIVERT_READ_CLOSED 0x00000002
76 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
79 #define FLOW_DIVERT_TRANSFERRED 0x00000020
80 #define FLOW_DIVERT_HAS_HMAC 0x00000040
81
82 #define FDLOG(level, pcb, format, ...) \
83 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
84
85 #define FDLOG0(level, pcb, msg) \
86 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
87
88 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
89 #define FDRELEASE(pcb) \
90 do { \
91 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
92 flow_divert_pcb_destroy(pcb); \
93 } \
94 } while (0)
95
96 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
97 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
98
99 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
100 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
101
102 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
103
104 #define GROUP_COUNT_MAX 32
105 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
106 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
107 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
108
109 struct flow_divert_trie_node {
110 uint16_t start;
111 uint16_t length;
112 uint16_t child_map;
113 };
114
115 #define CHILD_MAP_SIZE 256
116 #define NULL_TRIE_IDX 0xffff
117 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
118 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
119 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
120
121 static struct flow_divert_pcb nil_pcb;
122
123 decl_lck_rw_data(static, g_flow_divert_group_lck);
124 static struct flow_divert_group **g_flow_divert_groups = NULL;
125 static uint32_t g_active_group_count = 0;
126
127 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
128 static lck_attr_t *flow_divert_mtx_attr = NULL;
129 static lck_grp_t *flow_divert_mtx_grp = NULL;
130 static errno_t g_init_result = 0;
131
132 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
133
134 static struct protosw g_flow_divert_in_protosw;
135 static struct pr_usrreqs g_flow_divert_in_usrreqs;
136 static struct protosw g_flow_divert_in_udp_protosw;
137 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
138 #if INET6
139 static struct ip6protosw g_flow_divert_in6_protosw;
140 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
141 static struct ip6protosw g_flow_divert_in6_udp_protosw;
142 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
143 #endif /* INET6 */
144
145 static struct protosw *g_tcp_protosw = NULL;
146 static struct ip6protosw *g_tcp6_protosw = NULL;
147 static struct protosw *g_udp_protosw = NULL;
148 static struct ip6protosw *g_udp6_protosw = NULL;
149
150 static errno_t
151 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
152
153 static errno_t
154 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
155
156 static boolean_t
157 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
158
159 static int
160 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
161
162 struct sockaddr *
163 flow_divert_get_buffered_target_address(mbuf_t buffer);
164
165 static boolean_t
166 flow_divert_has_pcb_local_address(const struct inpcb *inp);
167
168 static void
169 flow_divert_disconnect_socket(struct socket *so);
170
171 static inline uint8_t
172 flow_divert_syslog_type_to_oslog_type(int syslog_type)
173 {
174 switch (syslog_type) {
175 case LOG_ERR: return OS_LOG_TYPE_ERROR;
176 case LOG_INFO: return OS_LOG_TYPE_INFO;
177 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
178 default: return OS_LOG_TYPE_DEFAULT;
179 }
180 }
181
182 static inline int
183 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
184 {
185 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
186 }
187
188 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
189 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
190
191 static const char *
192 flow_divert_packet_type2str(uint8_t packet_type)
193 {
194 switch (packet_type) {
195 case FLOW_DIVERT_PKT_CONNECT:
196 return "connect";
197 case FLOW_DIVERT_PKT_CONNECT_RESULT:
198 return "connect result";
199 case FLOW_DIVERT_PKT_DATA:
200 return "data";
201 case FLOW_DIVERT_PKT_CLOSE:
202 return "close";
203 case FLOW_DIVERT_PKT_READ_NOTIFY:
204 return "read notification";
205 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
206 return "properties update";
207 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
208 return "app map create";
209 default:
210 return "unknown";
211 }
212 }
213
214 static struct flow_divert_pcb *
215 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
216 {
217 struct flow_divert_pcb key_item;
218 struct flow_divert_pcb *fd_cb = NULL;
219
220 key_item.hash = hash;
221
222 lck_rw_lock_shared(&group->lck);
223 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
224 FDRETAIN(fd_cb);
225 lck_rw_done(&group->lck);
226
227 return fd_cb;
228 }
229
230 static errno_t
231 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
232 {
233 errno_t error = 0;
234 struct flow_divert_pcb *exist = NULL;
235 struct flow_divert_group *group;
236 static uint32_t g_nextkey = 1;
237 static uint32_t g_hash_seed = 0;
238 int try_count = 0;
239
240 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
241 return EINVAL;
242 }
243
244 socket_unlock(fd_cb->so, 0);
245 lck_rw_lock_shared(&g_flow_divert_group_lck);
246
247 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
248 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
249 error = ENETUNREACH;
250 goto done;
251 }
252
253 group = g_flow_divert_groups[ctl_unit];
254 if (group == NULL) {
255 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
256 error = ENETUNREACH;
257 goto done;
258 }
259
260 socket_lock(fd_cb->so, 0);
261
262 do {
263 uint32_t key[2];
264 uint32_t idx;
265
266 key[0] = g_nextkey++;
267 key[1] = RandomULong();
268
269 if (g_hash_seed == 0) {
270 g_hash_seed = RandomULong();
271 }
272
273 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
274
275 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
276 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
277 if (curr_group != NULL && curr_group != group) {
278 lck_rw_lock_shared(&curr_group->lck);
279 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
280 lck_rw_done(&curr_group->lck);
281 if (exist != NULL) {
282 break;
283 }
284 }
285 }
286
287 if (exist == NULL) {
288 lck_rw_lock_exclusive(&group->lck);
289 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
290 lck_rw_done(&group->lck);
291 }
292 } while (exist != NULL && try_count++ < 3);
293
294 if (exist == NULL) {
295 fd_cb->group = group;
296 FDRETAIN(fd_cb); /* The group now has a reference */
297 } else {
298 fd_cb->hash = 0;
299 error = EEXIST;
300 }
301
302 socket_unlock(fd_cb->so, 0);
303
304 done:
305 lck_rw_done(&g_flow_divert_group_lck);
306 socket_lock(fd_cb->so, 0);
307
308 return error;
309 }
310
311 static struct flow_divert_pcb *
312 flow_divert_pcb_create(socket_t so)
313 {
314 struct flow_divert_pcb *new_pcb = NULL;
315
316 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
317 if (new_pcb == NULL) {
318 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
319 return NULL;
320 }
321
322 memset(new_pcb, 0, sizeof(*new_pcb));
323
324 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
325 new_pcb->so = so;
326 new_pcb->log_level = nil_pcb.log_level;
327
328 FDRETAIN(new_pcb); /* Represents the socket's reference */
329
330 return new_pcb;
331 }
332
333 static void
334 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
335 {
336 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
337 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
338
339 if (fd_cb->local_address != NULL) {
340 FREE(fd_cb->local_address, M_SONAME);
341 }
342 if (fd_cb->remote_address != NULL) {
343 FREE(fd_cb->remote_address, M_SONAME);
344 }
345 if (fd_cb->connect_token != NULL) {
346 mbuf_freem(fd_cb->connect_token);
347 }
348 if (fd_cb->connect_packet != NULL) {
349 mbuf_freem(fd_cb->connect_packet);
350 }
351 if (fd_cb->app_data != NULL) {
352 FREE(fd_cb->app_data, M_TEMP);
353 }
354 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
355 }
356
357 static void
358 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
359 {
360 if (fd_cb->group != NULL) {
361 struct flow_divert_group *group = fd_cb->group;
362 lck_rw_lock_exclusive(&group->lck);
363 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
364 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
365 fd_cb->group = NULL;
366 FDRELEASE(fd_cb); /* Release the group's reference */
367 lck_rw_done(&group->lck);
368 }
369 }
370
371 static int
372 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
373 {
374 struct flow_divert_packet_header hdr;
375 int error = 0;
376
377 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
378 if (error) {
379 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
380 return error;
381 }
382
383 hdr.packet_type = packet_type;
384 hdr.conn_id = htonl(fd_cb->hash);
385
386 /* Lay down the header */
387 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
388 if (error) {
389 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
390 mbuf_freem(*packet);
391 *packet = NULL;
392 return error;
393 }
394
395 return 0;
396 }
397
398 static int
399 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
400 {
401 uint32_t net_length = htonl(length);
402 int error = 0;
403
404 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
405 if (error) {
406 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
407 return error;
408 }
409
410 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
411 if (error) {
412 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
413 return error;
414 }
415
416 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
417 if (error) {
418 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
419 return error;
420 }
421
422 return error;
423 }
424
425 static int
426 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
427 {
428 size_t cursor = offset;
429 int error = 0;
430 uint32_t curr_length;
431 uint8_t curr_type;
432
433 *err = 0;
434
435 do {
436 if (!next) {
437 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
438 if (error) {
439 *err = ENOENT;
440 return -1;
441 }
442 } else {
443 next = 0;
444 curr_type = FLOW_DIVERT_TLV_NIL;
445 }
446
447 if (curr_type != type) {
448 cursor += sizeof(curr_type);
449 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
450 if (error) {
451 *err = error;
452 return -1;
453 }
454
455 cursor += (sizeof(curr_length) + ntohl(curr_length));
456 }
457 } while (curr_type != type);
458
459 return cursor;
460 }
461
462 static int
463 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
464 {
465 int error = 0;
466 uint32_t length;
467 int tlv_offset;
468
469 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
470 if (tlv_offset < 0) {
471 return error;
472 }
473
474 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
475 if (error) {
476 return error;
477 }
478
479 length = ntohl(length);
480
481 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
482
483 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
484 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
485 return EINVAL;
486 }
487
488 if (val_size != NULL) {
489 *val_size = length;
490 }
491
492 if (buff != NULL && buff_len > 0) {
493 memset(buff, 0, buff_len);
494 size_t to_copy = (length < buff_len) ? length : buff_len;
495 error = mbuf_copydata(packet, data_offset, to_copy, buff);
496 if (error) {
497 return error;
498 }
499 }
500
501 return 0;
502 }
503
504 static int
505 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
506 {
507 mbuf_t curr_mbuf = packet;
508
509 if (g_crypto_funcs == NULL || group->token_key == NULL) {
510 return ENOPROTOOPT;
511 }
512
513 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
514 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
515
516 while (curr_mbuf != NULL) {
517 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
518 curr_mbuf = mbuf_next(curr_mbuf);
519 }
520
521 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
522
523 return 0;
524 }
525
526 static int
527 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
528 {
529 int error = 0;
530 struct flow_divert_group *group = NULL;
531 int hmac_offset;
532 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
533 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
534 mbuf_t tail;
535
536 lck_rw_lock_shared(&g_flow_divert_group_lck);
537
538 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
539 group = g_flow_divert_groups[ctl_unit];
540 }
541
542 if (group == NULL) {
543 lck_rw_done(&g_flow_divert_group_lck);
544 return ENOPROTOOPT;
545 }
546
547 lck_rw_lock_shared(&group->lck);
548
549 if (group->token_key == NULL) {
550 error = ENOPROTOOPT;
551 goto done;
552 }
553
554 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
555 if (hmac_offset < 0) {
556 goto done;
557 }
558
559 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
560 if (error) {
561 goto done;
562 }
563
564 /* Chop off the HMAC TLV */
565 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
566 if (error) {
567 goto done;
568 }
569
570 mbuf_free(tail);
571
572 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
573 if (error) {
574 goto done;
575 }
576
577 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
578 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
579 error = EINVAL;
580 goto done;
581 }
582
583 done:
584 lck_rw_done(&group->lck);
585 lck_rw_done(&g_flow_divert_group_lck);
586 return error;
587 }
588
589 static void
590 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
591 {
592 struct inpcb *inp = NULL;
593 struct ifnet *ifp = NULL;
594 Boolean cell = FALSE;
595 Boolean wifi = FALSE;
596 Boolean wired = FALSE;
597
598 inp = sotoinpcb(fd_cb->so);
599 if (inp == NULL) {
600 return;
601 }
602
603 ifp = inp->inp_last_outifp;
604 if (ifp != NULL) {
605 cell = IFNET_IS_CELLULAR(ifp);
606 wifi = (!cell && IFNET_IS_WIFI(ifp));
607 wired = (!wifi && IFNET_IS_WIRED(ifp));
608 }
609
610 if (send) {
611 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
612 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
613 } else {
614 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
615 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
616 }
617 inp_set_activity_bitmap(inp);
618 }
619
620 static errno_t
621 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
622 {
623 struct inpcb *inp = NULL;
624
625 inp = sotoinpcb(fd_cb->so);
626 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
627 IFNET_IS_CELLULAR(inp->inp_last_outifp)) {
628 return EHOSTUNREACH;
629 }
630
631 return 0;
632 }
633
634 static errno_t
635 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
636 {
637 struct inpcb *inp = NULL;
638
639 inp = sotoinpcb(fd_cb->so);
640 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
641 IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) {
642 return EHOSTUNREACH;
643 }
644
645 return 0;
646 }
647
648 static errno_t
649 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
650 {
651 struct inpcb *inp = NULL;
652
653 inp = sotoinpcb(fd_cb->so);
654 if (inp && INP_NO_CONSTRAINED(inp) && inp->inp_last_outifp &&
655 IFNET_IS_CONSTRAINED(inp->inp_last_outifp)) {
656 return EHOSTUNREACH;
657 }
658
659 return 0;
660 }
661
662 static void
663 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
664 {
665 if (how != SHUT_RD) {
666 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
667 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
668 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
669 /* If the tunnel is not accepting writes any more, then flush the send buffer */
670 sbflush(&fd_cb->so->so_snd);
671 }
672 }
673 if (how != SHUT_WR) {
674 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
675 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
676 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
677 }
678 }
679 }
680
681 static uint16_t
682 trie_node_alloc(struct flow_divert_trie *trie)
683 {
684 if (trie->nodes_free_next < trie->nodes_count) {
685 uint16_t node_idx = trie->nodes_free_next++;
686 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
687 return node_idx;
688 } else {
689 return NULL_TRIE_IDX;
690 }
691 }
692
693 static uint16_t
694 trie_child_map_alloc(struct flow_divert_trie *trie)
695 {
696 if (trie->child_maps_free_next < trie->child_maps_count) {
697 return trie->child_maps_free_next++;
698 } else {
699 return NULL_TRIE_IDX;
700 }
701 }
702
703 static uint16_t
704 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
705 {
706 uint16_t start = trie->bytes_free_next;
707 if (start + bytes_size <= trie->bytes_count) {
708 if (start != bytes_idx) {
709 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
710 }
711 trie->bytes_free_next += bytes_size;
712 return start;
713 } else {
714 return NULL_TRIE_IDX;
715 }
716 }
717
718 static uint16_t
719 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
720 {
721 uint16_t current = trie->root;
722 uint16_t child = trie->root;
723 uint16_t string_end = string_start + string_len;
724 uint16_t string_idx = string_start;
725 uint16_t string_remainder = string_len;
726
727 while (child != NULL_TRIE_IDX) {
728 uint16_t parent = current;
729 uint16_t node_idx;
730 uint16_t current_end;
731
732 current = child;
733 child = NULL_TRIE_IDX;
734
735 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
736
737 for (node_idx = TRIE_NODE(trie, current).start;
738 node_idx < current_end &&
739 string_idx < string_end &&
740 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
741 node_idx++, string_idx++) {
742 ;
743 }
744
745 string_remainder = string_end - string_idx;
746
747 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
748 /*
749 * We did not reach the end of the current node's string.
750 * We need to split the current node into two:
751 * 1. A new node that contains the prefix of the node that matches
752 * the prefix of the string being inserted.
753 * 2. The current node modified to point to the remainder
754 * of the current node's string.
755 */
756 uint16_t prefix = trie_node_alloc(trie);
757 if (prefix == NULL_TRIE_IDX) {
758 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
759 return NULL_TRIE_IDX;
760 }
761
762 /*
763 * Prefix points to the portion of the current nodes's string that has matched
764 * the input string thus far.
765 */
766 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
767 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
768
769 /*
770 * Prefix has the current node as the child corresponding to the first byte
771 * after the split.
772 */
773 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
774 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
775 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
776 return NULL_TRIE_IDX;
777 }
778 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
779
780 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
781 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
782
783 /* Current node is adjusted to point to the remainder */
784 TRIE_NODE(trie, current).start = node_idx;
785 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
786
787 /* We want to insert the new leaf (if any) as a child of the prefix */
788 current = prefix;
789 }
790
791 if (string_remainder > 0) {
792 /*
793 * We still have bytes in the string that have not been matched yet.
794 * If the current node has children, iterate to the child corresponding
795 * to the next byte in the string.
796 */
797 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
798 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
799 }
800 }
801 } /* while (child != NULL_TRIE_IDX) */
802
803 if (string_remainder > 0) {
804 /* Add a new leaf containing the remainder of the string */
805 uint16_t leaf = trie_node_alloc(trie);
806 if (leaf == NULL_TRIE_IDX) {
807 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
808 return NULL_TRIE_IDX;
809 }
810
811 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
812 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
813 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
814 return NULL_TRIE_IDX;
815 }
816 TRIE_NODE(trie, leaf).length = string_remainder;
817
818 /* Set the new leaf as the child of the current node */
819 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
820 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
821 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
822 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
823 return NULL_TRIE_IDX;
824 }
825 }
826 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
827 current = leaf;
828 } /* else duplicate or this string is a prefix of one of the existing strings */
829
830 return current;
831 }
832
833 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
834 static uint16_t
835 flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
836 {
837 uint16_t current = trie->root;
838 uint16_t string_idx = 0;
839
840 while (current != NULL_TRIE_IDX) {
841 uint16_t next = NULL_TRIE_IDX;
842 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
843 uint16_t node_idx;
844
845 for (node_idx = TRIE_NODE(trie, current).start;
846 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
847 node_idx++, string_idx++) {
848 ;
849 }
850
851 if (node_idx == node_end) {
852 if (string_bytes[string_idx] == '\0') {
853 return current; /* Got an exact match */
854 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
855 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
856 string_bytes[string_idx] = '\0';
857 return current; /* Got an apple webclip id prefix match */
858 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
859 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
860 }
861 }
862 current = next;
863 }
864
865 return NULL_TRIE_IDX;
866 }
867
868 struct uuid_search_info {
869 uuid_t target_uuid;
870 char *found_signing_id;
871 boolean_t found_multiple_signing_ids;
872 proc_t found_proc;
873 };
874
875 static int
876 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
877 {
878 struct uuid_search_info *info = (struct uuid_search_info *)arg;
879 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
880
881 if (info->found_signing_id != NULL) {
882 if (!info->found_multiple_signing_ids) {
883 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
884 info->found_proc = p;
885 result = PROC_CLAIMED_DONE;
886 } else {
887 uuid_string_t uuid_str;
888 uuid_unparse(info->target_uuid, uuid_str);
889 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
890 }
891 FREE(info->found_signing_id, M_TEMP);
892 info->found_signing_id = NULL;
893 }
894
895 if (result == PROC_RETURNED_DONE) {
896 uuid_string_t uuid_str;
897 uuid_unparse(info->target_uuid, uuid_str);
898 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
899 }
900
901 return result;
902 }
903
904 static int
905 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
906 {
907 struct uuid_search_info *info = (struct uuid_search_info *)arg;
908 int include = 0;
909
910 if (info->found_multiple_signing_ids) {
911 return include;
912 }
913
914 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
915 if (include) {
916 const char *signing_id = cs_identity_get(p);
917 if (signing_id != NULL) {
918 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
919 size_t signing_id_size = strlen(signing_id) + 1;
920 if (info->found_signing_id == NULL) {
921 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
922 memcpy(info->found_signing_id, signing_id, signing_id_size);
923 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
924 info->found_multiple_signing_ids = TRUE;
925 }
926 } else {
927 info->found_multiple_signing_ids = TRUE;
928 }
929 include = !info->found_multiple_signing_ids;
930 }
931
932 return include;
933 }
934
935 static proc_t
936 flow_divert_find_proc_by_uuid(uuid_t uuid)
937 {
938 struct uuid_search_info info;
939
940 if (LOG_INFO <= nil_pcb.log_level) {
941 uuid_string_t uuid_str;
942 uuid_unparse(uuid, uuid_str);
943 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
944 }
945
946 memset(&info, 0, sizeof(info));
947 info.found_proc = PROC_NULL;
948 uuid_copy(info.target_uuid, uuid);
949
950 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
951
952 return info.found_proc;
953 }
954
955 static int
956 flow_divert_get_src_proc(struct socket *so, proc_t *proc)
957 {
958 int release = 0;
959
960 if (so->so_flags & SOF_DELEGATED) {
961 if ((*proc)->p_pid != so->e_pid) {
962 *proc = proc_find(so->e_pid);
963 release = 1;
964 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
965 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
966 release = 1;
967 }
968 } else if (*proc == PROC_NULL) {
969 *proc = current_proc();
970 }
971
972 if (*proc != PROC_NULL) {
973 if ((*proc)->p_pid == 0) {
974 if (release) {
975 proc_rele(*proc);
976 }
977 release = 0;
978 *proc = PROC_NULL;
979 }
980 }
981
982 return release;
983 }
984
985 static int
986 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
987 {
988 int error;
989
990 if (fd_cb->group == NULL) {
991 fd_cb->so->so_error = ECONNABORTED;
992 flow_divert_disconnect_socket(fd_cb->so);
993 return ECONNABORTED;
994 }
995
996 lck_rw_lock_shared(&fd_cb->group->lck);
997
998 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
999 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1000 } else {
1001 error = ENOBUFS;
1002 }
1003
1004 if (error == ENOBUFS) {
1005 if (enqueue) {
1006 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1007 lck_rw_lock_exclusive(&fd_cb->group->lck);
1008 }
1009 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1010 error = 0;
1011 }
1012 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1013 }
1014
1015 lck_rw_done(&fd_cb->group->lck);
1016
1017 return error;
1018 }
1019
1020 static int
1021 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
1022 {
1023 int error = 0;
1024 int flow_type = 0;
1025 char *signing_id = NULL;
1026 int free_signing_id = 0;
1027 mbuf_t connect_packet = NULL;
1028 proc_t src_proc = p;
1029 int release_proc = 0;
1030
1031 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1032 if (error) {
1033 goto done;
1034 }
1035
1036 error = EPERM;
1037
1038 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1039 uint32_t sid_size = 0;
1040 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1041 if (find_error == 0 && sid_size > 0) {
1042 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1043 if (signing_id != NULL) {
1044 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1045 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1046 free_signing_id = 1;
1047 }
1048 }
1049 }
1050
1051 socket_unlock(so, 0);
1052
1053 release_proc = flow_divert_get_src_proc(so, &src_proc);
1054 if (src_proc != PROC_NULL) {
1055 proc_lock(src_proc);
1056 if (signing_id == NULL) {
1057 if (src_proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
1058 const char * cs_id;
1059 cs_id = cs_identity_get(src_proc);
1060 signing_id = __DECONST(char *, cs_id);
1061 } else {
1062 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
1063 }
1064 }
1065 } else {
1066 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
1067 }
1068
1069 if (signing_id != NULL) {
1070 uint16_t result = NULL_TRIE_IDX;
1071 lck_rw_lock_shared(&fd_cb->group->lck);
1072 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1073 result = 1;
1074 } else {
1075 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1076 }
1077 lck_rw_done(&fd_cb->group->lck);
1078 if (result != NULL_TRIE_IDX) {
1079 error = 0;
1080 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
1081
1082 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1083 if (error == 0) {
1084 if (src_proc != PROC_NULL) {
1085 unsigned char cdhash[SHA1_RESULTLEN];
1086 error = proc_getcdhash(src_proc, cdhash);
1087 if (error == 0) {
1088 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1089 if (error) {
1090 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1091 }
1092 } else {
1093 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
1094 }
1095 }
1096 } else {
1097 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1098 }
1099 } else {
1100 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
1101 }
1102 } else {
1103 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1104 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1105 error = 0;
1106 }
1107 }
1108
1109 if (error == 0 && src_proc != PROC_NULL) {
1110 task_t task = proc_task(src_proc);
1111 if (task != TASK_NULL) {
1112 audit_token_t audit_token;
1113 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1114 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1115 if (rc == KERN_SUCCESS) {
1116 error = flow_divert_packet_append_tlv(connect_packet,
1117 FLOW_DIVERT_TLV_APP_AUDIT_TOKEN,
1118 sizeof(audit_token_t),
1119 &audit_token);
1120 if (error) {
1121 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", error);
1122 error = 0; /* do not treat this as fatal error, proceed */
1123 }
1124 } else {
1125 FDLOG(LOG_ERR, fd_cb, "failed to retrieve app audit token: %d", rc);
1126 }
1127 }
1128 }
1129
1130 if (src_proc != PROC_NULL) {
1131 proc_unlock(src_proc);
1132 if (release_proc) {
1133 proc_rele(src_proc);
1134 }
1135 }
1136 socket_lock(so, 0);
1137
1138 if (free_signing_id) {
1139 FREE(signing_id, M_TEMP);
1140 }
1141
1142 if (error) {
1143 goto done;
1144 }
1145
1146 error = flow_divert_packet_append_tlv(connect_packet,
1147 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1148 sizeof(fd_cb->so->so_traffic_class),
1149 &fd_cb->so->so_traffic_class);
1150 if (error) {
1151 goto done;
1152 }
1153
1154 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1155 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1156 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1157 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1158 } else {
1159 error = EINVAL;
1160 goto done;
1161 }
1162 error = flow_divert_packet_append_tlv(connect_packet,
1163 FLOW_DIVERT_TLV_FLOW_TYPE,
1164 sizeof(flow_type),
1165 &flow_type);
1166
1167 if (error) {
1168 goto done;
1169 }
1170
1171 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1172 error = flow_divert_packet_append_tlv(connect_packet,
1173 FLOW_DIVERT_TLV_PID,
1174 sizeof(fd_cb->so->e_pid),
1175 &fd_cb->so->e_pid);
1176 if (error) {
1177 goto done;
1178 }
1179
1180 error = flow_divert_packet_append_tlv(connect_packet,
1181 FLOW_DIVERT_TLV_UUID,
1182 sizeof(fd_cb->so->e_uuid),
1183 &fd_cb->so->e_uuid);
1184 if (error) {
1185 goto done;
1186 }
1187 } else {
1188 error = flow_divert_packet_append_tlv(connect_packet,
1189 FLOW_DIVERT_TLV_PID,
1190 sizeof(fd_cb->so->e_pid),
1191 &fd_cb->so->last_pid);
1192 if (error) {
1193 goto done;
1194 }
1195
1196 error = flow_divert_packet_append_tlv(connect_packet,
1197 FLOW_DIVERT_TLV_UUID,
1198 sizeof(fd_cb->so->e_uuid),
1199 &fd_cb->so->last_uuid);
1200 if (error) {
1201 goto done;
1202 }
1203 }
1204
1205 if (fd_cb->connect_token != NULL) {
1206 unsigned int token_len = m_length(fd_cb->connect_token);
1207 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1208 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1209 fd_cb->connect_token = NULL;
1210 } else {
1211 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
1212
1213 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1214 if (error) {
1215 goto done;
1216 }
1217
1218 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1219 if (error) {
1220 goto done;
1221 }
1222 }
1223
1224 if (fd_cb->local_address != NULL) {
1225 error = EALREADY;
1226 goto done;
1227 } else {
1228 struct inpcb *inp = sotoinpcb(so);
1229 if (flow_divert_has_pcb_local_address(inp)) {
1230 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1231 if (error) {
1232 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1233 goto done;
1234 }
1235 }
1236 }
1237
1238 if (fd_cb->local_address != NULL) {
1239 /* socket is bound. */
1240 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1241 fd_cb->local_address->sa_len, fd_cb->local_address);
1242 if (error) {
1243 goto done;
1244 }
1245 }
1246
1247 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1248 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1249 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1250 if (error) {
1251 goto done;
1252 }
1253 }
1254
1255 done:
1256 if (!error) {
1257 *out_connect_packet = connect_packet;
1258 } else if (connect_packet != NULL) {
1259 mbuf_freem(connect_packet);
1260 }
1261
1262 return error;
1263 }
1264
1265 static int
1266 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1267 {
1268 int error = 0;
1269 mbuf_t packet = NULL;
1270 int rbuff_space = 0;
1271
1272 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1273 if (error) {
1274 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1275 goto done;
1276 }
1277
1278 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1279 if (rbuff_space < 0) {
1280 rbuff_space = 0;
1281 }
1282 rbuff_space = htonl(rbuff_space);
1283 error = flow_divert_packet_append_tlv(packet,
1284 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1285 sizeof(rbuff_space),
1286 &rbuff_space);
1287 if (error) {
1288 goto done;
1289 }
1290
1291 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1292 if (error) {
1293 goto done;
1294 }
1295
1296 done:
1297 if (error && packet != NULL) {
1298 mbuf_freem(packet);
1299 }
1300
1301 return error;
1302 }
1303
1304 static int
1305 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1306 {
1307 int error = 0;
1308 mbuf_t packet = NULL;
1309 uint32_t zero = 0;
1310
1311 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1312 if (error) {
1313 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1314 goto done;
1315 }
1316
1317 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1318 if (error) {
1319 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1320 goto done;
1321 }
1322
1323 how = htonl(how);
1324 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1325 if (error) {
1326 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1327 goto done;
1328 }
1329
1330 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1331 if (error) {
1332 goto done;
1333 }
1334
1335 done:
1336 if (error && packet != NULL) {
1337 mbuf_free(packet);
1338 }
1339
1340 return error;
1341 }
1342
1343 static int
1344 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1345 {
1346 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1347 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1348 return SHUT_RDWR;
1349 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1350 return SHUT_RD;
1351 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1352 return SHUT_WR;
1353 }
1354
1355 return -1;
1356 }
1357
1358 /*
1359 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1360 * writes. Returns FALSE otherwise.
1361 */
1362 static void
1363 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1364 {
1365 int how = -1;
1366
1367 /* Do not send any close messages if there is still data in the send buffer */
1368 if (fd_cb->so->so_snd.sb_cc == 0) {
1369 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1370 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1371 how = SHUT_RD;
1372 }
1373 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1374 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1375 if (how == SHUT_RD) {
1376 how = SHUT_RDWR;
1377 } else {
1378 how = SHUT_WR;
1379 }
1380 }
1381 }
1382
1383 if (how != -1) {
1384 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1385 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1386 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1387 if (how != SHUT_RD) {
1388 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1389 }
1390 if (how != SHUT_WR) {
1391 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1392 }
1393 }
1394 }
1395
1396 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1397 flow_divert_disconnect_socket(fd_cb->so);
1398 }
1399 }
1400
1401 static errno_t
1402 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1403 {
1404 mbuf_t packet;
1405 mbuf_t last;
1406 int error = 0;
1407
1408 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1409 if (error) {
1410 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1411 return error;
1412 }
1413
1414 if (toaddr != NULL) {
1415 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1416 if (error) {
1417 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1418 return error;
1419 }
1420 }
1421
1422 if (data_len > 0 && data != NULL) {
1423 last = m_last(packet);
1424 mbuf_setnext(last, data);
1425 mbuf_pkthdr_adjustlen(packet, data_len);
1426 }
1427 error = flow_divert_send_packet(fd_cb, packet, force);
1428
1429 if (error) {
1430 mbuf_setnext(last, NULL);
1431 mbuf_freem(packet);
1432 } else {
1433 fd_cb->bytes_sent += data_len;
1434 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1435 }
1436
1437 return error;
1438 }
1439
1440 static void
1441 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1442 {
1443 size_t to_send;
1444 size_t sent = 0;
1445 int error = 0;
1446 mbuf_t buffer;
1447
1448 to_send = fd_cb->so->so_snd.sb_cc;
1449 buffer = fd_cb->so->so_snd.sb_mb;
1450
1451 if (buffer == NULL && to_send > 0) {
1452 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1453 return;
1454 }
1455
1456 /* Ignore the send window if force is enabled */
1457 if (!force && (to_send > fd_cb->send_window)) {
1458 to_send = fd_cb->send_window;
1459 }
1460
1461 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1462 while (sent < to_send) {
1463 mbuf_t data;
1464 size_t data_len;
1465
1466 data_len = to_send - sent;
1467 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1468 data_len = FLOW_DIVERT_CHUNK_SIZE;
1469 }
1470
1471 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1472 if (error) {
1473 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1474 break;
1475 }
1476
1477 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1478 if (error) {
1479 mbuf_freem(data);
1480 break;
1481 }
1482
1483 sent += data_len;
1484 }
1485 sbdrop(&fd_cb->so->so_snd, sent);
1486 sowwakeup(fd_cb->so);
1487 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1488 mbuf_t data;
1489 mbuf_t m;
1490 size_t data_len;
1491
1492 while (buffer) {
1493 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1494
1495 m = buffer;
1496 if (toaddr != NULL) {
1497 /* look for data in the chain */
1498 do {
1499 m = m->m_next;
1500 if (m != NULL && m->m_type == MT_DATA) {
1501 break;
1502 }
1503 } while (m);
1504 if (m == NULL) {
1505 /* unexpected */
1506 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1507 goto move_on;
1508 }
1509 }
1510 data_len = mbuf_pkthdr_len(m);
1511 if (data_len > 0) {
1512 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1513 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1514 if (error) {
1515 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1516 break;
1517 }
1518 } else {
1519 data = NULL;
1520 }
1521 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1522 if (error) {
1523 mbuf_freem(data);
1524 break;
1525 }
1526 sent += data_len;
1527 move_on:
1528 buffer = buffer->m_nextpkt;
1529 (void) sbdroprecord(&(fd_cb->so->so_snd));
1530 }
1531 }
1532
1533 if (sent > 0) {
1534 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1535 if (fd_cb->send_window >= sent) {
1536 fd_cb->send_window -= sent;
1537 } else {
1538 fd_cb->send_window = 0;
1539 }
1540 }
1541 }
1542
1543 static int
1544 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1545 {
1546 size_t to_send = mbuf_pkthdr_len(data);
1547 int error = 0;
1548
1549 if (to_send > fd_cb->send_window) {
1550 to_send = fd_cb->send_window;
1551 }
1552
1553 if (fd_cb->so->so_snd.sb_cc > 0) {
1554 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1555 }
1556
1557 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1558 size_t sent = 0;
1559 mbuf_t remaining_data = data;
1560 mbuf_t pkt_data = NULL;
1561 while (sent < to_send && remaining_data != NULL) {
1562 size_t pkt_data_len;
1563
1564 pkt_data = remaining_data;
1565
1566 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1567 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1568 } else {
1569 pkt_data_len = to_send - sent;
1570 }
1571
1572 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1573 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1574 if (error) {
1575 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1576 pkt_data = NULL;
1577 break;
1578 }
1579 } else {
1580 remaining_data = NULL;
1581 }
1582
1583 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1584
1585 if (error) {
1586 break;
1587 }
1588
1589 pkt_data = NULL;
1590 sent += pkt_data_len;
1591 }
1592
1593 fd_cb->send_window -= sent;
1594
1595 error = 0;
1596
1597 if (pkt_data != NULL) {
1598 if (sbspace(&fd_cb->so->so_snd) > 0) {
1599 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1600 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1601 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1602 }
1603 } else {
1604 error = ENOBUFS;
1605 }
1606 }
1607
1608 if (remaining_data != NULL) {
1609 if (sbspace(&fd_cb->so->so_snd) > 0) {
1610 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1611 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1612 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1613 }
1614 } else {
1615 error = ENOBUFS;
1616 }
1617 }
1618 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1619 if (to_send || mbuf_pkthdr_len(data) == 0) {
1620 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1621 if (error) {
1622 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1623 } else {
1624 fd_cb->send_window -= to_send;
1625 }
1626 } else {
1627 /* buffer it */
1628 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1629 if (toaddr != NULL) {
1630 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1631 FDLOG(LOG_ERR, fd_cb,
1632 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1633 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1634 }
1635 } else {
1636 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1637 FDLOG(LOG_ERR, fd_cb,
1638 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1639 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1640 }
1641 }
1642 } else {
1643 error = ENOBUFS;
1644 }
1645 }
1646 }
1647
1648 return error;
1649 }
1650
1651 static int
1652 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1653 {
1654 int error = 0;
1655 mbuf_t packet = NULL;
1656 uint32_t net_read_count = htonl(read_count);
1657
1658 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1659 if (error) {
1660 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1661 goto done;
1662 }
1663
1664 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1665 if (error) {
1666 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1667 goto done;
1668 }
1669
1670 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1671 if (error) {
1672 goto done;
1673 }
1674
1675 done:
1676 if (error && packet != NULL) {
1677 mbuf_free(packet);
1678 }
1679
1680 return error;
1681 }
1682
1683 static int
1684 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1685 {
1686 int error = 0;
1687 mbuf_t packet = NULL;
1688
1689 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1690 if (error) {
1691 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1692 goto done;
1693 }
1694
1695 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1696 if (error) {
1697 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1698 goto done;
1699 }
1700
1701 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1702 if (error) {
1703 goto done;
1704 }
1705
1706 done:
1707 if (error && packet != NULL) {
1708 mbuf_free(packet);
1709 }
1710
1711 return error;
1712 }
1713
1714 static void
1715 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1716 {
1717 uint32_t connect_error;
1718 uint32_t ctl_unit = 0;
1719 int error = 0;
1720 struct flow_divert_group *grp = NULL;
1721 struct sockaddr_storage local_address;
1722 int out_if_index = 0;
1723 struct sockaddr_storage remote_address;
1724 uint32_t send_window;
1725 uint32_t app_data_length = 0;
1726
1727 memset(&local_address, 0, sizeof(local_address));
1728 memset(&remote_address, 0, sizeof(remote_address));
1729
1730 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1731 if (error) {
1732 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1733 return;
1734 }
1735
1736 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1737
1738 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1739 if (error) {
1740 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1741 return;
1742 }
1743
1744 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1745 if (error) {
1746 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
1747 }
1748
1749 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1750 if (error) {
1751 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1752 }
1753
1754 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1755 if (error) {
1756 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1757 }
1758
1759 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1760 if (error) {
1761 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1762 }
1763
1764 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1765 if (error) {
1766 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
1767 }
1768
1769 error = 0;
1770 connect_error = ntohl(connect_error);
1771 ctl_unit = ntohl(ctl_unit);
1772
1773 lck_rw_lock_shared(&g_flow_divert_group_lck);
1774
1775 if (connect_error == 0 && ctl_unit > 0) {
1776 if (ctl_unit >= GROUP_COUNT_MAX) {
1777 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1778 error = EINVAL;
1779 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1780 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1781 error = EINVAL;
1782 } else {
1783 grp = g_flow_divert_groups[ctl_unit];
1784 if (grp == NULL) {
1785 error = ECONNRESET;
1786 }
1787 }
1788 }
1789
1790 FDLOCK(fd_cb);
1791 if (fd_cb->so != NULL) {
1792 struct inpcb *inp = NULL;
1793 struct ifnet *ifp = NULL;
1794 struct flow_divert_group *old_group;
1795
1796 socket_lock(fd_cb->so, 0);
1797
1798 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1799 goto done;
1800 }
1801
1802 inp = sotoinpcb(fd_cb->so);
1803
1804 if (connect_error || error) {
1805 goto set_socket_state;
1806 }
1807
1808 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1809 error = EINVAL;
1810 goto set_socket_state;
1811 }
1812 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1813 if (local_address.ss_len > sizeof(local_address)) {
1814 local_address.ss_len = sizeof(local_address);
1815 }
1816 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1817 }
1818 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&local_address)) {
1819 if (inp->inp_vflag & INP_IPV4 && local_address.ss_family == AF_INET) {
1820 struct sockaddr_in *local_in_address = (struct sockaddr_in *)&local_address;
1821 inp->inp_lport = local_in_address->sin_port;
1822 memcpy(&inp->inp_laddr, &local_in_address->sin_addr, sizeof(struct in_addr));
1823 } else if (inp->inp_vflag & INP_IPV6 && local_address.ss_family == AF_INET6) {
1824 struct sockaddr_in6 *local_in6_address = (struct sockaddr_in6 *)&local_address;
1825 inp->inp_lport = local_in6_address->sin6_port;
1826 memcpy(&inp->in6p_laddr, &local_in6_address->sin6_addr, sizeof(struct in6_addr));
1827 }
1828 }
1829
1830 if (remote_address.ss_family != 0) {
1831 if (fd_cb->remote_address != NULL) {
1832 FREE(fd_cb->remote_address, M_SONAME);
1833 fd_cb->remote_address = NULL;
1834 }
1835 if (remote_address.ss_len > sizeof(remote_address)) {
1836 remote_address.ss_len = sizeof(remote_address);
1837 }
1838 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1839 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1840 if (inp->inp_vflag & INP_IPV4 && remote_address.ss_family == AF_INET) {
1841 struct sockaddr_in *remote_in_address = (struct sockaddr_in *)&remote_address;
1842 inp->inp_fport = remote_in_address->sin_port;
1843 memcpy(&inp->inp_faddr, &remote_in_address->sin_addr, sizeof(struct in_addr));
1844 } else if (inp->inp_vflag & INP_IPV6 && remote_address.ss_family == AF_INET6) {
1845 struct sockaddr_in6 *remote_in6_address = (struct sockaddr_in6 *)&remote_address;
1846 inp->inp_fport = remote_in6_address->sin6_port;
1847 memcpy(&inp->in6p_faddr, &remote_in6_address->sin6_addr, sizeof(struct in6_addr));
1848 }
1849 }
1850 } else {
1851 error = EINVAL;
1852 goto set_socket_state;
1853 }
1854
1855 if (app_data_length > 0) {
1856 uint8_t *app_data = NULL;
1857 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1858 if (app_data != NULL) {
1859 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1860 if (error == 0) {
1861 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1862 if (fd_cb->app_data != NULL) {
1863 FREE(fd_cb->app_data, M_TEMP);
1864 }
1865 fd_cb->app_data = app_data;
1866 fd_cb->app_data_length = app_data_length;
1867 } else {
1868 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1869 FREE(app_data, M_TEMP);
1870 }
1871 } else {
1872 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1873 }
1874 }
1875
1876 ifnet_head_lock_shared();
1877 if (out_if_index > 0 && out_if_index <= if_index) {
1878 ifp = ifindex2ifnet[out_if_index];
1879 }
1880
1881 if (ifp != NULL) {
1882 inp->inp_last_outifp = ifp;
1883 } else {
1884 error = EINVAL;
1885 }
1886 ifnet_head_done();
1887
1888 if (error) {
1889 goto set_socket_state;
1890 }
1891
1892 if (fd_cb->group == NULL) {
1893 error = EINVAL;
1894 goto set_socket_state;
1895 }
1896
1897 if (grp != NULL) {
1898 old_group = fd_cb->group;
1899
1900 lck_rw_lock_exclusive(&old_group->lck);
1901 lck_rw_lock_exclusive(&grp->lck);
1902
1903 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1904 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1905 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1906 }
1907
1908 fd_cb->group = grp;
1909
1910 lck_rw_done(&grp->lck);
1911 lck_rw_done(&old_group->lck);
1912 }
1913
1914 fd_cb->send_window = ntohl(send_window);
1915
1916 set_socket_state:
1917 if (!connect_error && !error) {
1918 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1919 error = flow_divert_send_connect_result(fd_cb);
1920 }
1921
1922 if (connect_error || error) {
1923 if (!connect_error) {
1924 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1925 fd_cb->so->so_error = error;
1926 flow_divert_send_close_if_needed(fd_cb);
1927 } else {
1928 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1929 fd_cb->so->so_error = connect_error;
1930 }
1931 flow_divert_disconnect_socket(fd_cb->so);
1932 } else {
1933 #if NECP
1934 /* Update NECP client with connected five-tuple */
1935 if (!uuid_is_null(inp->necp_client_uuid)) {
1936 socket_unlock(fd_cb->so, 0);
1937 necp_client_assign_from_socket(fd_cb->so->last_pid, inp->necp_client_uuid, inp);
1938 socket_lock(fd_cb->so, 0);
1939 }
1940 #endif /* NECP */
1941
1942 flow_divert_send_buffered_data(fd_cb, FALSE);
1943 soisconnected(fd_cb->so);
1944 }
1945
1946 done:
1947 socket_unlock(fd_cb->so, 0);
1948 }
1949 FDUNLOCK(fd_cb);
1950
1951 lck_rw_done(&g_flow_divert_group_lck);
1952 }
1953
1954 static void
1955 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1956 {
1957 uint32_t close_error;
1958 int error = 0;
1959 int how;
1960
1961 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1962 if (error) {
1963 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1964 return;
1965 }
1966
1967 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1968 if (error) {
1969 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1970 return;
1971 }
1972
1973 how = ntohl(how);
1974
1975 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1976
1977 FDLOCK(fd_cb);
1978 if (fd_cb->so != NULL) {
1979 socket_lock(fd_cb->so, 0);
1980
1981 fd_cb->so->so_error = ntohl(close_error);
1982
1983 flow_divert_update_closed_state(fd_cb, how, TRUE);
1984
1985 how = flow_divert_tunnel_how_closed(fd_cb);
1986 if (how == SHUT_RDWR) {
1987 flow_divert_disconnect_socket(fd_cb->so);
1988 } else if (how == SHUT_RD) {
1989 socantrcvmore(fd_cb->so);
1990 } else if (how == SHUT_WR) {
1991 socantsendmore(fd_cb->so);
1992 }
1993
1994 socket_unlock(fd_cb->so, 0);
1995 }
1996 FDUNLOCK(fd_cb);
1997 }
1998
1999 static mbuf_t
2000 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
2001 {
2002 struct inpcb *inp = sotoinpcb(fd_cb->so);
2003 if ((inp->inp_vflag & INP_IPV4) && (inp->inp_flags & INP_RECVDSTADDR)) {
2004 struct in_addr ia = { };
2005
2006 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in)) {
2007 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
2008 bcopy(&sin->sin_addr, &ia, sizeof(struct in_addr));
2009 }
2010
2011 return sbcreatecontrol((caddr_t)&ia, sizeof(ia), IP_RECVDSTADDR, IPPROTO_IP);
2012 } else if ((inp->inp_vflag & INP_IPV6) && (inp->inp_flags & IN6P_PKTINFO)) {
2013 struct in6_pktinfo pi6;
2014 memset(&pi6, 0, sizeof(pi6));
2015
2016 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET6 && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in6)) {
2017 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
2018 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof(struct in6_addr));
2019 pi6.ipi6_ifindex = 0;
2020 }
2021
2022 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
2023 }
2024 return NULL;
2025 }
2026
2027 static void
2028 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2029 {
2030 FDLOCK(fd_cb);
2031 if (fd_cb->so != NULL) {
2032 int error = 0;
2033 mbuf_t data = NULL;
2034 size_t data_size;
2035 struct sockaddr_storage remote_address;
2036 boolean_t got_remote_sa = FALSE;
2037
2038 socket_lock(fd_cb->so, 0);
2039
2040 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2041 uint32_t val_size = 0;
2042
2043 /* check if we got remote address with data */
2044 memset(&remote_address, 0, sizeof(remote_address));
2045 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2046 if (error || val_size > sizeof(remote_address)) {
2047 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2048 error = 0;
2049 } else {
2050 /* validate the address */
2051 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2052 got_remote_sa = TRUE;
2053 }
2054 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2055 }
2056 }
2057
2058 data_size = (mbuf_pkthdr_len(packet) - offset);
2059
2060 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2061
2062 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2063 if (error || data == NULL) {
2064 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2065 } else {
2066 if (flow_divert_check_no_cellular(fd_cb) ||
2067 flow_divert_check_no_expensive(fd_cb) ||
2068 flow_divert_check_no_constrained(fd_cb)) {
2069 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2070 flow_divert_send_close(fd_cb, SHUT_RDWR);
2071 flow_divert_disconnect_socket(fd_cb->so);
2072 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
2073 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2074 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
2075 fd_cb->bytes_received += data_size;
2076 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2077 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2078 sorwakeup(fd_cb->so);
2079 data = NULL;
2080 } else {
2081 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
2082 }
2083 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2084 struct sockaddr *append_sa;
2085 mbuf_t mctl;
2086
2087 if (got_remote_sa == TRUE) {
2088 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2089 (struct sockaddr *)&remote_address, &append_sa);
2090 } else {
2091 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2092 fd_cb->remote_address, &append_sa);
2093 }
2094 if (error) {
2095 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2096 }
2097
2098 mctl = flow_divert_get_control_mbuf(fd_cb);
2099 int append_error = 0;
2100 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error)) {
2101 fd_cb->bytes_received += data_size;
2102 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2103 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2104 sorwakeup(fd_cb->so);
2105 data = NULL;
2106 } else if (append_error != EJUSTRETURN) {
2107 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
2108 }
2109 if (!error) {
2110 FREE(append_sa, M_TEMP);
2111 }
2112 }
2113 }
2114 }
2115 socket_unlock(fd_cb->so, 0);
2116 }
2117 FDUNLOCK(fd_cb);
2118 }
2119
2120 static void
2121 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2122 {
2123 uint32_t read_count;
2124 int error = 0;
2125
2126 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2127 if (error) {
2128 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2129 return;
2130 }
2131
2132 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2133
2134 FDLOCK(fd_cb);
2135 if (fd_cb->so != NULL) {
2136 socket_lock(fd_cb->so, 0);
2137 fd_cb->send_window += ntohl(read_count);
2138 flow_divert_send_buffered_data(fd_cb, FALSE);
2139 socket_unlock(fd_cb->so, 0);
2140 }
2141 FDUNLOCK(fd_cb);
2142 }
2143
2144 static void
2145 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2146 {
2147 int error = 0;
2148 uint32_t key_size = 0;
2149 int log_level;
2150 uint32_t flags = 0;
2151
2152 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2153 if (error) {
2154 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2155 return;
2156 }
2157
2158 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2159 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2160 return;
2161 }
2162
2163 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2164 if (!error) {
2165 nil_pcb.log_level = log_level;
2166 }
2167
2168 lck_rw_lock_exclusive(&group->lck);
2169
2170 if (group->token_key != NULL) {
2171 FREE(group->token_key, M_TEMP);
2172 group->token_key = NULL;
2173 }
2174
2175 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2176 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2177 if (error) {
2178 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2179 FREE(group->token_key, M_TEMP);
2180 group->token_key = NULL;
2181 lck_rw_done(&group->lck);
2182 return;
2183 }
2184
2185 group->token_key_size = key_size;
2186
2187 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2188 if (!error) {
2189 group->flags = flags;
2190 }
2191
2192 lck_rw_done(&group->lck);
2193 }
2194
2195 static void
2196 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2197 {
2198 int error = 0;
2199 struct sockaddr_storage local_address;
2200 int out_if_index = 0;
2201 struct sockaddr_storage remote_address;
2202 uint32_t app_data_length = 0;
2203
2204 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2205
2206 memset(&local_address, 0, sizeof(local_address));
2207 memset(&remote_address, 0, sizeof(remote_address));
2208
2209 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2210 if (error) {
2211 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
2212 }
2213
2214 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2215 if (error) {
2216 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
2217 }
2218
2219 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2220 if (error) {
2221 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2222 }
2223
2224 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2225 if (error) {
2226 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2227 }
2228
2229 FDLOCK(fd_cb);
2230 if (fd_cb->so != NULL) {
2231 socket_lock(fd_cb->so, 0);
2232
2233 if (local_address.ss_family != 0) {
2234 if (local_address.ss_len > sizeof(local_address)) {
2235 local_address.ss_len = sizeof(local_address);
2236 }
2237 if (fd_cb->local_address != NULL) {
2238 FREE(fd_cb->local_address, M_SONAME);
2239 fd_cb->local_address = NULL;
2240 }
2241 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2242 }
2243
2244 if (remote_address.ss_family != 0) {
2245 if (remote_address.ss_len > sizeof(remote_address)) {
2246 remote_address.ss_len = sizeof(remote_address);
2247 }
2248 if (fd_cb->remote_address != NULL) {
2249 FREE(fd_cb->remote_address, M_SONAME);
2250 fd_cb->remote_address = NULL;
2251 }
2252 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2253 }
2254
2255 if (out_if_index > 0) {
2256 struct inpcb *inp = NULL;
2257 struct ifnet *ifp = NULL;
2258
2259 inp = sotoinpcb(fd_cb->so);
2260
2261 ifnet_head_lock_shared();
2262 if (out_if_index <= if_index) {
2263 ifp = ifindex2ifnet[out_if_index];
2264 }
2265
2266 if (ifp != NULL) {
2267 inp->inp_last_outifp = ifp;
2268 }
2269 ifnet_head_done();
2270 }
2271
2272 if (app_data_length > 0) {
2273 uint8_t *app_data = NULL;
2274 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2275 if (app_data != NULL) {
2276 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2277 if (error == 0) {
2278 if (fd_cb->app_data != NULL) {
2279 FREE(fd_cb->app_data, M_TEMP);
2280 }
2281 fd_cb->app_data = app_data;
2282 fd_cb->app_data_length = app_data_length;
2283 } else {
2284 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2285 FREE(app_data, M_TEMP);
2286 }
2287 } else {
2288 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2289 }
2290 }
2291
2292 socket_unlock(fd_cb->so, 0);
2293 }
2294 FDUNLOCK(fd_cb);
2295 }
2296
2297 static void
2298 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2299 {
2300 size_t bytes_mem_size;
2301 size_t child_maps_mem_size;
2302 int cursor;
2303 int error = 0;
2304 struct flow_divert_trie new_trie;
2305 int insert_error = 0;
2306 size_t nodes_mem_size;
2307 int prefix_count = 0;
2308 int signing_id_count = 0;
2309 size_t trie_memory_size = 0;
2310
2311 lck_rw_lock_exclusive(&group->lck);
2312
2313 /* Re-set the current trie */
2314 if (group->signing_id_trie.memory != NULL) {
2315 FREE(group->signing_id_trie.memory, M_TEMP);
2316 }
2317 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2318 group->signing_id_trie.root = NULL_TRIE_IDX;
2319
2320 memset(&new_trie, 0, sizeof(new_trie));
2321
2322 /* Get the number of shared prefixes in the new set of signing ID strings */
2323 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2324
2325 if (prefix_count < 0) {
2326 lck_rw_done(&group->lck);
2327 return;
2328 }
2329
2330 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2331 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2332 cursor >= 0;
2333 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2334 uint32_t sid_size = 0;
2335 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2336 new_trie.bytes_count += sid_size;
2337 signing_id_count++;
2338 }
2339
2340 if (signing_id_count == 0) {
2341 lck_rw_done(&group->lck);
2342 return;
2343 }
2344
2345 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2346 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2347
2348 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2349 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2350
2351 if (os_mul_overflow(sizeof(*new_trie.nodes), new_trie.nodes_count, &nodes_mem_size) ||
2352 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, new_trie.child_maps_count, &child_maps_mem_size) ||
2353 os_mul_overflow(sizeof(*new_trie.bytes), new_trie.bytes_count, &bytes_mem_size) ||
2354 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2355 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2356 lck_rw_done(&group->lck);
2357 return;
2358 }
2359
2360 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2361 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2362 lck_rw_done(&group->lck);
2363 return;
2364 }
2365
2366 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2367 if (new_trie.memory == NULL) {
2368 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2369 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2370 lck_rw_done(&group->lck);
2371 return;
2372 }
2373
2374 /* Initialize the free lists */
2375 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2376 new_trie.nodes_free_next = 0;
2377 memset(new_trie.nodes, 0, nodes_mem_size);
2378
2379 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2380 new_trie.child_maps_free_next = 0;
2381 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2382
2383 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2384 new_trie.bytes_free_next = 0;
2385
2386 /* The root is an empty node */
2387 new_trie.root = trie_node_alloc(&new_trie);
2388
2389 /* Add each signing ID to the trie */
2390 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2391 cursor >= 0;
2392 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2393 uint32_t sid_size = 0;
2394 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2395 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2396 uint16_t new_node_idx;
2397 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2398 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2399 if (new_node_idx == NULL_TRIE_IDX) {
2400 insert_error = EINVAL;
2401 break;
2402 }
2403 } else {
2404 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2405 insert_error = ENOBUFS;
2406 break;
2407 }
2408 }
2409
2410 if (!insert_error) {
2411 group->signing_id_trie = new_trie;
2412 } else {
2413 FREE(new_trie.memory, M_TEMP);
2414 }
2415
2416 lck_rw_done(&group->lck);
2417 }
2418
2419 static int
2420 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2421 {
2422 struct flow_divert_packet_header hdr;
2423 int error = 0;
2424 struct flow_divert_pcb *fd_cb;
2425
2426 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2427 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2428 error = EINVAL;
2429 goto done;
2430 }
2431
2432 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2433 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2434 error = EINVAL;
2435 goto done;
2436 }
2437
2438 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2439 if (error) {
2440 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2441 error = ENOBUFS;
2442 goto done;
2443 }
2444
2445 hdr.conn_id = ntohl(hdr.conn_id);
2446
2447 if (hdr.conn_id == 0) {
2448 switch (hdr.packet_type) {
2449 case FLOW_DIVERT_PKT_GROUP_INIT:
2450 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2451 break;
2452 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2453 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2454 break;
2455 default:
2456 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2457 break;
2458 }
2459 goto done;
2460 }
2461
2462 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2463 if (fd_cb == NULL) {
2464 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2465 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2466 }
2467 goto done;
2468 }
2469
2470 switch (hdr.packet_type) {
2471 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2472 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2473 break;
2474 case FLOW_DIVERT_PKT_CLOSE:
2475 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2476 break;
2477 case FLOW_DIVERT_PKT_DATA:
2478 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2479 break;
2480 case FLOW_DIVERT_PKT_READ_NOTIFY:
2481 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2482 break;
2483 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2484 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2485 break;
2486 default:
2487 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2488 break;
2489 }
2490
2491 FDRELEASE(fd_cb);
2492
2493 done:
2494 mbuf_freem(packet);
2495 return error;
2496 }
2497
2498 static void
2499 flow_divert_close_all(struct flow_divert_group *group)
2500 {
2501 struct flow_divert_pcb *fd_cb;
2502 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2503
2504 SLIST_INIT(&tmp_list);
2505
2506 lck_rw_lock_exclusive(&group->lck);
2507
2508 MBUFQ_DRAIN(&group->send_queue);
2509
2510 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2511 FDRETAIN(fd_cb);
2512 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2513 }
2514
2515 lck_rw_done(&group->lck);
2516
2517 while (!SLIST_EMPTY(&tmp_list)) {
2518 fd_cb = SLIST_FIRST(&tmp_list);
2519 FDLOCK(fd_cb);
2520 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2521 if (fd_cb->so != NULL) {
2522 socket_lock(fd_cb->so, 0);
2523 flow_divert_pcb_remove(fd_cb);
2524 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2525 fd_cb->so->so_error = ECONNABORTED;
2526 flow_divert_disconnect_socket(fd_cb->so);
2527 socket_unlock(fd_cb->so, 0);
2528 }
2529 FDUNLOCK(fd_cb);
2530 FDRELEASE(fd_cb);
2531 }
2532 }
2533
2534 void
2535 flow_divert_detach(struct socket *so)
2536 {
2537 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2538
2539 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2540
2541 so->so_flags &= ~SOF_FLOW_DIVERT;
2542 so->so_fd_pcb = NULL;
2543
2544 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2545
2546 if (fd_cb->group != NULL) {
2547 /* Last-ditch effort to send any buffered data */
2548 flow_divert_send_buffered_data(fd_cb, TRUE);
2549
2550 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2551 flow_divert_send_close_if_needed(fd_cb);
2552 /* Remove from the group */
2553 flow_divert_pcb_remove(fd_cb);
2554 }
2555
2556 socket_unlock(so, 0);
2557 FDLOCK(fd_cb);
2558 fd_cb->so = NULL;
2559 FDUNLOCK(fd_cb);
2560 socket_lock(so, 0);
2561
2562 FDRELEASE(fd_cb); /* Release the socket's reference */
2563 }
2564
2565 static int
2566 flow_divert_close(struct socket *so)
2567 {
2568 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2569
2570 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2571
2572 FDLOG0(LOG_INFO, fd_cb, "Closing");
2573
2574 if (SOCK_TYPE(so) == SOCK_STREAM) {
2575 soisdisconnecting(so);
2576 sbflush(&so->so_rcv);
2577 }
2578
2579 flow_divert_send_buffered_data(fd_cb, TRUE);
2580 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2581 flow_divert_send_close_if_needed(fd_cb);
2582
2583 /* Remove from the group */
2584 flow_divert_pcb_remove(fd_cb);
2585
2586 return 0;
2587 }
2588
2589 static int
2590 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2591 sae_connid_t cid __unused)
2592 {
2593 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2594 return EINVAL;
2595 }
2596
2597 return flow_divert_close(so);
2598 }
2599
2600 static int
2601 flow_divert_shutdown(struct socket *so)
2602 {
2603 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2604
2605 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2606
2607 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2608
2609 socantsendmore(so);
2610
2611 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2612 flow_divert_send_close_if_needed(fd_cb);
2613
2614 return 0;
2615 }
2616
2617 static int
2618 flow_divert_rcvd(struct socket *so, int flags __unused)
2619 {
2620 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2621 uint32_t latest_sb_size;
2622 uint32_t read_count;
2623
2624 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2625
2626 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2627
2628 if (fd_cb->sb_size < latest_sb_size) {
2629 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2630 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2631 }
2632
2633 read_count = fd_cb->sb_size - latest_sb_size;
2634
2635 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2636
2637 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2638 fd_cb->bytes_read_by_app += read_count;
2639 fd_cb->sb_size = latest_sb_size;
2640 }
2641
2642 return 0;
2643 }
2644
2645 static int
2646 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2647 {
2648 int error = 0;
2649 int port = 0;
2650
2651 if (!flow_divert_is_sockaddr_valid(toaddr)) {
2652 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
2653 error = EINVAL;
2654 goto done;
2655 }
2656
2657 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2658 if (error) {
2659 goto done;
2660 }
2661
2662 if (toaddr->sa_family == AF_INET) {
2663 port = ntohs((satosin(toaddr))->sin_port);
2664 }
2665 #if INET6
2666 else {
2667 port = ntohs((satosin6(toaddr))->sin6_port);
2668 }
2669 #endif
2670
2671 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2672 if (error) {
2673 goto done;
2674 }
2675
2676 done:
2677 return error;
2678 }
2679
2680 struct sockaddr *
2681 flow_divert_get_buffered_target_address(mbuf_t buffer)
2682 {
2683 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2684 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2685 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2686 return toaddr;
2687 }
2688 }
2689 return NULL;
2690 }
2691
2692 static boolean_t
2693 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2694 {
2695 switch (addr->sa_family) {
2696 case AF_INET:
2697 if (addr->sa_len < sizeof(struct sockaddr_in)) {
2698 return FALSE;
2699 }
2700 break;
2701 #if INET6
2702 case AF_INET6:
2703 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
2704 return FALSE;
2705 }
2706 break;
2707 #endif /* INET6 */
2708 default:
2709 return FALSE;
2710 }
2711 return TRUE;
2712 }
2713
2714 static errno_t
2715 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2716 {
2717 int error = 0;
2718 union sockaddr_in_4_6 sin46;
2719
2720 bzero(&sin46, sizeof(sin46));
2721 if (inp->inp_vflag & INP_IPV4) {
2722 struct sockaddr_in *sin = &sin46.sin;
2723
2724 sin->sin_family = AF_INET;
2725 sin->sin_len = sizeof(*sin);
2726 sin->sin_port = inp->inp_lport;
2727 sin->sin_addr = inp->inp_laddr;
2728 } else if (inp->inp_vflag & INP_IPV6) {
2729 struct sockaddr_in6 *sin6 = &sin46.sin6;
2730
2731 sin6->sin6_len = sizeof(*sin6);
2732 sin6->sin6_family = AF_INET6;
2733 sin6->sin6_port = inp->inp_lport;
2734 sin6->sin6_addr = inp->in6p_laddr;
2735 }
2736 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2737 if (*local_socket == NULL) {
2738 error = ENOBUFS;
2739 }
2740 return error;
2741 }
2742
2743 static boolean_t
2744 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2745 {
2746 return inp->inp_lport != 0
2747 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr));
2748 }
2749
2750 static errno_t
2751 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2752 struct sockaddr **dup)
2753 {
2754 int error = 0;
2755 struct sockaddr *result;
2756 struct sockaddr_storage ss;
2757
2758 if (addr != NULL) {
2759 result = addr;
2760 } else {
2761 memset(&ss, 0, sizeof(ss));
2762 ss.ss_family = family;
2763 if (ss.ss_family == AF_INET) {
2764 ss.ss_len = sizeof(struct sockaddr_in);
2765 }
2766 #if INET6
2767 else if (ss.ss_family == AF_INET6) {
2768 ss.ss_len = sizeof(struct sockaddr_in6);
2769 }
2770 #endif /* INET6 */
2771 else {
2772 error = EINVAL;
2773 }
2774 result = (struct sockaddr *)&ss;
2775 }
2776
2777 if (!error) {
2778 *dup = dup_sockaddr(result, 1);
2779 if (*dup == NULL) {
2780 error = ENOBUFS;
2781 }
2782 }
2783
2784 return error;
2785 }
2786
2787 static void
2788 flow_divert_disconnect_socket(struct socket *so)
2789 {
2790 soisdisconnected(so);
2791 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2792 struct inpcb *inp = NULL;
2793
2794 inp = sotoinpcb(so);
2795 if (inp != NULL) {
2796 #if INET6
2797 if (SOCK_CHECK_DOM(so, PF_INET6)) {
2798 in6_pcbdetach(inp);
2799 } else
2800 #endif /* INET6 */
2801 in_pcbdetach(inp);
2802 }
2803 }
2804 }
2805
2806 static errno_t
2807 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2808 {
2809 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2810
2811 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2812
2813 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2814 fd_cb->remote_address,
2815 sa);
2816 }
2817
2818 static errno_t
2819 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2820 {
2821 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2822
2823 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2824
2825 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2826 fd_cb->local_address,
2827 sa);
2828 }
2829
2830 static errno_t
2831 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2832 {
2833 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2834
2835 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2836
2837 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2838 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2839 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2840 }
2841 }
2842
2843 if (SOCK_DOM(so) == PF_INET) {
2844 return g_tcp_protosw->pr_ctloutput(so, sopt);
2845 }
2846 #if INET6
2847 else if (SOCK_DOM(so) == PF_INET6) {
2848 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2849 }
2850 #endif
2851 return 0;
2852 }
2853
2854 errno_t
2855 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2856 {
2857 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2858 int error = 0;
2859 struct inpcb *inp = sotoinpcb(so);
2860 struct sockaddr_in *sinp;
2861 mbuf_t connect_packet = NULL;
2862 int do_send = 1;
2863
2864 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2865
2866 if (fd_cb->group == NULL) {
2867 error = ENETUNREACH;
2868 goto done;
2869 }
2870
2871 if (inp == NULL) {
2872 error = EINVAL;
2873 goto done;
2874 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2875 if (so->so_error) {
2876 error = so->so_error;
2877 so->so_error = 0;
2878 } else {
2879 error = EINVAL;
2880 }
2881 goto done;
2882 }
2883
2884 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2885 error = EALREADY;
2886 goto done;
2887 }
2888
2889 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2890 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2891 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2892 if (fd_cb->remote_address != NULL) {
2893 soisconnected(fd_cb->so);
2894 goto done;
2895 }
2896 }
2897
2898 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2899
2900 if (fd_cb->connect_packet == NULL) {
2901 if (to == NULL) {
2902 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2903 error = EINVAL;
2904 goto done;
2905 }
2906
2907 sinp = (struct sockaddr_in *)(void *)to;
2908 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2909 error = EAFNOSUPPORT;
2910 goto done;
2911 }
2912
2913 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2914 if (error) {
2915 goto done;
2916 }
2917
2918 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2919 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2920 do_send = 0;
2921 }
2922 } else {
2923 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2924 connect_packet = fd_cb->connect_packet;
2925 fd_cb->connect_packet = NULL;
2926 }
2927
2928 if (do_send) {
2929 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2930 if (error) {
2931 goto done;
2932 }
2933
2934 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2935 } else {
2936 fd_cb->connect_packet = connect_packet;
2937 connect_packet = NULL;
2938 }
2939
2940 soisconnecting(so);
2941
2942 done:
2943 if (error && connect_packet != NULL) {
2944 mbuf_freem(connect_packet);
2945 }
2946 return error;
2947 }
2948
2949 static int
2950 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2951 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
2952 {
2953 struct inpcb *inp = sotoinpcb(so);
2954 int error;
2955
2956 if (inp == NULL) {
2957 return EINVAL;
2958 }
2959
2960 VERIFY(dst != NULL);
2961
2962 error = flow_divert_connect_out(so, dst, p);
2963
2964 if (error != 0) {
2965 return error;
2966 }
2967
2968 /* if there is data, send it */
2969 if (auio != NULL) {
2970 user_ssize_t datalen = 0;
2971
2972 socket_unlock(so, 0);
2973
2974 VERIFY(bytes_written != NULL);
2975
2976 datalen = uio_resid(auio);
2977 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2978 socket_lock(so, 0);
2979
2980 if (error == 0 || error == EWOULDBLOCK) {
2981 *bytes_written = datalen - uio_resid(auio);
2982 }
2983
2984 /*
2985 * sosend returns EWOULDBLOCK if it's a non-blocking
2986 * socket or a timeout occured (this allows to return
2987 * the amount of queued data through sendit()).
2988 *
2989 * However, connectx() returns EINPROGRESS in case of a
2990 * blocking socket. So we change the return value here.
2991 */
2992 if (error == EWOULDBLOCK) {
2993 error = EINPROGRESS;
2994 }
2995 }
2996
2997 if (error == 0 && pcid != NULL) {
2998 *pcid = 1; /* there is only 1 connection for a TCP */
2999 }
3000
3001 return error;
3002 }
3003
3004 static int
3005 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3006 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3007 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3008 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3009 {
3010 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3011 }
3012
3013 #if INET6
3014 static int
3015 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3016 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3017 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3018 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3019 {
3020 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3021 }
3022 #endif /* INET6 */
3023
3024 static int
3025 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
3026 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
3027 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
3028 user_addr_t aux_data __unused, uint32_t *aux_len)
3029 {
3030 int error = 0;
3031 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3032 struct ifnet *ifp = NULL;
3033 struct inpcb *inp = sotoinpcb(so);
3034
3035 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
3036
3037 if (so->so_fd_pcb == NULL || inp == NULL) {
3038 error = EINVAL;
3039 goto out;
3040 }
3041
3042 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
3043 error = EINVAL;
3044 goto out;
3045 }
3046
3047 ifp = inp->inp_last_outifp;
3048 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
3049 *soerror = so->so_error;
3050 *flags = 0;
3051
3052 if (so->so_state & SS_ISCONNECTED) {
3053 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
3054 }
3055
3056 if (fd_cb->local_address == NULL) {
3057 struct sockaddr_in sin;
3058 bzero(&sin, sizeof(sin));
3059 sin.sin_len = sizeof(sin);
3060 sin.sin_family = AF_INET;
3061 *src_len = sin.sin_len;
3062 if (src != USER_ADDR_NULL) {
3063 error = copyout(&sin, src, sin.sin_len);
3064 if (error != 0) {
3065 goto out;
3066 }
3067 }
3068 } else {
3069 *src_len = fd_cb->local_address->sa_len;
3070 if (src != USER_ADDR_NULL) {
3071 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
3072 if (error != 0) {
3073 goto out;
3074 }
3075 }
3076 }
3077
3078 if (fd_cb->remote_address == NULL) {
3079 struct sockaddr_in sin;
3080 bzero(&sin, sizeof(sin));
3081 sin.sin_len = sizeof(sin);
3082 sin.sin_family = AF_INET;
3083 *dst_len = sin.sin_len;
3084 if (dst != USER_ADDR_NULL) {
3085 error = copyout(&sin, dst, sin.sin_len);
3086 if (error != 0) {
3087 goto out;
3088 }
3089 }
3090 } else {
3091 *dst_len = fd_cb->remote_address->sa_len;
3092 if (dst != USER_ADDR_NULL) {
3093 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
3094 if (error != 0) {
3095 goto out;
3096 }
3097 }
3098 }
3099
3100 *aux_type = 0;
3101 *aux_len = 0;
3102
3103 out:
3104 return error;
3105 }
3106
3107 static int
3108 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3109 {
3110 int error = 0;
3111
3112 switch (cmd) {
3113 case SIOCGCONNINFO32: {
3114 struct so_cinforeq32 cifr;
3115 bcopy(data, &cifr, sizeof(cifr));
3116 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3117 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3118 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3119 &cifr.scir_aux_type, cifr.scir_aux_data,
3120 &cifr.scir_aux_len);
3121 if (error == 0) {
3122 bcopy(&cifr, data, sizeof(cifr));
3123 }
3124 break;
3125 }
3126
3127 case SIOCGCONNINFO64: {
3128 struct so_cinforeq64 cifr;
3129 bcopy(data, &cifr, sizeof(cifr));
3130 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3131 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3132 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3133 &cifr.scir_aux_type, cifr.scir_aux_data,
3134 &cifr.scir_aux_len);
3135 if (error == 0) {
3136 bcopy(&cifr, data, sizeof(cifr));
3137 }
3138 break;
3139 }
3140
3141 default:
3142 error = EOPNOTSUPP;
3143 }
3144
3145 return error;
3146 }
3147
3148 static int
3149 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3150 {
3151 int error = flow_divert_control(so, cmd, data, ifp, p);
3152
3153 if (error == EOPNOTSUPP) {
3154 error = in_control(so, cmd, data, ifp, p);
3155 }
3156
3157 return error;
3158 }
3159
3160 static int
3161 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3162 {
3163 int error = flow_divert_control(so, cmd, data, ifp, p);
3164
3165 if (error == EOPNOTSUPP) {
3166 error = in6_control(so, cmd, data, ifp, p);
3167 }
3168
3169 return error;
3170 }
3171
3172 static errno_t
3173 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3174 {
3175 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3176 int error = 0;
3177 struct inpcb *inp;
3178
3179 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3180
3181 inp = sotoinpcb(so);
3182 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3183 error = ECONNRESET;
3184 goto done;
3185 }
3186
3187 if (control && mbuf_len(control) > 0) {
3188 error = EINVAL;
3189 goto done;
3190 }
3191
3192 if (flags & MSG_OOB) {
3193 error = EINVAL;
3194 goto done; /* We don't support OOB data */
3195 }
3196
3197 error = flow_divert_check_no_cellular(fd_cb) ||
3198 flow_divert_check_no_expensive(fd_cb) ||
3199 flow_divert_check_no_constrained(fd_cb);
3200 if (error) {
3201 goto done;
3202 }
3203
3204 /* Implicit connect */
3205 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3206 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3207
3208 #if CONTENT_FILTER
3209 /*
3210 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3211 * retrieve the CFIL saved remote address from the mbuf and use it.
3212 */
3213 if (to == NULL && so->so_cfil_db) {
3214 struct sockaddr *cfil_faddr = NULL;
3215 struct m_tag *cfil_tag = cfil_udp_get_socket_state(data, NULL, NULL, &cfil_faddr);
3216 if (cfil_tag) {
3217 to = (struct sockaddr *)(void *)cfil_faddr;
3218 }
3219 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3220 }
3221 #endif
3222 error = flow_divert_connect_out(so, to, p);
3223 if (error) {
3224 goto done;
3225 }
3226
3227 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3228 /* Open up the send window so that the data will get sent right away */
3229 fd_cb->send_window = mbuf_pkthdr_len(data);
3230 }
3231 }
3232
3233 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3234
3235 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3236 error = flow_divert_send_app_data(fd_cb, data, to);
3237 if (error) {
3238 goto done;
3239 }
3240
3241 data = NULL;
3242
3243 if (flags & PRUS_EOF) {
3244 flow_divert_shutdown(so);
3245 }
3246
3247 done:
3248 if (data) {
3249 mbuf_freem(data);
3250 }
3251 if (control) {
3252 mbuf_free(control);
3253 }
3254 return error;
3255 }
3256
3257 static int
3258 flow_divert_preconnect(struct socket *so)
3259 {
3260 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3261 int error = 0;
3262
3263 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3264 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3265 mbuf_t connect_packet = fd_cb->connect_packet;
3266 fd_cb->connect_packet = NULL;
3267
3268 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3269 if (error) {
3270 mbuf_freem(connect_packet);
3271 }
3272
3273 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3274 }
3275
3276 soclearfastopen(so);
3277
3278 return error;
3279 }
3280
3281 static void
3282 flow_divert_set_protosw(struct socket *so)
3283 {
3284 so->so_flags |= SOF_FLOW_DIVERT;
3285 if (SOCK_DOM(so) == PF_INET) {
3286 so->so_proto = &g_flow_divert_in_protosw;
3287 }
3288 #if INET6
3289 else {
3290 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3291 }
3292 #endif /* INET6 */
3293 }
3294
3295 static void
3296 flow_divert_set_udp_protosw(struct socket *so)
3297 {
3298 so->so_flags |= SOF_FLOW_DIVERT;
3299 if (SOCK_DOM(so) == PF_INET) {
3300 so->so_proto = &g_flow_divert_in_udp_protosw;
3301 }
3302 #if INET6
3303 else {
3304 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3305 }
3306 #endif /* INET6 */
3307 }
3308
3309 static errno_t
3310 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3311 {
3312 int error = 0;
3313 struct flow_divert_pcb *fd_cb = NULL;
3314 struct ifnet *ifp = NULL;
3315 struct inpcb *inp = NULL;
3316 struct socket *old_so;
3317 mbuf_t recv_data = NULL;
3318
3319 socket_unlock(so, 0);
3320
3321 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3322
3323 /* Find the flow divert control block */
3324 lck_rw_lock_shared(&g_flow_divert_group_lck);
3325 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3326 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3327 if (group != NULL) {
3328 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3329 }
3330 }
3331 lck_rw_done(&g_flow_divert_group_lck);
3332
3333 if (fd_cb == NULL) {
3334 error = ENOENT;
3335 goto done;
3336 }
3337
3338 FDLOCK(fd_cb);
3339
3340 /* Dis-associate the flow divert control block from its current socket */
3341 old_so = fd_cb->so;
3342
3343 inp = sotoinpcb(old_so);
3344
3345 VERIFY(inp != NULL);
3346
3347 socket_lock(old_so, 0);
3348 flow_divert_disconnect_socket(old_so);
3349 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3350 old_so->so_fd_pcb = NULL;
3351 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3352 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3353 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3354 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3355 }
3356 fd_cb->so = NULL;
3357 /* Save the output interface */
3358 ifp = inp->inp_last_outifp;
3359 if (old_so->so_rcv.sb_cc > 0) {
3360 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3361 sbflush(&old_so->so_rcv);
3362 }
3363 socket_unlock(old_so, 0);
3364
3365 /* Associate the new socket with the flow divert control block */
3366 socket_lock(so, 0);
3367 so->so_fd_pcb = fd_cb;
3368 inp = sotoinpcb(so);
3369 inp->inp_last_outifp = ifp;
3370 if (recv_data != NULL) {
3371 if (sbappendstream(&so->so_rcv, recv_data)) {
3372 sorwakeup(so);
3373 }
3374 }
3375 flow_divert_set_protosw(so);
3376 socket_unlock(so, 0);
3377
3378 fd_cb->so = so;
3379 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3380
3381 FDUNLOCK(fd_cb);
3382
3383 done:
3384 socket_lock(so, 0);
3385
3386 if (fd_cb != NULL) {
3387 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3388 }
3389
3390 return error;
3391 }
3392
3393 errno_t
3394 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3395 {
3396 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3397 struct inpcb *inp;
3398 int error = 0;
3399
3400 inp = sotoinpcb(so);
3401 if (inp == NULL) {
3402 return EINVAL;
3403 }
3404
3405 if (fd_cb == NULL) {
3406 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3407 if (fd_ctl_unit > 0) {
3408 error = flow_divert_pcb_init(so, fd_ctl_unit);
3409 fd_cb = so->so_fd_pcb;
3410 if (error != 0 || fd_cb == NULL) {
3411 goto done;
3412 }
3413 } else {
3414 error = ENETDOWN;
3415 goto done;
3416 }
3417 }
3418 return flow_divert_data_out(so, flags, data, to, control, p);
3419
3420 done:
3421 if (data) {
3422 mbuf_freem(data);
3423 }
3424 if (control) {
3425 mbuf_free(control);
3426 }
3427
3428 return error;
3429 }
3430
3431 errno_t
3432 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3433 {
3434 errno_t error = 0;
3435 struct flow_divert_pcb *fd_cb;
3436
3437 if (so->so_flags & SOF_FLOW_DIVERT) {
3438 return EALREADY;
3439 }
3440
3441 fd_cb = flow_divert_pcb_create(so);
3442 if (fd_cb != NULL) {
3443 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3444 if (error) {
3445 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3446 FDRELEASE(fd_cb);
3447 } else {
3448 fd_cb->control_group_unit = ctl_unit;
3449 so->so_fd_pcb = fd_cb;
3450
3451 if (SOCK_TYPE(so) == SOCK_STREAM) {
3452 flow_divert_set_protosw(so);
3453 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3454 flow_divert_set_udp_protosw(so);
3455 }
3456
3457 FDLOG0(LOG_INFO, fd_cb, "Created");
3458 }
3459 } else {
3460 error = ENOMEM;
3461 }
3462
3463 return error;
3464 }
3465
3466 errno_t
3467 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3468 {
3469 uint32_t ctl_unit = 0;
3470 uint32_t key_unit = 0;
3471 uint32_t flow_id = 0;
3472 int error = 0;
3473 int hmac_error = 0;
3474 mbuf_t token = NULL;
3475
3476 if (so->so_flags & SOF_FLOW_DIVERT) {
3477 error = EALREADY;
3478 goto done;
3479 }
3480
3481 if (g_init_result) {
3482 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3483 error = ENOPROTOOPT;
3484 goto done;
3485 }
3486
3487 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3488 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3489 (SOCK_DOM(so) != PF_INET
3490 #if INET6
3491 && SOCK_DOM(so) != PF_INET6
3492 #endif
3493 )) {
3494 error = EINVAL;
3495 goto done;
3496 } else {
3497 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3498 struct tcpcb *tp = sototcpcb(so);
3499 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3500 error = EINVAL;
3501 goto done;
3502 }
3503 }
3504 }
3505
3506 error = soopt_getm(sopt, &token);
3507 if (error) {
3508 token = NULL;
3509 goto done;
3510 }
3511
3512 error = soopt_mcopyin(sopt, token);
3513 if (error) {
3514 token = NULL;
3515 goto done;
3516 }
3517
3518 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3519 if (!error) {
3520 key_unit = ntohl(key_unit);
3521 if (key_unit >= GROUP_COUNT_MAX) {
3522 key_unit = 0;
3523 }
3524 } else if (error != ENOENT) {
3525 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3526 goto done;
3527 } else {
3528 key_unit = 0;
3529 }
3530
3531 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3532 if (error) {
3533 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3534 goto done;
3535 }
3536
3537 /* A valid kernel control unit is required */
3538 ctl_unit = ntohl(ctl_unit);
3539 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3540 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3541 error = EINVAL;
3542 goto done;
3543 }
3544
3545 socket_unlock(so, 0);
3546 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3547 socket_lock(so, 0);
3548
3549 if (hmac_error && hmac_error != ENOENT) {
3550 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3551 error = hmac_error;
3552 goto done;
3553 }
3554
3555 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3556 if (error && error != ENOENT) {
3557 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3558 goto done;
3559 }
3560
3561 if (flow_id == 0) {
3562 error = flow_divert_pcb_init(so, ctl_unit);
3563 if (error == 0) {
3564 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3565 int log_level = LOG_NOTICE;
3566
3567 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3568 sizeof(log_level), &log_level, NULL);
3569 if (error == 0) {
3570 fd_cb->log_level = log_level;
3571 }
3572 error = 0;
3573
3574 fd_cb->connect_token = token;
3575 token = NULL;
3576 }
3577 } else {
3578 error = flow_divert_attach(so, flow_id, ctl_unit);
3579 }
3580
3581 if (hmac_error == 0) {
3582 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3583 if (fd_cb != NULL) {
3584 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3585 }
3586 }
3587
3588 done:
3589 if (token != NULL) {
3590 mbuf_freem(token);
3591 }
3592
3593 return error;
3594 }
3595
3596 errno_t
3597 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3598 {
3599 uint32_t ctl_unit;
3600 int error = 0;
3601 uint8_t hmac[SHA_DIGEST_LENGTH];
3602 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3603 mbuf_t token = NULL;
3604 struct flow_divert_group *control_group = NULL;
3605
3606 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3607 error = EINVAL;
3608 goto done;
3609 }
3610
3611 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3612
3613 if (fd_cb->group == NULL) {
3614 error = EINVAL;
3615 goto done;
3616 }
3617
3618 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3619 if (error) {
3620 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3621 goto done;
3622 }
3623
3624 ctl_unit = htonl(fd_cb->group->ctl_unit);
3625
3626 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3627 if (error) {
3628 goto done;
3629 }
3630
3631 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3632 if (error) {
3633 goto done;
3634 }
3635
3636 if (fd_cb->app_data != NULL) {
3637 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3638 if (error) {
3639 goto done;
3640 }
3641 }
3642
3643 socket_unlock(so, 0);
3644 lck_rw_lock_shared(&g_flow_divert_group_lck);
3645
3646 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3647 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
3648 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3649 }
3650
3651 if (control_group != NULL) {
3652 lck_rw_lock_shared(&control_group->lck);
3653 ctl_unit = htonl(control_group->ctl_unit);
3654 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3655 if (!error) {
3656 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3657 }
3658 lck_rw_done(&control_group->lck);
3659 } else {
3660 error = ENOPROTOOPT;
3661 }
3662
3663 lck_rw_done(&g_flow_divert_group_lck);
3664 socket_lock(so, 0);
3665
3666 if (error) {
3667 goto done;
3668 }
3669
3670 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3671 if (error) {
3672 goto done;
3673 }
3674
3675 if (sopt->sopt_val == USER_ADDR_NULL) {
3676 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3677 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3678 goto done;
3679 }
3680
3681 error = soopt_mcopyout(sopt, token);
3682 if (error) {
3683 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3684 goto done;
3685 }
3686
3687 done:
3688 if (token != NULL) {
3689 mbuf_freem(token);
3690 }
3691
3692 return error;
3693 }
3694
3695 static errno_t
3696 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3697 {
3698 struct flow_divert_group *new_group = NULL;
3699 int error = 0;
3700
3701 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3702 error = EINVAL;
3703 goto done;
3704 }
3705
3706 *unitinfo = NULL;
3707
3708 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3709 if (new_group == NULL) {
3710 error = ENOBUFS;
3711 goto done;
3712 }
3713
3714 memset(new_group, 0, sizeof(*new_group));
3715
3716 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3717 RB_INIT(&new_group->pcb_tree);
3718 new_group->ctl_unit = sac->sc_unit;
3719 MBUFQ_INIT(&new_group->send_queue);
3720 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3721
3722 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3723
3724 if (g_flow_divert_groups == NULL) {
3725 MALLOC(g_flow_divert_groups,
3726 struct flow_divert_group **,
3727 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3728 M_TEMP,
3729 M_WAITOK | M_ZERO);
3730 }
3731
3732 if (g_flow_divert_groups == NULL) {
3733 error = ENOBUFS;
3734 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3735 error = EALREADY;
3736 } else {
3737 g_flow_divert_groups[sac->sc_unit] = new_group;
3738 g_active_group_count++;
3739 }
3740
3741 lck_rw_done(&g_flow_divert_group_lck);
3742
3743 *unitinfo = new_group;
3744
3745 done:
3746 if (error != 0 && new_group != NULL) {
3747 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3748 }
3749 return error;
3750 }
3751
3752 static errno_t
3753 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3754 {
3755 struct flow_divert_group *group = NULL;
3756 errno_t error = 0;
3757
3758 if (unit >= GROUP_COUNT_MAX) {
3759 return EINVAL;
3760 }
3761
3762 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3763
3764 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3765
3766 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3767 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3768 g_flow_divert_groups, g_active_group_count);
3769 }
3770
3771 group = g_flow_divert_groups[unit];
3772
3773 if (group != (struct flow_divert_group *)unitinfo) {
3774 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3775 }
3776
3777 g_flow_divert_groups[unit] = NULL;
3778 g_active_group_count--;
3779
3780 if (g_active_group_count == 0) {
3781 FREE(g_flow_divert_groups, M_TEMP);
3782 g_flow_divert_groups = NULL;
3783 }
3784
3785 lck_rw_done(&g_flow_divert_group_lck);
3786
3787 if (group != NULL) {
3788 flow_divert_close_all(group);
3789
3790 lck_rw_lock_exclusive(&group->lck);
3791
3792 if (group->token_key != NULL) {
3793 memset(group->token_key, 0, group->token_key_size);
3794 FREE(group->token_key, M_TEMP);
3795 group->token_key = NULL;
3796 group->token_key_size = 0;
3797 }
3798
3799 /* Re-set the current trie */
3800 if (group->signing_id_trie.memory != NULL) {
3801 FREE(group->signing_id_trie.memory, M_TEMP);
3802 }
3803 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3804 group->signing_id_trie.root = NULL_TRIE_IDX;
3805
3806 lck_rw_done(&group->lck);
3807
3808 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3809 } else {
3810 error = EINVAL;
3811 }
3812
3813 return error;
3814 }
3815
3816 static errno_t
3817 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3818 {
3819 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3820 }
3821
3822 static void
3823 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3824 {
3825 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3826
3827 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3828 struct flow_divert_pcb *fd_cb;
3829 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3830
3831 lck_rw_lock_shared(&g_flow_divert_group_lck);
3832 lck_rw_lock_exclusive(&group->lck);
3833
3834 while (!MBUFQ_EMPTY(&group->send_queue)) {
3835 mbuf_t next_packet;
3836 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3837 next_packet = MBUFQ_FIRST(&group->send_queue);
3838 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3839 if (error) {
3840 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3841 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3842 lck_rw_done(&group->lck);
3843 lck_rw_done(&g_flow_divert_group_lck);
3844 return;
3845 }
3846 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3847 }
3848
3849 SLIST_INIT(&tmp_list);
3850
3851 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3852 FDRETAIN(fd_cb);
3853 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3854 }
3855
3856 lck_rw_done(&group->lck);
3857
3858 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3859 FDLOCK(fd_cb);
3860 if (fd_cb->so != NULL) {
3861 socket_lock(fd_cb->so, 0);
3862 if (fd_cb->group != NULL) {
3863 flow_divert_send_buffered_data(fd_cb, FALSE);
3864 }
3865 socket_unlock(fd_cb->so, 0);
3866 }
3867 FDUNLOCK(fd_cb);
3868 FDRELEASE(fd_cb);
3869 }
3870
3871 lck_rw_done(&g_flow_divert_group_lck);
3872 }
3873 }
3874
3875 static int
3876 flow_divert_kctl_init(void)
3877 {
3878 struct kern_ctl_reg ctl_reg;
3879 int result;
3880
3881 memset(&ctl_reg, 0, sizeof(ctl_reg));
3882
3883 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3884 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
3885 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3886 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3887 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3888
3889 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3890 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3891 ctl_reg.ctl_send = flow_divert_kctl_send;
3892 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3893
3894 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3895
3896 if (result) {
3897 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3898 return result;
3899 }
3900
3901 return 0;
3902 }
3903
3904 void
3905 flow_divert_init(void)
3906 {
3907 memset(&nil_pcb, 0, sizeof(nil_pcb));
3908 nil_pcb.log_level = LOG_NOTICE;
3909
3910 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3911
3912 VERIFY(g_tcp_protosw != NULL);
3913
3914 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3915 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3916
3917 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3918 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3919 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3920 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3921 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3922 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3923 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3924 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3925 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3926 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3927 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
3928
3929 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3930 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3931
3932 /*
3933 * Socket filters shouldn't attach/detach to/from this protosw
3934 * since pr_protosw is to be used instead, which points to the
3935 * real protocol; if they do, it is a bug and we should panic.
3936 */
3937 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3938 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3939 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3940 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3941
3942 /* UDP */
3943 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3944 VERIFY(g_udp_protosw != NULL);
3945
3946 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3947 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3948
3949 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3950 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3951 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3952 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3953 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3954 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3955 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3956 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3957 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3958 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3959 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3960 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3961 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3962
3963 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3964 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3965
3966 /*
3967 * Socket filters shouldn't attach/detach to/from this protosw
3968 * since pr_protosw is to be used instead, which points to the
3969 * real protocol; if they do, it is a bug and we should panic.
3970 */
3971 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3972 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3973 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3974 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3975
3976 #if INET6
3977 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3978
3979 VERIFY(g_tcp6_protosw != NULL);
3980
3981 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3982 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3983
3984 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3985 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3986 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3987 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3988 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3989 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3990 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3991 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3992 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3993 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3994 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
3995
3996 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3997 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3998 /*
3999 * Socket filters shouldn't attach/detach to/from this protosw
4000 * since pr_protosw is to be used instead, which points to the
4001 * real protocol; if they do, it is a bug and we should panic.
4002 */
4003 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4004 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4005 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4006 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4007
4008 /* UDP6 */
4009 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4010
4011 VERIFY(g_udp6_protosw != NULL);
4012
4013 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4014 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4015
4016 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4017 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4018 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
4019 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4020 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4021 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
4022 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4023 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4024 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4025 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4026 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4027 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4028 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4029
4030 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4031 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4032 /*
4033 * Socket filters shouldn't attach/detach to/from this protosw
4034 * since pr_protosw is to be used instead, which points to the
4035 * real protocol; if they do, it is a bug and we should panic.
4036 */
4037 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4038 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4039 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4040 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4041 #endif /* INET6 */
4042
4043 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4044 if (flow_divert_grp_attr == NULL) {
4045 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4046 g_init_result = ENOMEM;
4047 goto done;
4048 }
4049
4050 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4051 if (flow_divert_mtx_grp == NULL) {
4052 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4053 g_init_result = ENOMEM;
4054 goto done;
4055 }
4056
4057 flow_divert_mtx_attr = lck_attr_alloc_init();
4058 if (flow_divert_mtx_attr == NULL) {
4059 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4060 g_init_result = ENOMEM;
4061 goto done;
4062 }
4063
4064 g_init_result = flow_divert_kctl_init();
4065 if (g_init_result) {
4066 goto done;
4067 }
4068
4069 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4070
4071 done:
4072 if (g_init_result != 0) {
4073 if (flow_divert_mtx_attr != NULL) {
4074 lck_attr_free(flow_divert_mtx_attr);
4075 flow_divert_mtx_attr = NULL;
4076 }
4077 if (flow_divert_mtx_grp != NULL) {
4078 lck_grp_free(flow_divert_mtx_grp);
4079 flow_divert_mtx_grp = NULL;
4080 }
4081 if (flow_divert_grp_attr != NULL) {
4082 lck_grp_attr_free(flow_divert_grp_attr);
4083 flow_divert_grp_attr = NULL;
4084 }
4085
4086 if (g_flow_divert_kctl_ref != NULL) {
4087 ctl_deregister(g_flow_divert_kctl_ref);
4088 g_flow_divert_kctl_ref = NULL;
4089 }
4090 }
4091 }
mirror of XNU