]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4570.20.62.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
5ba3f43e 2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
48#include <net/if_var.h>
49#include <net/route.h>
50#include <net/flowhash.h>
51#include <net/ntstat.h>
52#include <netinet/in.h>
53#include <netinet/in_var.h>
54#include <netinet/tcp.h>
55#include <netinet/tcp_var.h>
56#include <netinet/tcp_fsm.h>
57#include <netinet/flow_divert.h>
58#include <netinet/flow_divert_proto.h>
59#if INET6
3e170ce0 60#include <netinet6/in6_pcb.h>
39236c6e
A		 61#include <netinet6/ip6protosw.h>
62#endif /* INET6 */
63#include <dev/random/randomdev.h>
64#include <libkern/crypto/sha1.h>
65#include <libkern/crypto/crypto_internal.h>
39037602 66#include <os/log.h>
39236c6e
A		 67
68#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69#define FLOW_DIVERT_READ_CLOSED 0x00000002
70#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73#define FLOW_DIVERT_TRANSFERRED 0x00000020
490019cf 74#define FLOW_DIVERT_HAS_HMAC 0x00000040
39236c6e 75
39037602
A		 76#define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
39236c6e 78
39037602
A		 79#define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
39236c6e
A		 81
82#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83#define FDRELEASE(pcb) \
84 do { \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
87 } \
88 } while (0)
89
90#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
92
39037602 93#define FD_CTL_SENDBUFF_SIZE (128 * 1024)
39236c6e
A		 94#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
95
96#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
97
98#define GROUP_COUNT_MAX 32
99#define FLOW_DIVERT_MAX_NAME_SIZE 4096
100#define FLOW_DIVERT_MAX_KEY_SIZE 1024
ecc0ceb4 101#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e 102
39236c6e
A		 103struct flow_divert_trie_node
104{
105 uint16_t start;
106 uint16_t length;
107 uint16_t child_map;
39236c6e
A		 108};
109
110#define CHILD_MAP_SIZE 256
111#define NULL_TRIE_IDX 0xffff
112#define TRIE_NODE(t, i) ((t)->nodes[(i)])
113#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
114#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
115
116static struct flow_divert_pcb nil_pcb;
117
118decl_lck_rw_data(static, g_flow_divert_group_lck);
119static struct flow_divert_group **g_flow_divert_groups = NULL;
120static uint32_t g_active_group_count = 0;
39236c6e
A		 121
122static lck_grp_attr_t *flow_divert_grp_attr = NULL;
123static lck_attr_t *flow_divert_mtx_attr = NULL;
124static lck_grp_t *flow_divert_mtx_grp = NULL;
125static errno_t g_init_result = 0;
126
127static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
128
129static struct protosw g_flow_divert_in_protosw;
130static struct pr_usrreqs g_flow_divert_in_usrreqs;
3e170ce0
A		 131static struct protosw g_flow_divert_in_udp_protosw;
132static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
39236c6e
A		 133#if INET6
134static struct ip6protosw g_flow_divert_in6_protosw;
135static struct pr_usrreqs g_flow_divert_in6_usrreqs;
3e170ce0
A		 136static struct ip6protosw g_flow_divert_in6_udp_protosw;
137static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
39236c6e
A		 138#endif /* INET6 */
139
140static struct protosw *g_tcp_protosw = NULL;
141static struct ip6protosw *g_tcp6_protosw = NULL;
3e170ce0
A		 142static struct protosw *g_udp_protosw = NULL;
143static struct ip6protosw *g_udp6_protosw = NULL;
144
145static errno_t
146flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
147
148static errno_t
149flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
150
151static boolean_t
152flow_divert_is_sockaddr_valid(struct sockaddr *addr);
153
154static int
155flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
156
157struct sockaddr *
158flow_divert_get_buffered_target_address(mbuf_t buffer);
159
160static boolean_t
161flow_divert_has_pcb_local_address(const struct inpcb *inp);
162
163static void
164flow_divert_disconnect_socket(struct socket *so);
39236c6e 165
39037602
A		 166static inline uint8_t
167flow_divert_syslog_type_to_oslog_type(int syslog_type)
168{
169 switch (syslog_type) {
170 case LOG_ERR: return OS_LOG_TYPE_ERROR;
171 case LOG_INFO: return OS_LOG_TYPE_INFO;
172 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
173 default: return OS_LOG_TYPE_DEFAULT;
174 }
175}
176
39236c6e
A		 177static inline int
178flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
179{
180 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
181}
182
183RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
184RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
185
186static const char *
187flow_divert_packet_type2str(uint8_t packet_type)
188{
189 switch (packet_type) {
190 case FLOW_DIVERT_PKT_CONNECT:
191 return "connect";
192 case FLOW_DIVERT_PKT_CONNECT_RESULT:
193 return "connect result";
194 case FLOW_DIVERT_PKT_DATA:
195 return "data";
196 case FLOW_DIVERT_PKT_CLOSE:
197 return "close";
198 case FLOW_DIVERT_PKT_READ_NOTIFY:
199 return "read notification";
200 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
201 return "properties update";
39236c6e
A		 202 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
203 return "app map create";
204 default:
205 return "unknown";
206 }
207}
208
209static struct flow_divert_pcb *
210flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
211{
212 struct flow_divert_pcb key_item;
213 struct flow_divert_pcb *fd_cb = NULL;
214
215 key_item.hash = hash;
216
217 lck_rw_lock_shared(&group->lck);
218 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
219 FDRETAIN(fd_cb);
220 lck_rw_done(&group->lck);
221
222 return fd_cb;
223}
224
225static errno_t
226flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
227{
3e170ce0 228 errno_t error = 0;
39236c6e
A		 229 struct flow_divert_pcb *exist = NULL;
230 struct flow_divert_group *group;
231 static uint32_t g_nextkey = 1;
232 static uint32_t g_hash_seed = 0;
39236c6e
A		 233 int try_count = 0;
234
235 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
236 return EINVAL;
237 }
238
239 socket_unlock(fd_cb->so, 0);
240 lck_rw_lock_shared(&g_flow_divert_group_lck);
241
242 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
243 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
244 error = ENETUNREACH;
245 goto done;
246 }
247
248 group = g_flow_divert_groups[ctl_unit];
249 if (group == NULL) {
250 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
251 error = ENETUNREACH;
252 goto done;
253 }
254
255 socket_lock(fd_cb->so, 0);
256
257 do {
258 uint32_t key[2];
259 uint32_t idx;
260
261 key[0] = g_nextkey++;
262 key[1] = RandomULong();
263
264 if (g_hash_seed == 0) {
265 g_hash_seed = RandomULong();
266 }
267
268 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
269
270 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
271 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
272 if (curr_group != NULL && curr_group != group) {
273 lck_rw_lock_shared(&curr_group->lck);
274 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
275 lck_rw_done(&curr_group->lck);
276 if (exist != NULL) {
277 break;
278 }
279 }
280 }
281
282 if (exist == NULL) {
283 lck_rw_lock_exclusive(&group->lck);
284 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
285 lck_rw_done(&group->lck);
286 }
287 } while (exist != NULL && try_count++ < 3);
288
289 if (exist == NULL) {
290 fd_cb->group = group;
291 FDRETAIN(fd_cb); /* The group now has a reference */
292 } else {
293 fd_cb->hash = 0;
3e170ce0 294 error = EEXIST;
39236c6e
A		 295 }
296
297 socket_unlock(fd_cb->so, 0);
298
299done:
300 lck_rw_done(&g_flow_divert_group_lck);
301 socket_lock(fd_cb->so, 0);
302
3e170ce0 303 return error;
39236c6e
A		 304}
305
306static struct flow_divert_pcb *
307flow_divert_pcb_create(socket_t so)
308{
309 struct flow_divert_pcb *new_pcb = NULL;
310
311 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
312 if (new_pcb == NULL) {
313 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
314 return NULL;
315 }
316
317 memset(new_pcb, 0, sizeof(*new_pcb));
318
319 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
320 new_pcb->so = so;
321 new_pcb->log_level = nil_pcb.log_level;
322
323 FDRETAIN(new_pcb); /* Represents the socket's reference */
324
325 return new_pcb;
326}
327
328static void
329flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
330{
331 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
332 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
333
334 if (fd_cb->local_address != NULL) {
335 FREE(fd_cb->local_address, M_SONAME);
336 }
337 if (fd_cb->remote_address != NULL) {
338 FREE(fd_cb->remote_address, M_SONAME);
339 }
340 if (fd_cb->connect_token != NULL) {
341 mbuf_freem(fd_cb->connect_token);
342 }
490019cf
A		 343 if (fd_cb->connect_packet != NULL) {
344 mbuf_freem(fd_cb->connect_packet);
345 }
346 if (fd_cb->app_data != NULL) {
347 FREE(fd_cb->app_data, M_TEMP);
348 }
39236c6e
A		 349 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
350}
351
352static void
353flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
354{
355 if (fd_cb->group != NULL) {
356 struct flow_divert_group *group = fd_cb->group;
357 lck_rw_lock_exclusive(&group->lck);
358 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
359 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
360 fd_cb->group = NULL;
361 FDRELEASE(fd_cb); /* Release the group's reference */
362 lck_rw_done(&group->lck);
363 }
364}
365
366static int
367flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
368{
369 struct flow_divert_packet_header hdr;
370 int error = 0;
371
372 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
373 if (error) {
374 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
375 return error;
376 }
377
378 hdr.packet_type = packet_type;
379 hdr.conn_id = htonl(fd_cb->hash);
380
381 /* Lay down the header */
382 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
383 if (error) {
384 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
385 mbuf_freem(*packet);
386 *packet = NULL;
387 return error;
388 }
389
390 return 0;
391}
392
393static int
3e170ce0 394flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 395{
3e170ce0
A		 396 uint32_t net_length = htonl(length);
397 int error = 0;
39236c6e
A		 398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
400 if (error) {
401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
406 if (error) {
39037602 407 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
39236c6e
A		 408 return error;
409 }
410
411 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
412 if (error) {
413 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
414 return error;
415 }
416
417 return error;
418}
419
420static int
421flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
422{
3e170ce0
A		 423 size_t cursor = offset;
424 int error = 0;
425 uint32_t curr_length;
426 uint8_t curr_type;
39236c6e
A		 427
428 *err = 0;
429
430 do {
431 if (!next) {
432 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
433 if (error) {
434 *err = ENOENT;
435 return -1;
436 }
437 } else {
438 next = 0;
439 curr_type = FLOW_DIVERT_TLV_NIL;
440 }
441
442 if (curr_type != type) {
443 cursor += sizeof(curr_type);
444 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
445 if (error) {
446 *err = error;
447 return -1;
448 }
449
450 cursor += (sizeof(curr_length) + ntohl(curr_length));
451 }
452 } while (curr_type != type);
453
454 return cursor;
455}
456
457static int
3e170ce0 458flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 459{
3e170ce0
A		 460 int error = 0;
461 uint32_t length;
462 int tlv_offset;
39236c6e
A		 463
464 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
465 if (tlv_offset < 0) {
466 return error;
467 }
468
469 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
470 if (error) {
471 return error;
472 }
473
474 length = ntohl(length);
475
476 if (val_size != NULL) {
477 *val_size = length;
478 }
479
480 if (buff != NULL && buff_len > 0) {
481 size_t to_copy = (length < buff_len) ? length : buff_len;
482 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
483 if (error) {
484 return error;
485 }
486 }
487
488 return 0;
489}
490
491static int
492flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
493{
494 mbuf_t curr_mbuf = packet;
495
496 if (g_crypto_funcs == NULL || group->token_key == NULL) {
497 return ENOPROTOOPT;
498 }
499
500 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
501 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
502
503 while (curr_mbuf != NULL) {
504 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
505 curr_mbuf = mbuf_next(curr_mbuf);
506 }
507
508 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
509
510 return 0;
511}
512
513static int
514flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
515{
516 int error = 0;
517 struct flow_divert_group *group = NULL;
518 int hmac_offset;
519 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
520 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
521 mbuf_t tail;
522
523 lck_rw_lock_shared(&g_flow_divert_group_lck);
524
525 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
526 group = g_flow_divert_groups[ctl_unit];
527 }
528
529 if (group == NULL) {
530 lck_rw_done(&g_flow_divert_group_lck);
531 return ENOPROTOOPT;
532 }
533
534 lck_rw_lock_shared(&group->lck);
535
536 if (group->token_key == NULL) {
537 error = ENOPROTOOPT;
538 goto done;
539 }
540
541 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
542 if (hmac_offset < 0) {
543 goto done;
544 }
545
546 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
547 if (error) {
548 goto done;
549 }
550
551 /* Chop off the HMAC TLV */
552 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
553 if (error) {
554 goto done;
555 }
556
557 mbuf_free(tail);
558
559 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
560 if (error) {
561 goto done;
562 }
563
564 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
565 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
566 error = EINVAL;
567 goto done;
568 }
569
570done:
571 lck_rw_done(&group->lck);
572 lck_rw_done(&g_flow_divert_group_lck);
573 return error;
574}
575
576static void
577flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
578{
579 struct inpcb *inp = NULL;
580 struct ifnet *ifp = NULL;
581 Boolean cell = FALSE;
582 Boolean wifi = FALSE;
fe8ab488 583 Boolean wired = FALSE;
39236c6e
A		 584
585 inp = sotoinpcb(fd_cb->so);
586 if (inp == NULL) {
587 return;
588 }
589
590 ifp = inp->inp_last_outifp;
591 if (ifp != NULL) {
592 cell = IFNET_IS_CELLULAR(ifp);
593 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 594 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e
A		 595 }
596
597 if (send) {
fe8ab488
A		 598 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
599 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 600 } else {
fe8ab488
A		 601 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
602 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e 603 }
5ba3f43e 604 inp_set_activity_bitmap(inp);
39236c6e
A		 605}
606
607static errno_t
608flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
609{
610 struct inpcb *inp = NULL;
39236c6e
A		 611
612 inp = sotoinpcb(fd_cb->so);
fe8ab488
A		 613 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
614 IFNET_IS_CELLULAR(inp->inp_last_outifp))
615 return EHOSTUNREACH;
616
617 return 0;
618}
619
620static errno_t
621flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
622{
623 struct inpcb *inp = NULL;
624
625 inp = sotoinpcb(fd_cb->so);
626 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
627 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
628 return EHOSTUNREACH;
39236c6e
A		 629
630 return 0;
631}
632
633static void
634flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
635{
636 if (how != SHUT_RD) {
637 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
638 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
639 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
640 /* If the tunnel is not accepting writes any more, then flush the send buffer */
641 sbflush(&fd_cb->so->so_snd);
642 }
643 }
644 if (how != SHUT_WR) {
645 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
646 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
647 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
648 }
649 }
650}
651
652static uint16_t
653trie_node_alloc(struct flow_divert_trie *trie)
654{
655 if (trie->nodes_free_next < trie->nodes_count) {
656 uint16_t node_idx = trie->nodes_free_next++;
657 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
658 return node_idx;
659 } else {
660 return NULL_TRIE_IDX;
661 }
662}
663
664static uint16_t
665trie_child_map_alloc(struct flow_divert_trie *trie)
666{
667 if (trie->child_maps_free_next < trie->child_maps_count) {
668 return trie->child_maps_free_next++;
669 } else {
670 return NULL_TRIE_IDX;
671 }
672}
673
674static uint16_t
675trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
676{
677 uint16_t start = trie->bytes_free_next;
678 if (start + bytes_size <= trie->bytes_count) {
679 if (start != bytes_idx) {
680 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
681 }
682 trie->bytes_free_next += bytes_size;
683 return start;
684 } else {
685 return NULL_TRIE_IDX;
686 }
687}
688
689static uint16_t
690flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
691{
692 uint16_t current = trie->root;
693 uint16_t child = trie->root;
694 uint16_t string_end = string_start + string_len;
695 uint16_t string_idx = string_start;
696 uint16_t string_remainder = string_len;
697
698 while (child != NULL_TRIE_IDX) {
699 uint16_t parent = current;
700 uint16_t node_idx;
701 uint16_t current_end;
702
703 current = child;
704 child = NULL_TRIE_IDX;
705
706 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
707
708 for (node_idx = TRIE_NODE(trie, current).start;
709 node_idx < current_end &&
710 string_idx < string_end &&
711 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
712 node_idx++, string_idx++);
713
714 string_remainder = string_end - string_idx;
715
716 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
717 /*
718 * We did not reach the end of the current node's string.
719 * We need to split the current node into two:
720 * 1. A new node that contains the prefix of the node that matches
721 * the prefix of the string being inserted.
722 * 2. The current node modified to point to the remainder
723 * of the current node's string.
724 */
725 uint16_t prefix = trie_node_alloc(trie);
726 if (prefix == NULL_TRIE_IDX) {
727 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
728 return NULL_TRIE_IDX;
729 }
730
731 /*
732 * Prefix points to the portion of the current nodes's string that has matched
733 * the input string thus far.
734 */
735 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
736 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
737
738 /*
739 * Prefix has the current node as the child corresponding to the first byte
740 * after the split.
741 */
742 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
743 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
744 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
745 return NULL_TRIE_IDX;
746 }
747 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
748
749 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
750 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
751
752 /* Current node is adjusted to point to the remainder */
753 TRIE_NODE(trie, current).start = node_idx;
754 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
755
756 /* We want to insert the new leaf (if any) as a child of the prefix */
757 current = prefix;
758 }
759
760 if (string_remainder > 0) {
761 /*
762 * We still have bytes in the string that have not been matched yet.
763 * If the current node has children, iterate to the child corresponding
764 * to the next byte in the string.
765 */
766 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
767 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
768 }
769 }
770 } /* while (child != NULL_TRIE_IDX) */
771
772 if (string_remainder > 0) {
773 /* Add a new leaf containing the remainder of the string */
774 uint16_t leaf = trie_node_alloc(trie);
775 if (leaf == NULL_TRIE_IDX) {
776 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
777 return NULL_TRIE_IDX;
778 }
779
780 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
781 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
782 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
783 return NULL_TRIE_IDX;
784 }
785 TRIE_NODE(trie, leaf).length = string_remainder;
786
787 /* Set the new leaf as the child of the current node */
788 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
789 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
790 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
791 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
792 return NULL_TRIE_IDX;
793 }
794 }
795 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
796 current = leaf;
797 } /* else duplicate or this string is a prefix of one of the existing strings */
798
799 return current;
800}
801
3e170ce0 802#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 803static uint16_t
3e170ce0 804flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
39236c6e
A		 805{
806 uint16_t current = trie->root;
807 uint16_t string_idx = 0;
808
809 while (current != NULL_TRIE_IDX) {
810 uint16_t next = NULL_TRIE_IDX;
811 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
812 uint16_t node_idx;
813
814 for (node_idx = TRIE_NODE(trie, current).start;
815 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
816 node_idx++, string_idx++);
817
818 if (node_idx == node_end) {
819 if (string_bytes[string_idx] == '\0') {
820 return current; /* Got an exact match */
3e170ce0
A		 821 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
822 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
823 string_bytes[string_idx] = '\0';
824 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 825 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
826 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
827 }
828 }
829 current = next;
830 }
831
832 return NULL_TRIE_IDX;
833}
834
490019cf
A		 835struct uuid_search_info {
836 uuid_t target_uuid;
837 char *found_signing_id;
838 boolean_t found_multiple_signing_ids;
839 proc_t found_proc;
840};
841
842static int
843flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
844{
845 struct uuid_search_info *info = (struct uuid_search_info *)arg;
846 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
847
848 if (info->found_signing_id != NULL) {
849 if (!info->found_multiple_signing_ids) {
850 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
851 info->found_proc = p;
852 result = PROC_CLAIMED_DONE;
853 } else {
854 uuid_string_t uuid_str;
855 uuid_unparse(info->target_uuid, uuid_str);
856 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
857 }
858 FREE(info->found_signing_id, M_TEMP);
859 info->found_signing_id = NULL;
860 }
861
862 if (result == PROC_RETURNED_DONE) {
863 uuid_string_t uuid_str;
864 uuid_unparse(info->target_uuid, uuid_str);
865 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
866 }
867
868 return result;
869}
870
871static int
872flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
873{
874 struct uuid_search_info *info = (struct uuid_search_info *)arg;
875 int include = 0;
876
877 if (info->found_multiple_signing_ids) {
878 return include;
879 }
880
881 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
882 if (include) {
883 const char *signing_id = cs_identity_get(p);
884 if (signing_id != NULL) {
885 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
886 size_t signing_id_size = strlen(signing_id) + 1;
887 if (info->found_signing_id == NULL) {
888 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
889 memcpy(info->found_signing_id, signing_id, signing_id_size);
890 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
891 info->found_multiple_signing_ids = TRUE;
892 }
893 } else {
894 info->found_multiple_signing_ids = TRUE;
895 }
896 include = !info->found_multiple_signing_ids;
897 }
898
899 return include;
900}
901
902static proc_t
903flow_divert_find_proc_by_uuid(uuid_t uuid)
904{
905 struct uuid_search_info info;
906
907 if (LOG_INFO <= nil_pcb.log_level) {
908 uuid_string_t uuid_str;
909 uuid_unparse(uuid, uuid_str);
910 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
911 }
912
913 memset(&info, 0, sizeof(info));
914 info.found_proc = PROC_NULL;
915 uuid_copy(info.target_uuid, uuid);
916
917 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
918
919 return info.found_proc;
920}
921
39236c6e 922static int
490019cf 923flow_divert_get_src_proc(struct socket *so, proc_t *proc)
39236c6e
A		 924{
925 int release = 0;
926
490019cf
A		 927 if (so->so_flags & SOF_DELEGATED) {
928 if ((*proc)->p_pid != so->e_pid) {
929 *proc = proc_find(so->e_pid);
930 release = 1;
931 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
932 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
933 release = 1;
934 }
39236c6e
A		 935 } else if (*proc == PROC_NULL) {
936 *proc = current_proc();
937 }
938
939 if (*proc != PROC_NULL) {
940 if ((*proc)->p_pid == 0) {
941 if (release) {
942 proc_rele(*proc);
943 }
944 release = 0;
945 *proc = PROC_NULL;
946 }
947 }
948
949 return release;
950}
951
952static int
953flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
954{
955 int error;
956
957 if (fd_cb->group == NULL) {
958 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 959 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 960 return ECONNABORTED;
961 }
962
963 lck_rw_lock_shared(&fd_cb->group->lck);
964
965 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
966 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
967 } else {
968 error = ENOBUFS;
969 }
970
971 if (error == ENOBUFS) {
972 if (enqueue) {
973 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
974 lck_rw_lock_exclusive(&fd_cb->group->lck);
975 }
976 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
977 error = 0;
978 }
979 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
980 }
981
982 lck_rw_done(&fd_cb->group->lck);
983
984 return error;
985}
986
987static int
490019cf 988flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 989{
39236c6e 990 int error = 0;
3e170ce0 991 int flow_type = 0;
490019cf
A		 992 char *signing_id = NULL;
993 int free_signing_id = 0;
994 mbuf_t connect_packet = NULL;
39037602
A		 995 proc_t src_proc = p;
996 int release_proc = 0;
490019cf
A		 997
998 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
999 if (error) {
1000 goto done;
1001 }
1002
1003 error = EPERM;
1004
1005 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1006 uint32_t sid_size = 0;
1007 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1008 if (find_error == 0 && sid_size > 0) {
1009 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1010 if (signing_id != NULL) {
1011 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1012 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1013 free_signing_id = 1;
1014 }
1015 }
1016 }
1017
1018 socket_unlock(so, 0);
39037602
A		 1019
1020 if (signing_id == NULL) {
1021 release_proc = flow_divert_get_src_proc(so, &src_proc);
1022 if (src_proc != PROC_NULL) {
1023 proc_lock(src_proc);
1024 if (src_proc->p_csflags & (CS_VALID|CS_DEBUGGED)) {
1025 const char * cs_id;
1026 cs_id = cs_identity_get(src_proc);
1027 signing_id = __DECONST(char *, cs_id);
490019cf 1028 } else {
39037602 1029 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
490019cf
A		 1030 }
1031 } else {
39037602 1032 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
490019cf 1033 }
39037602
A		 1034 } else {
1035 src_proc = PROC_NULL;
1036 }
490019cf 1037
39037602
A		 1038 if (signing_id != NULL) {
1039 uint16_t result = NULL_TRIE_IDX;
1040 lck_rw_lock_shared(&fd_cb->group->lck);
5ba3f43e
A		 1041 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1042 result = 1;
1043 } else {
1044 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1045 }
39037602
A		 1046 lck_rw_done(&fd_cb->group->lck);
1047 if (result != NULL_TRIE_IDX) {
1048 error = 0;
1049 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
490019cf 1050
39037602
A		 1051 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1052 if (error == 0) {
1053 if (src_proc != PROC_NULL) {
1054 unsigned char cdhash[SHA1_RESULTLEN];
1055 error = proc_getcdhash(src_proc, cdhash);
1056 if (error == 0) {
1057 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1058 if (error) {
1059 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
490019cf 1060 }
39037602
A		 1061 } else {
1062 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
490019cf 1063 }
490019cf
A		 1064 }
1065 } else {
39037602 1066 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
490019cf
A		 1067 }
1068 } else {
39037602 1069 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
490019cf 1070 }
39037602
A		 1071 } else {
1072 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1073 }
490019cf 1074
39037602
A		 1075 if (src_proc != PROC_NULL) {
1076 proc_unlock(src_proc);
1077 if (release_proc) {
1078 proc_rele(src_proc);
490019cf 1079 }
490019cf
A		 1080 }
1081 socket_lock(so, 0);
1082
1083 if (free_signing_id) {
1084 FREE(signing_id, M_TEMP);
1085 }
1086
1087 if (error) {
1088 goto done;
1089 }
39236c6e 1090
39236c6e
A		 1091 error = flow_divert_packet_append_tlv(connect_packet,
1092 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1093 sizeof(fd_cb->so->so_traffic_class),
1094 &fd_cb->so->so_traffic_class);
1095 if (error) {
1096 goto done;
1097 }
1098
3e170ce0
A		 1099 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1100 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1101 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1102 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1103 } else {
1104 error = EINVAL;
1105 goto done;
1106 }
1107 error = flow_divert_packet_append_tlv(connect_packet,
1108 FLOW_DIVERT_TLV_FLOW_TYPE,
1109 sizeof(flow_type),
1110 &flow_type);
1111
1112 if (error) {
1113 goto done;
1114 }
1115
39236c6e
A		 1116 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1117 error = flow_divert_packet_append_tlv(connect_packet,
1118 FLOW_DIVERT_TLV_PID,
1119 sizeof(fd_cb->so->e_pid),
1120 &fd_cb->so->e_pid);
1121 if (error) {
1122 goto done;
1123 }
1124
1125 error = flow_divert_packet_append_tlv(connect_packet,
1126 FLOW_DIVERT_TLV_UUID,
1127 sizeof(fd_cb->so->e_uuid),
1128 &fd_cb->so->e_uuid);
1129 if (error) {
1130 goto done;
1131 }
1132 } else {
1133 error = flow_divert_packet_append_tlv(connect_packet,
1134 FLOW_DIVERT_TLV_PID,
1135 sizeof(fd_cb->so->e_pid),
1136 &fd_cb->so->last_pid);
1137 if (error) {
1138 goto done;
1139 }
1140
1141 error = flow_divert_packet_append_tlv(connect_packet,
1142 FLOW_DIVERT_TLV_UUID,
1143 sizeof(fd_cb->so->e_uuid),
1144 &fd_cb->so->last_uuid);
1145 if (error) {
1146 goto done;
1147 }
1148 }
1149
1150 if (fd_cb->connect_token != NULL) {
1151 unsigned int token_len = m_length(fd_cb->connect_token);
1152 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1153 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1154 fd_cb->connect_token = NULL;
1155 } else {
1156 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
39236c6e
A		 1157
1158 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1159 if (error) {
1160 goto done;
1161 }
1162
3e170ce0 1163 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1164 if (error) {
1165 goto done;
1166 }
39236c6e
A		 1167 }
1168
490019cf
A		 1169 if (fd_cb->local_address != NULL) {
1170 error = EALREADY;
1171 goto done;
1172 } else {
1173 struct inpcb *inp = sotoinpcb(so);
1174 if (flow_divert_has_pcb_local_address(inp)) {
1175 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1176 if (error) {
1177 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1178 goto done;
1179 }
1180 }
1181 }
1182
3e170ce0
A		 1183 if (fd_cb->local_address != NULL) {
1184 /* socket is bound. */
490019cf
A		 1185 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1186 sizeof(struct sockaddr_storage), fd_cb->local_address);
1187 if (error) {
1188 goto done;
1189 }
1190 }
3e170ce0 1191
490019cf
A		 1192 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1193 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1194 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1195 if (error) {
1196 goto done;
1197 }
39236c6e
A		 1198 }
1199
1200done:
490019cf
A		 1201 if (!error) {
1202 *out_connect_packet = connect_packet;
1203 } else if (connect_packet != NULL) {
1204 mbuf_freem(connect_packet);
1205 }
1206
39236c6e
A		 1207 return error;
1208}
1209
1210static int
1211flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1212{
1213 int error = 0;
1214 mbuf_t packet = NULL;
1215 int rbuff_space = 0;
1216
1217 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1218 if (error) {
1219 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1220 goto done;
1221 }
1222
3e170ce0 1223 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1224 if (rbuff_space < 0) {
1225 rbuff_space = 0;
1226 }
1227 rbuff_space = htonl(rbuff_space);
1228 error = flow_divert_packet_append_tlv(packet,
1229 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1230 sizeof(rbuff_space),
1231 &rbuff_space);
1232 if (error) {
1233 goto done;
1234 }
1235
1236 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1237 if (error) {
1238 goto done;
1239 }
1240
1241done:
1242 if (error && packet != NULL) {
3e170ce0 1243 mbuf_freem(packet);
39236c6e
A		 1244 }
1245
1246 return error;
1247}
1248
1249static int
1250flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1251{
1252 int error = 0;
1253 mbuf_t packet = NULL;
1254 uint32_t zero = 0;
1255
1256 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1257 if (error) {
1258 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1259 goto done;
1260 }
1261
1262 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1263 if (error) {
1264 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1265 goto done;
1266 }
1267
1268 how = htonl(how);
1269 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1270 if (error) {
1271 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1272 goto done;
1273 }
1274
1275 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1276 if (error) {
1277 goto done;
1278 }
1279
1280done:
1281 if (error && packet != NULL) {
1282 mbuf_free(packet);
1283 }
1284
1285 return error;
1286}
1287
1288static int
1289flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1290{
1291 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1292 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1293 {
1294 return SHUT_RDWR;
1295 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1296 return SHUT_RD;
1297 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1298 return SHUT_WR;
1299 }
1300
1301 return -1;
1302}
1303
1304/*
1305 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1306 * writes. Returns FALSE otherwise.
1307 */
1308static void
1309flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1310{
1311 int how = -1;
1312
1313 /* Do not send any close messages if there is still data in the send buffer */
1314 if (fd_cb->so->so_snd.sb_cc == 0) {
1315 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1316 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1317 how = SHUT_RD;
1318 }
1319 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1320 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1321 if (how == SHUT_RD) {
1322 how = SHUT_RDWR;
1323 } else {
1324 how = SHUT_WR;
1325 }
1326 }
1327 }
1328
1329 if (how != -1) {
1330 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1331 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1332 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1333 if (how != SHUT_RD) {
1334 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1335 }
1336 if (how != SHUT_WR) {
1337 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1338 }
1339 }
1340 }
1341
1342 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1343 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1344 }
1345}
1346
1347static errno_t
3e170ce0 1348flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e
A		 1349{
1350 mbuf_t packet;
1351 mbuf_t last;
1352 int error = 0;
1353
1354 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1355 if (error) {
1356 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1357 return error;
1358 }
1359
3e170ce0
A		 1360 if (toaddr != NULL) {
1361 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1362 if (error) {
1363 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1364 return error;
1365 }
1366 }
1367
5ba3f43e
A		 1368 if (data_len > 0 && data != NULL) {
1369 last = m_last(packet);
1370 mbuf_setnext(last, data);
1371 mbuf_pkthdr_adjustlen(packet, data_len);
1372 }
39236c6e
A		 1373 error = flow_divert_send_packet(fd_cb, packet, force);
1374
1375 if (error) {
1376 mbuf_setnext(last, NULL);
3e170ce0 1377 mbuf_freem(packet);
39236c6e
A		 1378 } else {
1379 fd_cb->bytes_sent += data_len;
1380 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1381 }
1382
1383 return error;
1384}
1385
1386static void
1387flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1388{
1389 size_t to_send;
1390 size_t sent = 0;
1391 int error = 0;
1392 mbuf_t buffer;
1393
1394 to_send = fd_cb->so->so_snd.sb_cc;
1395 buffer = fd_cb->so->so_snd.sb_mb;
1396
1397 if (buffer == NULL && to_send > 0) {
1398 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1399 return;
1400 }
1401
1402 /* Ignore the send window if force is enabled */
1403 if (!force && (to_send > fd_cb->send_window)) {
1404 to_send = fd_cb->send_window;
1405 }
1406
3e170ce0
A		 1407 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1408 while (sent < to_send) {
1409 mbuf_t data;
1410 size_t data_len;
39236c6e 1411
3e170ce0
A		 1412 data_len = to_send - sent;
1413 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1414 data_len = FLOW_DIVERT_CHUNK_SIZE;
1415 }
39236c6e 1416
3e170ce0
A		 1417 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1418 if (error) {
1419 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1420 break;
1421 }
39236c6e 1422
3e170ce0
A		 1423 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1424 if (error) {
1425 mbuf_freem(data);
1426 break;
1427 }
39236c6e 1428
3e170ce0
A		 1429 sent += data_len;
1430 }
1431 sbdrop(&fd_cb->so->so_snd, sent);
1432 sowwakeup(fd_cb->so);
1433 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1434 mbuf_t data;
1435 mbuf_t m;
1436 size_t data_len;
1437
1438 while(buffer) {
1439 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1440
1441 m = buffer;
1442 if (toaddr != NULL) {
1443 /* look for data in the chain */
1444 do {
1445 m = m->m_next;
1446 if (m != NULL && m->m_type == MT_DATA) {
1447 break;
1448 }
1449 } while(m);
1450 if (m == NULL) {
1451 /* unexpected */
1452 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1453 goto move_on;
1454 }
1455 }
1456 data_len = mbuf_pkthdr_len(m);
5ba3f43e
A		 1457 if (data_len > 0) {
1458 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1459 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1460 if (error) {
1461 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1462 break;
1463 }
1464 } else {
1465 data = NULL;
3e170ce0
A		 1466 }
1467 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1468 if (error) {
1469 mbuf_freem(data);
1470 break;
1471 }
1472 sent += data_len;
1473move_on:
1474 buffer = buffer->m_nextpkt;
1475 (void) sbdroprecord(&(fd_cb->so->so_snd));
1476 }
39236c6e
A		 1477 }
1478
1479 if (sent > 0) {
1480 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1481 if (fd_cb->send_window >= sent) {
1482 fd_cb->send_window -= sent;
1483 } else {
1484 fd_cb->send_window = 0;
1485 }
39236c6e
A		 1486 }
1487}
1488
1489static int
3e170ce0 1490flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e
A		 1491{
1492 size_t to_send = mbuf_pkthdr_len(data);
3e170ce0 1493 int error = 0;
39236c6e
A		 1494
1495 if (to_send > fd_cb->send_window) {
1496 to_send = fd_cb->send_window;
1497 }
1498
1499 if (fd_cb->so->so_snd.sb_cc > 0) {
1500 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1501 }
1502
3e170ce0
A		 1503 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1504 size_t sent = 0;
1505 mbuf_t remaining_data = data;
1506 mbuf_t pkt_data = NULL;
490019cf 1507 while (sent < to_send && remaining_data != NULL) {
3e170ce0
A		 1508 size_t pkt_data_len;
1509
1510 pkt_data = remaining_data;
39236c6e 1511
3e170ce0
A		 1512 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1513 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1514 } else {
1515 pkt_data_len = to_send - sent;
1516 }
1517
1518 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1519 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1520 if (error) {
1521 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1522 pkt_data = NULL;
1523 break;
1524 }
1525 } else {
1526 remaining_data = NULL;
1527 }
1528
1529 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1530
39236c6e 1531 if (error) {
39236c6e
A		 1532 break;
1533 }
39236c6e 1534
3e170ce0
A		 1535 pkt_data = NULL;
1536 sent += pkt_data_len;
39236c6e
A		 1537 }
1538
3e170ce0 1539 fd_cb->send_window -= sent;
39236c6e 1540
3e170ce0 1541 error = 0;
39236c6e 1542
3e170ce0
A		 1543 if (pkt_data != NULL) {
1544 if (sbspace(&fd_cb->so->so_snd) > 0) {
1545 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1546 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1547 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1548 }
1549 } else {
1550 error = ENOBUFS;
39236c6e 1551 }
39236c6e 1552 }
39236c6e 1553
3e170ce0
A		 1554 if (remaining_data != NULL) {
1555 if (sbspace(&fd_cb->so->so_snd) > 0) {
1556 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1557 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1558 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1559 }
1560 } else {
1561 error = ENOBUFS;
1562 }
1563 }
1564 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
5ba3f43e 1565 if (to_send || mbuf_pkthdr_len(data) == 0) {
3e170ce0
A		 1566 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1567 if (error) {
39037602 1568 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
3e170ce0
A		 1569 } else {
1570 fd_cb->send_window -= to_send;
39236c6e
A		 1571 }
1572 } else {
3e170ce0
A		 1573 /* buffer it */
1574 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1575 if (toaddr != NULL) {
1576 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1577 FDLOG(LOG_ERR, fd_cb,
1578 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1579 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1580 }
1581 } else {
1582 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1583 FDLOG(LOG_ERR, fd_cb,
1584 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1585 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1586 }
1587 }
1588 } else {
1589 error = ENOBUFS;
1590 }
39236c6e
A		 1591 }
1592 }
1593
1594 return error;
1595}
1596
1597static int
1598flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1599{
1600 int error = 0;
1601 mbuf_t packet = NULL;
1602 uint32_t net_read_count = htonl(read_count);
1603
1604 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1605 if (error) {
1606 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1607 goto done;
1608 }
1609
1610 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1611 if (error) {
1612 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1613 goto done;
1614 }
1615
1616 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1617 if (error) {
1618 goto done;
1619 }
1620
1621done:
1622 if (error && packet != NULL) {
1623 mbuf_free(packet);
1624 }
1625
1626 return error;
1627}
1628
1629static int
1630flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1631{
1632 int error = 0;
1633 mbuf_t packet = NULL;
1634
1635 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1636 if (error) {
1637 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1638 goto done;
1639 }
1640
1641 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1642 if (error) {
1643 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1644 goto done;
1645 }
1646
1647 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1648 if (error) {
1649 goto done;
1650 }
1651
1652done:
1653 if (error && packet != NULL) {
1654 mbuf_free(packet);
1655 }
1656
1657 return error;
1658}
1659
1660static void
1661flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1662{
1663 uint32_t connect_error;
1664 uint32_t ctl_unit = 0;
1665 int error = 0;
1666 struct flow_divert_group *grp = NULL;
1667 struct sockaddr_storage local_address;
1668 int out_if_index = 0;
1669 struct sockaddr_storage remote_address;
1670 uint32_t send_window;
39037602 1671 uint32_t app_data_length = 0;
39236c6e
A		 1672
1673 memset(&local_address, 0, sizeof(local_address));
1674 memset(&remote_address, 0, sizeof(remote_address));
1675
1676 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1677 if (error) {
1678 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1679 return;
1680 }
1681
1682 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1683
1684 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1685 if (error) {
1686 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1687 return;
1688 }
1689
1690 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1691 if (error) {
39037602 1692 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
39236c6e
A		 1693 }
1694
1695 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1696 if (error) {
39037602 1697 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
39236c6e
A		 1698 }
1699
1700 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1701 if (error) {
39037602 1702 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
39236c6e
A		 1703 }
1704
1705 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1706 if (error) {
39037602
A		 1707 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1708 }
1709
1710 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1711 if (error) {
1712 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
39236c6e
A		 1713 }
1714
39037602 1715 error = 0;
39236c6e
A		 1716 connect_error = ntohl(connect_error);
1717 ctl_unit = ntohl(ctl_unit);
1718
1719 lck_rw_lock_shared(&g_flow_divert_group_lck);
1720
39037602
A		 1721 if (connect_error == 0 && ctl_unit > 0) {
1722 if (ctl_unit >= GROUP_COUNT_MAX) {
39236c6e
A		 1723 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1724 error = EINVAL;
1725 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1726 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1727 error = EINVAL;
1728 } else {
1729 grp = g_flow_divert_groups[ctl_unit];
1730 if (grp == NULL) {
1731 error = ECONNRESET;
1732 }
1733 }
1734 }
1735
1736 FDLOCK(fd_cb);
1737 if (fd_cb->so != NULL) {
1738 struct inpcb *inp = NULL;
1739 struct ifnet *ifp = NULL;
1740 struct flow_divert_group *old_group;
1741
1742 socket_lock(fd_cb->so, 0);
1743
1744 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1745 goto done;
1746 }
1747
1748 inp = sotoinpcb(fd_cb->so);
1749
1750 if (connect_error || error) {
1751 goto set_socket_state;
1752 }
1753
3e170ce0
A		 1754 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1755 error = EINVAL;
1756 goto set_socket_state;
1757 }
1758 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
39236c6e
A		 1759 if (local_address.ss_len > sizeof(local_address)) {
1760 local_address.ss_len = sizeof(local_address);
1761 }
1762 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
39236c6e
A		 1763 }
1764
1765 if (remote_address.ss_family != 0) {
1766 if (remote_address.ss_len > sizeof(remote_address)) {
1767 remote_address.ss_len = sizeof(remote_address);
1768 }
1769 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1770 } else {
1771 error = EINVAL;
1772 goto set_socket_state;
1773 }
1774
39037602
A		 1775 if (app_data_length > 0) {
1776 uint8_t *app_data = NULL;
1777 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1778 if (app_data != NULL) {
1779 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1780 if (error == 0) {
1781 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1782 if (fd_cb->app_data != NULL) {
1783 FREE(fd_cb->app_data, M_TEMP);
1784 }
1785 fd_cb->app_data = app_data;
1786 fd_cb->app_data_length = app_data_length;
1787 } else {
1788 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1789 FREE(app_data, M_TEMP);
1790 }
1791 } else {
1792 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1793 }
1794 }
1795
39236c6e
A		 1796 ifnet_head_lock_shared();
1797 if (out_if_index > 0 && out_if_index <= if_index) {
1798 ifp = ifindex2ifnet[out_if_index];
1799 }
1800
1801 if (ifp != NULL) {
1802 inp->inp_last_outifp = ifp;
1803 } else {
1804 error = EINVAL;
1805 }
1806 ifnet_head_done();
1807
1808 if (error) {
1809 goto set_socket_state;
1810 }
1811
1812 if (fd_cb->group == NULL) {
1813 error = EINVAL;
1814 goto set_socket_state;
1815 }
1816
39037602
A		 1817 if (grp != NULL) {
1818 old_group = fd_cb->group;
39236c6e 1819
39037602
A		 1820 lck_rw_lock_exclusive(&old_group->lck);
1821 lck_rw_lock_exclusive(&grp->lck);
39236c6e 1822
39037602
A		 1823 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1824 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1825 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1826 }
39236c6e 1827
39037602 1828 fd_cb->group = grp;
39236c6e 1829
39037602
A		 1830 lck_rw_done(&grp->lck);
1831 lck_rw_done(&old_group->lck);
1832 }
39236c6e
A		 1833
1834 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 1835
1836set_socket_state:
1837 if (!connect_error && !error) {
1838 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1839 error = flow_divert_send_connect_result(fd_cb);
1840 }
1841
1842 if (connect_error || error) {
1843 if (!connect_error) {
1844 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1845 fd_cb->so->so_error = error;
1846 flow_divert_send_close_if_needed(fd_cb);
1847 } else {
1848 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1849 fd_cb->so->so_error = connect_error;
1850 }
3e170ce0 1851 flow_divert_disconnect_socket(fd_cb->so);
39236c6e 1852 } else {
490019cf 1853 flow_divert_send_buffered_data(fd_cb, FALSE);
39236c6e
A		 1854 soisconnected(fd_cb->so);
1855 }
1856
1857done:
1858 socket_unlock(fd_cb->so, 0);
1859 }
1860 FDUNLOCK(fd_cb);
1861
1862 lck_rw_done(&g_flow_divert_group_lck);
1863}
1864
1865static void
1866flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1867{
1868 uint32_t close_error;
1869 int error = 0;
1870 int how;
1871
1872 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1873 if (error) {
1874 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1875 return;
1876 }
1877
1878 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1879 if (error) {
1880 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1881 return;
1882 }
1883
1884 how = ntohl(how);
1885
1886 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1887
1888 FDLOCK(fd_cb);
1889 if (fd_cb->so != NULL) {
1890 socket_lock(fd_cb->so, 0);
1891
1892 fd_cb->so->so_error = ntohl(close_error);
1893
1894 flow_divert_update_closed_state(fd_cb, how, TRUE);
1895
1896 how = flow_divert_tunnel_how_closed(fd_cb);
1897 if (how == SHUT_RDWR) {
3e170ce0 1898 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1899 } else if (how == SHUT_RD) {
1900 socantrcvmore(fd_cb->so);
1901 } else if (how == SHUT_WR) {
1902 socantsendmore(fd_cb->so);
1903 }
1904
1905 socket_unlock(fd_cb->so, 0);
1906 }
1907 FDUNLOCK(fd_cb);
1908}
1909
3e170ce0
A		 1910static mbuf_t
1911flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 1912{
3e170ce0
A		 1913 struct inpcb *inp = sotoinpcb(fd_cb->so);
1914 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1915 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
39236c6e 1916
3e170ce0
A		 1917 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1918 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1919 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1920 struct in6_pktinfo pi6;
39236c6e 1921
3e170ce0
A		 1922 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1923 pi6.ipi6_ifindex = 0;
1924 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 1925 }
3e170ce0
A		 1926 return (NULL);
1927}
39236c6e 1928
3e170ce0
A		 1929static void
1930flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1931{
39236c6e
A		 1932 FDLOCK(fd_cb);
1933 if (fd_cb->so != NULL) {
3e170ce0
A		 1934 int error = 0;
1935 mbuf_t data = NULL;
1936 size_t data_size;
1937 struct sockaddr_storage remote_address;
1938 boolean_t got_remote_sa = FALSE;
1939
39236c6e 1940 socket_lock(fd_cb->so, 0);
3e170ce0
A		 1941
1942 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1943 uint32_t val_size = 0;
1944
1945 /* check if we got remote address with data */
1946 memset(&remote_address, 0, sizeof(remote_address));
1947 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1948 if (error || val_size > sizeof(remote_address)) {
1949 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1950 error = 0;
39236c6e 1951 } else {
3e170ce0
A		 1952 /* validate the address */
1953 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1954 got_remote_sa = TRUE;
1955 }
1956 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1957 }
1958 }
1959
1960 data_size = (mbuf_pkthdr_len(packet) - offset);
1961
1962 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1963
1964 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1965 if (error || data == NULL) {
1966 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1967 } else {
1968 if (flow_divert_check_no_cellular(fd_cb) ||
1969 flow_divert_check_no_expensive(fd_cb))
1970 {
1971 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1972 flow_divert_send_close(fd_cb, SHUT_RDWR);
1973 flow_divert_disconnect_socket(fd_cb->so);
1974 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1975 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1976 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1977 fd_cb->bytes_received += data_size;
1978 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1979 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1980 sorwakeup(fd_cb->so);
1981 data = NULL;
1982 } else {
1983 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1984 }
1985 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1986 struct sockaddr *append_sa;
1987 mbuf_t mctl;
1988
1989 if (got_remote_sa == TRUE) {
1990 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1991 (struct sockaddr *)&remote_address, &append_sa);
1992 } else {
1993 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1994 fd_cb->remote_address, &append_sa);
1995 }
1996 if (error) {
1997 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
1998 }
1999
2000 mctl = flow_divert_get_control_mbuf(fd_cb);
2001 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
2002 fd_cb->bytes_received += data_size;
2003 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2004 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2005 sorwakeup(fd_cb->so);
2006 data = NULL;
2007 } else {
2008 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
2009 }
2010 if (!error) {
2011 FREE(append_sa, M_TEMP);
2012 }
2013 }
39236c6e
A		 2014 }
2015 }
2016 socket_unlock(fd_cb->so, 0);
39236c6e 2017
3e170ce0
A		 2018 if (data != NULL) {
2019 mbuf_freem(data);
2020 }
39236c6e 2021 }
3e170ce0 2022 FDUNLOCK(fd_cb);
39236c6e
A		 2023}
2024
2025static void
2026flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2027{
2028 uint32_t read_count;
2029 int error = 0;
2030
2031 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2032 if (error) {
2033 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2034 return;
2035 }
2036
3e170ce0 2037 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2038
2039 FDLOCK(fd_cb);
2040 if (fd_cb->so != NULL) {
2041 socket_lock(fd_cb->so, 0);
2042 fd_cb->send_window += ntohl(read_count);
2043 flow_divert_send_buffered_data(fd_cb, FALSE);
2044 socket_unlock(fd_cb->so, 0);
2045 }
2046 FDUNLOCK(fd_cb);
2047}
2048
2049static void
2050flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2051{
2052 int error = 0;
3e170ce0 2053 uint32_t key_size = 0;
39236c6e 2054 int log_level;
5ba3f43e 2055 uint32_t flags = 0;
39236c6e
A		 2056
2057 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2058 if (error) {
2059 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2060 return;
2061 }
2062
2063 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
39037602 2064 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
39236c6e
A		 2065 return;
2066 }
2067
2068 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2069 if (!error) {
2070 nil_pcb.log_level = log_level;
2071 }
2072
2073 lck_rw_lock_exclusive(&group->lck);
2074
2075 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2076 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2077 if (error) {
2078 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2079 FREE(group->token_key, M_TEMP);
2080 group->token_key = NULL;
2081 lck_rw_done(&group->lck);
2082 return;
2083 }
2084
2085 group->token_key_size = key_size;
2086
5ba3f43e
A		 2087 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2088 if (!error) {
2089 group->flags = flags;
2090 }
2091
39236c6e
A		 2092 lck_rw_done(&group->lck);
2093}
2094
2095static void
2096flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2097{
2098 int error = 0;
2099 struct sockaddr_storage local_address;
2100 int out_if_index = 0;
2101 struct sockaddr_storage remote_address;
490019cf 2102 uint32_t app_data_length = 0;
39236c6e
A		 2103
2104 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2105
2106 memset(&local_address, 0, sizeof(local_address));
2107 memset(&remote_address, 0, sizeof(remote_address));
2108
2109 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2110 if (error) {
490019cf 2111 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
39236c6e
A		 2112 }
2113
2114 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2115 if (error) {
490019cf 2116 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
39236c6e
A		 2117 }
2118
2119 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2120 if (error) {
490019cf
A		 2121 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2122 }
2123
2124 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2125 if (error) {
2126 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2127 }
2128
2129 FDLOCK(fd_cb);
2130 if (fd_cb->so != NULL) {
39236c6e
A		 2131 socket_lock(fd_cb->so, 0);
2132
39236c6e
A		 2133 if (local_address.ss_family != 0) {
2134 if (local_address.ss_len > sizeof(local_address)) {
2135 local_address.ss_len = sizeof(local_address);
2136 }
490019cf
A		 2137 if (fd_cb->local_address != NULL) {
2138 FREE(fd_cb->local_address, M_SONAME);
2139 fd_cb->local_address = NULL;
2140 }
39236c6e
A		 2141 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2142 }
2143
2144 if (remote_address.ss_family != 0) {
2145 if (remote_address.ss_len > sizeof(remote_address)) {
2146 remote_address.ss_len = sizeof(remote_address);
2147 }
490019cf
A		 2148 if (fd_cb->remote_address != NULL) {
2149 FREE(fd_cb->remote_address, M_SONAME);
2150 fd_cb->remote_address = NULL;
2151 }
39236c6e
A		 2152 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2153 }
2154
490019cf
A		 2155 if (out_if_index > 0) {
2156 struct inpcb *inp = NULL;
2157 struct ifnet *ifp = NULL;
2158
2159 inp = sotoinpcb(fd_cb->so);
2160
2161 ifnet_head_lock_shared();
2162 if (out_if_index <= if_index) {
2163 ifp = ifindex2ifnet[out_if_index];
2164 }
2165
2166 if (ifp != NULL) {
2167 inp->inp_last_outifp = ifp;
2168 }
2169 ifnet_head_done();
39236c6e
A		 2170 }
2171
490019cf
A		 2172 if (app_data_length > 0) {
2173 uint8_t *app_data = NULL;
2174 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2175 if (app_data != NULL) {
2176 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2177 if (error == 0) {
2178 if (fd_cb->app_data != NULL) {
2179 FREE(fd_cb->app_data, M_TEMP);
2180 }
2181 fd_cb->app_data = app_data;
2182 fd_cb->app_data_length = app_data_length;
2183 } else {
2184 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2185 FREE(app_data, M_TEMP);
2186 }
2187 } else {
2188 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2189 }
39236c6e 2190 }
39236c6e
A		 2191
2192 socket_unlock(fd_cb->so, 0);
2193 }
2194 FDUNLOCK(fd_cb);
2195}
2196
2197static void
39037602 2198flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
39236c6e
A		 2199{
2200 size_t bytes_mem_size;
2201 size_t child_maps_mem_size;
2202 int cursor;
2203 int error = 0;
2204 struct flow_divert_trie new_trie;
2205 int insert_error = 0;
2206 size_t nodes_mem_size;
2207 int prefix_count = 0;
2208 int signing_id_count = 0;
ecc0ceb4 2209 size_t trie_memory_size = 0;
39236c6e 2210
39037602 2211 lck_rw_lock_exclusive(&group->lck);
39236c6e
A		 2212
2213 /* Re-set the current trie */
39037602
A		 2214 if (group->signing_id_trie.memory != NULL) {
2215 FREE(group->signing_id_trie.memory, M_TEMP);
39236c6e 2216 }
39037602
A		 2217 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2218 group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 2219
2220 memset(&new_trie, 0, sizeof(new_trie));
2221
2222 /* Get the number of shared prefixes in the new set of signing ID strings */
2223 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2224
ecc0ceb4 2225 if (prefix_count < 0) {
39037602 2226 lck_rw_done(&group->lck);
ecc0ceb4
A		 2227 return;
2228 }
2229
39236c6e
A		 2230 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2231 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2232 cursor >= 0;
2233 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2234 {
3e170ce0 2235 uint32_t sid_size = 0;
39236c6e
A		 2236 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2237 new_trie.bytes_count += sid_size;
2238 signing_id_count++;
2239 }
2240
2241 if (signing_id_count == 0) {
39037602 2242 lck_rw_done(&group->lck);
39236c6e
A		 2243 return;
2244 }
2245
2246 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2247 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2248
2249 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2250 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2251
2252 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2253 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2254 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2255
ecc0ceb4
A		 2256 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2257 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
39037602
A		 2258 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2259 lck_rw_done(&group->lck);
ecc0ceb4
A		 2260 return;
2261 }
2262
2263 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2264 if (new_trie.memory == NULL) {
2265 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2266 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
39037602 2267 lck_rw_done(&group->lck);
39236c6e
A		 2268 return;
2269 }
2270
2271 /* Initialize the free lists */
2272 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2273 new_trie.nodes_free_next = 0;
2274 memset(new_trie.nodes, 0, nodes_mem_size);
2275
2276 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2277 new_trie.child_maps_free_next = 0;
2278 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2279
2280 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2281 new_trie.bytes_free_next = 0;
2282
2283 /* The root is an empty node */
2284 new_trie.root = trie_node_alloc(&new_trie);
2285
2286 /* Add each signing ID to the trie */
2287 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2288 cursor >= 0;
2289 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2290 {
3e170ce0 2291 uint32_t sid_size = 0;
39236c6e
A		 2292 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2293 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
39236c6e
A		 2294 uint16_t new_node_idx;
2295 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
39236c6e 2296 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
39037602 2297 if (new_node_idx == NULL_TRIE_IDX) {
39236c6e
A		 2298 insert_error = EINVAL;
2299 break;
2300 }
2301 } else {
2302 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2303 insert_error = ENOBUFS;
2304 break;
2305 }
2306 }
2307
2308 if (!insert_error) {
39037602 2309 group->signing_id_trie = new_trie;
39236c6e
A		 2310 } else {
2311 FREE(new_trie.memory, M_TEMP);
2312 }
2313
39236c6e 2314 lck_rw_done(&group->lck);
39236c6e
A		 2315}
2316
2317static int
2318flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2319{
2320 struct flow_divert_packet_header hdr;
2321 int error = 0;
2322 struct flow_divert_pcb *fd_cb;
2323
2324 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2325 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2326 error = EINVAL;
2327 goto done;
2328 }
2329
3e170ce0 2330 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
39037602 2331 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
3e170ce0
A		 2332 error = EINVAL;
2333 goto done;
2334 }
2335
39236c6e
A		 2336 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2337 if (error) {
2338 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2339 error = ENOBUFS;
2340 goto done;
2341 }
2342
2343 hdr.conn_id = ntohl(hdr.conn_id);
2344
2345 if (hdr.conn_id == 0) {
2346 switch (hdr.packet_type) {
2347 case FLOW_DIVERT_PKT_GROUP_INIT:
2348 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2349 break;
2350 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
39037602 2351 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
39236c6e
A		 2352 break;
2353 default:
2354 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2355 break;
2356 }
2357 goto done;
2358 }
2359
2360 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2361 if (fd_cb == NULL) {
2362 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2363 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2364 }
2365 goto done;
2366 }
2367
2368 switch (hdr.packet_type) {
2369 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2370 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2371 break;
2372 case FLOW_DIVERT_PKT_CLOSE:
2373 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2374 break;
2375 case FLOW_DIVERT_PKT_DATA:
2376 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2377 break;
2378 case FLOW_DIVERT_PKT_READ_NOTIFY:
2379 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2380 break;
2381 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2382 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2383 break;
2384 default:
2385 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2386 break;
2387 }
2388
2389 FDRELEASE(fd_cb);
2390
2391done:
3e170ce0 2392 mbuf_freem(packet);
39236c6e
A		 2393 return error;
2394}
2395
2396static void
2397flow_divert_close_all(struct flow_divert_group *group)
2398{
2399 struct flow_divert_pcb *fd_cb;
2400 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2401
2402 SLIST_INIT(&tmp_list);
2403
2404 lck_rw_lock_exclusive(&group->lck);
2405
2406 MBUFQ_DRAIN(&group->send_queue);
2407
2408 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2409 FDRETAIN(fd_cb);
2410 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2411 }
2412
2413 lck_rw_done(&group->lck);
2414
2415 while (!SLIST_EMPTY(&tmp_list)) {
2416 fd_cb = SLIST_FIRST(&tmp_list);
2417 FDLOCK(fd_cb);
2418 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2419 if (fd_cb->so != NULL) {
2420 socket_lock(fd_cb->so, 0);
2421 flow_divert_pcb_remove(fd_cb);
2422 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2423 fd_cb->so->so_error = ECONNABORTED;
39037602 2424 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2425 socket_unlock(fd_cb->so, 0);
2426 }
2427 FDUNLOCK(fd_cb);
2428 FDRELEASE(fd_cb);
2429 }
2430}
2431
2432void
2433flow_divert_detach(struct socket *so)
2434{
2435 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2436
2437 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2438
2439 so->so_flags &= ~SOF_FLOW_DIVERT;
2440 so->so_fd_pcb = NULL;
2441
2442 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2443
2444 if (fd_cb->group != NULL) {
2445 /* Last-ditch effort to send any buffered data */
2446 flow_divert_send_buffered_data(fd_cb, TRUE);
2447
3e170ce0
A		 2448 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2449 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 2450 /* Remove from the group */
2451 flow_divert_pcb_remove(fd_cb);
2452 }
2453
2454 socket_unlock(so, 0);
2455 FDLOCK(fd_cb);
2456 fd_cb->so = NULL;
2457 FDUNLOCK(fd_cb);
2458 socket_lock(so, 0);
2459
2460 FDRELEASE(fd_cb); /* Release the socket's reference */
2461}
2462
2463static int
2464flow_divert_close(struct socket *so)
2465{
2466 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2467
2468 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2469
2470 FDLOG0(LOG_INFO, fd_cb, "Closing");
2471
3e170ce0
A		 2472 if (SOCK_TYPE(so) == SOCK_STREAM) {
2473 soisdisconnecting(so);
2474 sbflush(&so->so_rcv);
2475 }
39236c6e
A		 2476
2477 flow_divert_send_buffered_data(fd_cb, TRUE);
2478 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2479 flow_divert_send_close_if_needed(fd_cb);
2480
2481 /* Remove from the group */
2482 flow_divert_pcb_remove(fd_cb);
2483
2484 return 0;
2485}
2486
2487static int
3e170ce0
A		 2488flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2489 sae_connid_t cid __unused)
39236c6e 2490{
3e170ce0 2491 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
39236c6e
A		 2492 return (EINVAL);
2493 }
2494
2495 return (flow_divert_close(so));
2496}
2497
2498static int
2499flow_divert_shutdown(struct socket *so)
2500{
2501 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2502
2503 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2504
2505 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2506
2507 socantsendmore(so);
2508
2509 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2510 flow_divert_send_close_if_needed(fd_cb);
2511
2512 return 0;
2513}
2514
2515static int
2516flow_divert_rcvd(struct socket *so, int flags __unused)
2517{
2518 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2519 uint32_t latest_sb_size;
2520 uint32_t read_count;
2521
2522 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2523
2524 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2525
2526 if (fd_cb->sb_size < latest_sb_size) {
2527 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2528 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2529 }
2530
2531 read_count = fd_cb->sb_size - latest_sb_size;
2532
2533 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2534
2535 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2536 fd_cb->bytes_read_by_app += read_count;
2537 fd_cb->sb_size = latest_sb_size;
2538 }
2539
2540 return 0;
2541}
2542
3e170ce0
A		 2543static int
2544flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2545{
2546 int error = 0;
2547 int port = 0;
2548
2549 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2550 if (error) {
2551 goto done;
2552 }
2553
2554 if (toaddr->sa_family == AF_INET) {
2555 port = ntohs((satosin(toaddr))->sin_port);
2556 }
2557#if INET6
2558 else {
2559 port = ntohs((satosin6(toaddr))->sin6_port);
2560 }
2561#endif
2562
2563 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2564 if (error) {
2565 goto done;
2566 }
2567
2568done:
2569 return error;
2570}
2571
2572struct sockaddr *
2573flow_divert_get_buffered_target_address(mbuf_t buffer)
2574{
2575 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2576 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2577 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2578 return toaddr;
2579 }
2580 }
2581 return NULL;
2582}
2583
2584static boolean_t
2585flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2586{
2587 switch(addr->sa_family)
2588 {
2589 case AF_INET:
2590 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2591 return FALSE;
2592 }
2593 break;
2594#if INET6
2595 case AF_INET6:
2596 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2597 return FALSE;
2598 }
2599 break;
2600#endif /* INET6 */
2601 default:
2602 return FALSE;
2603 }
2604 return TRUE;
2605}
2606
2607static errno_t
2608flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2609{
2610 int error = 0;
2611 union sockaddr_in_4_6 sin46;
2612
2613 bzero(&sin46, sizeof(sin46));
2614 if (inp->inp_vflag & INP_IPV4) {
2615 struct sockaddr_in *sin = &sin46.sin;
2616
2617 sin->sin_family = AF_INET;
2618 sin->sin_len = sizeof(*sin);
2619 sin->sin_port = inp->inp_lport;
2620 sin->sin_addr = inp->inp_laddr;
2621 } else if (inp->inp_vflag & INP_IPV6) {
2622 struct sockaddr_in6 *sin6 = &sin46.sin6;
2623
2624 sin6->sin6_len = sizeof(*sin6);
2625 sin6->sin6_family = AF_INET6;
2626 sin6->sin6_port = inp->inp_lport;
2627 sin6->sin6_addr = inp->in6p_laddr;
2628 }
2629 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2630 if (*local_socket == NULL) {
2631 error = ENOBUFS;
2632 }
2633 return (error);
2634}
2635
2636static boolean_t
2637flow_divert_has_pcb_local_address(const struct inpcb *inp)
2638{
2639 return (inp->inp_lport != 0
2640 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2641}
2642
39236c6e
A		 2643static errno_t
2644flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2645 struct sockaddr **dup)
2646{
2647 int error = 0;
2648 struct sockaddr *result;
2649 struct sockaddr_storage ss;
2650
2651 if (addr != NULL) {
2652 result = addr;
2653 } else {
2654 memset(&ss, 0, sizeof(ss));
2655 ss.ss_family = family;
2656 if (ss.ss_family == AF_INET) {
2657 ss.ss_len = sizeof(struct sockaddr_in);
2658 }
2659#if INET6
2660 else if (ss.ss_family == AF_INET6) {
2661 ss.ss_len = sizeof(struct sockaddr_in6);
2662 }
2663#endif /* INET6 */
2664 else {
2665 error = EINVAL;
2666 }
2667 result = (struct sockaddr *)&ss;
2668 }
2669
2670 if (!error) {
2671 *dup = dup_sockaddr(result, 1);
2672 if (*dup == NULL) {
2673 error = ENOBUFS;
2674 }
2675 }
2676
2677 return error;
2678}
2679
3e170ce0
A		 2680static void
2681flow_divert_disconnect_socket(struct socket *so)
2682{
2683 soisdisconnected(so);
2684 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2685 struct inpcb *inp = NULL;
2686
2687 inp = sotoinpcb(so);
2688 if (inp != NULL) {
2689#if INET6
2690 if (SOCK_CHECK_DOM(so, PF_INET6))
2691 in6_pcbdetach(inp);
2692 else
2693#endif /* INET6 */
2694 in_pcbdetach(inp);
2695 }
2696 }
2697}
2698
39236c6e
A		 2699static errno_t
2700flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2701{
2702 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2703
2704 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2705
2706 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2707 fd_cb->remote_address,
2708 sa);
2709}
2710
2711static errno_t
2712flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2713{
2714 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2715
2716 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2717
2718 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2719 fd_cb->local_address,
2720 sa);
2721}
2722
2723static errno_t
2724flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2725{
2726 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2727
2728 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2729
2730 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2731 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2732 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2733 }
2734 }
2735
2736 if (SOCK_DOM(so) == PF_INET) {
2737 return g_tcp_protosw->pr_ctloutput(so, sopt);
2738 }
2739#if INET6
2740 else if (SOCK_DOM(so) == PF_INET6) {
2741 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2742 }
2743#endif
2744 return 0;
2745}
2746
2747errno_t
2748flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2749{
2750 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2751 int error = 0;
2752 struct inpcb *inp = sotoinpcb(so);
2753 struct sockaddr_in *sinp;
fe8ab488 2754 mbuf_t connect_packet = NULL;
490019cf 2755 int do_send = 1;
39236c6e
A		 2756
2757 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2758
2759 if (fd_cb->group == NULL) {
2760 error = ENETUNREACH;
2761 goto done;
2762 }
2763
2764 if (inp == NULL) {
2765 error = EINVAL;
2766 goto done;
2767 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2768 if (so->so_error) {
2769 error = so->so_error;
2770 so->so_error = 0;
2771 } else {
2772 error = EINVAL;
2773 }
2774 goto done;
2775 }
2776
39236c6e
A		 2777 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2778 error = EALREADY;
2779 goto done;
2780 }
2781
2782 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2783 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2784 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2785 if (fd_cb->remote_address != NULL) {
2786 soisconnected(fd_cb->so);
2787 goto done;
2788 }
2789 }
2790
490019cf 2791 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 2792
490019cf
A		 2793 if (fd_cb->connect_packet == NULL) {
2794 if (to == NULL) {
2795 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2796 error = EINVAL;
2797 goto done;
fe8ab488 2798 }
fe8ab488 2799
490019cf
A		 2800 sinp = (struct sockaddr_in *)(void *)to;
2801 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2802 error = EAFNOSUPPORT;
2803 goto done;
fe8ab488
A		 2804 }
2805
490019cf
A		 2806 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2807 if (error) {
2808 goto done;
fe8ab488
A		 2809 }
2810
490019cf
A		 2811 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2812 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2813 do_send = 0;
fe8ab488
A		 2814 }
2815 } else {
490019cf
A		 2816 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2817 connect_packet = fd_cb->connect_packet;
2818 fd_cb->connect_packet = NULL;
fe8ab488
A		 2819 }
2820
490019cf
A		 2821 if (do_send) {
2822 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2823 if (error) {
2824 goto done;
2825 }
39236c6e 2826
490019cf
A		 2827 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2828 } else {
2829 fd_cb->connect_packet = connect_packet;
2830 connect_packet = NULL;
39236c6e
A		 2831 }
2832
39236c6e
A		 2833 soisconnecting(so);
2834
2835done:
fe8ab488 2836 if (error && connect_packet != NULL) {
3e170ce0 2837 mbuf_freem(connect_packet);
fe8ab488 2838 }
39236c6e
A		 2839 return error;
2840}
2841
2842static int
813fb2f6
A		 2843flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2844 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
39236c6e 2845{
39236c6e
A		 2846 struct inpcb *inp = sotoinpcb(so);
2847 int error;
2848
2849 if (inp == NULL) {
2850 return (EINVAL);
2851 }
2852
813fb2f6 2853 VERIFY(dst != NULL);
39236c6e 2854
813fb2f6 2855 error = flow_divert_connect_out(so, dst, p);
39236c6e 2856
490019cf
A		 2857 if (error != 0) {
2858 return error;
2859 }
2860
2861 /* if there is data, send it */
2862 if (auio != NULL) {
2863 user_ssize_t datalen = 0;
2864
2865 socket_unlock(so, 0);
2866
2867 VERIFY(bytes_written != NULL);
2868
2869 datalen = uio_resid(auio);
2870 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2871 socket_lock(so, 0);
2872
2873 if (error == 0 || error == EWOULDBLOCK) {
2874 *bytes_written = datalen - uio_resid(auio);
2875 }
2876
2877 /*
2878 * sosend returns EWOULDBLOCK if it's a non-blocking
2879 * socket or a timeout occured (this allows to return
2880 * the amount of queued data through sendit()).
2881 *
2882 * However, connectx() returns EINPROGRESS in case of a
2883 * blocking socket. So we change the return value here.
2884 */
2885 if (error == EWOULDBLOCK) {
2886 error = EINPROGRESS;
2887 }
2888 }
2889
39236c6e
A		 2890 if (error == 0 && pcid != NULL) {
2891 *pcid = 1; /* there is only 1 connection for a TCP */
2892 }
2893
2894 return (error);
2895}
2896
2897static int
813fb2f6
A		 2898flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
2899 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2900 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2901 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 2902{
813fb2f6 2903 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
39236c6e
A		 2904}
2905
2906#if INET6
2907static int
813fb2f6
A		 2908flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
2909 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2910 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2911 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2912{
2913 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
39236c6e
A		 2914}
2915#endif /* INET6 */
2916
2917static int
3e170ce0 2918flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
39236c6e
A		 2919 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2920 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2921 user_addr_t aux_data __unused, uint32_t *aux_len)
2922{
2923 int error = 0;
2924 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2925 struct ifnet *ifp = NULL;
2926 struct inpcb *inp = sotoinpcb(so);
2927
2928 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2929
2930 if (so->so_fd_pcb == NULL || inp == NULL) {
2931 error = EINVAL;
2932 goto out;
2933 }
2934
3e170ce0 2935 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
39236c6e
A		 2936 error = EINVAL;
2937 goto out;
2938 }
2939
2940 ifp = inp->inp_last_outifp;
2941 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2942 *soerror = so->so_error;
2943 *flags = 0;
2944
2945 if (so->so_state & SS_ISCONNECTED) {
2946 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2947 }
2948
2949 if (fd_cb->local_address == NULL) {
2950 struct sockaddr_in sin;
2951 bzero(&sin, sizeof(sin));
2952 sin.sin_len = sizeof(sin);
2953 sin.sin_family = AF_INET;
2954 *src_len = sin.sin_len;
2955 if (src != USER_ADDR_NULL) {
2956 error = copyout(&sin, src, sin.sin_len);
2957 if (error != 0) {
2958 goto out;
2959 }
2960 }
2961 } else {
2962 *src_len = fd_cb->local_address->sa_len;
2963 if (src != USER_ADDR_NULL) {
2964 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2965 if (error != 0) {
2966 goto out;
2967 }
2968 }
2969 }
2970
2971 if (fd_cb->remote_address == NULL) {
2972 struct sockaddr_in sin;
2973 bzero(&sin, sizeof(sin));
2974 sin.sin_len = sizeof(sin);
2975 sin.sin_family = AF_INET;
2976 *dst_len = sin.sin_len;
2977 if (dst != USER_ADDR_NULL) {
2978 error = copyout(&sin, dst, sin.sin_len);
2979 if (error != 0) {
2980 goto out;
2981 }
2982 }
2983 } else {
2984 *dst_len = fd_cb->remote_address->sa_len;
2985 if (dst != USER_ADDR_NULL) {
2986 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2987 if (error != 0) {
2988 goto out;
2989 }
2990 }
2991 }
2992
2993 *aux_type = 0;
2994 *aux_len = 0;
2995
2996out:
2997 return error;
2998}
2999
3000static int
3001flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3002{
3003 int error = 0;
3004
3005 switch (cmd) {
3006 case SIOCGCONNINFO32: {
3007 struct so_cinforeq32 cifr;
3008 bcopy(data, &cifr, sizeof (cifr));
3009 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3010 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3011 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3012 &cifr.scir_aux_type, cifr.scir_aux_data,
3013 &cifr.scir_aux_len);
3014 if (error == 0) {
3015 bcopy(&cifr, data, sizeof (cifr));
3016 }
3017 break;
3018 }
3019
3020 case SIOCGCONNINFO64: {
3021 struct so_cinforeq64 cifr;
3022 bcopy(data, &cifr, sizeof (cifr));
3023 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3024 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3025 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3026 &cifr.scir_aux_type, cifr.scir_aux_data,
3027 &cifr.scir_aux_len);
3028 if (error == 0) {
3029 bcopy(&cifr, data, sizeof (cifr));
3030 }
3031 break;
3032 }
3033
3034 default:
3035 error = EOPNOTSUPP;
3036 }
3037
3038 return error;
3039}
3040
3041static int
3042flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3043{
3044 int error = flow_divert_control(so, cmd, data, ifp, p);
3045
3046 if (error == EOPNOTSUPP) {
3047 error = in_control(so, cmd, data, ifp, p);
3048 }
3049
3050 return error;
3051}
3052
3053static int
3054flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3055{
3056 int error = flow_divert_control(so, cmd, data, ifp, p);
3057
3058 if (error == EOPNOTSUPP) {
3059 error = in6_control(so, cmd, data, ifp, p);
3060 }
3061
3062 return error;
3063}
3064
3065static errno_t
490019cf 3066flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e
A		 3067{
3068 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3069 int error = 0;
3070 struct inpcb *inp;
3071
3072 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3073
3074 inp = sotoinpcb(so);
3075 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3076 error = ECONNRESET;
3077 goto done;
3078 }
3079
3080 if (control && mbuf_len(control) > 0) {
3081 error = EINVAL;
3082 goto done;
3083 }
3084
3085 if (flags & MSG_OOB) {
3086 error = EINVAL;
3087 goto done; /* We don't support OOB data */
3088 }
3089
fe8ab488
A		 3090 error = flow_divert_check_no_cellular(fd_cb) ||
3091 flow_divert_check_no_expensive(fd_cb);
39236c6e
A		 3092 if (error) {
3093 goto done;
3094 }
3095
3096 /* Implicit connect */
3097 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3098 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
490019cf 3099 error = flow_divert_connect_out(so, to, p);
39236c6e
A		 3100 if (error) {
3101 goto done;
3102 }
490019cf
A		 3103
3104 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3105 /* Open up the send window so that the data will get sent right away */
3106 fd_cb->send_window = mbuf_pkthdr_len(data);
3107 }
39236c6e
A		 3108 }
3109
3110 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3111
3112 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3113 error = flow_divert_send_app_data(fd_cb, data, to);
39236c6e
A		 3114 if (error) {
3115 goto done;
3116 }
3117
3118 data = NULL;
3119
3120 if (flags & PRUS_EOF) {
3121 flow_divert_shutdown(so);
3122 }
3123
3124done:
3125 if (data) {
3e170ce0 3126 mbuf_freem(data);
39236c6e
A		 3127 }
3128 if (control) {
3129 mbuf_free(control);
3130 }
3131 return error;
3132}
3133
490019cf
A		 3134static int
3135flow_divert_preconnect(struct socket *so)
3136{
3137 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3138 int error = 0;
3139
3140 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3141 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3142 mbuf_t connect_packet = fd_cb->connect_packet;
3143 fd_cb->connect_packet = NULL;
3144
3145 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3146 if (error) {
3147 mbuf_freem(connect_packet);
3148 }
3149
3150 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3151 }
3152
5ba3f43e 3153 soclearfastopen(so);
490019cf
A		 3154
3155 return error;
3156}
3157
39236c6e
A		 3158static void
3159flow_divert_set_protosw(struct socket *so)
3160{
3161 so->so_flags |= SOF_FLOW_DIVERT;
3162 if (SOCK_DOM(so) == PF_INET) {
3163 so->so_proto = &g_flow_divert_in_protosw;
3164 }
3165#if INET6
3166 else {
3167 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3168 }
3169#endif /* INET6 */
3170}
3171
3e170ce0
A		 3172static void
3173flow_divert_set_udp_protosw(struct socket *so)
3174{
3175 so->so_flags |= SOF_FLOW_DIVERT;
3176 if (SOCK_DOM(so) == PF_INET) {
3177 so->so_proto = &g_flow_divert_in_udp_protosw;
3178 }
3179#if INET6
3180 else {
3181 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3182 }
3183#endif /* INET6 */
3184}
3185
39236c6e
A		 3186static errno_t
3187flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3188{
3189 int error = 0;
3190 struct flow_divert_pcb *fd_cb = NULL;
3191 struct ifnet *ifp = NULL;
3192 struct inpcb *inp = NULL;
3193 struct socket *old_so;
3194 mbuf_t recv_data = NULL;
3195
3196 socket_unlock(so, 0);
3197
3198 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3199
3200 /* Find the flow divert control block */
3201 lck_rw_lock_shared(&g_flow_divert_group_lck);
3202 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3203 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3204 if (group != NULL) {
3205 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3206 }
3207 }
3208 lck_rw_done(&g_flow_divert_group_lck);
3209
3210 if (fd_cb == NULL) {
3211 error = ENOENT;
3212 goto done;
3213 }
3214
3215 FDLOCK(fd_cb);
3216
3217 /* Dis-associate the flow divert control block from its current socket */
3218 old_so = fd_cb->so;
3219
3220 inp = sotoinpcb(old_so);
3221
3222 VERIFY(inp != NULL);
3223
3224 socket_lock(old_so, 0);
3e170ce0 3225 flow_divert_disconnect_socket(old_so);
39236c6e
A		 3226 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3227 old_so->so_fd_pcb = NULL;
3e170ce0
A		 3228 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3229 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3230 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3231 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3232 }
39236c6e
A		 3233 fd_cb->so = NULL;
3234 /* Save the output interface */
3235 ifp = inp->inp_last_outifp;
3236 if (old_so->so_rcv.sb_cc > 0) {
3237 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3238 sbflush(&old_so->so_rcv);
3239 }
3240 socket_unlock(old_so, 0);
3241
3242 /* Associate the new socket with the flow divert control block */
3243 socket_lock(so, 0);
3244 so->so_fd_pcb = fd_cb;
3245 inp = sotoinpcb(so);
3246 inp->inp_last_outifp = ifp;
3247 if (recv_data != NULL) {
3248 if (sbappendstream(&so->so_rcv, recv_data)) {
3249 sorwakeup(so);
3250 }
3251 }
3252 flow_divert_set_protosw(so);
3253 socket_unlock(so, 0);
3254
3255 fd_cb->so = so;
3256 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3257
3258 FDUNLOCK(fd_cb);
3259
3260done:
3261 socket_lock(so, 0);
3262
3263 if (fd_cb != NULL) {
3264 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3265 }
3266
3267 return error;
3268}
3269
3e170ce0
A		 3270errno_t
3271flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3272{
3273 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3274 struct inpcb *inp;
3275 int error = 0;
3276
3277 inp = sotoinpcb(so);
3278 if (inp == NULL) {
3279 return (EINVAL);
3280 }
3281
3282 if (fd_cb == NULL) {
3283 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3284 if (fd_ctl_unit > 0) {
3285 error = flow_divert_pcb_init(so, fd_ctl_unit);
3286 fd_cb = so->so_fd_pcb;
3287 if (error != 0 || fd_cb == NULL) {
3288 goto done;
3289 }
3290 } else {
3291 error = ENETDOWN;
3292 goto done;
3293 }
3294 }
3295 return flow_divert_data_out(so, flags, data, to, control, p);
3296
3297done:
3298 if (data) {
3299 mbuf_freem(data);
3300 }
3301 if (control) {
3302 mbuf_free(control);
3303 }
3304
3305 return error;
3306}
3307
39236c6e
A		 3308errno_t
3309flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3310{
3311 errno_t error = 0;
3312 struct flow_divert_pcb *fd_cb;
3313
3314 if (so->so_flags & SOF_FLOW_DIVERT) {
3315 return EALREADY;
3316 }
3317
3318 fd_cb = flow_divert_pcb_create(so);
3319 if (fd_cb != NULL) {
3320 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3321 if (error) {
3322 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3323 FDRELEASE(fd_cb);
3324 } else {
39236c6e
A		 3325 fd_cb->control_group_unit = ctl_unit;
3326 so->so_fd_pcb = fd_cb;
3327
3e170ce0
A		 3328 if (SOCK_TYPE(so) == SOCK_STREAM) {
3329 flow_divert_set_protosw(so);
3330 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3331 flow_divert_set_udp_protosw(so);
3332 }
39236c6e
A		 3333
3334 FDLOG0(LOG_INFO, fd_cb, "Created");
3335 }
3336 } else {
3337 error = ENOMEM;
3338 }
3339
3340 return error;
3341}
3342
3343errno_t
3344flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3345{
3346 uint32_t ctl_unit = 0;
3347 uint32_t key_unit = 0;
3348 uint32_t flow_id = 0;
3349 int error = 0;
490019cf 3350 int hmac_error = 0;
39236c6e
A		 3351 mbuf_t token = NULL;
3352
3353 if (so->so_flags & SOF_FLOW_DIVERT) {
3354 error = EALREADY;
3355 goto done;
3356 }
3357
3358 if (g_init_result) {
3359 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3360 error = ENOPROTOOPT;
3361 goto done;
3362 }
3363
3e170ce0
A		 3364 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3365 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
39236c6e
A		 3366 (SOCK_DOM(so) != PF_INET
3367#if INET6
3368 && SOCK_DOM(so) != PF_INET6
3369#endif
3370 ))
3371 {
3372 error = EINVAL;
3373 goto done;
3374 } else {
3e170ce0
A		 3375 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3376 struct tcpcb *tp = sototcpcb(so);
3377 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3378 error = EINVAL;
3379 goto done;
3380 }
39236c6e
A		 3381 }
3382 }
3383
3384 error = soopt_getm(sopt, &token);
3385 if (error) {
3386 goto done;
3387 }
3388
3389 error = soopt_mcopyin(sopt, token);
3390 if (error) {
3391 goto done;
3392 }
3393
3394 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3395 if (!error) {
3396 key_unit = ntohl(key_unit);
4d15aeb1
A		 3397 if (key_unit >= GROUP_COUNT_MAX) {
3398 key_unit = 0;
3399 }
39236c6e
A		 3400 } else if (error != ENOENT) {
3401 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3402 goto done;
3403 } else {
3404 key_unit = 0;
3405 }
3406
3407 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3408 if (error) {
3409 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3410 goto done;
3411 }
3412
3413 /* A valid kernel control unit is required */
3414 ctl_unit = ntohl(ctl_unit);
3415 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3416 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3417 error = EINVAL;
3418 goto done;
3419 }
3420
3421 socket_unlock(so, 0);
490019cf 3422 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
39236c6e
A		 3423 socket_lock(so, 0);
3424
490019cf
A		 3425 if (hmac_error && hmac_error != ENOENT) {
3426 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3427 error = hmac_error;
39236c6e
A		 3428 goto done;
3429 }
3430
3431 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3432 if (error && error != ENOENT) {
3433 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3434 goto done;
3435 }
3436
3437 if (flow_id == 0) {
3438 error = flow_divert_pcb_init(so, ctl_unit);
3439 if (error == 0) {
3440 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3441 int log_level = LOG_NOTICE;
3442
3443 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3444 sizeof(log_level), &log_level, NULL);
3445 if (error == 0) {
3446 fd_cb->log_level = log_level;
3447 }
3448 error = 0;
3449
3450 fd_cb->connect_token = token;
3451 token = NULL;
3452 }
3453 } else {
3454 error = flow_divert_attach(so, flow_id, ctl_unit);
3455 }
3456
490019cf
A		 3457 if (hmac_error == 0) {
3458 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3459 if (fd_cb != NULL) {
3460 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3461 }
3462 }
3463
39236c6e
A		 3464done:
3465 if (token != NULL) {
3466 mbuf_freem(token);
3467 }
3468
3469 return error;
3470}
3471
3472errno_t
3473flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3474{
3475 uint32_t ctl_unit;
3476 int error = 0;
3477 uint8_t hmac[SHA_DIGEST_LENGTH];
3478 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3479 mbuf_t token = NULL;
3480 struct flow_divert_group *control_group = NULL;
3481
3482 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3483 error = EINVAL;
3484 goto done;
3485 }
3486
3487 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3488
3489 if (fd_cb->group == NULL) {
3490 error = EINVAL;
3491 goto done;
3492 }
3493
3494 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3495 if (error) {
3496 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3497 goto done;
3498 }
3499
3500 ctl_unit = htonl(fd_cb->group->ctl_unit);
3501
3502 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3503 if (error) {
3504 goto done;
3505 }
3506
3507 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3508 if (error) {
3509 goto done;
3510 }
3511
490019cf
A		 3512 if (fd_cb->app_data != NULL) {
3513 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3514 if (error) {
3515 goto done;
3516 }
3517 }
3518
39236c6e
A		 3519 socket_unlock(so, 0);
3520 lck_rw_lock_shared(&g_flow_divert_group_lck);
3521
3522 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3523 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3524 {
3525 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3526 }
3527
3528 if (control_group != NULL) {
3529 lck_rw_lock_shared(&control_group->lck);
3530 ctl_unit = htonl(control_group->ctl_unit);
3531 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3532 if (!error) {
3533 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3534 }
3535 lck_rw_done(&control_group->lck);
3536 } else {
3537 error = ENOPROTOOPT;
3538 }
3539
3540 lck_rw_done(&g_flow_divert_group_lck);
3541 socket_lock(so, 0);
3542
3543 if (error) {
3544 goto done;
3545 }
3546
3547 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3548 if (error) {
3549 goto done;
3550 }
3551
39037602
A		 3552 if (sopt->sopt_val == USER_ADDR_NULL) {
3553 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3554 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3555 goto done;
3556 }
3557
39236c6e
A		 3558 error = soopt_mcopyout(sopt, token);
3559 if (error) {
3560 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3561 goto done;
3562 }
3563
3564done:
3565 if (token != NULL) {
3566 mbuf_freem(token);
3567 }
3568
3569 return error;
3570}
3571
3572static errno_t
3573flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3574{
3e170ce0 3575 struct flow_divert_group *new_group = NULL;
39236c6e
A		 3576 int error = 0;
3577
3578 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3579 error = EINVAL;
3580 goto done;
3581 }
3582
3583 *unitinfo = NULL;
3584
3585 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3586 if (new_group == NULL) {
3587 error = ENOBUFS;
3588 goto done;
3589 }
3590
3591 memset(new_group, 0, sizeof(*new_group));
3592
3593 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3594 RB_INIT(&new_group->pcb_tree);
3595 new_group->ctl_unit = sac->sc_unit;
3596 MBUFQ_INIT(&new_group->send_queue);
39037602 3597 new_group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 3598
3599 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3600
3601 if (g_flow_divert_groups == NULL) {
3602 MALLOC(g_flow_divert_groups,
3603 struct flow_divert_group **,
3604 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3605 M_TEMP,
3606 M_WAITOK | M_ZERO);
3607 }
3608
3609 if (g_flow_divert_groups == NULL) {
3610 error = ENOBUFS;
3611 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3612 error = EALREADY;
3613 } else {
3614 g_flow_divert_groups[sac->sc_unit] = new_group;
3615 g_active_group_count++;
3616 }
3617
3618 lck_rw_done(&g_flow_divert_group_lck);
3619
3620 *unitinfo = new_group;
3621
3622done:
3623 if (error != 0 && new_group != NULL) {
3624 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3625 }
3626 return error;
3627}
3628
3629static errno_t
3630flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3631{
3632 struct flow_divert_group *group = NULL;
3633 errno_t error = 0;
39236c6e
A		 3634
3635 if (unit >= GROUP_COUNT_MAX) {
3636 return EINVAL;
3637 }
3638
3639 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3640
3641 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3642
3643 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3644 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3645 g_flow_divert_groups, g_active_group_count);
3646 }
3647
3648 group = g_flow_divert_groups[unit];
3649
3650 if (group != (struct flow_divert_group *)unitinfo) {
3651 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3652 }
3653
3654 if (group != NULL) {
3655 flow_divert_close_all(group);
3656 if (group->token_key != NULL) {
3657 memset(group->token_key, 0, group->token_key_size);
3658 FREE(group->token_key, M_TEMP);
3659 group->token_key = NULL;
3660 group->token_key_size = 0;
3661 }
39037602
A		 3662
3663 /* Re-set the current trie */
3664 if (group->signing_id_trie.memory != NULL) {
3665 FREE(group->signing_id_trie.memory, M_TEMP);
3666 }
3667 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3668 group->signing_id_trie.root = NULL_TRIE_IDX;
3669
39236c6e
A		 3670 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3671 g_flow_divert_groups[unit] = NULL;
3672 g_active_group_count--;
3673 } else {
3674 error = EINVAL;
3675 }
3676
3677 if (g_active_group_count == 0) {
3678 FREE(g_flow_divert_groups, M_TEMP);
3679 g_flow_divert_groups = NULL;
3680 }
3681
39236c6e
A		 3682 lck_rw_done(&g_flow_divert_group_lck);
3683
3684 return error;
3685}
3686
3687static errno_t
3688flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3689{
3690 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3691}
3692
3693static void
3694flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3695{
3696 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3697
3698 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3699 struct flow_divert_pcb *fd_cb;
3700 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3701
3702 lck_rw_lock_shared(&g_flow_divert_group_lck);
3703 lck_rw_lock_exclusive(&group->lck);
3704
3705 while (!MBUFQ_EMPTY(&group->send_queue)) {
3706 mbuf_t next_packet;
3707 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3708 next_packet = MBUFQ_FIRST(&group->send_queue);
3709 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3710 if (error) {
3711 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3712 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3713 lck_rw_done(&group->lck);
3714 lck_rw_done(&g_flow_divert_group_lck);
3715 return;
3716 }
3717 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3718 }
3719
3720 SLIST_INIT(&tmp_list);
3721
3722 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3723 FDRETAIN(fd_cb);
3724 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3725 }
3726
3727 lck_rw_done(&group->lck);
3728
3729 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3730 FDLOCK(fd_cb);
3731 if (fd_cb->so != NULL) {
3732 socket_lock(fd_cb->so, 0);
3733 if (fd_cb->group != NULL) {
3734 flow_divert_send_buffered_data(fd_cb, FALSE);
3735 }
3736 socket_unlock(fd_cb->so, 0);
3737 }
3738 FDUNLOCK(fd_cb);
3739 FDRELEASE(fd_cb);
3740 }
3741
3742 lck_rw_done(&g_flow_divert_group_lck);
3743 }
3744}
3745
3746static int
3747flow_divert_kctl_init(void)
3748{
3749 struct kern_ctl_reg ctl_reg;
3750 int result;
3751
3752 memset(&ctl_reg, 0, sizeof(ctl_reg));
3753
fe8ab488 3754 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
39236c6e
A		 3755 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3756 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3757 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3758 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3759
3760 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3761 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3762 ctl_reg.ctl_send = flow_divert_kctl_send;
3763 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3764
3765 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3766
3767 if (result) {
3768 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3769 return result;
3770 }
3771
3772 return 0;
3773}
3774
3775void
3776flow_divert_init(void)
3777{
3778 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 3779 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 3780
3781 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3782
3783 VERIFY(g_tcp_protosw != NULL);
3784
3785 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3786 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3787
3788 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3789 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3790 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3791 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3792 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3793 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3794 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3795 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3796 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3797 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3798 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3799
3800 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3801 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3802
3803 /*
3804 * Socket filters shouldn't attach/detach to/from this protosw
3805 * since pr_protosw is to be used instead, which points to the
3806 * real protocol; if they do, it is a bug and we should panic.
3807 */
3808 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3809 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3810 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3811 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3812
3e170ce0
A		 3813 /* UDP */
3814 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3815 VERIFY(g_udp_protosw != NULL);
3816
3817 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3818 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3819
3820 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3821 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3822 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3823 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3824 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3825 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3826 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3827 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3828 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3829 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3830 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3831 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3832 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3833
3834 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3835 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3836
3837 /*
3838 * Socket filters shouldn't attach/detach to/from this protosw
3839 * since pr_protosw is to be used instead, which points to the
3840 * real protocol; if they do, it is a bug and we should panic.
3841 */
3842 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3843 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3844 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3845 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3846
39236c6e
A		 3847#if INET6
3848 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3849
3850 VERIFY(g_tcp6_protosw != NULL);
3851
3852 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3853 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3854
3855 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3856 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3857 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3858 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3859 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3860 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3861 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3862 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3863 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3864 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3865 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3866
3867 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3868 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3869 /*
3870 * Socket filters shouldn't attach/detach to/from this protosw
3871 * since pr_protosw is to be used instead, which points to the
3872 * real protocol; if they do, it is a bug and we should panic.
3873 */
3874 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3875 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3876 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3877 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 3878
3879 /* UDP6 */
3880 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3881
3882 VERIFY(g_udp6_protosw != NULL);
3883
3884 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3885 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3886
3887 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3888 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3889 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3890 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3891 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3892 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3893 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3894 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3895 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3896 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3897 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3898 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3899 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3900
3901 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3902 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3903 /*
3904 * Socket filters shouldn't attach/detach to/from this protosw
3905 * since pr_protosw is to be used instead, which points to the
3906 * real protocol; if they do, it is a bug and we should panic.
3907 */
3908 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3909 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3910 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3911 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
39236c6e
A		 3912#endif /* INET6 */
3913
3914 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3915 if (flow_divert_grp_attr == NULL) {
3916 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3917 g_init_result = ENOMEM;
3918 goto done;
3919 }
3920
3921 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3922 if (flow_divert_mtx_grp == NULL) {
3923 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3924 g_init_result = ENOMEM;
3925 goto done;
3926 }
3927
3928 flow_divert_mtx_attr = lck_attr_alloc_init();
3929 if (flow_divert_mtx_attr == NULL) {
3930 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3931 g_init_result = ENOMEM;
3932 goto done;
3933 }
3934
3935 g_init_result = flow_divert_kctl_init();
3936 if (g_init_result) {
3937 goto done;
3938 }
3939
3940 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3941
39236c6e
A		 3942done:
3943 if (g_init_result != 0) {
3944 if (flow_divert_mtx_attr != NULL) {
3945 lck_attr_free(flow_divert_mtx_attr);
3946 flow_divert_mtx_attr = NULL;
3947 }
3948 if (flow_divert_mtx_grp != NULL) {
3949 lck_grp_free(flow_divert_mtx_grp);
3950 flow_divert_mtx_grp = NULL;
3951 }
3952 if (flow_divert_grp_attr != NULL) {
3953 lck_grp_attr_free(flow_divert_grp_attr);
3954 flow_divert_grp_attr = NULL;
3955 }
3956
3957 if (g_flow_divert_kctl_ref != NULL) {
3958 ctl_deregister(g_flow_divert_kctl_ref);
3959 g_flow_divert_kctl_ref = NULL;
3960 }
3961 }
3962}
mirror of XNU