]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-4903.231.4.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
5ba3f43e 2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
48#include <net/if_var.h>
49#include <net/route.h>
50#include <net/flowhash.h>
51#include <net/ntstat.h>
52#include <netinet/in.h>
53#include <netinet/in_var.h>
54#include <netinet/tcp.h>
55#include <netinet/tcp_var.h>
56#include <netinet/tcp_fsm.h>
57#include <netinet/flow_divert.h>
58#include <netinet/flow_divert_proto.h>
59#if INET6
3e170ce0 60#include <netinet6/in6_pcb.h>
39236c6e
A		 61#include <netinet6/ip6protosw.h>
62#endif /* INET6 */
63#include <dev/random/randomdev.h>
64#include <libkern/crypto/sha1.h>
65#include <libkern/crypto/crypto_internal.h>
39037602 66#include <os/log.h>
39236c6e
A		 67
68#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69#define FLOW_DIVERT_READ_CLOSED 0x00000002
70#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73#define FLOW_DIVERT_TRANSFERRED 0x00000020
490019cf 74#define FLOW_DIVERT_HAS_HMAC 0x00000040
39236c6e 75
39037602
A		 76#define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
39236c6e 78
39037602
A		 79#define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
39236c6e
A		 81
82#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83#define FDRELEASE(pcb) \
84 do { \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
87 } \
88 } while (0)
89
90#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
92
39037602 93#define FD_CTL_SENDBUFF_SIZE (128 * 1024)
39236c6e
A		 94#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
95
96#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
97
98#define GROUP_COUNT_MAX 32
99#define FLOW_DIVERT_MAX_NAME_SIZE 4096
100#define FLOW_DIVERT_MAX_KEY_SIZE 1024
ecc0ceb4 101#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e 102
39236c6e
A		 103struct flow_divert_trie_node
104{
105 uint16_t start;
106 uint16_t length;
107 uint16_t child_map;
39236c6e
A		 108};
109
110#define CHILD_MAP_SIZE 256
111#define NULL_TRIE_IDX 0xffff
112#define TRIE_NODE(t, i) ((t)->nodes[(i)])
113#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
114#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
115
116static struct flow_divert_pcb nil_pcb;
117
118decl_lck_rw_data(static, g_flow_divert_group_lck);
119static struct flow_divert_group **g_flow_divert_groups = NULL;
120static uint32_t g_active_group_count = 0;
39236c6e
A		 121
122static lck_grp_attr_t *flow_divert_grp_attr = NULL;
123static lck_attr_t *flow_divert_mtx_attr = NULL;
124static lck_grp_t *flow_divert_mtx_grp = NULL;
125static errno_t g_init_result = 0;
126
127static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
128
129static struct protosw g_flow_divert_in_protosw;
130static struct pr_usrreqs g_flow_divert_in_usrreqs;
3e170ce0
A		 131static struct protosw g_flow_divert_in_udp_protosw;
132static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
39236c6e
A		 133#if INET6
134static struct ip6protosw g_flow_divert_in6_protosw;
135static struct pr_usrreqs g_flow_divert_in6_usrreqs;
3e170ce0
A		 136static struct ip6protosw g_flow_divert_in6_udp_protosw;
137static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
39236c6e
A		 138#endif /* INET6 */
139
140static struct protosw *g_tcp_protosw = NULL;
141static struct ip6protosw *g_tcp6_protosw = NULL;
3e170ce0
A		 142static struct protosw *g_udp_protosw = NULL;
143static struct ip6protosw *g_udp6_protosw = NULL;
144
145static errno_t
146flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
147
148static errno_t
149flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
150
151static boolean_t
152flow_divert_is_sockaddr_valid(struct sockaddr *addr);
153
154static int
155flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
156
157struct sockaddr *
158flow_divert_get_buffered_target_address(mbuf_t buffer);
159
160static boolean_t
161flow_divert_has_pcb_local_address(const struct inpcb *inp);
162
163static void
164flow_divert_disconnect_socket(struct socket *so);
39236c6e 165
39037602
A		 166static inline uint8_t
167flow_divert_syslog_type_to_oslog_type(int syslog_type)
168{
169 switch (syslog_type) {
170 case LOG_ERR: return OS_LOG_TYPE_ERROR;
171 case LOG_INFO: return OS_LOG_TYPE_INFO;
172 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
173 default: return OS_LOG_TYPE_DEFAULT;
174 }
175}
176
39236c6e
A		 177static inline int
178flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
179{
180 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
181}
182
183RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
184RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
185
186static const char *
187flow_divert_packet_type2str(uint8_t packet_type)
188{
189 switch (packet_type) {
190 case FLOW_DIVERT_PKT_CONNECT:
191 return "connect";
192 case FLOW_DIVERT_PKT_CONNECT_RESULT:
193 return "connect result";
194 case FLOW_DIVERT_PKT_DATA:
195 return "data";
196 case FLOW_DIVERT_PKT_CLOSE:
197 return "close";
198 case FLOW_DIVERT_PKT_READ_NOTIFY:
199 return "read notification";
200 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
201 return "properties update";
39236c6e
A		 202 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
203 return "app map create";
204 default:
205 return "unknown";
206 }
207}
208
209static struct flow_divert_pcb *
210flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
211{
212 struct flow_divert_pcb key_item;
213 struct flow_divert_pcb *fd_cb = NULL;
214
215 key_item.hash = hash;
216
217 lck_rw_lock_shared(&group->lck);
218 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
219 FDRETAIN(fd_cb);
220 lck_rw_done(&group->lck);
221
222 return fd_cb;
223}
224
225static errno_t
226flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
227{
3e170ce0 228 errno_t error = 0;
39236c6e
A		 229 struct flow_divert_pcb *exist = NULL;
230 struct flow_divert_group *group;
231 static uint32_t g_nextkey = 1;
232 static uint32_t g_hash_seed = 0;
39236c6e
A		 233 int try_count = 0;
234
235 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
236 return EINVAL;
237 }
238
239 socket_unlock(fd_cb->so, 0);
240 lck_rw_lock_shared(&g_flow_divert_group_lck);
241
242 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
243 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
244 error = ENETUNREACH;
245 goto done;
246 }
247
248 group = g_flow_divert_groups[ctl_unit];
249 if (group == NULL) {
250 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
251 error = ENETUNREACH;
252 goto done;
253 }
254
255 socket_lock(fd_cb->so, 0);
256
257 do {
258 uint32_t key[2];
259 uint32_t idx;
260
261 key[0] = g_nextkey++;
262 key[1] = RandomULong();
263
264 if (g_hash_seed == 0) {
265 g_hash_seed = RandomULong();
266 }
267
268 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
269
270 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
271 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
272 if (curr_group != NULL && curr_group != group) {
273 lck_rw_lock_shared(&curr_group->lck);
274 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
275 lck_rw_done(&curr_group->lck);
276 if (exist != NULL) {
277 break;
278 }
279 }
280 }
281
282 if (exist == NULL) {
283 lck_rw_lock_exclusive(&group->lck);
284 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
285 lck_rw_done(&group->lck);
286 }
287 } while (exist != NULL && try_count++ < 3);
288
289 if (exist == NULL) {
290 fd_cb->group = group;
291 FDRETAIN(fd_cb); /* The group now has a reference */
292 } else {
293 fd_cb->hash = 0;
3e170ce0 294 error = EEXIST;
39236c6e
A		 295 }
296
297 socket_unlock(fd_cb->so, 0);
298
299done:
300 lck_rw_done(&g_flow_divert_group_lck);
301 socket_lock(fd_cb->so, 0);
302
3e170ce0 303 return error;
39236c6e
A		 304}
305
306static struct flow_divert_pcb *
307flow_divert_pcb_create(socket_t so)
308{
309 struct flow_divert_pcb *new_pcb = NULL;
310
311 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
312 if (new_pcb == NULL) {
313 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
314 return NULL;
315 }
316
317 memset(new_pcb, 0, sizeof(*new_pcb));
318
319 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
320 new_pcb->so = so;
321 new_pcb->log_level = nil_pcb.log_level;
322
323 FDRETAIN(new_pcb); /* Represents the socket's reference */
324
325 return new_pcb;
326}
327
328static void
329flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
330{
331 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
332 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
333
334 if (fd_cb->local_address != NULL) {
335 FREE(fd_cb->local_address, M_SONAME);
336 }
337 if (fd_cb->remote_address != NULL) {
338 FREE(fd_cb->remote_address, M_SONAME);
339 }
340 if (fd_cb->connect_token != NULL) {
341 mbuf_freem(fd_cb->connect_token);
342 }
490019cf
A		 343 if (fd_cb->connect_packet != NULL) {
344 mbuf_freem(fd_cb->connect_packet);
345 }
346 if (fd_cb->app_data != NULL) {
347 FREE(fd_cb->app_data, M_TEMP);
348 }
39236c6e
A		 349 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
350}
351
352static void
353flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
354{
355 if (fd_cb->group != NULL) {
356 struct flow_divert_group *group = fd_cb->group;
357 lck_rw_lock_exclusive(&group->lck);
358 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
359 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
360 fd_cb->group = NULL;
361 FDRELEASE(fd_cb); /* Release the group's reference */
362 lck_rw_done(&group->lck);
363 }
364}
365
366static int
367flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
368{
369 struct flow_divert_packet_header hdr;
370 int error = 0;
371
372 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
373 if (error) {
374 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
375 return error;
376 }
377
378 hdr.packet_type = packet_type;
379 hdr.conn_id = htonl(fd_cb->hash);
380
381 /* Lay down the header */
382 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
383 if (error) {
384 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
385 mbuf_freem(*packet);
386 *packet = NULL;
387 return error;
388 }
389
390 return 0;
391}
392
393static int
3e170ce0 394flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 395{
3e170ce0
A		 396 uint32_t net_length = htonl(length);
397 int error = 0;
39236c6e
A		 398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
400 if (error) {
401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
406 if (error) {
39037602 407 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
39236c6e
A		 408 return error;
409 }
410
411 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
412 if (error) {
413 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
414 return error;
415 }
416
417 return error;
418}
419
420static int
421flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
422{
3e170ce0
A		 423 size_t cursor = offset;
424 int error = 0;
425 uint32_t curr_length;
426 uint8_t curr_type;
39236c6e
A		 427
428 *err = 0;
429
430 do {
431 if (!next) {
432 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
433 if (error) {
434 *err = ENOENT;
435 return -1;
436 }
437 } else {
438 next = 0;
439 curr_type = FLOW_DIVERT_TLV_NIL;
440 }
441
442 if (curr_type != type) {
443 cursor += sizeof(curr_type);
444 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
445 if (error) {
446 *err = error;
447 return -1;
448 }
449
450 cursor += (sizeof(curr_length) + ntohl(curr_length));
451 }
452 } while (curr_type != type);
453
454 return cursor;
455}
456
457static int
3e170ce0 458flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 459{
3e170ce0
A		 460 int error = 0;
461 uint32_t length;
462 int tlv_offset;
39236c6e
A		 463
464 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
465 if (tlv_offset < 0) {
466 return error;
467 }
468
469 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
470 if (error) {
471 return error;
472 }
473
474 length = ntohl(length);
475
476 if (val_size != NULL) {
477 *val_size = length;
478 }
479
480 if (buff != NULL && buff_len > 0) {
481 size_t to_copy = (length < buff_len) ? length : buff_len;
482 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
483 if (error) {
484 return error;
485 }
486 }
487
488 return 0;
489}
490
491static int
492flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
493{
494 mbuf_t curr_mbuf = packet;
495
496 if (g_crypto_funcs == NULL || group->token_key == NULL) {
497 return ENOPROTOOPT;
498 }
499
500 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
501 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
502
503 while (curr_mbuf != NULL) {
504 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
505 curr_mbuf = mbuf_next(curr_mbuf);
506 }
507
508 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
509
510 return 0;
511}
512
513static int
514flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
515{
516 int error = 0;
517 struct flow_divert_group *group = NULL;
518 int hmac_offset;
519 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
520 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
521 mbuf_t tail;
522
523 lck_rw_lock_shared(&g_flow_divert_group_lck);
524
525 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
526 group = g_flow_divert_groups[ctl_unit];
527 }
528
529 if (group == NULL) {
530 lck_rw_done(&g_flow_divert_group_lck);
531 return ENOPROTOOPT;
532 }
533
534 lck_rw_lock_shared(&group->lck);
535
536 if (group->token_key == NULL) {
537 error = ENOPROTOOPT;
538 goto done;
539 }
540
541 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
542 if (hmac_offset < 0) {
543 goto done;
544 }
545
546 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
547 if (error) {
548 goto done;
549 }
550
551 /* Chop off the HMAC TLV */
552 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
553 if (error) {
554 goto done;
555 }
556
557 mbuf_free(tail);
558
559 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
560 if (error) {
561 goto done;
562 }
563
564 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
565 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
566 error = EINVAL;
567 goto done;
568 }
569
570done:
571 lck_rw_done(&group->lck);
572 lck_rw_done(&g_flow_divert_group_lck);
573 return error;
574}
575
576static void
577flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
578{
579 struct inpcb *inp = NULL;
580 struct ifnet *ifp = NULL;
581 Boolean cell = FALSE;
582 Boolean wifi = FALSE;
fe8ab488 583 Boolean wired = FALSE;
39236c6e
A		 584
585 inp = sotoinpcb(fd_cb->so);
586 if (inp == NULL) {
587 return;
588 }
589
590 ifp = inp->inp_last_outifp;
591 if (ifp != NULL) {
592 cell = IFNET_IS_CELLULAR(ifp);
593 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 594 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e
A		 595 }
596
597 if (send) {
fe8ab488
A		 598 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
599 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 600 } else {
fe8ab488
A		 601 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
602 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e 603 }
5ba3f43e 604 inp_set_activity_bitmap(inp);
39236c6e
A		 605}
606
607static errno_t
608flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
609{
610 struct inpcb *inp = NULL;
39236c6e
A		 611
612 inp = sotoinpcb(fd_cb->so);
fe8ab488
A		 613 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
614 IFNET_IS_CELLULAR(inp->inp_last_outifp))
615 return EHOSTUNREACH;
616
617 return 0;
618}
619
620static errno_t
621flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
622{
623 struct inpcb *inp = NULL;
624
625 inp = sotoinpcb(fd_cb->so);
626 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
627 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
628 return EHOSTUNREACH;
39236c6e
A		 629
630 return 0;
631}
632
633static void
634flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
635{
636 if (how != SHUT_RD) {
637 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
638 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
639 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
640 /* If the tunnel is not accepting writes any more, then flush the send buffer */
641 sbflush(&fd_cb->so->so_snd);
642 }
643 }
644 if (how != SHUT_WR) {
645 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
646 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
647 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
648 }
649 }
650}
651
652static uint16_t
653trie_node_alloc(struct flow_divert_trie *trie)
654{
655 if (trie->nodes_free_next < trie->nodes_count) {
656 uint16_t node_idx = trie->nodes_free_next++;
657 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
658 return node_idx;
659 } else {
660 return NULL_TRIE_IDX;
661 }
662}
663
664static uint16_t
665trie_child_map_alloc(struct flow_divert_trie *trie)
666{
667 if (trie->child_maps_free_next < trie->child_maps_count) {
668 return trie->child_maps_free_next++;
669 } else {
670 return NULL_TRIE_IDX;
671 }
672}
673
674static uint16_t
675trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
676{
677 uint16_t start = trie->bytes_free_next;
678 if (start + bytes_size <= trie->bytes_count) {
679 if (start != bytes_idx) {
680 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
681 }
682 trie->bytes_free_next += bytes_size;
683 return start;
684 } else {
685 return NULL_TRIE_IDX;
686 }
687}
688
689static uint16_t
690flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
691{
692 uint16_t current = trie->root;
693 uint16_t child = trie->root;
694 uint16_t string_end = string_start + string_len;
695 uint16_t string_idx = string_start;
696 uint16_t string_remainder = string_len;
697
698 while (child != NULL_TRIE_IDX) {
699 uint16_t parent = current;
700 uint16_t node_idx;
701 uint16_t current_end;
702
703 current = child;
704 child = NULL_TRIE_IDX;
705
706 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
707
708 for (node_idx = TRIE_NODE(trie, current).start;
709 node_idx < current_end &&
710 string_idx < string_end &&
711 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
712 node_idx++, string_idx++);
713
714 string_remainder = string_end - string_idx;
715
716 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
717 /*
718 * We did not reach the end of the current node's string.
719 * We need to split the current node into two:
720 * 1. A new node that contains the prefix of the node that matches
721 * the prefix of the string being inserted.
722 * 2. The current node modified to point to the remainder
723 * of the current node's string.
724 */
725 uint16_t prefix = trie_node_alloc(trie);
726 if (prefix == NULL_TRIE_IDX) {
727 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
728 return NULL_TRIE_IDX;
729 }
730
731 /*
732 * Prefix points to the portion of the current nodes's string that has matched
733 * the input string thus far.
734 */
735 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
736 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
737
738 /*
739 * Prefix has the current node as the child corresponding to the first byte
740 * after the split.
741 */
742 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
743 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
744 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
745 return NULL_TRIE_IDX;
746 }
747 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
748
749 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
750 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
751
752 /* Current node is adjusted to point to the remainder */
753 TRIE_NODE(trie, current).start = node_idx;
754 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
755
756 /* We want to insert the new leaf (if any) as a child of the prefix */
757 current = prefix;
758 }
759
760 if (string_remainder > 0) {
761 /*
762 * We still have bytes in the string that have not been matched yet.
763 * If the current node has children, iterate to the child corresponding
764 * to the next byte in the string.
765 */
766 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
767 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
768 }
769 }
770 } /* while (child != NULL_TRIE_IDX) */
771
772 if (string_remainder > 0) {
773 /* Add a new leaf containing the remainder of the string */
774 uint16_t leaf = trie_node_alloc(trie);
775 if (leaf == NULL_TRIE_IDX) {
776 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
777 return NULL_TRIE_IDX;
778 }
779
780 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
781 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
782 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
783 return NULL_TRIE_IDX;
784 }
785 TRIE_NODE(trie, leaf).length = string_remainder;
786
787 /* Set the new leaf as the child of the current node */
788 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
789 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
790 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
791 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
792 return NULL_TRIE_IDX;
793 }
794 }
795 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
796 current = leaf;
797 } /* else duplicate or this string is a prefix of one of the existing strings */
798
799 return current;
800}
801
3e170ce0 802#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 803static uint16_t
3e170ce0 804flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
39236c6e
A		 805{
806 uint16_t current = trie->root;
807 uint16_t string_idx = 0;
808
809 while (current != NULL_TRIE_IDX) {
810 uint16_t next = NULL_TRIE_IDX;
811 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
812 uint16_t node_idx;
813
814 for (node_idx = TRIE_NODE(trie, current).start;
815 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
816 node_idx++, string_idx++);
817
818 if (node_idx == node_end) {
819 if (string_bytes[string_idx] == '\0') {
820 return current; /* Got an exact match */
3e170ce0
A		 821 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
822 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
823 string_bytes[string_idx] = '\0';
824 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 825 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
826 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
827 }
828 }
829 current = next;
830 }
831
832 return NULL_TRIE_IDX;
833}
834
490019cf
A		 835struct uuid_search_info {
836 uuid_t target_uuid;
837 char *found_signing_id;
838 boolean_t found_multiple_signing_ids;
839 proc_t found_proc;
840};
841
842static int
843flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
844{
845 struct uuid_search_info *info = (struct uuid_search_info *)arg;
846 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
847
848 if (info->found_signing_id != NULL) {
849 if (!info->found_multiple_signing_ids) {
850 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
851 info->found_proc = p;
852 result = PROC_CLAIMED_DONE;
853 } else {
854 uuid_string_t uuid_str;
855 uuid_unparse(info->target_uuid, uuid_str);
856 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
857 }
858 FREE(info->found_signing_id, M_TEMP);
859 info->found_signing_id = NULL;
860 }
861
862 if (result == PROC_RETURNED_DONE) {
863 uuid_string_t uuid_str;
864 uuid_unparse(info->target_uuid, uuid_str);
865 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
866 }
867
868 return result;
869}
870
871static int
872flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
873{
874 struct uuid_search_info *info = (struct uuid_search_info *)arg;
875 int include = 0;
876
877 if (info->found_multiple_signing_ids) {
878 return include;
879 }
880
881 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
882 if (include) {
883 const char *signing_id = cs_identity_get(p);
884 if (signing_id != NULL) {
885 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
886 size_t signing_id_size = strlen(signing_id) + 1;
887 if (info->found_signing_id == NULL) {
888 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
889 memcpy(info->found_signing_id, signing_id, signing_id_size);
890 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
891 info->found_multiple_signing_ids = TRUE;
892 }
893 } else {
894 info->found_multiple_signing_ids = TRUE;
895 }
896 include = !info->found_multiple_signing_ids;
897 }
898
899 return include;
900}
901
902static proc_t
903flow_divert_find_proc_by_uuid(uuid_t uuid)
904{
905 struct uuid_search_info info;
906
907 if (LOG_INFO <= nil_pcb.log_level) {
908 uuid_string_t uuid_str;
909 uuid_unparse(uuid, uuid_str);
910 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
911 }
912
913 memset(&info, 0, sizeof(info));
914 info.found_proc = PROC_NULL;
915 uuid_copy(info.target_uuid, uuid);
916
917 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
918
919 return info.found_proc;
920}
921
39236c6e 922static int
490019cf 923flow_divert_get_src_proc(struct socket *so, proc_t *proc)
39236c6e
A		 924{
925 int release = 0;
926
490019cf
A		 927 if (so->so_flags & SOF_DELEGATED) {
928 if ((*proc)->p_pid != so->e_pid) {
929 *proc = proc_find(so->e_pid);
930 release = 1;
931 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
932 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
933 release = 1;
934 }
39236c6e
A		 935 } else if (*proc == PROC_NULL) {
936 *proc = current_proc();
937 }
938
939 if (*proc != PROC_NULL) {
940 if ((*proc)->p_pid == 0) {
941 if (release) {
942 proc_rele(*proc);
943 }
944 release = 0;
945 *proc = PROC_NULL;
946 }
947 }
948
949 return release;
950}
951
952static int
953flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
954{
955 int error;
956
957 if (fd_cb->group == NULL) {
958 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 959 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 960 return ECONNABORTED;
961 }
962
963 lck_rw_lock_shared(&fd_cb->group->lck);
964
965 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
966 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
967 } else {
968 error = ENOBUFS;
969 }
970
971 if (error == ENOBUFS) {
972 if (enqueue) {
973 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
974 lck_rw_lock_exclusive(&fd_cb->group->lck);
975 }
976 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
977 error = 0;
978 }
979 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
980 }
981
982 lck_rw_done(&fd_cb->group->lck);
983
984 return error;
985}
986
987static int
490019cf 988flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 989{
39236c6e 990 int error = 0;
3e170ce0 991 int flow_type = 0;
490019cf
A		 992 char *signing_id = NULL;
993 int free_signing_id = 0;
994 mbuf_t connect_packet = NULL;
39037602
A		 995 proc_t src_proc = p;
996 int release_proc = 0;
490019cf
A		 997
998 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
999 if (error) {
1000 goto done;
1001 }
1002
1003 error = EPERM;
1004
1005 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1006 uint32_t sid_size = 0;
1007 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1008 if (find_error == 0 && sid_size > 0) {
1009 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1010 if (signing_id != NULL) {
1011 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1012 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1013 free_signing_id = 1;
1014 }
1015 }
1016 }
1017
1018 socket_unlock(so, 0);
39037602
A		 1019
1020 if (signing_id == NULL) {
1021 release_proc = flow_divert_get_src_proc(so, &src_proc);
1022 if (src_proc != PROC_NULL) {
1023 proc_lock(src_proc);
1024 if (src_proc->p_csflags & (CS_VALID|CS_DEBUGGED)) {
1025 const char * cs_id;
1026 cs_id = cs_identity_get(src_proc);
1027 signing_id = __DECONST(char *, cs_id);
490019cf 1028 } else {
39037602 1029 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
490019cf
A		 1030 }
1031 } else {
39037602 1032 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
490019cf 1033 }
39037602
A		 1034 } else {
1035 src_proc = PROC_NULL;
1036 }
490019cf 1037
39037602
A		 1038 if (signing_id != NULL) {
1039 uint16_t result = NULL_TRIE_IDX;
1040 lck_rw_lock_shared(&fd_cb->group->lck);
5ba3f43e
A		 1041 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1042 result = 1;
1043 } else {
1044 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1045 }
39037602
A		 1046 lck_rw_done(&fd_cb->group->lck);
1047 if (result != NULL_TRIE_IDX) {
1048 error = 0;
1049 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
490019cf 1050
39037602
A		 1051 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1052 if (error == 0) {
1053 if (src_proc != PROC_NULL) {
1054 unsigned char cdhash[SHA1_RESULTLEN];
1055 error = proc_getcdhash(src_proc, cdhash);
1056 if (error == 0) {
1057 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1058 if (error) {
1059 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
490019cf 1060 }
39037602
A		 1061 } else {
1062 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
490019cf 1063 }
490019cf
A		 1064 }
1065 } else {
39037602 1066 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
490019cf
A		 1067 }
1068 } else {
39037602 1069 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
490019cf 1070 }
39037602
A		 1071 } else {
1072 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
5c9f4661
A		 1073 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1074 error = 0;
1075 }
39037602 1076 }
490019cf 1077
39037602
A		 1078 if (src_proc != PROC_NULL) {
1079 proc_unlock(src_proc);
1080 if (release_proc) {
1081 proc_rele(src_proc);
490019cf 1082 }
490019cf
A		 1083 }
1084 socket_lock(so, 0);
1085
1086 if (free_signing_id) {
1087 FREE(signing_id, M_TEMP);
1088 }
1089
1090 if (error) {
1091 goto done;
1092 }
39236c6e 1093
39236c6e
A		 1094 error = flow_divert_packet_append_tlv(connect_packet,
1095 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1096 sizeof(fd_cb->so->so_traffic_class),
1097 &fd_cb->so->so_traffic_class);
1098 if (error) {
1099 goto done;
1100 }
1101
3e170ce0
A		 1102 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1103 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1104 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1105 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1106 } else {
1107 error = EINVAL;
1108 goto done;
1109 }
1110 error = flow_divert_packet_append_tlv(connect_packet,
1111 FLOW_DIVERT_TLV_FLOW_TYPE,
1112 sizeof(flow_type),
1113 &flow_type);
1114
1115 if (error) {
1116 goto done;
1117 }
1118
39236c6e
A		 1119 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1120 error = flow_divert_packet_append_tlv(connect_packet,
1121 FLOW_DIVERT_TLV_PID,
1122 sizeof(fd_cb->so->e_pid),
1123 &fd_cb->so->e_pid);
1124 if (error) {
1125 goto done;
1126 }
1127
1128 error = flow_divert_packet_append_tlv(connect_packet,
1129 FLOW_DIVERT_TLV_UUID,
1130 sizeof(fd_cb->so->e_uuid),
1131 &fd_cb->so->e_uuid);
1132 if (error) {
1133 goto done;
1134 }
1135 } else {
1136 error = flow_divert_packet_append_tlv(connect_packet,
1137 FLOW_DIVERT_TLV_PID,
1138 sizeof(fd_cb->so->e_pid),
1139 &fd_cb->so->last_pid);
1140 if (error) {
1141 goto done;
1142 }
1143
1144 error = flow_divert_packet_append_tlv(connect_packet,
1145 FLOW_DIVERT_TLV_UUID,
1146 sizeof(fd_cb->so->e_uuid),
1147 &fd_cb->so->last_uuid);
1148 if (error) {
1149 goto done;
1150 }
1151 }
1152
1153 if (fd_cb->connect_token != NULL) {
1154 unsigned int token_len = m_length(fd_cb->connect_token);
1155 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1156 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1157 fd_cb->connect_token = NULL;
1158 } else {
1159 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
39236c6e
A		 1160
1161 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1162 if (error) {
1163 goto done;
1164 }
1165
3e170ce0 1166 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1167 if (error) {
1168 goto done;
1169 }
39236c6e
A		 1170 }
1171
490019cf
A		 1172 if (fd_cb->local_address != NULL) {
1173 error = EALREADY;
1174 goto done;
1175 } else {
1176 struct inpcb *inp = sotoinpcb(so);
1177 if (flow_divert_has_pcb_local_address(inp)) {
1178 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1179 if (error) {
1180 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1181 goto done;
1182 }
1183 }
1184 }
1185
3e170ce0
A		 1186 if (fd_cb->local_address != NULL) {
1187 /* socket is bound. */
490019cf 1188 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
e8c3f781 1189 fd_cb->local_address->sa_len, fd_cb->local_address);
490019cf
A		 1190 if (error) {
1191 goto done;
1192 }
1193 }
3e170ce0 1194
490019cf
A		 1195 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1196 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1197 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1198 if (error) {
1199 goto done;
1200 }
39236c6e
A		 1201 }
1202
1203done:
490019cf
A		 1204 if (!error) {
1205 *out_connect_packet = connect_packet;
1206 } else if (connect_packet != NULL) {
1207 mbuf_freem(connect_packet);
1208 }
1209
39236c6e
A		 1210 return error;
1211}
1212
1213static int
1214flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1215{
1216 int error = 0;
1217 mbuf_t packet = NULL;
1218 int rbuff_space = 0;
1219
1220 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1221 if (error) {
1222 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1223 goto done;
1224 }
1225
3e170ce0 1226 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1227 if (rbuff_space < 0) {
1228 rbuff_space = 0;
1229 }
1230 rbuff_space = htonl(rbuff_space);
1231 error = flow_divert_packet_append_tlv(packet,
1232 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1233 sizeof(rbuff_space),
1234 &rbuff_space);
1235 if (error) {
1236 goto done;
1237 }
1238
1239 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1240 if (error) {
1241 goto done;
1242 }
1243
1244done:
1245 if (error && packet != NULL) {
3e170ce0 1246 mbuf_freem(packet);
39236c6e
A		 1247 }
1248
1249 return error;
1250}
1251
1252static int
1253flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1254{
1255 int error = 0;
1256 mbuf_t packet = NULL;
1257 uint32_t zero = 0;
1258
1259 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1260 if (error) {
1261 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1262 goto done;
1263 }
1264
1265 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1266 if (error) {
1267 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1268 goto done;
1269 }
1270
1271 how = htonl(how);
1272 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1273 if (error) {
1274 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1275 goto done;
1276 }
1277
1278 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1279 if (error) {
1280 goto done;
1281 }
1282
1283done:
1284 if (error && packet != NULL) {
1285 mbuf_free(packet);
1286 }
1287
1288 return error;
1289}
1290
1291static int
1292flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1293{
1294 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1295 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1296 {
1297 return SHUT_RDWR;
1298 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1299 return SHUT_RD;
1300 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1301 return SHUT_WR;
1302 }
1303
1304 return -1;
1305}
1306
1307/*
1308 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1309 * writes. Returns FALSE otherwise.
1310 */
1311static void
1312flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1313{
1314 int how = -1;
1315
1316 /* Do not send any close messages if there is still data in the send buffer */
1317 if (fd_cb->so->so_snd.sb_cc == 0) {
1318 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1319 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1320 how = SHUT_RD;
1321 }
1322 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1323 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1324 if (how == SHUT_RD) {
1325 how = SHUT_RDWR;
1326 } else {
1327 how = SHUT_WR;
1328 }
1329 }
1330 }
1331
1332 if (how != -1) {
1333 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1334 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1335 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1336 if (how != SHUT_RD) {
1337 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1338 }
1339 if (how != SHUT_WR) {
1340 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1341 }
1342 }
1343 }
1344
1345 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1346 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1347 }
1348}
1349
1350static errno_t
3e170ce0 1351flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e
A		 1352{
1353 mbuf_t packet;
1354 mbuf_t last;
1355 int error = 0;
1356
1357 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1358 if (error) {
1359 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1360 return error;
1361 }
1362
3e170ce0
A		 1363 if (toaddr != NULL) {
1364 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1365 if (error) {
1366 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1367 return error;
1368 }
1369 }
1370
5ba3f43e
A		 1371 if (data_len > 0 && data != NULL) {
1372 last = m_last(packet);
1373 mbuf_setnext(last, data);
1374 mbuf_pkthdr_adjustlen(packet, data_len);
1375 }
39236c6e
A		 1376 error = flow_divert_send_packet(fd_cb, packet, force);
1377
1378 if (error) {
1379 mbuf_setnext(last, NULL);
3e170ce0 1380 mbuf_freem(packet);
39236c6e
A		 1381 } else {
1382 fd_cb->bytes_sent += data_len;
1383 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1384 }
1385
1386 return error;
1387}
1388
1389static void
1390flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1391{
1392 size_t to_send;
1393 size_t sent = 0;
1394 int error = 0;
1395 mbuf_t buffer;
1396
1397 to_send = fd_cb->so->so_snd.sb_cc;
1398 buffer = fd_cb->so->so_snd.sb_mb;
1399
1400 if (buffer == NULL && to_send > 0) {
1401 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1402 return;
1403 }
1404
1405 /* Ignore the send window if force is enabled */
1406 if (!force && (to_send > fd_cb->send_window)) {
1407 to_send = fd_cb->send_window;
1408 }
1409
3e170ce0
A		 1410 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1411 while (sent < to_send) {
1412 mbuf_t data;
1413 size_t data_len;
39236c6e 1414
3e170ce0
A		 1415 data_len = to_send - sent;
1416 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1417 data_len = FLOW_DIVERT_CHUNK_SIZE;
1418 }
39236c6e 1419
3e170ce0
A		 1420 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1421 if (error) {
1422 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1423 break;
1424 }
39236c6e 1425
3e170ce0
A		 1426 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1427 if (error) {
1428 mbuf_freem(data);
1429 break;
1430 }
39236c6e 1431
3e170ce0
A		 1432 sent += data_len;
1433 }
1434 sbdrop(&fd_cb->so->so_snd, sent);
1435 sowwakeup(fd_cb->so);
1436 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1437 mbuf_t data;
1438 mbuf_t m;
1439 size_t data_len;
1440
1441 while(buffer) {
1442 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1443
1444 m = buffer;
1445 if (toaddr != NULL) {
1446 /* look for data in the chain */
1447 do {
1448 m = m->m_next;
1449 if (m != NULL && m->m_type == MT_DATA) {
1450 break;
1451 }
1452 } while(m);
1453 if (m == NULL) {
1454 /* unexpected */
1455 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1456 goto move_on;
1457 }
1458 }
1459 data_len = mbuf_pkthdr_len(m);
5ba3f43e
A		 1460 if (data_len > 0) {
1461 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1462 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1463 if (error) {
1464 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1465 break;
1466 }
1467 } else {
1468 data = NULL;
3e170ce0
A		 1469 }
1470 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1471 if (error) {
1472 mbuf_freem(data);
1473 break;
1474 }
1475 sent += data_len;
1476move_on:
1477 buffer = buffer->m_nextpkt;
1478 (void) sbdroprecord(&(fd_cb->so->so_snd));
1479 }
39236c6e
A		 1480 }
1481
1482 if (sent > 0) {
1483 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1484 if (fd_cb->send_window >= sent) {
1485 fd_cb->send_window -= sent;
1486 } else {
1487 fd_cb->send_window = 0;
1488 }
39236c6e
A		 1489 }
1490}
1491
1492static int
3e170ce0 1493flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e
A		 1494{
1495 size_t to_send = mbuf_pkthdr_len(data);
3e170ce0 1496 int error = 0;
39236c6e
A		 1497
1498 if (to_send > fd_cb->send_window) {
1499 to_send = fd_cb->send_window;
1500 }
1501
1502 if (fd_cb->so->so_snd.sb_cc > 0) {
1503 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1504 }
1505
3e170ce0
A		 1506 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1507 size_t sent = 0;
1508 mbuf_t remaining_data = data;
1509 mbuf_t pkt_data = NULL;
490019cf 1510 while (sent < to_send && remaining_data != NULL) {
3e170ce0
A		 1511 size_t pkt_data_len;
1512
1513 pkt_data = remaining_data;
39236c6e 1514
3e170ce0
A		 1515 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1516 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1517 } else {
1518 pkt_data_len = to_send - sent;
1519 }
1520
1521 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1522 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1523 if (error) {
1524 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1525 pkt_data = NULL;
1526 break;
1527 }
1528 } else {
1529 remaining_data = NULL;
1530 }
1531
1532 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1533
39236c6e 1534 if (error) {
39236c6e
A		 1535 break;
1536 }
39236c6e 1537
3e170ce0
A		 1538 pkt_data = NULL;
1539 sent += pkt_data_len;
39236c6e
A		 1540 }
1541
3e170ce0 1542 fd_cb->send_window -= sent;
39236c6e 1543
3e170ce0 1544 error = 0;
39236c6e 1545
3e170ce0
A		 1546 if (pkt_data != NULL) {
1547 if (sbspace(&fd_cb->so->so_snd) > 0) {
1548 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1549 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1550 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1551 }
1552 } else {
1553 error = ENOBUFS;
39236c6e 1554 }
39236c6e 1555 }
39236c6e 1556
3e170ce0
A		 1557 if (remaining_data != NULL) {
1558 if (sbspace(&fd_cb->so->so_snd) > 0) {
1559 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1560 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1561 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1562 }
1563 } else {
1564 error = ENOBUFS;
1565 }
1566 }
1567 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
5ba3f43e 1568 if (to_send || mbuf_pkthdr_len(data) == 0) {
3e170ce0
A		 1569 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1570 if (error) {
39037602 1571 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
3e170ce0
A		 1572 } else {
1573 fd_cb->send_window -= to_send;
39236c6e
A		 1574 }
1575 } else {
3e170ce0
A		 1576 /* buffer it */
1577 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1578 if (toaddr != NULL) {
1579 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1580 FDLOG(LOG_ERR, fd_cb,
1581 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1582 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1583 }
1584 } else {
1585 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1586 FDLOG(LOG_ERR, fd_cb,
1587 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1588 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1589 }
1590 }
1591 } else {
1592 error = ENOBUFS;
1593 }
39236c6e
A		 1594 }
1595 }
1596
1597 return error;
1598}
1599
1600static int
1601flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1602{
1603 int error = 0;
1604 mbuf_t packet = NULL;
1605 uint32_t net_read_count = htonl(read_count);
1606
1607 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1608 if (error) {
1609 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1610 goto done;
1611 }
1612
1613 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1614 if (error) {
1615 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1616 goto done;
1617 }
1618
1619 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1620 if (error) {
1621 goto done;
1622 }
1623
1624done:
1625 if (error && packet != NULL) {
1626 mbuf_free(packet);
1627 }
1628
1629 return error;
1630}
1631
1632static int
1633flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1634{
1635 int error = 0;
1636 mbuf_t packet = NULL;
1637
1638 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1639 if (error) {
1640 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1641 goto done;
1642 }
1643
1644 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1645 if (error) {
1646 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1647 goto done;
1648 }
1649
1650 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1651 if (error) {
1652 goto done;
1653 }
1654
1655done:
1656 if (error && packet != NULL) {
1657 mbuf_free(packet);
1658 }
1659
1660 return error;
1661}
1662
1663static void
1664flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1665{
1666 uint32_t connect_error;
1667 uint32_t ctl_unit = 0;
1668 int error = 0;
1669 struct flow_divert_group *grp = NULL;
1670 struct sockaddr_storage local_address;
1671 int out_if_index = 0;
1672 struct sockaddr_storage remote_address;
1673 uint32_t send_window;
39037602 1674 uint32_t app_data_length = 0;
39236c6e
A		 1675
1676 memset(&local_address, 0, sizeof(local_address));
1677 memset(&remote_address, 0, sizeof(remote_address));
1678
1679 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1680 if (error) {
1681 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1682 return;
1683 }
1684
1685 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1686
1687 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1688 if (error) {
1689 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1690 return;
1691 }
1692
1693 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1694 if (error) {
39037602 1695 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
39236c6e
A		 1696 }
1697
1698 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1699 if (error) {
39037602 1700 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
39236c6e
A		 1701 }
1702
1703 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1704 if (error) {
39037602 1705 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
39236c6e
A		 1706 }
1707
1708 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1709 if (error) {
39037602
A		 1710 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1711 }
1712
1713 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1714 if (error) {
1715 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
39236c6e
A		 1716 }
1717
39037602 1718 error = 0;
39236c6e
A		 1719 connect_error = ntohl(connect_error);
1720 ctl_unit = ntohl(ctl_unit);
1721
1722 lck_rw_lock_shared(&g_flow_divert_group_lck);
1723
39037602
A		 1724 if (connect_error == 0 && ctl_unit > 0) {
1725 if (ctl_unit >= GROUP_COUNT_MAX) {
39236c6e
A		 1726 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1727 error = EINVAL;
1728 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1729 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1730 error = EINVAL;
1731 } else {
1732 grp = g_flow_divert_groups[ctl_unit];
1733 if (grp == NULL) {
1734 error = ECONNRESET;
1735 }
1736 }
1737 }
1738
1739 FDLOCK(fd_cb);
1740 if (fd_cb->so != NULL) {
1741 struct inpcb *inp = NULL;
1742 struct ifnet *ifp = NULL;
1743 struct flow_divert_group *old_group;
1744
1745 socket_lock(fd_cb->so, 0);
1746
1747 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1748 goto done;
1749 }
1750
1751 inp = sotoinpcb(fd_cb->so);
1752
1753 if (connect_error || error) {
1754 goto set_socket_state;
1755 }
1756
3e170ce0
A		 1757 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1758 error = EINVAL;
1759 goto set_socket_state;
1760 }
1761 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
39236c6e
A		 1762 if (local_address.ss_len > sizeof(local_address)) {
1763 local_address.ss_len = sizeof(local_address);
1764 }
1765 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
39236c6e
A		 1766 }
1767
1768 if (remote_address.ss_family != 0) {
1769 if (remote_address.ss_len > sizeof(remote_address)) {
1770 remote_address.ss_len = sizeof(remote_address);
1771 }
1772 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1773 } else {
1774 error = EINVAL;
1775 goto set_socket_state;
1776 }
1777
39037602
A		 1778 if (app_data_length > 0) {
1779 uint8_t *app_data = NULL;
1780 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1781 if (app_data != NULL) {
1782 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1783 if (error == 0) {
1784 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1785 if (fd_cb->app_data != NULL) {
1786 FREE(fd_cb->app_data, M_TEMP);
1787 }
1788 fd_cb->app_data = app_data;
1789 fd_cb->app_data_length = app_data_length;
1790 } else {
1791 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1792 FREE(app_data, M_TEMP);
1793 }
1794 } else {
1795 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1796 }
1797 }
1798
39236c6e
A		 1799 ifnet_head_lock_shared();
1800 if (out_if_index > 0 && out_if_index <= if_index) {
1801 ifp = ifindex2ifnet[out_if_index];
1802 }
1803
1804 if (ifp != NULL) {
1805 inp->inp_last_outifp = ifp;
1806 } else {
1807 error = EINVAL;
1808 }
1809 ifnet_head_done();
1810
1811 if (error) {
1812 goto set_socket_state;
1813 }
1814
1815 if (fd_cb->group == NULL) {
1816 error = EINVAL;
1817 goto set_socket_state;
1818 }
1819
39037602
A		 1820 if (grp != NULL) {
1821 old_group = fd_cb->group;
39236c6e 1822
39037602
A		 1823 lck_rw_lock_exclusive(&old_group->lck);
1824 lck_rw_lock_exclusive(&grp->lck);
39236c6e 1825
39037602
A		 1826 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1827 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1828 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1829 }
39236c6e 1830
39037602 1831 fd_cb->group = grp;
39236c6e 1832
39037602
A		 1833 lck_rw_done(&grp->lck);
1834 lck_rw_done(&old_group->lck);
1835 }
39236c6e
A		 1836
1837 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 1838
1839set_socket_state:
1840 if (!connect_error && !error) {
1841 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1842 error = flow_divert_send_connect_result(fd_cb);
1843 }
1844
1845 if (connect_error || error) {
1846 if (!connect_error) {
1847 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1848 fd_cb->so->so_error = error;
1849 flow_divert_send_close_if_needed(fd_cb);
1850 } else {
1851 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1852 fd_cb->so->so_error = connect_error;
1853 }
3e170ce0 1854 flow_divert_disconnect_socket(fd_cb->so);
39236c6e 1855 } else {
490019cf 1856 flow_divert_send_buffered_data(fd_cb, FALSE);
39236c6e
A		 1857 soisconnected(fd_cb->so);
1858 }
1859
1860done:
1861 socket_unlock(fd_cb->so, 0);
1862 }
1863 FDUNLOCK(fd_cb);
1864
1865 lck_rw_done(&g_flow_divert_group_lck);
1866}
1867
1868static void
1869flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1870{
1871 uint32_t close_error;
1872 int error = 0;
1873 int how;
1874
1875 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1876 if (error) {
1877 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1878 return;
1879 }
1880
1881 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1882 if (error) {
1883 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1884 return;
1885 }
1886
1887 how = ntohl(how);
1888
1889 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1890
1891 FDLOCK(fd_cb);
1892 if (fd_cb->so != NULL) {
1893 socket_lock(fd_cb->so, 0);
1894
1895 fd_cb->so->so_error = ntohl(close_error);
1896
1897 flow_divert_update_closed_state(fd_cb, how, TRUE);
1898
1899 how = flow_divert_tunnel_how_closed(fd_cb);
1900 if (how == SHUT_RDWR) {
3e170ce0 1901 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1902 } else if (how == SHUT_RD) {
1903 socantrcvmore(fd_cb->so);
1904 } else if (how == SHUT_WR) {
1905 socantsendmore(fd_cb->so);
1906 }
1907
1908 socket_unlock(fd_cb->so, 0);
1909 }
1910 FDUNLOCK(fd_cb);
1911}
1912
3e170ce0
A		 1913static mbuf_t
1914flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 1915{
3e170ce0
A		 1916 struct inpcb *inp = sotoinpcb(fd_cb->so);
1917 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1918 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
39236c6e 1919
3e170ce0
A		 1920 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1921 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1922 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1923 struct in6_pktinfo pi6;
39236c6e 1924
3e170ce0
A		 1925 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1926 pi6.ipi6_ifindex = 0;
1927 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 1928 }
3e170ce0
A		 1929 return (NULL);
1930}
39236c6e 1931
3e170ce0
A		 1932static void
1933flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1934{
39236c6e
A		 1935 FDLOCK(fd_cb);
1936 if (fd_cb->so != NULL) {
3e170ce0
A		 1937 int error = 0;
1938 mbuf_t data = NULL;
1939 size_t data_size;
1940 struct sockaddr_storage remote_address;
1941 boolean_t got_remote_sa = FALSE;
1942
39236c6e 1943 socket_lock(fd_cb->so, 0);
3e170ce0
A		 1944
1945 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1946 uint32_t val_size = 0;
1947
1948 /* check if we got remote address with data */
1949 memset(&remote_address, 0, sizeof(remote_address));
1950 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1951 if (error || val_size > sizeof(remote_address)) {
1952 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1953 error = 0;
39236c6e 1954 } else {
3e170ce0
A		 1955 /* validate the address */
1956 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1957 got_remote_sa = TRUE;
1958 }
1959 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1960 }
1961 }
1962
1963 data_size = (mbuf_pkthdr_len(packet) - offset);
1964
1965 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1966
1967 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1968 if (error || data == NULL) {
1969 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1970 } else {
1971 if (flow_divert_check_no_cellular(fd_cb) ||
1972 flow_divert_check_no_expensive(fd_cb))
1973 {
1974 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1975 flow_divert_send_close(fd_cb, SHUT_RDWR);
1976 flow_divert_disconnect_socket(fd_cb->so);
1977 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1978 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1979 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1980 fd_cb->bytes_received += data_size;
1981 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1982 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1983 sorwakeup(fd_cb->so);
1984 data = NULL;
1985 } else {
1986 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1987 }
1988 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1989 struct sockaddr *append_sa;
1990 mbuf_t mctl;
1991
1992 if (got_remote_sa == TRUE) {
1993 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1994 (struct sockaddr *)&remote_address, &append_sa);
1995 } else {
1996 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1997 fd_cb->remote_address, &append_sa);
1998 }
1999 if (error) {
2000 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2001 }
2002
2003 mctl = flow_divert_get_control_mbuf(fd_cb);
2004 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
2005 fd_cb->bytes_received += data_size;
2006 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2007 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2008 sorwakeup(fd_cb->so);
2009 data = NULL;
2010 } else {
2011 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
2012 }
2013 if (!error) {
2014 FREE(append_sa, M_TEMP);
2015 }
2016 }
39236c6e
A		 2017 }
2018 }
2019 socket_unlock(fd_cb->so, 0);
39236c6e 2020 }
3e170ce0 2021 FDUNLOCK(fd_cb);
39236c6e
A		 2022}
2023
2024static void
2025flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2026{
2027 uint32_t read_count;
2028 int error = 0;
2029
2030 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2031 if (error) {
2032 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2033 return;
2034 }
2035
3e170ce0 2036 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2037
2038 FDLOCK(fd_cb);
2039 if (fd_cb->so != NULL) {
2040 socket_lock(fd_cb->so, 0);
2041 fd_cb->send_window += ntohl(read_count);
2042 flow_divert_send_buffered_data(fd_cb, FALSE);
2043 socket_unlock(fd_cb->so, 0);
2044 }
2045 FDUNLOCK(fd_cb);
2046}
2047
2048static void
2049flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2050{
2051 int error = 0;
3e170ce0 2052 uint32_t key_size = 0;
39236c6e 2053 int log_level;
5ba3f43e 2054 uint32_t flags = 0;
39236c6e
A		 2055
2056 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2057 if (error) {
2058 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2059 return;
2060 }
2061
2062 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
39037602 2063 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
39236c6e
A		 2064 return;
2065 }
2066
2067 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2068 if (!error) {
2069 nil_pcb.log_level = log_level;
2070 }
2071
2072 lck_rw_lock_exclusive(&group->lck);
2073
2074 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2075 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2076 if (error) {
2077 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2078 FREE(group->token_key, M_TEMP);
2079 group->token_key = NULL;
2080 lck_rw_done(&group->lck);
2081 return;
2082 }
2083
2084 group->token_key_size = key_size;
2085
5ba3f43e
A		 2086 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2087 if (!error) {
2088 group->flags = flags;
2089 }
2090
39236c6e
A		 2091 lck_rw_done(&group->lck);
2092}
2093
2094static void
2095flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2096{
2097 int error = 0;
2098 struct sockaddr_storage local_address;
2099 int out_if_index = 0;
2100 struct sockaddr_storage remote_address;
490019cf 2101 uint32_t app_data_length = 0;
39236c6e
A		 2102
2103 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2104
2105 memset(&local_address, 0, sizeof(local_address));
2106 memset(&remote_address, 0, sizeof(remote_address));
2107
2108 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2109 if (error) {
490019cf 2110 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
39236c6e
A		 2111 }
2112
2113 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2114 if (error) {
490019cf 2115 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
39236c6e
A		 2116 }
2117
2118 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2119 if (error) {
490019cf
A		 2120 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2121 }
2122
2123 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2124 if (error) {
2125 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2126 }
2127
2128 FDLOCK(fd_cb);
2129 if (fd_cb->so != NULL) {
39236c6e
A		 2130 socket_lock(fd_cb->so, 0);
2131
39236c6e
A		 2132 if (local_address.ss_family != 0) {
2133 if (local_address.ss_len > sizeof(local_address)) {
2134 local_address.ss_len = sizeof(local_address);
2135 }
490019cf
A		 2136 if (fd_cb->local_address != NULL) {
2137 FREE(fd_cb->local_address, M_SONAME);
2138 fd_cb->local_address = NULL;
2139 }
39236c6e
A		 2140 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2141 }
2142
2143 if (remote_address.ss_family != 0) {
2144 if (remote_address.ss_len > sizeof(remote_address)) {
2145 remote_address.ss_len = sizeof(remote_address);
2146 }
490019cf
A		 2147 if (fd_cb->remote_address != NULL) {
2148 FREE(fd_cb->remote_address, M_SONAME);
2149 fd_cb->remote_address = NULL;
2150 }
39236c6e
A		 2151 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2152 }
2153
490019cf
A		 2154 if (out_if_index > 0) {
2155 struct inpcb *inp = NULL;
2156 struct ifnet *ifp = NULL;
2157
2158 inp = sotoinpcb(fd_cb->so);
2159
2160 ifnet_head_lock_shared();
2161 if (out_if_index <= if_index) {
2162 ifp = ifindex2ifnet[out_if_index];
2163 }
2164
2165 if (ifp != NULL) {
2166 inp->inp_last_outifp = ifp;
2167 }
2168 ifnet_head_done();
39236c6e
A		 2169 }
2170
490019cf
A		 2171 if (app_data_length > 0) {
2172 uint8_t *app_data = NULL;
2173 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2174 if (app_data != NULL) {
2175 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2176 if (error == 0) {
2177 if (fd_cb->app_data != NULL) {
2178 FREE(fd_cb->app_data, M_TEMP);
2179 }
2180 fd_cb->app_data = app_data;
2181 fd_cb->app_data_length = app_data_length;
2182 } else {
2183 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2184 FREE(app_data, M_TEMP);
2185 }
2186 } else {
2187 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2188 }
39236c6e 2189 }
39236c6e
A		 2190
2191 socket_unlock(fd_cb->so, 0);
2192 }
2193 FDUNLOCK(fd_cb);
2194}
2195
2196static void
39037602 2197flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
39236c6e
A		 2198{
2199 size_t bytes_mem_size;
2200 size_t child_maps_mem_size;
2201 int cursor;
2202 int error = 0;
2203 struct flow_divert_trie new_trie;
2204 int insert_error = 0;
2205 size_t nodes_mem_size;
2206 int prefix_count = 0;
2207 int signing_id_count = 0;
ecc0ceb4 2208 size_t trie_memory_size = 0;
39236c6e 2209
39037602 2210 lck_rw_lock_exclusive(&group->lck);
39236c6e
A		 2211
2212 /* Re-set the current trie */
39037602
A		 2213 if (group->signing_id_trie.memory != NULL) {
2214 FREE(group->signing_id_trie.memory, M_TEMP);
39236c6e 2215 }
39037602
A		 2216 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2217 group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 2218
2219 memset(&new_trie, 0, sizeof(new_trie));
2220
2221 /* Get the number of shared prefixes in the new set of signing ID strings */
2222 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2223
ecc0ceb4 2224 if (prefix_count < 0) {
39037602 2225 lck_rw_done(&group->lck);
ecc0ceb4
A		 2226 return;
2227 }
2228
39236c6e
A		 2229 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2230 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2231 cursor >= 0;
2232 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2233 {
3e170ce0 2234 uint32_t sid_size = 0;
39236c6e
A		 2235 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2236 new_trie.bytes_count += sid_size;
2237 signing_id_count++;
2238 }
2239
2240 if (signing_id_count == 0) {
39037602 2241 lck_rw_done(&group->lck);
39236c6e
A		 2242 return;
2243 }
2244
2245 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2246 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2247
2248 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2249 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2250
2251 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2252 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2253 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2254
ecc0ceb4
A		 2255 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2256 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
39037602
A		 2257 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2258 lck_rw_done(&group->lck);
ecc0ceb4
A		 2259 return;
2260 }
2261
2262 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2263 if (new_trie.memory == NULL) {
2264 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2265 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
39037602 2266 lck_rw_done(&group->lck);
39236c6e
A		 2267 return;
2268 }
2269
2270 /* Initialize the free lists */
2271 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2272 new_trie.nodes_free_next = 0;
2273 memset(new_trie.nodes, 0, nodes_mem_size);
2274
2275 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2276 new_trie.child_maps_free_next = 0;
2277 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2278
2279 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2280 new_trie.bytes_free_next = 0;
2281
2282 /* The root is an empty node */
2283 new_trie.root = trie_node_alloc(&new_trie);
2284
2285 /* Add each signing ID to the trie */
2286 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2287 cursor >= 0;
2288 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2289 {
3e170ce0 2290 uint32_t sid_size = 0;
39236c6e
A		 2291 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2292 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
39236c6e
A		 2293 uint16_t new_node_idx;
2294 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
39236c6e 2295 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
39037602 2296 if (new_node_idx == NULL_TRIE_IDX) {
39236c6e
A		 2297 insert_error = EINVAL;
2298 break;
2299 }
2300 } else {
2301 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2302 insert_error = ENOBUFS;
2303 break;
2304 }
2305 }
2306
2307 if (!insert_error) {
39037602 2308 group->signing_id_trie = new_trie;
39236c6e
A		 2309 } else {
2310 FREE(new_trie.memory, M_TEMP);
2311 }
2312
39236c6e 2313 lck_rw_done(&group->lck);
39236c6e
A		 2314}
2315
2316static int
2317flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2318{
2319 struct flow_divert_packet_header hdr;
2320 int error = 0;
2321 struct flow_divert_pcb *fd_cb;
2322
2323 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2324 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2325 error = EINVAL;
2326 goto done;
2327 }
2328
3e170ce0 2329 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
39037602 2330 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
3e170ce0
A		 2331 error = EINVAL;
2332 goto done;
2333 }
2334
39236c6e
A		 2335 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2336 if (error) {
2337 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2338 error = ENOBUFS;
2339 goto done;
2340 }
2341
2342 hdr.conn_id = ntohl(hdr.conn_id);
2343
2344 if (hdr.conn_id == 0) {
2345 switch (hdr.packet_type) {
2346 case FLOW_DIVERT_PKT_GROUP_INIT:
2347 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2348 break;
2349 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
39037602 2350 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
39236c6e
A		 2351 break;
2352 default:
2353 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2354 break;
2355 }
2356 goto done;
2357 }
2358
2359 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2360 if (fd_cb == NULL) {
2361 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2362 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2363 }
2364 goto done;
2365 }
2366
2367 switch (hdr.packet_type) {
2368 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2369 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2370 break;
2371 case FLOW_DIVERT_PKT_CLOSE:
2372 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2373 break;
2374 case FLOW_DIVERT_PKT_DATA:
2375 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2376 break;
2377 case FLOW_DIVERT_PKT_READ_NOTIFY:
2378 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2379 break;
2380 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2381 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2382 break;
2383 default:
2384 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2385 break;
2386 }
2387
2388 FDRELEASE(fd_cb);
2389
2390done:
3e170ce0 2391 mbuf_freem(packet);
39236c6e
A		 2392 return error;
2393}
2394
2395static void
2396flow_divert_close_all(struct flow_divert_group *group)
2397{
2398 struct flow_divert_pcb *fd_cb;
2399 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2400
2401 SLIST_INIT(&tmp_list);
2402
2403 lck_rw_lock_exclusive(&group->lck);
2404
2405 MBUFQ_DRAIN(&group->send_queue);
2406
2407 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2408 FDRETAIN(fd_cb);
2409 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2410 }
2411
2412 lck_rw_done(&group->lck);
2413
2414 while (!SLIST_EMPTY(&tmp_list)) {
2415 fd_cb = SLIST_FIRST(&tmp_list);
2416 FDLOCK(fd_cb);
2417 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2418 if (fd_cb->so != NULL) {
2419 socket_lock(fd_cb->so, 0);
2420 flow_divert_pcb_remove(fd_cb);
2421 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2422 fd_cb->so->so_error = ECONNABORTED;
39037602 2423 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2424 socket_unlock(fd_cb->so, 0);
2425 }
2426 FDUNLOCK(fd_cb);
2427 FDRELEASE(fd_cb);
2428 }
2429}
2430
2431void
2432flow_divert_detach(struct socket *so)
2433{
2434 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2435
2436 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2437
2438 so->so_flags &= ~SOF_FLOW_DIVERT;
2439 so->so_fd_pcb = NULL;
2440
2441 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2442
2443 if (fd_cb->group != NULL) {
2444 /* Last-ditch effort to send any buffered data */
2445 flow_divert_send_buffered_data(fd_cb, TRUE);
2446
3e170ce0
A		 2447 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2448 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 2449 /* Remove from the group */
2450 flow_divert_pcb_remove(fd_cb);
2451 }
2452
2453 socket_unlock(so, 0);
2454 FDLOCK(fd_cb);
2455 fd_cb->so = NULL;
2456 FDUNLOCK(fd_cb);
2457 socket_lock(so, 0);
2458
2459 FDRELEASE(fd_cb); /* Release the socket's reference */
2460}
2461
2462static int
2463flow_divert_close(struct socket *so)
2464{
2465 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2466
2467 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2468
2469 FDLOG0(LOG_INFO, fd_cb, "Closing");
2470
3e170ce0
A		 2471 if (SOCK_TYPE(so) == SOCK_STREAM) {
2472 soisdisconnecting(so);
2473 sbflush(&so->so_rcv);
2474 }
39236c6e
A		 2475
2476 flow_divert_send_buffered_data(fd_cb, TRUE);
2477 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2478 flow_divert_send_close_if_needed(fd_cb);
2479
2480 /* Remove from the group */
2481 flow_divert_pcb_remove(fd_cb);
2482
2483 return 0;
2484}
2485
2486static int
3e170ce0
A		 2487flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2488 sae_connid_t cid __unused)
39236c6e 2489{
3e170ce0 2490 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
39236c6e
A		 2491 return (EINVAL);
2492 }
2493
2494 return (flow_divert_close(so));
2495}
2496
2497static int
2498flow_divert_shutdown(struct socket *so)
2499{
2500 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2501
2502 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2503
2504 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2505
2506 socantsendmore(so);
2507
2508 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2509 flow_divert_send_close_if_needed(fd_cb);
2510
2511 return 0;
2512}
2513
2514static int
2515flow_divert_rcvd(struct socket *so, int flags __unused)
2516{
2517 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2518 uint32_t latest_sb_size;
2519 uint32_t read_count;
2520
2521 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2522
2523 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2524
2525 if (fd_cb->sb_size < latest_sb_size) {
2526 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2527 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2528 }
2529
2530 read_count = fd_cb->sb_size - latest_sb_size;
2531
2532 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2533
2534 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2535 fd_cb->bytes_read_by_app += read_count;
2536 fd_cb->sb_size = latest_sb_size;
2537 }
2538
2539 return 0;
2540}
2541
3e170ce0
A		 2542static int
2543flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2544{
2545 int error = 0;
2546 int port = 0;
2547
2548 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2549 if (error) {
2550 goto done;
2551 }
2552
2553 if (toaddr->sa_family == AF_INET) {
2554 port = ntohs((satosin(toaddr))->sin_port);
2555 }
2556#if INET6
2557 else {
2558 port = ntohs((satosin6(toaddr))->sin6_port);
2559 }
2560#endif
2561
2562 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2563 if (error) {
2564 goto done;
2565 }
2566
2567done:
2568 return error;
2569}
2570
2571struct sockaddr *
2572flow_divert_get_buffered_target_address(mbuf_t buffer)
2573{
2574 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2575 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2576 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2577 return toaddr;
2578 }
2579 }
2580 return NULL;
2581}
2582
2583static boolean_t
2584flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2585{
2586 switch(addr->sa_family)
2587 {
2588 case AF_INET:
2589 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2590 return FALSE;
2591 }
2592 break;
2593#if INET6
2594 case AF_INET6:
2595 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2596 return FALSE;
2597 }
2598 break;
2599#endif /* INET6 */
2600 default:
2601 return FALSE;
2602 }
2603 return TRUE;
2604}
2605
2606static errno_t
2607flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2608{
2609 int error = 0;
2610 union sockaddr_in_4_6 sin46;
2611
2612 bzero(&sin46, sizeof(sin46));
2613 if (inp->inp_vflag & INP_IPV4) {
2614 struct sockaddr_in *sin = &sin46.sin;
2615
2616 sin->sin_family = AF_INET;
2617 sin->sin_len = sizeof(*sin);
2618 sin->sin_port = inp->inp_lport;
2619 sin->sin_addr = inp->inp_laddr;
2620 } else if (inp->inp_vflag & INP_IPV6) {
2621 struct sockaddr_in6 *sin6 = &sin46.sin6;
2622
2623 sin6->sin6_len = sizeof(*sin6);
2624 sin6->sin6_family = AF_INET6;
2625 sin6->sin6_port = inp->inp_lport;
2626 sin6->sin6_addr = inp->in6p_laddr;
2627 }
2628 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2629 if (*local_socket == NULL) {
2630 error = ENOBUFS;
2631 }
2632 return (error);
2633}
2634
2635static boolean_t
2636flow_divert_has_pcb_local_address(const struct inpcb *inp)
2637{
2638 return (inp->inp_lport != 0
2639 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2640}
2641
39236c6e
A		 2642static errno_t
2643flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2644 struct sockaddr **dup)
2645{
2646 int error = 0;
2647 struct sockaddr *result;
2648 struct sockaddr_storage ss;
2649
2650 if (addr != NULL) {
2651 result = addr;
2652 } else {
2653 memset(&ss, 0, sizeof(ss));
2654 ss.ss_family = family;
2655 if (ss.ss_family == AF_INET) {
2656 ss.ss_len = sizeof(struct sockaddr_in);
2657 }
2658#if INET6
2659 else if (ss.ss_family == AF_INET6) {
2660 ss.ss_len = sizeof(struct sockaddr_in6);
2661 }
2662#endif /* INET6 */
2663 else {
2664 error = EINVAL;
2665 }
2666 result = (struct sockaddr *)&ss;
2667 }
2668
2669 if (!error) {
2670 *dup = dup_sockaddr(result, 1);
2671 if (*dup == NULL) {
2672 error = ENOBUFS;
2673 }
2674 }
2675
2676 return error;
2677}
2678
3e170ce0
A		 2679static void
2680flow_divert_disconnect_socket(struct socket *so)
2681{
2682 soisdisconnected(so);
2683 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2684 struct inpcb *inp = NULL;
2685
2686 inp = sotoinpcb(so);
2687 if (inp != NULL) {
2688#if INET6
2689 if (SOCK_CHECK_DOM(so, PF_INET6))
2690 in6_pcbdetach(inp);
2691 else
2692#endif /* INET6 */
2693 in_pcbdetach(inp);
2694 }
2695 }
2696}
2697
39236c6e
A		 2698static errno_t
2699flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2700{
2701 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2702
2703 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2704
2705 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2706 fd_cb->remote_address,
2707 sa);
2708}
2709
2710static errno_t
2711flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2712{
2713 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2714
2715 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2716
2717 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2718 fd_cb->local_address,
2719 sa);
2720}
2721
2722static errno_t
2723flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2724{
2725 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2726
2727 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2728
2729 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2730 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2731 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2732 }
2733 }
2734
2735 if (SOCK_DOM(so) == PF_INET) {
2736 return g_tcp_protosw->pr_ctloutput(so, sopt);
2737 }
2738#if INET6
2739 else if (SOCK_DOM(so) == PF_INET6) {
2740 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2741 }
2742#endif
2743 return 0;
2744}
2745
2746errno_t
2747flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2748{
2749 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2750 int error = 0;
2751 struct inpcb *inp = sotoinpcb(so);
2752 struct sockaddr_in *sinp;
fe8ab488 2753 mbuf_t connect_packet = NULL;
490019cf 2754 int do_send = 1;
39236c6e
A		 2755
2756 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2757
2758 if (fd_cb->group == NULL) {
2759 error = ENETUNREACH;
2760 goto done;
2761 }
2762
2763 if (inp == NULL) {
2764 error = EINVAL;
2765 goto done;
2766 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2767 if (so->so_error) {
2768 error = so->so_error;
2769 so->so_error = 0;
2770 } else {
2771 error = EINVAL;
2772 }
2773 goto done;
2774 }
2775
39236c6e
A		 2776 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2777 error = EALREADY;
2778 goto done;
2779 }
2780
2781 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2782 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2783 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2784 if (fd_cb->remote_address != NULL) {
2785 soisconnected(fd_cb->so);
2786 goto done;
2787 }
2788 }
2789
490019cf 2790 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 2791
490019cf
A		 2792 if (fd_cb->connect_packet == NULL) {
2793 if (to == NULL) {
2794 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2795 error = EINVAL;
2796 goto done;
fe8ab488 2797 }
fe8ab488 2798
490019cf
A		 2799 sinp = (struct sockaddr_in *)(void *)to;
2800 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2801 error = EAFNOSUPPORT;
2802 goto done;
fe8ab488
A		 2803 }
2804
490019cf
A		 2805 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2806 if (error) {
2807 goto done;
fe8ab488
A		 2808 }
2809
490019cf
A		 2810 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2811 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2812 do_send = 0;
fe8ab488
A		 2813 }
2814 } else {
490019cf
A		 2815 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2816 connect_packet = fd_cb->connect_packet;
2817 fd_cb->connect_packet = NULL;
fe8ab488
A		 2818 }
2819
490019cf
A		 2820 if (do_send) {
2821 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2822 if (error) {
2823 goto done;
2824 }
39236c6e 2825
490019cf
A		 2826 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2827 } else {
2828 fd_cb->connect_packet = connect_packet;
2829 connect_packet = NULL;
39236c6e
A		 2830 }
2831
39236c6e
A		 2832 soisconnecting(so);
2833
2834done:
fe8ab488 2835 if (error && connect_packet != NULL) {
3e170ce0 2836 mbuf_freem(connect_packet);
fe8ab488 2837 }
39236c6e
A		 2838 return error;
2839}
2840
2841static int
813fb2f6
A		 2842flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2843 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
39236c6e 2844{
39236c6e
A		 2845 struct inpcb *inp = sotoinpcb(so);
2846 int error;
2847
2848 if (inp == NULL) {
2849 return (EINVAL);
2850 }
2851
813fb2f6 2852 VERIFY(dst != NULL);
39236c6e 2853
813fb2f6 2854 error = flow_divert_connect_out(so, dst, p);
39236c6e 2855
490019cf
A		 2856 if (error != 0) {
2857 return error;
2858 }
2859
2860 /* if there is data, send it */
2861 if (auio != NULL) {
2862 user_ssize_t datalen = 0;
2863
2864 socket_unlock(so, 0);
2865
2866 VERIFY(bytes_written != NULL);
2867
2868 datalen = uio_resid(auio);
2869 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2870 socket_lock(so, 0);
2871
2872 if (error == 0 || error == EWOULDBLOCK) {
2873 *bytes_written = datalen - uio_resid(auio);
2874 }
2875
2876 /*
2877 * sosend returns EWOULDBLOCK if it's a non-blocking
2878 * socket or a timeout occured (this allows to return
2879 * the amount of queued data through sendit()).
2880 *
2881 * However, connectx() returns EINPROGRESS in case of a
2882 * blocking socket. So we change the return value here.
2883 */
2884 if (error == EWOULDBLOCK) {
2885 error = EINPROGRESS;
2886 }
2887 }
2888
39236c6e
A		 2889 if (error == 0 && pcid != NULL) {
2890 *pcid = 1; /* there is only 1 connection for a TCP */
2891 }
2892
2893 return (error);
2894}
2895
2896static int
813fb2f6
A		 2897flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
2898 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2899 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2900 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 2901{
813fb2f6 2902 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
39236c6e
A		 2903}
2904
2905#if INET6
2906static int
813fb2f6
A		 2907flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
2908 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2909 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2910 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2911{
2912 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
39236c6e
A		 2913}
2914#endif /* INET6 */
2915
2916static int
3e170ce0 2917flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
39236c6e
A		 2918 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2919 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2920 user_addr_t aux_data __unused, uint32_t *aux_len)
2921{
2922 int error = 0;
2923 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2924 struct ifnet *ifp = NULL;
2925 struct inpcb *inp = sotoinpcb(so);
2926
2927 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2928
2929 if (so->so_fd_pcb == NULL || inp == NULL) {
2930 error = EINVAL;
2931 goto out;
2932 }
2933
3e170ce0 2934 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
39236c6e
A		 2935 error = EINVAL;
2936 goto out;
2937 }
2938
2939 ifp = inp->inp_last_outifp;
2940 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2941 *soerror = so->so_error;
2942 *flags = 0;
2943
2944 if (so->so_state & SS_ISCONNECTED) {
2945 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2946 }
2947
2948 if (fd_cb->local_address == NULL) {
2949 struct sockaddr_in sin;
2950 bzero(&sin, sizeof(sin));
2951 sin.sin_len = sizeof(sin);
2952 sin.sin_family = AF_INET;
2953 *src_len = sin.sin_len;
2954 if (src != USER_ADDR_NULL) {
2955 error = copyout(&sin, src, sin.sin_len);
2956 if (error != 0) {
2957 goto out;
2958 }
2959 }
2960 } else {
2961 *src_len = fd_cb->local_address->sa_len;
2962 if (src != USER_ADDR_NULL) {
2963 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2964 if (error != 0) {
2965 goto out;
2966 }
2967 }
2968 }
2969
2970 if (fd_cb->remote_address == NULL) {
2971 struct sockaddr_in sin;
2972 bzero(&sin, sizeof(sin));
2973 sin.sin_len = sizeof(sin);
2974 sin.sin_family = AF_INET;
2975 *dst_len = sin.sin_len;
2976 if (dst != USER_ADDR_NULL) {
2977 error = copyout(&sin, dst, sin.sin_len);
2978 if (error != 0) {
2979 goto out;
2980 }
2981 }
2982 } else {
2983 *dst_len = fd_cb->remote_address->sa_len;
2984 if (dst != USER_ADDR_NULL) {
2985 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2986 if (error != 0) {
2987 goto out;
2988 }
2989 }
2990 }
2991
2992 *aux_type = 0;
2993 *aux_len = 0;
2994
2995out:
2996 return error;
2997}
2998
2999static int
3000flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3001{
3002 int error = 0;
3003
3004 switch (cmd) {
3005 case SIOCGCONNINFO32: {
3006 struct so_cinforeq32 cifr;
3007 bcopy(data, &cifr, sizeof (cifr));
3008 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3009 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3010 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3011 &cifr.scir_aux_type, cifr.scir_aux_data,
3012 &cifr.scir_aux_len);
3013 if (error == 0) {
3014 bcopy(&cifr, data, sizeof (cifr));
3015 }
3016 break;
3017 }
3018
3019 case SIOCGCONNINFO64: {
3020 struct so_cinforeq64 cifr;
3021 bcopy(data, &cifr, sizeof (cifr));
3022 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3023 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3024 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3025 &cifr.scir_aux_type, cifr.scir_aux_data,
3026 &cifr.scir_aux_len);
3027 if (error == 0) {
3028 bcopy(&cifr, data, sizeof (cifr));
3029 }
3030 break;
3031 }
3032
3033 default:
3034 error = EOPNOTSUPP;
3035 }
3036
3037 return error;
3038}
3039
3040static int
3041flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3042{
3043 int error = flow_divert_control(so, cmd, data, ifp, p);
3044
3045 if (error == EOPNOTSUPP) {
3046 error = in_control(so, cmd, data, ifp, p);
3047 }
3048
3049 return error;
3050}
3051
3052static int
3053flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3054{
3055 int error = flow_divert_control(so, cmd, data, ifp, p);
3056
3057 if (error == EOPNOTSUPP) {
3058 error = in6_control(so, cmd, data, ifp, p);
3059 }
3060
3061 return error;
3062}
3063
3064static errno_t
490019cf 3065flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e
A		 3066{
3067 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3068 int error = 0;
3069 struct inpcb *inp;
3070
3071 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3072
3073 inp = sotoinpcb(so);
3074 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3075 error = ECONNRESET;
3076 goto done;
3077 }
3078
3079 if (control && mbuf_len(control) > 0) {
3080 error = EINVAL;
3081 goto done;
3082 }
3083
3084 if (flags & MSG_OOB) {
3085 error = EINVAL;
3086 goto done; /* We don't support OOB data */
3087 }
3088
fe8ab488
A		 3089 error = flow_divert_check_no_cellular(fd_cb) ||
3090 flow_divert_check_no_expensive(fd_cb);
39236c6e
A		 3091 if (error) {
3092 goto done;
3093 }
3094
3095 /* Implicit connect */
3096 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3097 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
490019cf 3098 error = flow_divert_connect_out(so, to, p);
39236c6e
A		 3099 if (error) {
3100 goto done;
3101 }
490019cf
A		 3102
3103 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3104 /* Open up the send window so that the data will get sent right away */
3105 fd_cb->send_window = mbuf_pkthdr_len(data);
3106 }
39236c6e
A		 3107 }
3108
3109 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3110
3111 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3112 error = flow_divert_send_app_data(fd_cb, data, to);
39236c6e
A		 3113 if (error) {
3114 goto done;
3115 }
3116
3117 data = NULL;
3118
3119 if (flags & PRUS_EOF) {
3120 flow_divert_shutdown(so);
3121 }
3122
3123done:
3124 if (data) {
3e170ce0 3125 mbuf_freem(data);
39236c6e
A		 3126 }
3127 if (control) {
3128 mbuf_free(control);
3129 }
3130 return error;
3131}
3132
490019cf
A		 3133static int
3134flow_divert_preconnect(struct socket *so)
3135{
3136 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3137 int error = 0;
3138
3139 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3140 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3141 mbuf_t connect_packet = fd_cb->connect_packet;
3142 fd_cb->connect_packet = NULL;
3143
3144 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3145 if (error) {
3146 mbuf_freem(connect_packet);
3147 }
3148
3149 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3150 }
3151
5ba3f43e 3152 soclearfastopen(so);
490019cf
A		 3153
3154 return error;
3155}
3156
39236c6e
A		 3157static void
3158flow_divert_set_protosw(struct socket *so)
3159{
3160 so->so_flags |= SOF_FLOW_DIVERT;
3161 if (SOCK_DOM(so) == PF_INET) {
3162 so->so_proto = &g_flow_divert_in_protosw;
3163 }
3164#if INET6
3165 else {
3166 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3167 }
3168#endif /* INET6 */
3169}
3170
3e170ce0
A		 3171static void
3172flow_divert_set_udp_protosw(struct socket *so)
3173{
3174 so->so_flags |= SOF_FLOW_DIVERT;
3175 if (SOCK_DOM(so) == PF_INET) {
3176 so->so_proto = &g_flow_divert_in_udp_protosw;
3177 }
3178#if INET6
3179 else {
3180 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3181 }
3182#endif /* INET6 */
3183}
3184
39236c6e
A		 3185static errno_t
3186flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3187{
3188 int error = 0;
3189 struct flow_divert_pcb *fd_cb = NULL;
3190 struct ifnet *ifp = NULL;
3191 struct inpcb *inp = NULL;
3192 struct socket *old_so;
3193 mbuf_t recv_data = NULL;
3194
3195 socket_unlock(so, 0);
3196
3197 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3198
3199 /* Find the flow divert control block */
3200 lck_rw_lock_shared(&g_flow_divert_group_lck);
3201 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3202 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3203 if (group != NULL) {
3204 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3205 }
3206 }
3207 lck_rw_done(&g_flow_divert_group_lck);
3208
3209 if (fd_cb == NULL) {
3210 error = ENOENT;
3211 goto done;
3212 }
3213
3214 FDLOCK(fd_cb);
3215
3216 /* Dis-associate the flow divert control block from its current socket */
3217 old_so = fd_cb->so;
3218
3219 inp = sotoinpcb(old_so);
3220
3221 VERIFY(inp != NULL);
3222
3223 socket_lock(old_so, 0);
3e170ce0 3224 flow_divert_disconnect_socket(old_so);
39236c6e
A		 3225 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3226 old_so->so_fd_pcb = NULL;
3e170ce0
A		 3227 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3228 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3229 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3230 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3231 }
39236c6e
A		 3232 fd_cb->so = NULL;
3233 /* Save the output interface */
3234 ifp = inp->inp_last_outifp;
3235 if (old_so->so_rcv.sb_cc > 0) {
3236 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3237 sbflush(&old_so->so_rcv);
3238 }
3239 socket_unlock(old_so, 0);
3240
3241 /* Associate the new socket with the flow divert control block */
3242 socket_lock(so, 0);
3243 so->so_fd_pcb = fd_cb;
3244 inp = sotoinpcb(so);
3245 inp->inp_last_outifp = ifp;
3246 if (recv_data != NULL) {
3247 if (sbappendstream(&so->so_rcv, recv_data)) {
3248 sorwakeup(so);
3249 }
3250 }
3251 flow_divert_set_protosw(so);
3252 socket_unlock(so, 0);
3253
3254 fd_cb->so = so;
3255 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3256
3257 FDUNLOCK(fd_cb);
3258
3259done:
3260 socket_lock(so, 0);
3261
3262 if (fd_cb != NULL) {
3263 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3264 }
3265
3266 return error;
3267}
3268
3e170ce0
A		 3269errno_t
3270flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3271{
3272 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3273 struct inpcb *inp;
3274 int error = 0;
3275
3276 inp = sotoinpcb(so);
3277 if (inp == NULL) {
3278 return (EINVAL);
3279 }
3280
3281 if (fd_cb == NULL) {
3282 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3283 if (fd_ctl_unit > 0) {
3284 error = flow_divert_pcb_init(so, fd_ctl_unit);
3285 fd_cb = so->so_fd_pcb;
3286 if (error != 0 || fd_cb == NULL) {
3287 goto done;
3288 }
3289 } else {
3290 error = ENETDOWN;
3291 goto done;
3292 }
3293 }
3294 return flow_divert_data_out(so, flags, data, to, control, p);
3295
3296done:
3297 if (data) {
3298 mbuf_freem(data);
3299 }
3300 if (control) {
3301 mbuf_free(control);
3302 }
3303
3304 return error;
3305}
3306
39236c6e
A		 3307errno_t
3308flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3309{
3310 errno_t error = 0;
3311 struct flow_divert_pcb *fd_cb;
3312
3313 if (so->so_flags & SOF_FLOW_DIVERT) {
3314 return EALREADY;
3315 }
3316
3317 fd_cb = flow_divert_pcb_create(so);
3318 if (fd_cb != NULL) {
3319 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3320 if (error) {
3321 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3322 FDRELEASE(fd_cb);
3323 } else {
39236c6e
A		 3324 fd_cb->control_group_unit = ctl_unit;
3325 so->so_fd_pcb = fd_cb;
3326
3e170ce0
A		 3327 if (SOCK_TYPE(so) == SOCK_STREAM) {
3328 flow_divert_set_protosw(so);
3329 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3330 flow_divert_set_udp_protosw(so);
3331 }
39236c6e
A		 3332
3333 FDLOG0(LOG_INFO, fd_cb, "Created");
3334 }
3335 } else {
3336 error = ENOMEM;
3337 }
3338
3339 return error;
3340}
3341
3342errno_t
3343flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3344{
3345 uint32_t ctl_unit = 0;
3346 uint32_t key_unit = 0;
3347 uint32_t flow_id = 0;
3348 int error = 0;
490019cf 3349 int hmac_error = 0;
39236c6e
A		 3350 mbuf_t token = NULL;
3351
3352 if (so->so_flags & SOF_FLOW_DIVERT) {
3353 error = EALREADY;
3354 goto done;
3355 }
3356
3357 if (g_init_result) {
3358 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3359 error = ENOPROTOOPT;
3360 goto done;
3361 }
3362
3e170ce0
A		 3363 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3364 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
39236c6e
A		 3365 (SOCK_DOM(so) != PF_INET
3366#if INET6
3367 && SOCK_DOM(so) != PF_INET6
3368#endif
3369 ))
3370 {
3371 error = EINVAL;
3372 goto done;
3373 } else {
3e170ce0
A		 3374 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3375 struct tcpcb *tp = sototcpcb(so);
3376 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3377 error = EINVAL;
3378 goto done;
3379 }
39236c6e
A		 3380 }
3381 }
3382
3383 error = soopt_getm(sopt, &token);
3384 if (error) {
5c9f4661 3385 token = NULL;
39236c6e
A		 3386 goto done;
3387 }
3388
3389 error = soopt_mcopyin(sopt, token);
3390 if (error) {
5c9f4661 3391 token = NULL;
39236c6e
A		 3392 goto done;
3393 }
3394
3395 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3396 if (!error) {
3397 key_unit = ntohl(key_unit);
4d15aeb1
A		 3398 if (key_unit >= GROUP_COUNT_MAX) {
3399 key_unit = 0;
3400 }
39236c6e
A		 3401 } else if (error != ENOENT) {
3402 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3403 goto done;
3404 } else {
3405 key_unit = 0;
3406 }
3407
3408 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3409 if (error) {
3410 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3411 goto done;
3412 }
3413
3414 /* A valid kernel control unit is required */
3415 ctl_unit = ntohl(ctl_unit);
3416 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3417 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3418 error = EINVAL;
3419 goto done;
3420 }
3421
3422 socket_unlock(so, 0);
490019cf 3423 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
39236c6e
A		 3424 socket_lock(so, 0);
3425
490019cf
A		 3426 if (hmac_error && hmac_error != ENOENT) {
3427 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3428 error = hmac_error;
39236c6e
A		 3429 goto done;
3430 }
3431
3432 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3433 if (error && error != ENOENT) {
3434 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3435 goto done;
3436 }
3437
3438 if (flow_id == 0) {
3439 error = flow_divert_pcb_init(so, ctl_unit);
3440 if (error == 0) {
3441 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3442 int log_level = LOG_NOTICE;
3443
3444 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3445 sizeof(log_level), &log_level, NULL);
3446 if (error == 0) {
3447 fd_cb->log_level = log_level;
3448 }
3449 error = 0;
3450
3451 fd_cb->connect_token = token;
3452 token = NULL;
3453 }
3454 } else {
3455 error = flow_divert_attach(so, flow_id, ctl_unit);
3456 }
3457
490019cf
A		 3458 if (hmac_error == 0) {
3459 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3460 if (fd_cb != NULL) {
3461 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3462 }
3463 }
3464
39236c6e
A		 3465done:
3466 if (token != NULL) {
3467 mbuf_freem(token);
3468 }
3469
3470 return error;
3471}
3472
3473errno_t
3474flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3475{
3476 uint32_t ctl_unit;
3477 int error = 0;
3478 uint8_t hmac[SHA_DIGEST_LENGTH];
3479 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3480 mbuf_t token = NULL;
3481 struct flow_divert_group *control_group = NULL;
3482
3483 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3484 error = EINVAL;
3485 goto done;
3486 }
3487
3488 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3489
3490 if (fd_cb->group == NULL) {
3491 error = EINVAL;
3492 goto done;
3493 }
3494
3495 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3496 if (error) {
3497 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3498 goto done;
3499 }
3500
3501 ctl_unit = htonl(fd_cb->group->ctl_unit);
3502
3503 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3504 if (error) {
3505 goto done;
3506 }
3507
3508 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3509 if (error) {
3510 goto done;
3511 }
3512
490019cf
A		 3513 if (fd_cb->app_data != NULL) {
3514 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3515 if (error) {
3516 goto done;
3517 }
3518 }
3519
39236c6e
A		 3520 socket_unlock(so, 0);
3521 lck_rw_lock_shared(&g_flow_divert_group_lck);
3522
3523 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3524 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3525 {
3526 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3527 }
3528
3529 if (control_group != NULL) {
3530 lck_rw_lock_shared(&control_group->lck);
3531 ctl_unit = htonl(control_group->ctl_unit);
3532 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3533 if (!error) {
3534 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3535 }
3536 lck_rw_done(&control_group->lck);
3537 } else {
3538 error = ENOPROTOOPT;
3539 }
3540
3541 lck_rw_done(&g_flow_divert_group_lck);
3542 socket_lock(so, 0);
3543
3544 if (error) {
3545 goto done;
3546 }
3547
3548 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3549 if (error) {
3550 goto done;
3551 }
3552
39037602
A		 3553 if (sopt->sopt_val == USER_ADDR_NULL) {
3554 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3555 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3556 goto done;
3557 }
3558
39236c6e
A		 3559 error = soopt_mcopyout(sopt, token);
3560 if (error) {
3561 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3562 goto done;
3563 }
3564
3565done:
3566 if (token != NULL) {
3567 mbuf_freem(token);
3568 }
3569
3570 return error;
3571}
3572
3573static errno_t
3574flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3575{
3e170ce0 3576 struct flow_divert_group *new_group = NULL;
39236c6e
A		 3577 int error = 0;
3578
3579 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3580 error = EINVAL;
3581 goto done;
3582 }
3583
3584 *unitinfo = NULL;
3585
3586 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3587 if (new_group == NULL) {
3588 error = ENOBUFS;
3589 goto done;
3590 }
3591
3592 memset(new_group, 0, sizeof(*new_group));
3593
3594 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3595 RB_INIT(&new_group->pcb_tree);
3596 new_group->ctl_unit = sac->sc_unit;
3597 MBUFQ_INIT(&new_group->send_queue);
39037602 3598 new_group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 3599
3600 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3601
3602 if (g_flow_divert_groups == NULL) {
3603 MALLOC(g_flow_divert_groups,
3604 struct flow_divert_group **,
3605 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3606 M_TEMP,
3607 M_WAITOK | M_ZERO);
3608 }
3609
3610 if (g_flow_divert_groups == NULL) {
3611 error = ENOBUFS;
3612 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3613 error = EALREADY;
3614 } else {
3615 g_flow_divert_groups[sac->sc_unit] = new_group;
3616 g_active_group_count++;
3617 }
3618
3619 lck_rw_done(&g_flow_divert_group_lck);
3620
3621 *unitinfo = new_group;
3622
3623done:
3624 if (error != 0 && new_group != NULL) {
3625 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3626 }
3627 return error;
3628}
3629
3630static errno_t
3631flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3632{
3633 struct flow_divert_group *group = NULL;
3634 errno_t error = 0;
39236c6e
A		 3635
3636 if (unit >= GROUP_COUNT_MAX) {
3637 return EINVAL;
3638 }
3639
3640 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3641
3642 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3643
3644 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3645 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3646 g_flow_divert_groups, g_active_group_count);
3647 }
3648
3649 group = g_flow_divert_groups[unit];
3650
3651 if (group != (struct flow_divert_group *)unitinfo) {
3652 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3653 }
3654
3655 if (group != NULL) {
3656 flow_divert_close_all(group);
3657 if (group->token_key != NULL) {
3658 memset(group->token_key, 0, group->token_key_size);
3659 FREE(group->token_key, M_TEMP);
3660 group->token_key = NULL;
3661 group->token_key_size = 0;
3662 }
39037602
A		 3663
3664 /* Re-set the current trie */
3665 if (group->signing_id_trie.memory != NULL) {
3666 FREE(group->signing_id_trie.memory, M_TEMP);
3667 }
3668 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3669 group->signing_id_trie.root = NULL_TRIE_IDX;
3670
39236c6e
A		 3671 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3672 g_flow_divert_groups[unit] = NULL;
3673 g_active_group_count--;
3674 } else {
3675 error = EINVAL;
3676 }
3677
3678 if (g_active_group_count == 0) {
3679 FREE(g_flow_divert_groups, M_TEMP);
3680 g_flow_divert_groups = NULL;
3681 }
3682
39236c6e
A		 3683 lck_rw_done(&g_flow_divert_group_lck);
3684
3685 return error;
3686}
3687
3688static errno_t
3689flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3690{
3691 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3692}
3693
3694static void
3695flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3696{
3697 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3698
3699 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3700 struct flow_divert_pcb *fd_cb;
3701 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3702
3703 lck_rw_lock_shared(&g_flow_divert_group_lck);
3704 lck_rw_lock_exclusive(&group->lck);
3705
3706 while (!MBUFQ_EMPTY(&group->send_queue)) {
3707 mbuf_t next_packet;
3708 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3709 next_packet = MBUFQ_FIRST(&group->send_queue);
3710 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3711 if (error) {
3712 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3713 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3714 lck_rw_done(&group->lck);
3715 lck_rw_done(&g_flow_divert_group_lck);
3716 return;
3717 }
3718 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3719 }
3720
3721 SLIST_INIT(&tmp_list);
3722
3723 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3724 FDRETAIN(fd_cb);
3725 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3726 }
3727
3728 lck_rw_done(&group->lck);
3729
3730 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3731 FDLOCK(fd_cb);
3732 if (fd_cb->so != NULL) {
3733 socket_lock(fd_cb->so, 0);
3734 if (fd_cb->group != NULL) {
3735 flow_divert_send_buffered_data(fd_cb, FALSE);
3736 }
3737 socket_unlock(fd_cb->so, 0);
3738 }
3739 FDUNLOCK(fd_cb);
3740 FDRELEASE(fd_cb);
3741 }
3742
3743 lck_rw_done(&g_flow_divert_group_lck);
3744 }
3745}
3746
3747static int
3748flow_divert_kctl_init(void)
3749{
3750 struct kern_ctl_reg ctl_reg;
3751 int result;
3752
3753 memset(&ctl_reg, 0, sizeof(ctl_reg));
3754
fe8ab488 3755 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
39236c6e
A		 3756 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3757 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3758 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3759 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3760
3761 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3762 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3763 ctl_reg.ctl_send = flow_divert_kctl_send;
3764 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3765
3766 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3767
3768 if (result) {
3769 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3770 return result;
3771 }
3772
3773 return 0;
3774}
3775
3776void
3777flow_divert_init(void)
3778{
3779 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 3780 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 3781
3782 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3783
3784 VERIFY(g_tcp_protosw != NULL);
3785
3786 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3787 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3788
3789 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3790 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3791 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3792 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3793 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3794 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3795 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3796 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3797 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3798 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3799 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3800
3801 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3802 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3803
3804 /*
3805 * Socket filters shouldn't attach/detach to/from this protosw
3806 * since pr_protosw is to be used instead, which points to the
3807 * real protocol; if they do, it is a bug and we should panic.
3808 */
3809 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3810 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3811 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3812 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3813
3e170ce0
A		 3814 /* UDP */
3815 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3816 VERIFY(g_udp_protosw != NULL);
3817
3818 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3819 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3820
3821 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3822 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3823 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3824 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3825 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3826 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3827 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3828 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3829 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3830 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3831 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3832 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3833 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3834
3835 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3836 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3837
3838 /*
3839 * Socket filters shouldn't attach/detach to/from this protosw
3840 * since pr_protosw is to be used instead, which points to the
3841 * real protocol; if they do, it is a bug and we should panic.
3842 */
3843 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3844 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3845 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3846 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3847
39236c6e
A		 3848#if INET6
3849 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3850
3851 VERIFY(g_tcp6_protosw != NULL);
3852
3853 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3854 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3855
3856 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3857 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3858 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3859 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3860 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3861 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3862 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3863 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3864 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3865 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3866 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3867
3868 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3869 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3870 /*
3871 * Socket filters shouldn't attach/detach to/from this protosw
3872 * since pr_protosw is to be used instead, which points to the
3873 * real protocol; if they do, it is a bug and we should panic.
3874 */
3875 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3876 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3877 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3878 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 3879
3880 /* UDP6 */
3881 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3882
3883 VERIFY(g_udp6_protosw != NULL);
3884
3885 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3886 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3887
3888 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3889 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3890 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3891 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3892 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3893 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3894 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3895 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3896 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3897 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3898 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3899 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3900 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3901
3902 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3903 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3904 /*
3905 * Socket filters shouldn't attach/detach to/from this protosw
3906 * since pr_protosw is to be used instead, which points to the
3907 * real protocol; if they do, it is a bug and we should panic.
3908 */
3909 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3910 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3911 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3912 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
39236c6e
A		 3913#endif /* INET6 */
3914
3915 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3916 if (flow_divert_grp_attr == NULL) {
3917 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3918 g_init_result = ENOMEM;
3919 goto done;
3920 }
3921
3922 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3923 if (flow_divert_mtx_grp == NULL) {
3924 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3925 g_init_result = ENOMEM;
3926 goto done;
3927 }
3928
3929 flow_divert_mtx_attr = lck_attr_alloc_init();
3930 if (flow_divert_mtx_attr == NULL) {
3931 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3932 g_init_result = ENOMEM;
3933 goto done;
3934 }
3935
3936 g_init_result = flow_divert_kctl_init();
3937 if (g_init_result) {
3938 goto done;
3939 }
3940
3941 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3942
39236c6e
A		 3943done:
3944 if (g_init_result != 0) {
3945 if (flow_divert_mtx_attr != NULL) {
3946 lck_attr_free(flow_divert_mtx_attr);
3947 flow_divert_mtx_attr = NULL;
3948 }
3949 if (flow_divert_mtx_grp != NULL) {
3950 lck_grp_free(flow_divert_mtx_grp);
3951 flow_divert_mtx_grp = NULL;
3952 }
3953 if (flow_divert_grp_attr != NULL) {
3954 lck_grp_attr_free(flow_divert_grp_attr);
3955 flow_divert_grp_attr = NULL;
3956 }
3957
3958 if (g_flow_divert_kctl_ref != NULL) {
3959 ctl_deregister(g_flow_divert_kctl_ref);
3960 g_flow_divert_kctl_ref = NULL;
3961 }
3962 }
3963}
mirror of XNU