]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3248.50.21.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
490019cf 2 * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
48#include <net/if_var.h>
49#include <net/route.h>
50#include <net/flowhash.h>
51#include <net/ntstat.h>
52#include <netinet/in.h>
53#include <netinet/in_var.h>
54#include <netinet/tcp.h>
55#include <netinet/tcp_var.h>
56#include <netinet/tcp_fsm.h>
57#include <netinet/flow_divert.h>
58#include <netinet/flow_divert_proto.h>
59#if INET6
3e170ce0 60#include <netinet6/in6_pcb.h>
39236c6e
A		 61#include <netinet6/ip6protosw.h>
62#endif /* INET6 */
63#include <dev/random/randomdev.h>
64#include <libkern/crypto/sha1.h>
65#include <libkern/crypto/crypto_internal.h>
66
67#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
68#define FLOW_DIVERT_READ_CLOSED 0x00000002
69#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
70#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
71#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
72#define FLOW_DIVERT_TRANSFERRED 0x00000020
490019cf 73#define FLOW_DIVERT_HAS_HMAC 0x00000040
39236c6e
A		 74
75#define FDLOG(level, pcb, format, ...) do { \
76 if (level <= (pcb)->log_level) { \
77 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): " format "\n", __FUNCTION__, (pcb)->hash, __VA_ARGS__); \
78 } \
79} while (0)
80
81#define FDLOG0(level, pcb, msg) do { \
82 if (level <= (pcb)->log_level) { \
83 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): %s\n", __FUNCTION__, (pcb)->hash, msg); \
84 } \
85} while (0)
86
87#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
88#define FDRELEASE(pcb) \
89 do { \
90 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
91 flow_divert_pcb_destroy(pcb); \
92 } \
93 } while (0)
94
95#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
96#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
97
98#define FD_CTL_SENDBUFF_SIZE (2 * FLOW_DIVERT_CHUNK_SIZE)
99#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
100
101#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
102
103#define GROUP_COUNT_MAX 32
104#define FLOW_DIVERT_MAX_NAME_SIZE 4096
105#define FLOW_DIVERT_MAX_KEY_SIZE 1024
ecc0ceb4 106#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e
A		 107
108#define DNS_SERVICE_GROUP_UNIT (GROUP_COUNT_MAX + 1)
109
110struct flow_divert_trie_node
111{
112 uint16_t start;
113 uint16_t length;
114 uint16_t child_map;
115 uint32_t group_unit;
116};
117
118struct flow_divert_trie
119{
120 struct flow_divert_trie_node *nodes;
121 uint16_t *child_maps;
122 uint8_t *bytes;
123 void *memory;
124 size_t nodes_count;
125 size_t child_maps_count;
126 size_t bytes_count;
127 size_t nodes_free_next;
128 size_t child_maps_free_next;
129 size_t bytes_free_next;
130 uint16_t root;
131};
132
133#define CHILD_MAP_SIZE 256
134#define NULL_TRIE_IDX 0xffff
135#define TRIE_NODE(t, i) ((t)->nodes[(i)])
136#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
137#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
138
139static struct flow_divert_pcb nil_pcb;
140
141decl_lck_rw_data(static, g_flow_divert_group_lck);
142static struct flow_divert_group **g_flow_divert_groups = NULL;
143static uint32_t g_active_group_count = 0;
144static struct flow_divert_trie g_signing_id_trie;
145
146static lck_grp_attr_t *flow_divert_grp_attr = NULL;
147static lck_attr_t *flow_divert_mtx_attr = NULL;
148static lck_grp_t *flow_divert_mtx_grp = NULL;
149static errno_t g_init_result = 0;
150
151static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
152
153static struct protosw g_flow_divert_in_protosw;
154static struct pr_usrreqs g_flow_divert_in_usrreqs;
3e170ce0
A		 155static struct protosw g_flow_divert_in_udp_protosw;
156static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
39236c6e
A		 157#if INET6
158static struct ip6protosw g_flow_divert_in6_protosw;
159static struct pr_usrreqs g_flow_divert_in6_usrreqs;
3e170ce0
A		 160static struct ip6protosw g_flow_divert_in6_udp_protosw;
161static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
39236c6e
A		 162#endif /* INET6 */
163
164static struct protosw *g_tcp_protosw = NULL;
165static struct ip6protosw *g_tcp6_protosw = NULL;
3e170ce0
A		 166static struct protosw *g_udp_protosw = NULL;
167static struct ip6protosw *g_udp6_protosw = NULL;
168
169static errno_t
170flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
171
172static errno_t
173flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
174
175static boolean_t
176flow_divert_is_sockaddr_valid(struct sockaddr *addr);
177
178static int
179flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
180
181struct sockaddr *
182flow_divert_get_buffered_target_address(mbuf_t buffer);
183
184static boolean_t
185flow_divert_has_pcb_local_address(const struct inpcb *inp);
186
187static void
188flow_divert_disconnect_socket(struct socket *so);
39236c6e
A		 189
190static inline int
191flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
192{
193 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
194}
195
196RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
197RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
198
199static const char *
200flow_divert_packet_type2str(uint8_t packet_type)
201{
202 switch (packet_type) {
203 case FLOW_DIVERT_PKT_CONNECT:
204 return "connect";
205 case FLOW_DIVERT_PKT_CONNECT_RESULT:
206 return "connect result";
207 case FLOW_DIVERT_PKT_DATA:
208 return "data";
209 case FLOW_DIVERT_PKT_CLOSE:
210 return "close";
211 case FLOW_DIVERT_PKT_READ_NOTIFY:
212 return "read notification";
213 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
214 return "properties update";
215 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
216 return "app map update";
217 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
218 return "app map create";
219 default:
220 return "unknown";
221 }
222}
223
224static struct flow_divert_pcb *
225flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
226{
227 struct flow_divert_pcb key_item;
228 struct flow_divert_pcb *fd_cb = NULL;
229
230 key_item.hash = hash;
231
232 lck_rw_lock_shared(&group->lck);
233 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
234 FDRETAIN(fd_cb);
235 lck_rw_done(&group->lck);
236
237 return fd_cb;
238}
239
240static errno_t
241flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
242{
3e170ce0 243 errno_t error = 0;
39236c6e
A		 244 struct flow_divert_pcb *exist = NULL;
245 struct flow_divert_group *group;
246 static uint32_t g_nextkey = 1;
247 static uint32_t g_hash_seed = 0;
39236c6e
A		 248 int try_count = 0;
249
250 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
251 return EINVAL;
252 }
253
254 socket_unlock(fd_cb->so, 0);
255 lck_rw_lock_shared(&g_flow_divert_group_lck);
256
257 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
258 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
259 error = ENETUNREACH;
260 goto done;
261 }
262
263 group = g_flow_divert_groups[ctl_unit];
264 if (group == NULL) {
265 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
266 error = ENETUNREACH;
267 goto done;
268 }
269
270 socket_lock(fd_cb->so, 0);
271
272 do {
273 uint32_t key[2];
274 uint32_t idx;
275
276 key[0] = g_nextkey++;
277 key[1] = RandomULong();
278
279 if (g_hash_seed == 0) {
280 g_hash_seed = RandomULong();
281 }
282
283 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
284
285 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
286 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
287 if (curr_group != NULL && curr_group != group) {
288 lck_rw_lock_shared(&curr_group->lck);
289 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
290 lck_rw_done(&curr_group->lck);
291 if (exist != NULL) {
292 break;
293 }
294 }
295 }
296
297 if (exist == NULL) {
298 lck_rw_lock_exclusive(&group->lck);
299 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
300 lck_rw_done(&group->lck);
301 }
302 } while (exist != NULL && try_count++ < 3);
303
304 if (exist == NULL) {
305 fd_cb->group = group;
306 FDRETAIN(fd_cb); /* The group now has a reference */
307 } else {
308 fd_cb->hash = 0;
3e170ce0 309 error = EEXIST;
39236c6e
A		 310 }
311
312 socket_unlock(fd_cb->so, 0);
313
314done:
315 lck_rw_done(&g_flow_divert_group_lck);
316 socket_lock(fd_cb->so, 0);
317
3e170ce0 318 return error;
39236c6e
A		 319}
320
321static struct flow_divert_pcb *
322flow_divert_pcb_create(socket_t so)
323{
324 struct flow_divert_pcb *new_pcb = NULL;
325
326 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
327 if (new_pcb == NULL) {
328 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
329 return NULL;
330 }
331
332 memset(new_pcb, 0, sizeof(*new_pcb));
333
334 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
335 new_pcb->so = so;
336 new_pcb->log_level = nil_pcb.log_level;
337
338 FDRETAIN(new_pcb); /* Represents the socket's reference */
339
340 return new_pcb;
341}
342
343static void
344flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
345{
346 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
347 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
348
349 if (fd_cb->local_address != NULL) {
350 FREE(fd_cb->local_address, M_SONAME);
351 }
352 if (fd_cb->remote_address != NULL) {
353 FREE(fd_cb->remote_address, M_SONAME);
354 }
355 if (fd_cb->connect_token != NULL) {
356 mbuf_freem(fd_cb->connect_token);
357 }
490019cf
A		 358 if (fd_cb->connect_packet != NULL) {
359 mbuf_freem(fd_cb->connect_packet);
360 }
361 if (fd_cb->app_data != NULL) {
362 FREE(fd_cb->app_data, M_TEMP);
363 }
39236c6e
A		 364 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
365}
366
367static void
368flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
369{
370 if (fd_cb->group != NULL) {
371 struct flow_divert_group *group = fd_cb->group;
372 lck_rw_lock_exclusive(&group->lck);
373 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
374 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
375 fd_cb->group = NULL;
376 FDRELEASE(fd_cb); /* Release the group's reference */
377 lck_rw_done(&group->lck);
378 }
379}
380
381static int
382flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
383{
384 struct flow_divert_packet_header hdr;
385 int error = 0;
386
387 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
388 if (error) {
389 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
390 return error;
391 }
392
393 hdr.packet_type = packet_type;
394 hdr.conn_id = htonl(fd_cb->hash);
395
396 /* Lay down the header */
397 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
398 if (error) {
399 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
400 mbuf_freem(*packet);
401 *packet = NULL;
402 return error;
403 }
404
405 return 0;
406}
407
408static int
3e170ce0 409flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 410{
3e170ce0
A		 411 uint32_t net_length = htonl(length);
412 int error = 0;
39236c6e
A		 413
414 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
415 if (error) {
416 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
417 return error;
418 }
419
420 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
421 if (error) {
422 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%lu)", length);
423 return error;
424 }
425
426 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
427 if (error) {
428 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
429 return error;
430 }
431
432 return error;
433}
434
435static int
436flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
437{
3e170ce0
A		 438 size_t cursor = offset;
439 int error = 0;
440 uint32_t curr_length;
441 uint8_t curr_type;
39236c6e
A		 442
443 *err = 0;
444
445 do {
446 if (!next) {
447 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
448 if (error) {
449 *err = ENOENT;
450 return -1;
451 }
452 } else {
453 next = 0;
454 curr_type = FLOW_DIVERT_TLV_NIL;
455 }
456
457 if (curr_type != type) {
458 cursor += sizeof(curr_type);
459 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
460 if (error) {
461 *err = error;
462 return -1;
463 }
464
465 cursor += (sizeof(curr_length) + ntohl(curr_length));
466 }
467 } while (curr_type != type);
468
469 return cursor;
470}
471
472static int
3e170ce0 473flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 474{
3e170ce0
A		 475 int error = 0;
476 uint32_t length;
477 int tlv_offset;
39236c6e
A		 478
479 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
480 if (tlv_offset < 0) {
481 return error;
482 }
483
484 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
485 if (error) {
486 return error;
487 }
488
489 length = ntohl(length);
490
491 if (val_size != NULL) {
492 *val_size = length;
493 }
494
495 if (buff != NULL && buff_len > 0) {
496 size_t to_copy = (length < buff_len) ? length : buff_len;
497 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
498 if (error) {
499 return error;
500 }
501 }
502
503 return 0;
504}
505
506static int
507flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
508{
509 mbuf_t curr_mbuf = packet;
510
511 if (g_crypto_funcs == NULL || group->token_key == NULL) {
512 return ENOPROTOOPT;
513 }
514
515 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
516 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
517
518 while (curr_mbuf != NULL) {
519 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
520 curr_mbuf = mbuf_next(curr_mbuf);
521 }
522
523 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
524
525 return 0;
526}
527
528static int
529flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
530{
531 int error = 0;
532 struct flow_divert_group *group = NULL;
533 int hmac_offset;
534 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
535 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
536 mbuf_t tail;
537
538 lck_rw_lock_shared(&g_flow_divert_group_lck);
539
540 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
541 group = g_flow_divert_groups[ctl_unit];
542 }
543
544 if (group == NULL) {
545 lck_rw_done(&g_flow_divert_group_lck);
546 return ENOPROTOOPT;
547 }
548
549 lck_rw_lock_shared(&group->lck);
550
551 if (group->token_key == NULL) {
552 error = ENOPROTOOPT;
553 goto done;
554 }
555
556 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
557 if (hmac_offset < 0) {
558 goto done;
559 }
560
561 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
562 if (error) {
563 goto done;
564 }
565
566 /* Chop off the HMAC TLV */
567 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
568 if (error) {
569 goto done;
570 }
571
572 mbuf_free(tail);
573
574 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
575 if (error) {
576 goto done;
577 }
578
579 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
580 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
581 error = EINVAL;
582 goto done;
583 }
584
585done:
586 lck_rw_done(&group->lck);
587 lck_rw_done(&g_flow_divert_group_lck);
588 return error;
589}
590
591static void
592flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
593{
594 struct inpcb *inp = NULL;
595 struct ifnet *ifp = NULL;
596 Boolean cell = FALSE;
597 Boolean wifi = FALSE;
fe8ab488 598 Boolean wired = FALSE;
39236c6e
A		 599
600 inp = sotoinpcb(fd_cb->so);
601 if (inp == NULL) {
602 return;
603 }
604
605 ifp = inp->inp_last_outifp;
606 if (ifp != NULL) {
607 cell = IFNET_IS_CELLULAR(ifp);
608 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 609 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e
A		 610 }
611
612 if (send) {
fe8ab488
A		 613 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
614 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 615 } else {
fe8ab488
A		 616 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
617 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e
A		 618 }
619}
620
621static errno_t
622flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
623{
624 struct inpcb *inp = NULL;
39236c6e
A		 625
626 inp = sotoinpcb(fd_cb->so);
fe8ab488
A		 627 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
628 IFNET_IS_CELLULAR(inp->inp_last_outifp))
629 return EHOSTUNREACH;
630
631 return 0;
632}
633
634static errno_t
635flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
636{
637 struct inpcb *inp = NULL;
638
639 inp = sotoinpcb(fd_cb->so);
640 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
641 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
642 return EHOSTUNREACH;
39236c6e
A		 643
644 return 0;
645}
646
647static void
648flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
649{
650 if (how != SHUT_RD) {
651 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
652 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
653 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
654 /* If the tunnel is not accepting writes any more, then flush the send buffer */
655 sbflush(&fd_cb->so->so_snd);
656 }
657 }
658 if (how != SHUT_WR) {
659 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
660 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
661 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
662 }
663 }
664}
665
666static uint16_t
667trie_node_alloc(struct flow_divert_trie *trie)
668{
669 if (trie->nodes_free_next < trie->nodes_count) {
670 uint16_t node_idx = trie->nodes_free_next++;
671 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
672 return node_idx;
673 } else {
674 return NULL_TRIE_IDX;
675 }
676}
677
678static uint16_t
679trie_child_map_alloc(struct flow_divert_trie *trie)
680{
681 if (trie->child_maps_free_next < trie->child_maps_count) {
682 return trie->child_maps_free_next++;
683 } else {
684 return NULL_TRIE_IDX;
685 }
686}
687
688static uint16_t
689trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
690{
691 uint16_t start = trie->bytes_free_next;
692 if (start + bytes_size <= trie->bytes_count) {
693 if (start != bytes_idx) {
694 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
695 }
696 trie->bytes_free_next += bytes_size;
697 return start;
698 } else {
699 return NULL_TRIE_IDX;
700 }
701}
702
703static uint16_t
704flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
705{
706 uint16_t current = trie->root;
707 uint16_t child = trie->root;
708 uint16_t string_end = string_start + string_len;
709 uint16_t string_idx = string_start;
710 uint16_t string_remainder = string_len;
711
712 while (child != NULL_TRIE_IDX) {
713 uint16_t parent = current;
714 uint16_t node_idx;
715 uint16_t current_end;
716
717 current = child;
718 child = NULL_TRIE_IDX;
719
720 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
721
722 for (node_idx = TRIE_NODE(trie, current).start;
723 node_idx < current_end &&
724 string_idx < string_end &&
725 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
726 node_idx++, string_idx++);
727
728 string_remainder = string_end - string_idx;
729
730 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
731 /*
732 * We did not reach the end of the current node's string.
733 * We need to split the current node into two:
734 * 1. A new node that contains the prefix of the node that matches
735 * the prefix of the string being inserted.
736 * 2. The current node modified to point to the remainder
737 * of the current node's string.
738 */
739 uint16_t prefix = trie_node_alloc(trie);
740 if (prefix == NULL_TRIE_IDX) {
741 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
742 return NULL_TRIE_IDX;
743 }
744
745 /*
746 * Prefix points to the portion of the current nodes's string that has matched
747 * the input string thus far.
748 */
749 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
750 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
751
752 /*
753 * Prefix has the current node as the child corresponding to the first byte
754 * after the split.
755 */
756 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
757 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
758 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
759 return NULL_TRIE_IDX;
760 }
761 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
762
763 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
764 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
765
766 /* Current node is adjusted to point to the remainder */
767 TRIE_NODE(trie, current).start = node_idx;
768 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
769
770 /* We want to insert the new leaf (if any) as a child of the prefix */
771 current = prefix;
772 }
773
774 if (string_remainder > 0) {
775 /*
776 * We still have bytes in the string that have not been matched yet.
777 * If the current node has children, iterate to the child corresponding
778 * to the next byte in the string.
779 */
780 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
781 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
782 }
783 }
784 } /* while (child != NULL_TRIE_IDX) */
785
786 if (string_remainder > 0) {
787 /* Add a new leaf containing the remainder of the string */
788 uint16_t leaf = trie_node_alloc(trie);
789 if (leaf == NULL_TRIE_IDX) {
790 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
791 return NULL_TRIE_IDX;
792 }
793
794 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
795 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
796 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
797 return NULL_TRIE_IDX;
798 }
799 TRIE_NODE(trie, leaf).length = string_remainder;
800
801 /* Set the new leaf as the child of the current node */
802 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
803 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
804 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
805 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
806 return NULL_TRIE_IDX;
807 }
808 }
809 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
810 current = leaf;
811 } /* else duplicate or this string is a prefix of one of the existing strings */
812
813 return current;
814}
815
3e170ce0 816#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 817static uint16_t
3e170ce0 818flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
39236c6e
A		 819{
820 uint16_t current = trie->root;
821 uint16_t string_idx = 0;
822
823 while (current != NULL_TRIE_IDX) {
824 uint16_t next = NULL_TRIE_IDX;
825 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
826 uint16_t node_idx;
827
828 for (node_idx = TRIE_NODE(trie, current).start;
829 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
830 node_idx++, string_idx++);
831
832 if (node_idx == node_end) {
833 if (string_bytes[string_idx] == '\0') {
834 return current; /* Got an exact match */
3e170ce0
A		 835 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
836 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
837 string_bytes[string_idx] = '\0';
838 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 839 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
840 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
841 }
842 }
843 current = next;
844 }
845
846 return NULL_TRIE_IDX;
847}
848
490019cf
A		 849struct uuid_search_info {
850 uuid_t target_uuid;
851 char *found_signing_id;
852 boolean_t found_multiple_signing_ids;
853 proc_t found_proc;
854};
855
856static int
857flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
858{
859 struct uuid_search_info *info = (struct uuid_search_info *)arg;
860 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
861
862 if (info->found_signing_id != NULL) {
863 if (!info->found_multiple_signing_ids) {
864 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
865 info->found_proc = p;
866 result = PROC_CLAIMED_DONE;
867 } else {
868 uuid_string_t uuid_str;
869 uuid_unparse(info->target_uuid, uuid_str);
870 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
871 }
872 FREE(info->found_signing_id, M_TEMP);
873 info->found_signing_id = NULL;
874 }
875
876 if (result == PROC_RETURNED_DONE) {
877 uuid_string_t uuid_str;
878 uuid_unparse(info->target_uuid, uuid_str);
879 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
880 }
881
882 return result;
883}
884
885static int
886flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
887{
888 struct uuid_search_info *info = (struct uuid_search_info *)arg;
889 int include = 0;
890
891 if (info->found_multiple_signing_ids) {
892 return include;
893 }
894
895 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
896 if (include) {
897 const char *signing_id = cs_identity_get(p);
898 if (signing_id != NULL) {
899 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
900 size_t signing_id_size = strlen(signing_id) + 1;
901 if (info->found_signing_id == NULL) {
902 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
903 memcpy(info->found_signing_id, signing_id, signing_id_size);
904 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
905 info->found_multiple_signing_ids = TRUE;
906 }
907 } else {
908 info->found_multiple_signing_ids = TRUE;
909 }
910 include = !info->found_multiple_signing_ids;
911 }
912
913 return include;
914}
915
916static proc_t
917flow_divert_find_proc_by_uuid(uuid_t uuid)
918{
919 struct uuid_search_info info;
920
921 if (LOG_INFO <= nil_pcb.log_level) {
922 uuid_string_t uuid_str;
923 uuid_unparse(uuid, uuid_str);
924 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
925 }
926
927 memset(&info, 0, sizeof(info));
928 info.found_proc = PROC_NULL;
929 uuid_copy(info.target_uuid, uuid);
930
931 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
932
933 return info.found_proc;
934}
935
39236c6e 936static int
490019cf 937flow_divert_get_src_proc(struct socket *so, proc_t *proc)
39236c6e
A		 938{
939 int release = 0;
940
490019cf
A		 941 if (so->so_flags & SOF_DELEGATED) {
942 if ((*proc)->p_pid != so->e_pid) {
943 *proc = proc_find(so->e_pid);
944 release = 1;
945 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
946 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
947 release = 1;
948 }
39236c6e
A		 949 } else if (*proc == PROC_NULL) {
950 *proc = current_proc();
951 }
952
953 if (*proc != PROC_NULL) {
954 if ((*proc)->p_pid == 0) {
955 if (release) {
956 proc_rele(*proc);
957 }
958 release = 0;
959 *proc = PROC_NULL;
960 }
961 }
962
963 return release;
964}
965
966static int
967flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
968{
969 int error;
970
971 if (fd_cb->group == NULL) {
972 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 973 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 974 return ECONNABORTED;
975 }
976
977 lck_rw_lock_shared(&fd_cb->group->lck);
978
979 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
980 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
981 } else {
982 error = ENOBUFS;
983 }
984
985 if (error == ENOBUFS) {
986 if (enqueue) {
987 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
988 lck_rw_lock_exclusive(&fd_cb->group->lck);
989 }
990 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
991 error = 0;
992 }
993 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
994 }
995
996 lck_rw_done(&fd_cb->group->lck);
997
998 return error;
999}
1000
1001static int
490019cf 1002flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 1003{
39236c6e 1004 int error = 0;
3e170ce0 1005 int flow_type = 0;
490019cf
A		 1006 char *signing_id = NULL;
1007 int free_signing_id = 0;
1008 mbuf_t connect_packet = NULL;
1009
1010 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1011 if (error) {
1012 goto done;
1013 }
1014
1015 error = EPERM;
1016
1017 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1018 uint32_t sid_size = 0;
1019 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1020 if (find_error == 0 && sid_size > 0) {
1021 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1022 if (signing_id != NULL) {
1023 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1024 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1025 free_signing_id = 1;
1026 }
1027 }
1028 }
1029
1030 socket_unlock(so, 0);
1031 if (g_signing_id_trie.root != NULL_TRIE_IDX) {
1032 proc_t src_proc = p;
1033 int release_proc = 0;
1034
1035 if (signing_id == NULL) {
1036 release_proc = flow_divert_get_src_proc(so, &src_proc);
1037 if (src_proc != PROC_NULL) {
1038 proc_lock(src_proc);
1039 if (src_proc->p_csflags & CS_VALID) {
1040 const char * cs_id;
1041 cs_id = cs_identity_get(src_proc);
1042 signing_id = __DECONST(char *, cs_id);
1043 } else {
1044 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
1045 }
1046 } else {
1047 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
1048 }
1049 } else {
1050 src_proc = PROC_NULL;
1051 }
1052
1053 if (signing_id != NULL) {
1054 uint16_t result = NULL_TRIE_IDX;
1055 lck_rw_lock_shared(&g_flow_divert_group_lck);
1056 result = flow_divert_trie_search(&g_signing_id_trie, (uint8_t *)signing_id);
1057 lck_rw_done(&g_flow_divert_group_lck);
1058 if (result != NULL_TRIE_IDX) {
1059 error = 0;
1060 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
1061
1062 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1063 if (error == 0) {
1064 if (src_proc != PROC_NULL) {
1065 unsigned char cdhash[SHA1_RESULTLEN];
1066 error = proc_getcdhash(src_proc, cdhash);
1067 if (error == 0) {
1068 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1069 if (error) {
1070 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1071 }
1072 } else {
1073 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
1074 }
1075 }
1076 } else {
1077 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1078 }
1079 } else {
1080 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
1081 }
1082 } else {
1083 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1084 }
1085
1086 if (src_proc != PROC_NULL) {
1087 proc_unlock(src_proc);
1088 if (release_proc) {
1089 proc_rele(src_proc);
1090 }
1091 }
1092 } else {
1093 FDLOG0(LOG_WARNING, fd_cb, "The signing ID trie is empty");
1094 }
1095 socket_lock(so, 0);
1096
1097 if (free_signing_id) {
1098 FREE(signing_id, M_TEMP);
1099 }
1100
1101 if (error) {
1102 goto done;
1103 }
39236c6e 1104
39236c6e
A		 1105 error = flow_divert_packet_append_tlv(connect_packet,
1106 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1107 sizeof(fd_cb->so->so_traffic_class),
1108 &fd_cb->so->so_traffic_class);
1109 if (error) {
1110 goto done;
1111 }
1112
3e170ce0
A		 1113 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1114 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1115 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1116 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1117 } else {
1118 error = EINVAL;
1119 goto done;
1120 }
1121 error = flow_divert_packet_append_tlv(connect_packet,
1122 FLOW_DIVERT_TLV_FLOW_TYPE,
1123 sizeof(flow_type),
1124 &flow_type);
1125
1126 if (error) {
1127 goto done;
1128 }
1129
39236c6e
A		 1130 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1131 error = flow_divert_packet_append_tlv(connect_packet,
1132 FLOW_DIVERT_TLV_PID,
1133 sizeof(fd_cb->so->e_pid),
1134 &fd_cb->so->e_pid);
1135 if (error) {
1136 goto done;
1137 }
1138
1139 error = flow_divert_packet_append_tlv(connect_packet,
1140 FLOW_DIVERT_TLV_UUID,
1141 sizeof(fd_cb->so->e_uuid),
1142 &fd_cb->so->e_uuid);
1143 if (error) {
1144 goto done;
1145 }
1146 } else {
1147 error = flow_divert_packet_append_tlv(connect_packet,
1148 FLOW_DIVERT_TLV_PID,
1149 sizeof(fd_cb->so->e_pid),
1150 &fd_cb->so->last_pid);
1151 if (error) {
1152 goto done;
1153 }
1154
1155 error = flow_divert_packet_append_tlv(connect_packet,
1156 FLOW_DIVERT_TLV_UUID,
1157 sizeof(fd_cb->so->e_uuid),
1158 &fd_cb->so->last_uuid);
1159 if (error) {
1160 goto done;
1161 }
1162 }
1163
1164 if (fd_cb->connect_token != NULL) {
1165 unsigned int token_len = m_length(fd_cb->connect_token);
1166 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1167 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1168 fd_cb->connect_token = NULL;
1169 } else {
1170 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
39236c6e
A		 1171
1172 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1173 if (error) {
1174 goto done;
1175 }
1176
3e170ce0 1177 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1178 if (error) {
1179 goto done;
1180 }
39236c6e
A		 1181 }
1182
490019cf
A		 1183 if (fd_cb->local_address != NULL) {
1184 error = EALREADY;
1185 goto done;
1186 } else {
1187 struct inpcb *inp = sotoinpcb(so);
1188 if (flow_divert_has_pcb_local_address(inp)) {
1189 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1190 if (error) {
1191 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1192 goto done;
1193 }
1194 }
1195 }
1196
3e170ce0
A		 1197 if (fd_cb->local_address != NULL) {
1198 /* socket is bound. */
490019cf
A		 1199 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1200 sizeof(struct sockaddr_storage), fd_cb->local_address);
1201 if (error) {
1202 goto done;
1203 }
1204 }
3e170ce0 1205
490019cf
A		 1206 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1207 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1208 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1209 if (error) {
1210 goto done;
1211 }
39236c6e
A		 1212 }
1213
1214done:
490019cf
A		 1215 if (!error) {
1216 *out_connect_packet = connect_packet;
1217 } else if (connect_packet != NULL) {
1218 mbuf_freem(connect_packet);
1219 }
1220
39236c6e
A		 1221 return error;
1222}
1223
1224static int
1225flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1226{
1227 int error = 0;
1228 mbuf_t packet = NULL;
1229 int rbuff_space = 0;
1230
1231 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1232 if (error) {
1233 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1234 goto done;
1235 }
1236
3e170ce0 1237 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1238 if (rbuff_space < 0) {
1239 rbuff_space = 0;
1240 }
1241 rbuff_space = htonl(rbuff_space);
1242 error = flow_divert_packet_append_tlv(packet,
1243 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1244 sizeof(rbuff_space),
1245 &rbuff_space);
1246 if (error) {
1247 goto done;
1248 }
1249
1250 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1251 if (error) {
1252 goto done;
1253 }
1254
1255done:
1256 if (error && packet != NULL) {
3e170ce0 1257 mbuf_freem(packet);
39236c6e
A		 1258 }
1259
1260 return error;
1261}
1262
1263static int
1264flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1265{
1266 int error = 0;
1267 mbuf_t packet = NULL;
1268 uint32_t zero = 0;
1269
1270 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1271 if (error) {
1272 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1273 goto done;
1274 }
1275
1276 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1277 if (error) {
1278 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1279 goto done;
1280 }
1281
1282 how = htonl(how);
1283 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1284 if (error) {
1285 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1286 goto done;
1287 }
1288
1289 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1290 if (error) {
1291 goto done;
1292 }
1293
1294done:
1295 if (error && packet != NULL) {
1296 mbuf_free(packet);
1297 }
1298
1299 return error;
1300}
1301
1302static int
1303flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1304{
1305 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1306 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1307 {
1308 return SHUT_RDWR;
1309 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1310 return SHUT_RD;
1311 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1312 return SHUT_WR;
1313 }
1314
1315 return -1;
1316}
1317
1318/*
1319 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1320 * writes. Returns FALSE otherwise.
1321 */
1322static void
1323flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1324{
1325 int how = -1;
1326
1327 /* Do not send any close messages if there is still data in the send buffer */
1328 if (fd_cb->so->so_snd.sb_cc == 0) {
1329 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1330 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1331 how = SHUT_RD;
1332 }
1333 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1334 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1335 if (how == SHUT_RD) {
1336 how = SHUT_RDWR;
1337 } else {
1338 how = SHUT_WR;
1339 }
1340 }
1341 }
1342
1343 if (how != -1) {
1344 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1345 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1346 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1347 if (how != SHUT_RD) {
1348 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1349 }
1350 if (how != SHUT_WR) {
1351 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1352 }
1353 }
1354 }
1355
1356 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1357 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1358 }
1359}
1360
1361static errno_t
3e170ce0 1362flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e
A		 1363{
1364 mbuf_t packet;
1365 mbuf_t last;
1366 int error = 0;
1367
1368 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1369 if (error) {
1370 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1371 return error;
1372 }
1373
3e170ce0
A		 1374 if (toaddr != NULL) {
1375 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1376 if (error) {
1377 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1378 return error;
1379 }
1380 }
1381
39236c6e
A		 1382 last = m_last(packet);
1383 mbuf_setnext(last, data);
1384 mbuf_pkthdr_adjustlen(packet, data_len);
39236c6e
A		 1385 error = flow_divert_send_packet(fd_cb, packet, force);
1386
1387 if (error) {
1388 mbuf_setnext(last, NULL);
3e170ce0 1389 mbuf_freem(packet);
39236c6e
A		 1390 } else {
1391 fd_cb->bytes_sent += data_len;
1392 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1393 }
1394
1395 return error;
1396}
1397
1398static void
1399flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1400{
1401 size_t to_send;
1402 size_t sent = 0;
1403 int error = 0;
1404 mbuf_t buffer;
1405
1406 to_send = fd_cb->so->so_snd.sb_cc;
1407 buffer = fd_cb->so->so_snd.sb_mb;
1408
1409 if (buffer == NULL && to_send > 0) {
1410 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1411 return;
1412 }
1413
1414 /* Ignore the send window if force is enabled */
1415 if (!force && (to_send > fd_cb->send_window)) {
1416 to_send = fd_cb->send_window;
1417 }
1418
3e170ce0
A		 1419 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1420 while (sent < to_send) {
1421 mbuf_t data;
1422 size_t data_len;
39236c6e 1423
3e170ce0
A		 1424 data_len = to_send - sent;
1425 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1426 data_len = FLOW_DIVERT_CHUNK_SIZE;
1427 }
39236c6e 1428
3e170ce0
A		 1429 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1430 if (error) {
1431 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1432 break;
1433 }
39236c6e 1434
3e170ce0
A		 1435 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1436 if (error) {
1437 mbuf_freem(data);
1438 break;
1439 }
39236c6e 1440
3e170ce0
A		 1441 sent += data_len;
1442 }
1443 sbdrop(&fd_cb->so->so_snd, sent);
1444 sowwakeup(fd_cb->so);
1445 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1446 mbuf_t data;
1447 mbuf_t m;
1448 size_t data_len;
1449
1450 while(buffer) {
1451 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1452
1453 m = buffer;
1454 if (toaddr != NULL) {
1455 /* look for data in the chain */
1456 do {
1457 m = m->m_next;
1458 if (m != NULL && m->m_type == MT_DATA) {
1459 break;
1460 }
1461 } while(m);
1462 if (m == NULL) {
1463 /* unexpected */
1464 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1465 goto move_on;
1466 }
1467 }
1468 data_len = mbuf_pkthdr_len(m);
1469 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %u", data_len);
1470 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1471 if (error) {
1472 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1473 break;
1474 }
1475 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1476 if (error) {
1477 mbuf_freem(data);
1478 break;
1479 }
1480 sent += data_len;
1481move_on:
1482 buffer = buffer->m_nextpkt;
1483 (void) sbdroprecord(&(fd_cb->so->so_snd));
1484 }
39236c6e
A		 1485 }
1486
1487 if (sent > 0) {
1488 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1489 if (fd_cb->send_window >= sent) {
1490 fd_cb->send_window -= sent;
1491 } else {
1492 fd_cb->send_window = 0;
1493 }
39236c6e
A		 1494 }
1495}
1496
1497static int
3e170ce0 1498flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e
A		 1499{
1500 size_t to_send = mbuf_pkthdr_len(data);
3e170ce0 1501 int error = 0;
39236c6e
A		 1502
1503 if (to_send > fd_cb->send_window) {
1504 to_send = fd_cb->send_window;
1505 }
1506
1507 if (fd_cb->so->so_snd.sb_cc > 0) {
1508 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1509 }
1510
3e170ce0
A		 1511 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1512 size_t sent = 0;
1513 mbuf_t remaining_data = data;
1514 mbuf_t pkt_data = NULL;
490019cf 1515 while (sent < to_send && remaining_data != NULL) {
3e170ce0
A		 1516 size_t pkt_data_len;
1517
1518 pkt_data = remaining_data;
39236c6e 1519
3e170ce0
A		 1520 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1521 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1522 } else {
1523 pkt_data_len = to_send - sent;
1524 }
1525
1526 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1527 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1528 if (error) {
1529 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1530 pkt_data = NULL;
1531 break;
1532 }
1533 } else {
1534 remaining_data = NULL;
1535 }
1536
1537 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1538
39236c6e 1539 if (error) {
39236c6e
A		 1540 break;
1541 }
39236c6e 1542
3e170ce0
A		 1543 pkt_data = NULL;
1544 sent += pkt_data_len;
39236c6e
A		 1545 }
1546
3e170ce0 1547 fd_cb->send_window -= sent;
39236c6e 1548
3e170ce0 1549 error = 0;
39236c6e 1550
3e170ce0
A		 1551 if (pkt_data != NULL) {
1552 if (sbspace(&fd_cb->so->so_snd) > 0) {
1553 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1554 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1555 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1556 }
1557 } else {
1558 error = ENOBUFS;
39236c6e 1559 }
39236c6e 1560 }
39236c6e 1561
3e170ce0
A		 1562 if (remaining_data != NULL) {
1563 if (sbspace(&fd_cb->so->so_snd) > 0) {
1564 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1565 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1566 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1567 }
1568 } else {
1569 error = ENOBUFS;
1570 }
1571 }
1572 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1573 if (to_send) {
1574 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1575 if (error) {
1576 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %u", to_send);
1577 } else {
1578 fd_cb->send_window -= to_send;
39236c6e
A		 1579 }
1580 } else {
3e170ce0
A		 1581 /* buffer it */
1582 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1583 if (toaddr != NULL) {
1584 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1585 FDLOG(LOG_ERR, fd_cb,
1586 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1587 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1588 }
1589 } else {
1590 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1591 FDLOG(LOG_ERR, fd_cb,
1592 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1593 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1594 }
1595 }
1596 } else {
1597 error = ENOBUFS;
1598 }
39236c6e
A		 1599 }
1600 }
1601
1602 return error;
1603}
1604
1605static int
1606flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1607{
1608 int error = 0;
1609 mbuf_t packet = NULL;
1610 uint32_t net_read_count = htonl(read_count);
1611
1612 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1613 if (error) {
1614 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1615 goto done;
1616 }
1617
1618 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1619 if (error) {
1620 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1621 goto done;
1622 }
1623
1624 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1625 if (error) {
1626 goto done;
1627 }
1628
1629done:
1630 if (error && packet != NULL) {
1631 mbuf_free(packet);
1632 }
1633
1634 return error;
1635}
1636
1637static int
1638flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1639{
1640 int error = 0;
1641 mbuf_t packet = NULL;
1642
1643 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1644 if (error) {
1645 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1646 goto done;
1647 }
1648
1649 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1650 if (error) {
1651 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1652 goto done;
1653 }
1654
1655 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1656 if (error) {
1657 goto done;
1658 }
1659
1660done:
1661 if (error && packet != NULL) {
1662 mbuf_free(packet);
1663 }
1664
1665 return error;
1666}
1667
1668static void
1669flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1670{
1671 uint32_t connect_error;
1672 uint32_t ctl_unit = 0;
1673 int error = 0;
1674 struct flow_divert_group *grp = NULL;
1675 struct sockaddr_storage local_address;
1676 int out_if_index = 0;
1677 struct sockaddr_storage remote_address;
1678 uint32_t send_window;
1679
1680 memset(&local_address, 0, sizeof(local_address));
1681 memset(&remote_address, 0, sizeof(remote_address));
1682
1683 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1684 if (error) {
1685 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1686 return;
1687 }
1688
1689 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1690
1691 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1692 if (error) {
1693 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1694 return;
1695 }
1696
1697 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1698 if (error) {
1699 FDLOG(LOG_ERR, fd_cb, "failed to get the control unit: %d", error);
1700 return;
1701 }
1702
1703 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1704 if (error) {
1705 FDLOG0(LOG_NOTICE, fd_cb, "No local address provided");
1706 }
1707
1708 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1709 if (error) {
1710 FDLOG0(LOG_NOTICE, fd_cb, "No remote address provided");
1711 }
1712
1713 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1714 if (error) {
1715 FDLOG0(LOG_NOTICE, fd_cb, "No output if index provided");
1716 }
1717
1718 connect_error = ntohl(connect_error);
1719 ctl_unit = ntohl(ctl_unit);
1720
1721 lck_rw_lock_shared(&g_flow_divert_group_lck);
1722
1723 if (connect_error == 0) {
1724 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
1725 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1726 error = EINVAL;
1727 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1728 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1729 error = EINVAL;
1730 } else {
1731 grp = g_flow_divert_groups[ctl_unit];
1732 if (grp == NULL) {
1733 error = ECONNRESET;
1734 }
1735 }
1736 }
1737
1738 FDLOCK(fd_cb);
1739 if (fd_cb->so != NULL) {
1740 struct inpcb *inp = NULL;
1741 struct ifnet *ifp = NULL;
1742 struct flow_divert_group *old_group;
1743
1744 socket_lock(fd_cb->so, 0);
1745
1746 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1747 goto done;
1748 }
1749
1750 inp = sotoinpcb(fd_cb->so);
1751
1752 if (connect_error || error) {
1753 goto set_socket_state;
1754 }
1755
3e170ce0
A		 1756 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1757 error = EINVAL;
1758 goto set_socket_state;
1759 }
1760 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
39236c6e
A		 1761 if (local_address.ss_len > sizeof(local_address)) {
1762 local_address.ss_len = sizeof(local_address);
1763 }
1764 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
39236c6e
A		 1765 }
1766
1767 if (remote_address.ss_family != 0) {
1768 if (remote_address.ss_len > sizeof(remote_address)) {
1769 remote_address.ss_len = sizeof(remote_address);
1770 }
1771 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1772 } else {
1773 error = EINVAL;
1774 goto set_socket_state;
1775 }
1776
1777 ifnet_head_lock_shared();
1778 if (out_if_index > 0 && out_if_index <= if_index) {
1779 ifp = ifindex2ifnet[out_if_index];
1780 }
1781
1782 if (ifp != NULL) {
1783 inp->inp_last_outifp = ifp;
1784 } else {
1785 error = EINVAL;
1786 }
1787 ifnet_head_done();
1788
1789 if (error) {
1790 goto set_socket_state;
1791 }
1792
1793 if (fd_cb->group == NULL) {
1794 error = EINVAL;
1795 goto set_socket_state;
1796 }
1797
1798 old_group = fd_cb->group;
1799
1800 lck_rw_lock_exclusive(&old_group->lck);
1801 lck_rw_lock_exclusive(&grp->lck);
1802
1803 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1804 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1805 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1806 }
1807
1808 fd_cb->group = grp;
1809
1810 lck_rw_done(&grp->lck);
1811 lck_rw_done(&old_group->lck);
1812
1813 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 1814
1815set_socket_state:
1816 if (!connect_error && !error) {
1817 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1818 error = flow_divert_send_connect_result(fd_cb);
1819 }
1820
1821 if (connect_error || error) {
1822 if (!connect_error) {
1823 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1824 fd_cb->so->so_error = error;
1825 flow_divert_send_close_if_needed(fd_cb);
1826 } else {
1827 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1828 fd_cb->so->so_error = connect_error;
1829 }
3e170ce0 1830 flow_divert_disconnect_socket(fd_cb->so);
39236c6e 1831 } else {
490019cf 1832 flow_divert_send_buffered_data(fd_cb, FALSE);
39236c6e
A		 1833 soisconnected(fd_cb->so);
1834 }
1835
1836done:
1837 socket_unlock(fd_cb->so, 0);
1838 }
1839 FDUNLOCK(fd_cb);
1840
1841 lck_rw_done(&g_flow_divert_group_lck);
1842}
1843
1844static void
1845flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1846{
1847 uint32_t close_error;
1848 int error = 0;
1849 int how;
1850
1851 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1852 if (error) {
1853 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1854 return;
1855 }
1856
1857 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1858 if (error) {
1859 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1860 return;
1861 }
1862
1863 how = ntohl(how);
1864
1865 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1866
1867 FDLOCK(fd_cb);
1868 if (fd_cb->so != NULL) {
1869 socket_lock(fd_cb->so, 0);
1870
1871 fd_cb->so->so_error = ntohl(close_error);
1872
1873 flow_divert_update_closed_state(fd_cb, how, TRUE);
1874
1875 how = flow_divert_tunnel_how_closed(fd_cb);
1876 if (how == SHUT_RDWR) {
3e170ce0 1877 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1878 } else if (how == SHUT_RD) {
1879 socantrcvmore(fd_cb->so);
1880 } else if (how == SHUT_WR) {
1881 socantsendmore(fd_cb->so);
1882 }
1883
1884 socket_unlock(fd_cb->so, 0);
1885 }
1886 FDUNLOCK(fd_cb);
1887}
1888
3e170ce0
A		 1889static mbuf_t
1890flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 1891{
3e170ce0
A		 1892 struct inpcb *inp = sotoinpcb(fd_cb->so);
1893 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1894 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
39236c6e 1895
3e170ce0
A		 1896 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1897 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1898 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1899 struct in6_pktinfo pi6;
39236c6e 1900
3e170ce0
A		 1901 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1902 pi6.ipi6_ifindex = 0;
1903 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 1904 }
3e170ce0
A		 1905 return (NULL);
1906}
39236c6e 1907
3e170ce0
A		 1908static void
1909flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1910{
39236c6e
A		 1911 FDLOCK(fd_cb);
1912 if (fd_cb->so != NULL) {
3e170ce0
A		 1913 int error = 0;
1914 mbuf_t data = NULL;
1915 size_t data_size;
1916 struct sockaddr_storage remote_address;
1917 boolean_t got_remote_sa = FALSE;
1918
39236c6e 1919 socket_lock(fd_cb->so, 0);
3e170ce0
A		 1920
1921 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1922 uint32_t val_size = 0;
1923
1924 /* check if we got remote address with data */
1925 memset(&remote_address, 0, sizeof(remote_address));
1926 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1927 if (error || val_size > sizeof(remote_address)) {
1928 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1929 error = 0;
39236c6e 1930 } else {
3e170ce0
A		 1931 /* validate the address */
1932 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1933 got_remote_sa = TRUE;
1934 }
1935 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1936 }
1937 }
1938
1939 data_size = (mbuf_pkthdr_len(packet) - offset);
1940
1941 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1942
1943 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1944 if (error || data == NULL) {
1945 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1946 } else {
1947 if (flow_divert_check_no_cellular(fd_cb) ||
1948 flow_divert_check_no_expensive(fd_cb))
1949 {
1950 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1951 flow_divert_send_close(fd_cb, SHUT_RDWR);
1952 flow_divert_disconnect_socket(fd_cb->so);
1953 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1954 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1955 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1956 fd_cb->bytes_received += data_size;
1957 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1958 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1959 sorwakeup(fd_cb->so);
1960 data = NULL;
1961 } else {
1962 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1963 }
1964 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1965 struct sockaddr *append_sa;
1966 mbuf_t mctl;
1967
1968 if (got_remote_sa == TRUE) {
1969 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1970 (struct sockaddr *)&remote_address, &append_sa);
1971 } else {
1972 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1973 fd_cb->remote_address, &append_sa);
1974 }
1975 if (error) {
1976 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
1977 }
1978
1979 mctl = flow_divert_get_control_mbuf(fd_cb);
1980 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
1981 fd_cb->bytes_received += data_size;
1982 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1983 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1984 sorwakeup(fd_cb->so);
1985 data = NULL;
1986 } else {
1987 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
1988 }
1989 if (!error) {
1990 FREE(append_sa, M_TEMP);
1991 }
1992 }
39236c6e
A		 1993 }
1994 }
1995 socket_unlock(fd_cb->so, 0);
39236c6e 1996
3e170ce0
A		 1997 if (data != NULL) {
1998 mbuf_freem(data);
1999 }
39236c6e 2000 }
3e170ce0 2001 FDUNLOCK(fd_cb);
39236c6e
A		 2002}
2003
2004static void
2005flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2006{
2007 uint32_t read_count;
2008 int error = 0;
2009
2010 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2011 if (error) {
2012 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2013 return;
2014 }
2015
3e170ce0 2016 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2017
2018 FDLOCK(fd_cb);
2019 if (fd_cb->so != NULL) {
2020 socket_lock(fd_cb->so, 0);
2021 fd_cb->send_window += ntohl(read_count);
2022 flow_divert_send_buffered_data(fd_cb, FALSE);
2023 socket_unlock(fd_cb->so, 0);
2024 }
2025 FDUNLOCK(fd_cb);
2026}
2027
2028static void
2029flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2030{
2031 int error = 0;
3e170ce0 2032 uint32_t key_size = 0;
39236c6e
A		 2033 int log_level;
2034
2035 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2036 if (error) {
2037 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2038 return;
2039 }
2040
2041 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2042 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %lu", key_size);
2043 return;
2044 }
2045
2046 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2047 if (!error) {
2048 nil_pcb.log_level = log_level;
2049 }
2050
2051 lck_rw_lock_exclusive(&group->lck);
2052
2053 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2054 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2055 if (error) {
2056 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2057 FREE(group->token_key, M_TEMP);
2058 group->token_key = NULL;
2059 lck_rw_done(&group->lck);
2060 return;
2061 }
2062
2063 group->token_key_size = key_size;
2064
2065 lck_rw_done(&group->lck);
2066}
2067
2068static void
2069flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2070{
2071 int error = 0;
2072 struct sockaddr_storage local_address;
2073 int out_if_index = 0;
2074 struct sockaddr_storage remote_address;
490019cf 2075 uint32_t app_data_length = 0;
39236c6e
A		 2076
2077 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2078
2079 memset(&local_address, 0, sizeof(local_address));
2080 memset(&remote_address, 0, sizeof(remote_address));
2081
2082 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2083 if (error) {
490019cf 2084 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
39236c6e
A		 2085 }
2086
2087 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2088 if (error) {
490019cf 2089 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
39236c6e
A		 2090 }
2091
2092 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2093 if (error) {
490019cf
A		 2094 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2095 }
2096
2097 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2098 if (error) {
2099 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2100 }
2101
2102 FDLOCK(fd_cb);
2103 if (fd_cb->so != NULL) {
39236c6e
A		 2104 socket_lock(fd_cb->so, 0);
2105
39236c6e
A		 2106 if (local_address.ss_family != 0) {
2107 if (local_address.ss_len > sizeof(local_address)) {
2108 local_address.ss_len = sizeof(local_address);
2109 }
490019cf
A		 2110 if (fd_cb->local_address != NULL) {
2111 FREE(fd_cb->local_address, M_SONAME);
2112 fd_cb->local_address = NULL;
2113 }
39236c6e
A		 2114 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2115 }
2116
2117 if (remote_address.ss_family != 0) {
2118 if (remote_address.ss_len > sizeof(remote_address)) {
2119 remote_address.ss_len = sizeof(remote_address);
2120 }
490019cf
A		 2121 if (fd_cb->remote_address != NULL) {
2122 FREE(fd_cb->remote_address, M_SONAME);
2123 fd_cb->remote_address = NULL;
2124 }
39236c6e
A		 2125 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2126 }
2127
490019cf
A		 2128 if (out_if_index > 0) {
2129 struct inpcb *inp = NULL;
2130 struct ifnet *ifp = NULL;
2131
2132 inp = sotoinpcb(fd_cb->so);
2133
2134 ifnet_head_lock_shared();
2135 if (out_if_index <= if_index) {
2136 ifp = ifindex2ifnet[out_if_index];
2137 }
2138
2139 if (ifp != NULL) {
2140 inp->inp_last_outifp = ifp;
2141 }
2142 ifnet_head_done();
39236c6e
A		 2143 }
2144
490019cf
A		 2145 if (app_data_length > 0) {
2146 uint8_t *app_data = NULL;
2147 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2148 if (app_data != NULL) {
2149 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2150 if (error == 0) {
2151 if (fd_cb->app_data != NULL) {
2152 FREE(fd_cb->app_data, M_TEMP);
2153 }
2154 fd_cb->app_data = app_data;
2155 fd_cb->app_data_length = app_data_length;
2156 } else {
2157 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2158 FREE(app_data, M_TEMP);
2159 }
2160 } else {
2161 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2162 }
39236c6e 2163 }
39236c6e
A		 2164
2165 socket_unlock(fd_cb->so, 0);
2166 }
2167 FDUNLOCK(fd_cb);
2168}
2169
2170static void
2171flow_divert_handle_app_map_create(mbuf_t packet, int offset)
2172{
2173 size_t bytes_mem_size;
2174 size_t child_maps_mem_size;
2175 int cursor;
2176 int error = 0;
2177 struct flow_divert_trie new_trie;
2178 int insert_error = 0;
2179 size_t nodes_mem_size;
2180 int prefix_count = 0;
2181 int signing_id_count = 0;
ecc0ceb4 2182 size_t trie_memory_size = 0;
39236c6e
A		 2183
2184 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
2185
2186 /* Re-set the current trie */
2187 if (g_signing_id_trie.memory != NULL) {
2188 FREE(g_signing_id_trie.memory, M_TEMP);
2189 }
2190 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
2191 g_signing_id_trie.root = NULL_TRIE_IDX;
2192
2193 memset(&new_trie, 0, sizeof(new_trie));
2194
2195 /* Get the number of shared prefixes in the new set of signing ID strings */
2196 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2197
ecc0ceb4
A		 2198 if (prefix_count < 0) {
2199 lck_rw_done(&g_flow_divert_group_lck);
2200 return;
2201 }
2202
39236c6e
A		 2203 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2204 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2205 cursor >= 0;
2206 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2207 {
3e170ce0 2208 uint32_t sid_size = 0;
39236c6e
A		 2209 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2210 new_trie.bytes_count += sid_size;
2211 signing_id_count++;
2212 }
2213
2214 if (signing_id_count == 0) {
2215 lck_rw_done(&g_flow_divert_group_lck);
2216 return;
2217 }
2218
2219 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2220 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2221
2222 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2223 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2224
2225 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2226 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2227 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2228
ecc0ceb4
A		 2229 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2230 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2231 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%u) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2232 lck_rw_done(&g_flow_divert_group_lck);
2233 return;
2234 }
2235
2236 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2237 if (new_trie.memory == NULL) {
2238 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2239 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
ecc0ceb4 2240 lck_rw_done(&g_flow_divert_group_lck);
39236c6e
A		 2241 return;
2242 }
2243
2244 /* Initialize the free lists */
2245 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2246 new_trie.nodes_free_next = 0;
2247 memset(new_trie.nodes, 0, nodes_mem_size);
2248
2249 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2250 new_trie.child_maps_free_next = 0;
2251 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2252
2253 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2254 new_trie.bytes_free_next = 0;
2255
2256 /* The root is an empty node */
2257 new_trie.root = trie_node_alloc(&new_trie);
2258
2259 /* Add each signing ID to the trie */
2260 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2261 cursor >= 0;
2262 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2263 {
3e170ce0 2264 uint32_t sid_size = 0;
39236c6e
A		 2265 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2266 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2267 boolean_t is_dns;
2268 uint16_t new_node_idx;
2269 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2270 is_dns = (sid_size == sizeof(FLOW_DIVERT_DNS_SERVICE_SIGNING_ID) - 1 &&
2271 !memcmp(&TRIE_BYTE(&new_trie, new_trie.bytes_free_next),
2272 FLOW_DIVERT_DNS_SERVICE_SIGNING_ID,
2273 sid_size));
2274 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2275 if (new_node_idx != NULL_TRIE_IDX) {
2276 if (is_dns) {
fe8ab488 2277 FDLOG(LOG_INFO, &nil_pcb, "Setting group unit for %s to %d", FLOW_DIVERT_DNS_SERVICE_SIGNING_ID, DNS_SERVICE_GROUP_UNIT);
39236c6e
A		 2278 TRIE_NODE(&new_trie, new_node_idx).group_unit = DNS_SERVICE_GROUP_UNIT;
2279 }
2280 } else {
2281 insert_error = EINVAL;
2282 break;
2283 }
2284 } else {
2285 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2286 insert_error = ENOBUFS;
2287 break;
2288 }
2289 }
2290
2291 if (!insert_error) {
2292 g_signing_id_trie = new_trie;
2293 } else {
2294 FREE(new_trie.memory, M_TEMP);
2295 }
2296
2297 lck_rw_done(&g_flow_divert_group_lck);
2298}
2299
2300static void
2301flow_divert_handle_app_map_update(struct flow_divert_group *group, mbuf_t packet, int offset)
2302{
2303 int error = 0;
2304 int cursor;
2305 size_t max_size = 0;
2306 uint8_t *signing_id;
2307 uint32_t ctl_unit;
2308
2309 lck_rw_lock_shared(&group->lck);
2310 ctl_unit = group->ctl_unit;
2311 lck_rw_done(&group->lck);
2312
2313 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2314 cursor >= 0;
2315 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2316 {
3e170ce0 2317 uint32_t sid_size = 0;
39236c6e
A		 2318 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2319 if (sid_size > max_size) {
2320 max_size = sid_size;
2321 }
2322 }
2323
2324 MALLOC(signing_id, uint8_t *, max_size + 1, M_TEMP, M_WAITOK);
2325 if (signing_id == NULL) {
2326 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate a string to hold the signing ID (size %lu)", max_size);
2327 return;
2328 }
2329
2330 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2331 cursor >= 0;
2332 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2333 {
3e170ce0 2334 uint32_t signing_id_len = 0;
39236c6e
A		 2335 uint16_t node;
2336
2337 flow_divert_packet_get_tlv(packet,
2338 cursor, FLOW_DIVERT_TLV_SIGNING_ID, max_size, signing_id, &signing_id_len);
2339
2340 signing_id[signing_id_len] = '\0';
2341
2342 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
2343
2344 node = flow_divert_trie_search(&g_signing_id_trie, signing_id);
2345 if (node != NULL_TRIE_IDX) {
2346 if (TRIE_NODE(&g_signing_id_trie, node).group_unit != DNS_SERVICE_GROUP_UNIT) {
2347 FDLOG(LOG_INFO, &nil_pcb, "Setting %s to ctl unit %u", signing_id, group->ctl_unit);
2348 TRIE_NODE(&g_signing_id_trie, node).group_unit = ctl_unit;
2349 }
2350 } else {
2351 FDLOG(LOG_ERR, &nil_pcb, "Failed to find signing ID %s", signing_id);
2352 }
2353
2354 lck_rw_done(&g_flow_divert_group_lck);
2355 }
2356
2357 FREE(signing_id, M_TEMP);
2358}
2359
2360static int
2361flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2362{
2363 struct flow_divert_packet_header hdr;
2364 int error = 0;
2365 struct flow_divert_pcb *fd_cb;
2366
2367 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2368 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2369 error = EINVAL;
2370 goto done;
2371 }
2372
3e170ce0
A		 2373 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2374 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %lu", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2375 error = EINVAL;
2376 goto done;
2377 }
2378
39236c6e
A		 2379 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2380 if (error) {
2381 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2382 error = ENOBUFS;
2383 goto done;
2384 }
2385
2386 hdr.conn_id = ntohl(hdr.conn_id);
2387
2388 if (hdr.conn_id == 0) {
2389 switch (hdr.packet_type) {
2390 case FLOW_DIVERT_PKT_GROUP_INIT:
2391 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2392 break;
2393 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2394 flow_divert_handle_app_map_create(packet, sizeof(hdr));
2395 break;
2396 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
2397 flow_divert_handle_app_map_update(group, packet, sizeof(hdr));
2398 break;
2399 default:
2400 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2401 break;
2402 }
2403 goto done;
2404 }
2405
2406 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2407 if (fd_cb == NULL) {
2408 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2409 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2410 }
2411 goto done;
2412 }
2413
2414 switch (hdr.packet_type) {
2415 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2416 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2417 break;
2418 case FLOW_DIVERT_PKT_CLOSE:
2419 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2420 break;
2421 case FLOW_DIVERT_PKT_DATA:
2422 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2423 break;
2424 case FLOW_DIVERT_PKT_READ_NOTIFY:
2425 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2426 break;
2427 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2428 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2429 break;
2430 default:
2431 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2432 break;
2433 }
2434
2435 FDRELEASE(fd_cb);
2436
2437done:
3e170ce0 2438 mbuf_freem(packet);
39236c6e
A		 2439 return error;
2440}
2441
2442static void
2443flow_divert_close_all(struct flow_divert_group *group)
2444{
2445 struct flow_divert_pcb *fd_cb;
2446 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2447
2448 SLIST_INIT(&tmp_list);
2449
2450 lck_rw_lock_exclusive(&group->lck);
2451
2452 MBUFQ_DRAIN(&group->send_queue);
2453
2454 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2455 FDRETAIN(fd_cb);
2456 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2457 }
2458
2459 lck_rw_done(&group->lck);
2460
2461 while (!SLIST_EMPTY(&tmp_list)) {
2462 fd_cb = SLIST_FIRST(&tmp_list);
2463 FDLOCK(fd_cb);
2464 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2465 if (fd_cb->so != NULL) {
2466 socket_lock(fd_cb->so, 0);
2467 flow_divert_pcb_remove(fd_cb);
2468 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2469 fd_cb->so->so_error = ECONNABORTED;
2470 socket_unlock(fd_cb->so, 0);
2471 }
2472 FDUNLOCK(fd_cb);
2473 FDRELEASE(fd_cb);
2474 }
2475}
2476
2477void
2478flow_divert_detach(struct socket *so)
2479{
2480 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2481
2482 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2483
2484 so->so_flags &= ~SOF_FLOW_DIVERT;
2485 so->so_fd_pcb = NULL;
2486
2487 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2488
2489 if (fd_cb->group != NULL) {
2490 /* Last-ditch effort to send any buffered data */
2491 flow_divert_send_buffered_data(fd_cb, TRUE);
2492
3e170ce0
A		 2493 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2494 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 2495 /* Remove from the group */
2496 flow_divert_pcb_remove(fd_cb);
2497 }
2498
2499 socket_unlock(so, 0);
2500 FDLOCK(fd_cb);
2501 fd_cb->so = NULL;
2502 FDUNLOCK(fd_cb);
2503 socket_lock(so, 0);
2504
2505 FDRELEASE(fd_cb); /* Release the socket's reference */
2506}
2507
2508static int
2509flow_divert_close(struct socket *so)
2510{
2511 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2512
2513 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2514
2515 FDLOG0(LOG_INFO, fd_cb, "Closing");
2516
3e170ce0
A		 2517 if (SOCK_TYPE(so) == SOCK_STREAM) {
2518 soisdisconnecting(so);
2519 sbflush(&so->so_rcv);
2520 }
39236c6e
A		 2521
2522 flow_divert_send_buffered_data(fd_cb, TRUE);
2523 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2524 flow_divert_send_close_if_needed(fd_cb);
2525
2526 /* Remove from the group */
2527 flow_divert_pcb_remove(fd_cb);
2528
2529 return 0;
2530}
2531
2532static int
3e170ce0
A		 2533flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2534 sae_connid_t cid __unused)
39236c6e 2535{
3e170ce0 2536 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
39236c6e
A		 2537 return (EINVAL);
2538 }
2539
2540 return (flow_divert_close(so));
2541}
2542
2543static int
2544flow_divert_shutdown(struct socket *so)
2545{
2546 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2547
2548 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2549
2550 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2551
2552 socantsendmore(so);
2553
2554 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2555 flow_divert_send_close_if_needed(fd_cb);
2556
2557 return 0;
2558}
2559
2560static int
2561flow_divert_rcvd(struct socket *so, int flags __unused)
2562{
2563 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2564 uint32_t latest_sb_size;
2565 uint32_t read_count;
2566
2567 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2568
2569 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2570
2571 if (fd_cb->sb_size < latest_sb_size) {
2572 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2573 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2574 }
2575
2576 read_count = fd_cb->sb_size - latest_sb_size;
2577
2578 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2579
2580 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2581 fd_cb->bytes_read_by_app += read_count;
2582 fd_cb->sb_size = latest_sb_size;
2583 }
2584
2585 return 0;
2586}
2587
3e170ce0
A		 2588static int
2589flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2590{
2591 int error = 0;
2592 int port = 0;
2593
2594 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2595 if (error) {
2596 goto done;
2597 }
2598
2599 if (toaddr->sa_family == AF_INET) {
2600 port = ntohs((satosin(toaddr))->sin_port);
2601 }
2602#if INET6
2603 else {
2604 port = ntohs((satosin6(toaddr))->sin6_port);
2605 }
2606#endif
2607
2608 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2609 if (error) {
2610 goto done;
2611 }
2612
2613done:
2614 return error;
2615}
2616
2617struct sockaddr *
2618flow_divert_get_buffered_target_address(mbuf_t buffer)
2619{
2620 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2621 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2622 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2623 return toaddr;
2624 }
2625 }
2626 return NULL;
2627}
2628
2629static boolean_t
2630flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2631{
2632 switch(addr->sa_family)
2633 {
2634 case AF_INET:
2635 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2636 return FALSE;
2637 }
2638 break;
2639#if INET6
2640 case AF_INET6:
2641 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2642 return FALSE;
2643 }
2644 break;
2645#endif /* INET6 */
2646 default:
2647 return FALSE;
2648 }
2649 return TRUE;
2650}
2651
2652static errno_t
2653flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2654{
2655 int error = 0;
2656 union sockaddr_in_4_6 sin46;
2657
2658 bzero(&sin46, sizeof(sin46));
2659 if (inp->inp_vflag & INP_IPV4) {
2660 struct sockaddr_in *sin = &sin46.sin;
2661
2662 sin->sin_family = AF_INET;
2663 sin->sin_len = sizeof(*sin);
2664 sin->sin_port = inp->inp_lport;
2665 sin->sin_addr = inp->inp_laddr;
2666 } else if (inp->inp_vflag & INP_IPV6) {
2667 struct sockaddr_in6 *sin6 = &sin46.sin6;
2668
2669 sin6->sin6_len = sizeof(*sin6);
2670 sin6->sin6_family = AF_INET6;
2671 sin6->sin6_port = inp->inp_lport;
2672 sin6->sin6_addr = inp->in6p_laddr;
2673 }
2674 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2675 if (*local_socket == NULL) {
2676 error = ENOBUFS;
2677 }
2678 return (error);
2679}
2680
2681static boolean_t
2682flow_divert_has_pcb_local_address(const struct inpcb *inp)
2683{
2684 return (inp->inp_lport != 0
2685 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2686}
2687
39236c6e
A		 2688static errno_t
2689flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2690 struct sockaddr **dup)
2691{
2692 int error = 0;
2693 struct sockaddr *result;
2694 struct sockaddr_storage ss;
2695
2696 if (addr != NULL) {
2697 result = addr;
2698 } else {
2699 memset(&ss, 0, sizeof(ss));
2700 ss.ss_family = family;
2701 if (ss.ss_family == AF_INET) {
2702 ss.ss_len = sizeof(struct sockaddr_in);
2703 }
2704#if INET6
2705 else if (ss.ss_family == AF_INET6) {
2706 ss.ss_len = sizeof(struct sockaddr_in6);
2707 }
2708#endif /* INET6 */
2709 else {
2710 error = EINVAL;
2711 }
2712 result = (struct sockaddr *)&ss;
2713 }
2714
2715 if (!error) {
2716 *dup = dup_sockaddr(result, 1);
2717 if (*dup == NULL) {
2718 error = ENOBUFS;
2719 }
2720 }
2721
2722 return error;
2723}
2724
3e170ce0
A		 2725static void
2726flow_divert_disconnect_socket(struct socket *so)
2727{
2728 soisdisconnected(so);
2729 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2730 struct inpcb *inp = NULL;
2731
2732 inp = sotoinpcb(so);
2733 if (inp != NULL) {
2734#if INET6
2735 if (SOCK_CHECK_DOM(so, PF_INET6))
2736 in6_pcbdetach(inp);
2737 else
2738#endif /* INET6 */
2739 in_pcbdetach(inp);
2740 }
2741 }
2742}
2743
39236c6e
A		 2744static errno_t
2745flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2746{
2747 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2748
2749 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2750
2751 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2752 fd_cb->remote_address,
2753 sa);
2754}
2755
2756static errno_t
2757flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2758{
2759 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2760
2761 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2762
2763 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2764 fd_cb->local_address,
2765 sa);
2766}
2767
2768static errno_t
2769flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2770{
2771 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2772
2773 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2774
2775 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2776 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2777 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2778 }
2779 }
2780
2781 if (SOCK_DOM(so) == PF_INET) {
2782 return g_tcp_protosw->pr_ctloutput(so, sopt);
2783 }
2784#if INET6
2785 else if (SOCK_DOM(so) == PF_INET6) {
2786 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2787 }
2788#endif
2789 return 0;
2790}
2791
2792errno_t
2793flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2794{
2795 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2796 int error = 0;
2797 struct inpcb *inp = sotoinpcb(so);
2798 struct sockaddr_in *sinp;
fe8ab488 2799 mbuf_t connect_packet = NULL;
490019cf 2800 int do_send = 1;
39236c6e
A		 2801
2802 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2803
2804 if (fd_cb->group == NULL) {
2805 error = ENETUNREACH;
2806 goto done;
2807 }
2808
2809 if (inp == NULL) {
2810 error = EINVAL;
2811 goto done;
2812 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2813 if (so->so_error) {
2814 error = so->so_error;
2815 so->so_error = 0;
2816 } else {
2817 error = EINVAL;
2818 }
2819 goto done;
2820 }
2821
39236c6e
A		 2822 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2823 error = EALREADY;
2824 goto done;
2825 }
2826
2827 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2828 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2829 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2830 if (fd_cb->remote_address != NULL) {
2831 soisconnected(fd_cb->so);
2832 goto done;
2833 }
2834 }
2835
490019cf 2836 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 2837
490019cf
A		 2838 if (fd_cb->connect_packet == NULL) {
2839 if (to == NULL) {
2840 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2841 error = EINVAL;
2842 goto done;
fe8ab488 2843 }
fe8ab488 2844
490019cf
A		 2845 sinp = (struct sockaddr_in *)(void *)to;
2846 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2847 error = EAFNOSUPPORT;
2848 goto done;
fe8ab488
A		 2849 }
2850
490019cf
A		 2851 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2852 if (error) {
2853 goto done;
fe8ab488
A		 2854 }
2855
490019cf
A		 2856 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2857 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2858 do_send = 0;
fe8ab488
A		 2859 }
2860 } else {
490019cf
A		 2861 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2862 connect_packet = fd_cb->connect_packet;
2863 fd_cb->connect_packet = NULL;
fe8ab488
A		 2864 }
2865
490019cf
A		 2866 if (do_send) {
2867 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2868 if (error) {
2869 goto done;
2870 }
39236c6e 2871
490019cf
A		 2872 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2873 } else {
2874 fd_cb->connect_packet = connect_packet;
2875 connect_packet = NULL;
39236c6e
A		 2876 }
2877
39236c6e
A		 2878 soisconnecting(so);
2879
2880done:
fe8ab488 2881 if (error && connect_packet != NULL) {
3e170ce0 2882 mbuf_freem(connect_packet);
fe8ab488 2883 }
39236c6e
A		 2884 return error;
2885}
2886
2887static int
2888flow_divert_connectx_out_common(struct socket *so, int af,
2889 struct sockaddr_list **src_sl, struct sockaddr_list **dst_sl,
3e170ce0
A		 2890 struct proc *p, uint32_t ifscope __unused, sae_associd_t aid __unused,
2891 sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
490019cf 2892 uint32_t arglen __unused, struct uio *auio, user_ssize_t *bytes_written)
39236c6e
A		 2893{
2894 struct sockaddr_entry *src_se = NULL, *dst_se = NULL;
2895 struct inpcb *inp = sotoinpcb(so);
2896 int error;
2897
2898 if (inp == NULL) {
2899 return (EINVAL);
2900 }
2901
2902 VERIFY(dst_sl != NULL);
2903
2904 /* select source (if specified) and destination addresses */
2905 error = in_selectaddrs(af, src_sl, &src_se, dst_sl, &dst_se);
2906 if (error != 0) {
2907 return (error);
2908 }
2909
2910 VERIFY(*dst_sl != NULL && dst_se != NULL);
2911 VERIFY(src_se == NULL || *src_sl != NULL);
2912 VERIFY(dst_se->se_addr->sa_family == af);
2913 VERIFY(src_se == NULL || src_se->se_addr->sa_family == af);
2914
2915 error = flow_divert_connect_out(so, dst_se->se_addr, p);
2916
490019cf
A		 2917 if (error != 0) {
2918 return error;
2919 }
2920
2921 /* if there is data, send it */
2922 if (auio != NULL) {
2923 user_ssize_t datalen = 0;
2924
2925 socket_unlock(so, 0);
2926
2927 VERIFY(bytes_written != NULL);
2928
2929 datalen = uio_resid(auio);
2930 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2931 socket_lock(so, 0);
2932
2933 if (error == 0 || error == EWOULDBLOCK) {
2934 *bytes_written = datalen - uio_resid(auio);
2935 }
2936
2937 /*
2938 * sosend returns EWOULDBLOCK if it's a non-blocking
2939 * socket or a timeout occured (this allows to return
2940 * the amount of queued data through sendit()).
2941 *
2942 * However, connectx() returns EINPROGRESS in case of a
2943 * blocking socket. So we change the return value here.
2944 */
2945 if (error == EWOULDBLOCK) {
2946 error = EINPROGRESS;
2947 }
2948 }
2949
39236c6e
A		 2950 if (error == 0 && pcid != NULL) {
2951 *pcid = 1; /* there is only 1 connection for a TCP */
2952 }
2953
2954 return (error);
2955}
2956
2957static int
2958flow_divert_connectx_out(struct socket *so, struct sockaddr_list **src_sl,
2959 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
3e170ce0
A		 2960 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2961 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 2962{
3e170ce0 2963#pragma unused(uio, bytes_written)
39236c6e 2964 return (flow_divert_connectx_out_common(so, AF_INET, src_sl, dst_sl,
490019cf 2965 p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written));
39236c6e
A		 2966}
2967
2968#if INET6
2969static int
2970flow_divert_connectx6_out(struct socket *so, struct sockaddr_list **src_sl,
2971 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
3e170ce0
A		 2972 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2973 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 2974{
3e170ce0 2975#pragma unused(uio, bytes_written)
39236c6e 2976 return (flow_divert_connectx_out_common(so, AF_INET6, src_sl, dst_sl,
490019cf 2977 p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written));
39236c6e
A		 2978}
2979#endif /* INET6 */
2980
2981static int
3e170ce0 2982flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
39236c6e
A		 2983 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2984 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2985 user_addr_t aux_data __unused, uint32_t *aux_len)
2986{
2987 int error = 0;
2988 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2989 struct ifnet *ifp = NULL;
2990 struct inpcb *inp = sotoinpcb(so);
2991
2992 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2993
2994 if (so->so_fd_pcb == NULL || inp == NULL) {
2995 error = EINVAL;
2996 goto out;
2997 }
2998
3e170ce0 2999 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
39236c6e
A		 3000 error = EINVAL;
3001 goto out;
3002 }
3003
3004 ifp = inp->inp_last_outifp;
3005 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
3006 *soerror = so->so_error;
3007 *flags = 0;
3008
3009 if (so->so_state & SS_ISCONNECTED) {
3010 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
3011 }
3012
3013 if (fd_cb->local_address == NULL) {
3014 struct sockaddr_in sin;
3015 bzero(&sin, sizeof(sin));
3016 sin.sin_len = sizeof(sin);
3017 sin.sin_family = AF_INET;
3018 *src_len = sin.sin_len;
3019 if (src != USER_ADDR_NULL) {
3020 error = copyout(&sin, src, sin.sin_len);
3021 if (error != 0) {
3022 goto out;
3023 }
3024 }
3025 } else {
3026 *src_len = fd_cb->local_address->sa_len;
3027 if (src != USER_ADDR_NULL) {
3028 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
3029 if (error != 0) {
3030 goto out;
3031 }
3032 }
3033 }
3034
3035 if (fd_cb->remote_address == NULL) {
3036 struct sockaddr_in sin;
3037 bzero(&sin, sizeof(sin));
3038 sin.sin_len = sizeof(sin);
3039 sin.sin_family = AF_INET;
3040 *dst_len = sin.sin_len;
3041 if (dst != USER_ADDR_NULL) {
3042 error = copyout(&sin, dst, sin.sin_len);
3043 if (error != 0) {
3044 goto out;
3045 }
3046 }
3047 } else {
3048 *dst_len = fd_cb->remote_address->sa_len;
3049 if (dst != USER_ADDR_NULL) {
3050 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
3051 if (error != 0) {
3052 goto out;
3053 }
3054 }
3055 }
3056
3057 *aux_type = 0;
3058 *aux_len = 0;
3059
3060out:
3061 return error;
3062}
3063
3064static int
3065flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3066{
3067 int error = 0;
3068
3069 switch (cmd) {
3070 case SIOCGCONNINFO32: {
3071 struct so_cinforeq32 cifr;
3072 bcopy(data, &cifr, sizeof (cifr));
3073 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3074 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3075 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3076 &cifr.scir_aux_type, cifr.scir_aux_data,
3077 &cifr.scir_aux_len);
3078 if (error == 0) {
3079 bcopy(&cifr, data, sizeof (cifr));
3080 }
3081 break;
3082 }
3083
3084 case SIOCGCONNINFO64: {
3085 struct so_cinforeq64 cifr;
3086 bcopy(data, &cifr, sizeof (cifr));
3087 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3088 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3089 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3090 &cifr.scir_aux_type, cifr.scir_aux_data,
3091 &cifr.scir_aux_len);
3092 if (error == 0) {
3093 bcopy(&cifr, data, sizeof (cifr));
3094 }
3095 break;
3096 }
3097
3098 default:
3099 error = EOPNOTSUPP;
3100 }
3101
3102 return error;
3103}
3104
3105static int
3106flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3107{
3108 int error = flow_divert_control(so, cmd, data, ifp, p);
3109
3110 if (error == EOPNOTSUPP) {
3111 error = in_control(so, cmd, data, ifp, p);
3112 }
3113
3114 return error;
3115}
3116
3117static int
3118flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3119{
3120 int error = flow_divert_control(so, cmd, data, ifp, p);
3121
3122 if (error == EOPNOTSUPP) {
3123 error = in6_control(so, cmd, data, ifp, p);
3124 }
3125
3126 return error;
3127}
3128
3129static errno_t
490019cf 3130flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e
A		 3131{
3132 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3133 int error = 0;
3134 struct inpcb *inp;
3135
3136 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3137
3138 inp = sotoinpcb(so);
3139 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3140 error = ECONNRESET;
3141 goto done;
3142 }
3143
3144 if (control && mbuf_len(control) > 0) {
3145 error = EINVAL;
3146 goto done;
3147 }
3148
3149 if (flags & MSG_OOB) {
3150 error = EINVAL;
3151 goto done; /* We don't support OOB data */
3152 }
3153
fe8ab488
A		 3154 error = flow_divert_check_no_cellular(fd_cb) ||
3155 flow_divert_check_no_expensive(fd_cb);
39236c6e
A		 3156 if (error) {
3157 goto done;
3158 }
3159
3160 /* Implicit connect */
3161 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3162 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
490019cf 3163 error = flow_divert_connect_out(so, to, p);
39236c6e
A		 3164 if (error) {
3165 goto done;
3166 }
490019cf
A		 3167
3168 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3169 /* Open up the send window so that the data will get sent right away */
3170 fd_cb->send_window = mbuf_pkthdr_len(data);
3171 }
39236c6e
A		 3172 }
3173
3174 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3175
3176 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3177 error = flow_divert_send_app_data(fd_cb, data, to);
39236c6e
A		 3178 if (error) {
3179 goto done;
3180 }
3181
3182 data = NULL;
3183
3184 if (flags & PRUS_EOF) {
3185 flow_divert_shutdown(so);
3186 }
3187
3188done:
3189 if (data) {
3e170ce0 3190 mbuf_freem(data);
39236c6e
A		 3191 }
3192 if (control) {
3193 mbuf_free(control);
3194 }
3195 return error;
3196}
3197
490019cf
A		 3198static int
3199flow_divert_preconnect(struct socket *so)
3200{
3201 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3202 int error = 0;
3203
3204 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3205 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3206 mbuf_t connect_packet = fd_cb->connect_packet;
3207 fd_cb->connect_packet = NULL;
3208
3209 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3210 if (error) {
3211 mbuf_freem(connect_packet);
3212 }
3213
3214 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3215 }
3216
3217 so->so_flags1 &= ~SOF1_PRECONNECT_DATA;
3218
3219 return error;
3220}
3221
39236c6e
A		 3222static void
3223flow_divert_set_protosw(struct socket *so)
3224{
3225 so->so_flags |= SOF_FLOW_DIVERT;
3226 if (SOCK_DOM(so) == PF_INET) {
3227 so->so_proto = &g_flow_divert_in_protosw;
3228 }
3229#if INET6
3230 else {
3231 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3232 }
3233#endif /* INET6 */
3234}
3235
3e170ce0
A		 3236static void
3237flow_divert_set_udp_protosw(struct socket *so)
3238{
3239 so->so_flags |= SOF_FLOW_DIVERT;
3240 if (SOCK_DOM(so) == PF_INET) {
3241 so->so_proto = &g_flow_divert_in_udp_protosw;
3242 }
3243#if INET6
3244 else {
3245 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3246 }
3247#endif /* INET6 */
3248}
3249
39236c6e
A		 3250static errno_t
3251flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3252{
3253 int error = 0;
3254 struct flow_divert_pcb *fd_cb = NULL;
3255 struct ifnet *ifp = NULL;
3256 struct inpcb *inp = NULL;
3257 struct socket *old_so;
3258 mbuf_t recv_data = NULL;
3259
3260 socket_unlock(so, 0);
3261
3262 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3263
3264 /* Find the flow divert control block */
3265 lck_rw_lock_shared(&g_flow_divert_group_lck);
3266 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3267 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3268 if (group != NULL) {
3269 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3270 }
3271 }
3272 lck_rw_done(&g_flow_divert_group_lck);
3273
3274 if (fd_cb == NULL) {
3275 error = ENOENT;
3276 goto done;
3277 }
3278
3279 FDLOCK(fd_cb);
3280
3281 /* Dis-associate the flow divert control block from its current socket */
3282 old_so = fd_cb->so;
3283
3284 inp = sotoinpcb(old_so);
3285
3286 VERIFY(inp != NULL);
3287
3288 socket_lock(old_so, 0);
3e170ce0 3289 flow_divert_disconnect_socket(old_so);
39236c6e
A		 3290 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3291 old_so->so_fd_pcb = NULL;
3e170ce0
A		 3292 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3293 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3294 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3295 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3296 }
39236c6e
A		 3297 fd_cb->so = NULL;
3298 /* Save the output interface */
3299 ifp = inp->inp_last_outifp;
3300 if (old_so->so_rcv.sb_cc > 0) {
3301 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3302 sbflush(&old_so->so_rcv);
3303 }
3304 socket_unlock(old_so, 0);
3305
3306 /* Associate the new socket with the flow divert control block */
3307 socket_lock(so, 0);
3308 so->so_fd_pcb = fd_cb;
3309 inp = sotoinpcb(so);
3310 inp->inp_last_outifp = ifp;
3311 if (recv_data != NULL) {
3312 if (sbappendstream(&so->so_rcv, recv_data)) {
3313 sorwakeup(so);
3314 }
3315 }
3316 flow_divert_set_protosw(so);
3317 socket_unlock(so, 0);
3318
3319 fd_cb->so = so;
3320 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3321
3322 FDUNLOCK(fd_cb);
3323
3324done:
3325 socket_lock(so, 0);
3326
3327 if (fd_cb != NULL) {
3328 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3329 }
3330
3331 return error;
3332}
3333
3e170ce0
A		 3334errno_t
3335flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3336{
3337 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3338 struct inpcb *inp;
3339 int error = 0;
3340
3341 inp = sotoinpcb(so);
3342 if (inp == NULL) {
3343 return (EINVAL);
3344 }
3345
3346 if (fd_cb == NULL) {
3347 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3348 if (fd_ctl_unit > 0) {
3349 error = flow_divert_pcb_init(so, fd_ctl_unit);
3350 fd_cb = so->so_fd_pcb;
3351 if (error != 0 || fd_cb == NULL) {
3352 goto done;
3353 }
3354 } else {
3355 error = ENETDOWN;
3356 goto done;
3357 }
3358 }
3359 return flow_divert_data_out(so, flags, data, to, control, p);
3360
3361done:
3362 if (data) {
3363 mbuf_freem(data);
3364 }
3365 if (control) {
3366 mbuf_free(control);
3367 }
3368
3369 return error;
3370}
3371
39236c6e
A		 3372errno_t
3373flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3374{
3375 errno_t error = 0;
3376 struct flow_divert_pcb *fd_cb;
3377
3378 if (so->so_flags & SOF_FLOW_DIVERT) {
3379 return EALREADY;
3380 }
3381
3382 fd_cb = flow_divert_pcb_create(so);
3383 if (fd_cb != NULL) {
3384 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3385 if (error) {
3386 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3387 FDRELEASE(fd_cb);
3388 } else {
39236c6e
A		 3389 fd_cb->control_group_unit = ctl_unit;
3390 so->so_fd_pcb = fd_cb;
3391
3e170ce0
A		 3392 if (SOCK_TYPE(so) == SOCK_STREAM) {
3393 flow_divert_set_protosw(so);
3394 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3395 flow_divert_set_udp_protosw(so);
3396 }
39236c6e
A		 3397
3398 FDLOG0(LOG_INFO, fd_cb, "Created");
3399 }
3400 } else {
3401 error = ENOMEM;
3402 }
3403
3404 return error;
3405}
3406
3407errno_t
3408flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3409{
3410 uint32_t ctl_unit = 0;
3411 uint32_t key_unit = 0;
3412 uint32_t flow_id = 0;
3413 int error = 0;
490019cf 3414 int hmac_error = 0;
39236c6e
A		 3415 mbuf_t token = NULL;
3416
3417 if (so->so_flags & SOF_FLOW_DIVERT) {
3418 error = EALREADY;
3419 goto done;
3420 }
3421
3422 if (g_init_result) {
3423 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3424 error = ENOPROTOOPT;
3425 goto done;
3426 }
3427
3e170ce0
A		 3428 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3429 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
39236c6e
A		 3430 (SOCK_DOM(so) != PF_INET
3431#if INET6
3432 && SOCK_DOM(so) != PF_INET6
3433#endif
3434 ))
3435 {
3436 error = EINVAL;
3437 goto done;
3438 } else {
3e170ce0
A		 3439 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3440 struct tcpcb *tp = sototcpcb(so);
3441 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3442 error = EINVAL;
3443 goto done;
3444 }
39236c6e
A		 3445 }
3446 }
3447
3448 error = soopt_getm(sopt, &token);
3449 if (error) {
3450 goto done;
3451 }
3452
3453 error = soopt_mcopyin(sopt, token);
3454 if (error) {
3455 goto done;
3456 }
3457
3458 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3459 if (!error) {
3460 key_unit = ntohl(key_unit);
3461 } else if (error != ENOENT) {
3462 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3463 goto done;
3464 } else {
3465 key_unit = 0;
3466 }
3467
3468 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3469 if (error) {
3470 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3471 goto done;
3472 }
3473
3474 /* A valid kernel control unit is required */
3475 ctl_unit = ntohl(ctl_unit);
3476 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3477 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3478 error = EINVAL;
3479 goto done;
3480 }
3481
3482 socket_unlock(so, 0);
490019cf 3483 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
39236c6e
A		 3484 socket_lock(so, 0);
3485
490019cf
A		 3486 if (hmac_error && hmac_error != ENOENT) {
3487 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3488 error = hmac_error;
39236c6e
A		 3489 goto done;
3490 }
3491
3492 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3493 if (error && error != ENOENT) {
3494 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3495 goto done;
3496 }
3497
3498 if (flow_id == 0) {
3499 error = flow_divert_pcb_init(so, ctl_unit);
3500 if (error == 0) {
3501 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3502 int log_level = LOG_NOTICE;
3503
3504 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3505 sizeof(log_level), &log_level, NULL);
3506 if (error == 0) {
3507 fd_cb->log_level = log_level;
3508 }
3509 error = 0;
3510
3511 fd_cb->connect_token = token;
3512 token = NULL;
3513 }
3514 } else {
3515 error = flow_divert_attach(so, flow_id, ctl_unit);
3516 }
3517
490019cf
A		 3518 if (hmac_error == 0) {
3519 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3520 if (fd_cb != NULL) {
3521 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3522 }
3523 }
3524
39236c6e
A		 3525done:
3526 if (token != NULL) {
3527 mbuf_freem(token);
3528 }
3529
3530 return error;
3531}
3532
3533errno_t
3534flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3535{
3536 uint32_t ctl_unit;
3537 int error = 0;
3538 uint8_t hmac[SHA_DIGEST_LENGTH];
3539 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3540 mbuf_t token = NULL;
3541 struct flow_divert_group *control_group = NULL;
3542
3543 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3544 error = EINVAL;
3545 goto done;
3546 }
3547
3548 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3549
3550 if (fd_cb->group == NULL) {
3551 error = EINVAL;
3552 goto done;
3553 }
3554
3555 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3556 if (error) {
3557 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3558 goto done;
3559 }
3560
3561 ctl_unit = htonl(fd_cb->group->ctl_unit);
3562
3563 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3564 if (error) {
3565 goto done;
3566 }
3567
3568 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3569 if (error) {
3570 goto done;
3571 }
3572
490019cf
A		 3573 if (fd_cb->app_data != NULL) {
3574 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3575 if (error) {
3576 goto done;
3577 }
3578 }
3579
39236c6e
A		 3580 socket_unlock(so, 0);
3581 lck_rw_lock_shared(&g_flow_divert_group_lck);
3582
3583 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3584 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3585 {
3586 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3587 }
3588
3589 if (control_group != NULL) {
3590 lck_rw_lock_shared(&control_group->lck);
3591 ctl_unit = htonl(control_group->ctl_unit);
3592 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3593 if (!error) {
3594 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3595 }
3596 lck_rw_done(&control_group->lck);
3597 } else {
3598 error = ENOPROTOOPT;
3599 }
3600
3601 lck_rw_done(&g_flow_divert_group_lck);
3602 socket_lock(so, 0);
3603
3604 if (error) {
3605 goto done;
3606 }
3607
3608 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3609 if (error) {
3610 goto done;
3611 }
3612
3613 error = soopt_mcopyout(sopt, token);
3614 if (error) {
3615 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3616 goto done;
3617 }
3618
3619done:
3620 if (token != NULL) {
3621 mbuf_freem(token);
3622 }
3623
3624 return error;
3625}
3626
3627static errno_t
3628flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3629{
3e170ce0 3630 struct flow_divert_group *new_group = NULL;
39236c6e
A		 3631 int error = 0;
3632
3633 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3634 error = EINVAL;
3635 goto done;
3636 }
3637
3638 *unitinfo = NULL;
3639
3640 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3641 if (new_group == NULL) {
3642 error = ENOBUFS;
3643 goto done;
3644 }
3645
3646 memset(new_group, 0, sizeof(*new_group));
3647
3648 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3649 RB_INIT(&new_group->pcb_tree);
3650 new_group->ctl_unit = sac->sc_unit;
3651 MBUFQ_INIT(&new_group->send_queue);
3652
3653 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3654
3655 if (g_flow_divert_groups == NULL) {
3656 MALLOC(g_flow_divert_groups,
3657 struct flow_divert_group **,
3658 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3659 M_TEMP,
3660 M_WAITOK | M_ZERO);
3661 }
3662
3663 if (g_flow_divert_groups == NULL) {
3664 error = ENOBUFS;
3665 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3666 error = EALREADY;
3667 } else {
3668 g_flow_divert_groups[sac->sc_unit] = new_group;
3669 g_active_group_count++;
3670 }
3671
3672 lck_rw_done(&g_flow_divert_group_lck);
3673
3674 *unitinfo = new_group;
3675
3676done:
3677 if (error != 0 && new_group != NULL) {
3678 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3679 }
3680 return error;
3681}
3682
3683static errno_t
3684flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3685{
3686 struct flow_divert_group *group = NULL;
3687 errno_t error = 0;
3688 uint16_t node = 0;
3689
3690 if (unit >= GROUP_COUNT_MAX) {
3691 return EINVAL;
3692 }
3693
3694 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3695
3696 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3697
3698 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3699 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3700 g_flow_divert_groups, g_active_group_count);
3701 }
3702
3703 group = g_flow_divert_groups[unit];
3704
3705 if (group != (struct flow_divert_group *)unitinfo) {
3706 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3707 }
3708
3709 if (group != NULL) {
3710 flow_divert_close_all(group);
3711 if (group->token_key != NULL) {
3712 memset(group->token_key, 0, group->token_key_size);
3713 FREE(group->token_key, M_TEMP);
3714 group->token_key = NULL;
3715 group->token_key_size = 0;
3716 }
3717 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3718 g_flow_divert_groups[unit] = NULL;
3719 g_active_group_count--;
3720 } else {
3721 error = EINVAL;
3722 }
3723
3724 if (g_active_group_count == 0) {
3725 FREE(g_flow_divert_groups, M_TEMP);
3726 g_flow_divert_groups = NULL;
3727 }
3728
3729 /* Remove all signing IDs that point to this unit */
3730 for (node = 0; node < g_signing_id_trie.nodes_count; node++) {
3731 if (TRIE_NODE(&g_signing_id_trie, node).group_unit == unit) {
3732 TRIE_NODE(&g_signing_id_trie, node).group_unit = 0;
3733 }
3734 }
3735
3736 lck_rw_done(&g_flow_divert_group_lck);
3737
3738 return error;
3739}
3740
3741static errno_t
3742flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3743{
3744 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3745}
3746
3747static void
3748flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3749{
3750 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3751
3752 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3753 struct flow_divert_pcb *fd_cb;
3754 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3755
3756 lck_rw_lock_shared(&g_flow_divert_group_lck);
3757 lck_rw_lock_exclusive(&group->lck);
3758
3759 while (!MBUFQ_EMPTY(&group->send_queue)) {
3760 mbuf_t next_packet;
3761 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3762 next_packet = MBUFQ_FIRST(&group->send_queue);
3763 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3764 if (error) {
3765 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3766 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3767 lck_rw_done(&group->lck);
3768 lck_rw_done(&g_flow_divert_group_lck);
3769 return;
3770 }
3771 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3772 }
3773
3774 SLIST_INIT(&tmp_list);
3775
3776 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3777 FDRETAIN(fd_cb);
3778 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3779 }
3780
3781 lck_rw_done(&group->lck);
3782
3783 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3784 FDLOCK(fd_cb);
3785 if (fd_cb->so != NULL) {
3786 socket_lock(fd_cb->so, 0);
3787 if (fd_cb->group != NULL) {
3788 flow_divert_send_buffered_data(fd_cb, FALSE);
3789 }
3790 socket_unlock(fd_cb->so, 0);
3791 }
3792 FDUNLOCK(fd_cb);
3793 FDRELEASE(fd_cb);
3794 }
3795
3796 lck_rw_done(&g_flow_divert_group_lck);
3797 }
3798}
3799
3800static int
3801flow_divert_kctl_init(void)
3802{
3803 struct kern_ctl_reg ctl_reg;
3804 int result;
3805
3806 memset(&ctl_reg, 0, sizeof(ctl_reg));
3807
fe8ab488 3808 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
39236c6e
A		 3809 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3810 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3811 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3812 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3813
3814 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3815 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3816 ctl_reg.ctl_send = flow_divert_kctl_send;
3817 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3818
3819 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3820
3821 if (result) {
3822 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3823 return result;
3824 }
3825
3826 return 0;
3827}
3828
3829void
3830flow_divert_init(void)
3831{
3832 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 3833 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 3834
3835 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3836
3837 VERIFY(g_tcp_protosw != NULL);
3838
3839 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3840 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3841
3842 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3843 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3844 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3845 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3846 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3847 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3848 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3849 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3850 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3851 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3852 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3853
3854 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3855 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3856
3857 /*
3858 * Socket filters shouldn't attach/detach to/from this protosw
3859 * since pr_protosw is to be used instead, which points to the
3860 * real protocol; if they do, it is a bug and we should panic.
3861 */
3862 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3863 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3864 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3865 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3866
3e170ce0
A		 3867 /* UDP */
3868 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3869 VERIFY(g_udp_protosw != NULL);
3870
3871 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3872 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3873
3874 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3875 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3876 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3877 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3878 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3879 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3880 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3881 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3882 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3883 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3884 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3885 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3886 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3887
3888 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3889 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3890
3891 /*
3892 * Socket filters shouldn't attach/detach to/from this protosw
3893 * since pr_protosw is to be used instead, which points to the
3894 * real protocol; if they do, it is a bug and we should panic.
3895 */
3896 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3897 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3898 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3899 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3900
39236c6e
A		 3901#if INET6
3902 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3903
3904 VERIFY(g_tcp6_protosw != NULL);
3905
3906 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3907 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3908
3909 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3910 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3911 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3912 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3913 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3914 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3915 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3916 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3917 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3918 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3919 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3920
3921 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3922 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3923 /*
3924 * Socket filters shouldn't attach/detach to/from this protosw
3925 * since pr_protosw is to be used instead, which points to the
3926 * real protocol; if they do, it is a bug and we should panic.
3927 */
3928 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3929 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3930 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3931 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 3932
3933 /* UDP6 */
3934 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3935
3936 VERIFY(g_udp6_protosw != NULL);
3937
3938 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3939 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3940
3941 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3942 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3943 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3944 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3945 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3946 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3947 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3948 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3949 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3950 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3951 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3952 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3953 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3954
3955 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3956 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3957 /*
3958 * Socket filters shouldn't attach/detach to/from this protosw
3959 * since pr_protosw is to be used instead, which points to the
3960 * real protocol; if they do, it is a bug and we should panic.
3961 */
3962 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3963 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3964 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3965 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
39236c6e
A		 3966#endif /* INET6 */
3967
3968 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3969 if (flow_divert_grp_attr == NULL) {
3970 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3971 g_init_result = ENOMEM;
3972 goto done;
3973 }
3974
3975 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3976 if (flow_divert_mtx_grp == NULL) {
3977 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3978 g_init_result = ENOMEM;
3979 goto done;
3980 }
3981
3982 flow_divert_mtx_attr = lck_attr_alloc_init();
3983 if (flow_divert_mtx_attr == NULL) {
3984 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3985 g_init_result = ENOMEM;
3986 goto done;
3987 }
3988
3989 g_init_result = flow_divert_kctl_init();
3990 if (g_init_result) {
3991 goto done;
3992 }
3993
3994 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3995
3996 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
3997 g_signing_id_trie.root = NULL_TRIE_IDX;
3998
3999done:
4000 if (g_init_result != 0) {
4001 if (flow_divert_mtx_attr != NULL) {
4002 lck_attr_free(flow_divert_mtx_attr);
4003 flow_divert_mtx_attr = NULL;
4004 }
4005 if (flow_divert_mtx_grp != NULL) {
4006 lck_grp_free(flow_divert_mtx_grp);
4007 flow_divert_mtx_grp = NULL;
4008 }
4009 if (flow_divert_grp_attr != NULL) {
4010 lck_grp_attr_free(flow_divert_grp_attr);
4011 flow_divert_grp_attr = NULL;
4012 }
4013
4014 if (g_flow_divert_kctl_ref != NULL) {
4015 ctl_deregister(g_flow_divert_kctl_ref);
4016 g_flow_divert_kctl_ref = NULL;
4017 }
4018 }
4019}
mirror of XNU