]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:


697016c4996a806e54d433e36391091f817c69d4
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
59 #if INET6
60 #include <netinet6/ip6protosw.h>
61 #endif /* INET6 */
62 #include <dev/random/randomdev.h>
63 #include <libkern/crypto/sha1.h>
64 #include <libkern/crypto/crypto_internal.h>
65
66 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
67 #define FLOW_DIVERT_READ_CLOSED 0x00000002
68 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
69 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
70 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
71 #define FLOW_DIVERT_TRANSFERRED 0x00000020
72
73 #define FDLOG(level, pcb, format, ...) do { \
74 if (level <= (pcb)->log_level) { \
75 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): " format "\n", __FUNCTION__, (pcb)->hash, __VA_ARGS__); \
76 } \
77 } while (0)
78
79 #define FDLOG0(level, pcb, msg) do { \
80 if (level <= (pcb)->log_level) { \
81 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): %s\n", __FUNCTION__, (pcb)->hash, msg); \
82 } \
83 } while (0)
84
85 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
86 #define FDRELEASE(pcb) \
87 do { \
88 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
89 flow_divert_pcb_destroy(pcb); \
90 } \
91 } while (0)
92
93 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
94 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
95
96 #define FD_CTL_SENDBUFF_SIZE (2 * FLOW_DIVERT_CHUNK_SIZE)
97 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
98
99 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
100
101 #define GROUP_COUNT_MAX 32
102 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
103 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
104
105 #define DNS_SERVICE_GROUP_UNIT (GROUP_COUNT_MAX + 1)
106
107 struct flow_divert_trie_node
108 {
109 uint16_t start;
110 uint16_t length;
111 uint16_t child_map;
112 uint32_t group_unit;
113 };
114
115 struct flow_divert_trie
116 {
117 struct flow_divert_trie_node *nodes;
118 uint16_t *child_maps;
119 uint8_t *bytes;
120 void *memory;
121 size_t nodes_count;
122 size_t child_maps_count;
123 size_t bytes_count;
124 size_t nodes_free_next;
125 size_t child_maps_free_next;
126 size_t bytes_free_next;
127 uint16_t root;
128 };
129
130 #define CHILD_MAP_SIZE 256
131 #define NULL_TRIE_IDX 0xffff
132 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
133 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
134 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
135
136 static struct flow_divert_pcb nil_pcb;
137
138 decl_lck_rw_data(static, g_flow_divert_group_lck);
139 static struct flow_divert_group **g_flow_divert_groups = NULL;
140 static uint32_t g_active_group_count = 0;
141 static struct flow_divert_trie g_signing_id_trie;
142
143 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
144 static lck_attr_t *flow_divert_mtx_attr = NULL;
145 static lck_grp_t *flow_divert_mtx_grp = NULL;
146 static errno_t g_init_result = 0;
147
148 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
149
150 static struct protosw g_flow_divert_in_protosw;
151 static struct pr_usrreqs g_flow_divert_in_usrreqs;
152 #if INET6
153 static struct ip6protosw g_flow_divert_in6_protosw;
154 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
155 #endif /* INET6 */
156
157 static struct protosw *g_tcp_protosw = NULL;
158 static struct ip6protosw *g_tcp6_protosw = NULL;
159
160 static inline int
161 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
162 {
163 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
164 }
165
166 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
167 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
168
169 static const char *
170 flow_divert_packet_type2str(uint8_t packet_type)
171 {
172 switch (packet_type) {
173 case FLOW_DIVERT_PKT_CONNECT:
174 return "connect";
175 case FLOW_DIVERT_PKT_CONNECT_RESULT:
176 return "connect result";
177 case FLOW_DIVERT_PKT_DATA:
178 return "data";
179 case FLOW_DIVERT_PKT_CLOSE:
180 return "close";
181 case FLOW_DIVERT_PKT_READ_NOTIFY:
182 return "read notification";
183 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
184 return "properties update";
185 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
186 return "app map update";
187 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
188 return "app map create";
189 default:
190 return "unknown";
191 }
192 }
193
194 static struct flow_divert_pcb *
195 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
196 {
197 struct flow_divert_pcb key_item;
198 struct flow_divert_pcb *fd_cb = NULL;
199
200 key_item.hash = hash;
201
202 lck_rw_lock_shared(&group->lck);
203 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
204 FDRETAIN(fd_cb);
205 lck_rw_done(&group->lck);
206
207 return fd_cb;
208 }
209
210 static errno_t
211 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
212 {
213 int error = 0;
214 struct flow_divert_pcb *exist = NULL;
215 struct flow_divert_group *group;
216 static uint32_t g_nextkey = 1;
217 static uint32_t g_hash_seed = 0;
218 errno_t result = 0;
219 int try_count = 0;
220
221 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
222 return EINVAL;
223 }
224
225 socket_unlock(fd_cb->so, 0);
226 lck_rw_lock_shared(&g_flow_divert_group_lck);
227
228 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
229 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
230 error = ENETUNREACH;
231 goto done;
232 }
233
234 group = g_flow_divert_groups[ctl_unit];
235 if (group == NULL) {
236 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
237 error = ENETUNREACH;
238 goto done;
239 }
240
241 socket_lock(fd_cb->so, 0);
242
243 do {
244 uint32_t key[2];
245 uint32_t idx;
246
247 key[0] = g_nextkey++;
248 key[1] = RandomULong();
249
250 if (g_hash_seed == 0) {
251 g_hash_seed = RandomULong();
252 }
253
254 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
255
256 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
257 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
258 if (curr_group != NULL && curr_group != group) {
259 lck_rw_lock_shared(&curr_group->lck);
260 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
261 lck_rw_done(&curr_group->lck);
262 if (exist != NULL) {
263 break;
264 }
265 }
266 }
267
268 if (exist == NULL) {
269 lck_rw_lock_exclusive(&group->lck);
270 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
271 lck_rw_done(&group->lck);
272 }
273 } while (exist != NULL && try_count++ < 3);
274
275 if (exist == NULL) {
276 fd_cb->group = group;
277 FDRETAIN(fd_cb); /* The group now has a reference */
278 } else {
279 fd_cb->hash = 0;
280 result = EEXIST;
281 }
282
283 socket_unlock(fd_cb->so, 0);
284
285 done:
286 lck_rw_done(&g_flow_divert_group_lck);
287 socket_lock(fd_cb->so, 0);
288
289 return result;
290 }
291
292 static struct flow_divert_pcb *
293 flow_divert_pcb_create(socket_t so)
294 {
295 struct flow_divert_pcb *new_pcb = NULL;
296
297 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
298 if (new_pcb == NULL) {
299 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
300 return NULL;
301 }
302
303 memset(new_pcb, 0, sizeof(*new_pcb));
304
305 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
306 new_pcb->so = so;
307 new_pcb->log_level = nil_pcb.log_level;
308
309 FDRETAIN(new_pcb); /* Represents the socket's reference */
310
311 return new_pcb;
312 }
313
314 static void
315 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
316 {
317 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
318 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
319
320 if (fd_cb->local_address != NULL) {
321 FREE(fd_cb->local_address, M_SONAME);
322 }
323 if (fd_cb->remote_address != NULL) {
324 FREE(fd_cb->remote_address, M_SONAME);
325 }
326 if (fd_cb->connect_token != NULL) {
327 mbuf_freem(fd_cb->connect_token);
328 }
329 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
330 }
331
332 static void
333 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
334 {
335 if (fd_cb->group != NULL) {
336 struct flow_divert_group *group = fd_cb->group;
337 lck_rw_lock_exclusive(&group->lck);
338 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
339 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
340 fd_cb->group = NULL;
341 FDRELEASE(fd_cb); /* Release the group's reference */
342 lck_rw_done(&group->lck);
343 }
344 }
345
346 static int
347 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
348 {
349 struct flow_divert_packet_header hdr;
350 int error = 0;
351
352 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
353 if (error) {
354 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
355 return error;
356 }
357
358 hdr.packet_type = packet_type;
359 hdr.conn_id = htonl(fd_cb->hash);
360
361 /* Lay down the header */
362 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
363 if (error) {
364 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
365 mbuf_freem(*packet);
366 *packet = NULL;
367 return error;
368 }
369
370 return 0;
371 }
372
373 static int
374 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, size_t length, const void *value)
375 {
376 size_t net_length = htonl(length);
377 int error = 0;
378
379 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
380 if (error) {
381 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
382 return error;
383 }
384
385 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
386 if (error) {
387 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%lu)", length);
388 return error;
389 }
390
391 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
392 if (error) {
393 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
394 return error;
395 }
396
397 return error;
398 }
399
400 static int
401 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
402 {
403 size_t cursor = offset;
404 int error = 0;
405 size_t curr_length;
406 uint8_t curr_type;
407
408 *err = 0;
409
410 do {
411 if (!next) {
412 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
413 if (error) {
414 *err = ENOENT;
415 return -1;
416 }
417 } else {
418 next = 0;
419 curr_type = FLOW_DIVERT_TLV_NIL;
420 }
421
422 if (curr_type != type) {
423 cursor += sizeof(curr_type);
424 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
425 if (error) {
426 *err = error;
427 return -1;
428 }
429
430 cursor += (sizeof(curr_length) + ntohl(curr_length));
431 }
432 } while (curr_type != type);
433
434 return cursor;
435 }
436
437 static int
438 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, size_t *val_size)
439 {
440 int error = 0;
441 size_t length;
442 int tlv_offset;
443
444 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
445 if (tlv_offset < 0) {
446 return error;
447 }
448
449 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
450 if (error) {
451 return error;
452 }
453
454 length = ntohl(length);
455
456 if (val_size != NULL) {
457 *val_size = length;
458 }
459
460 if (buff != NULL && buff_len > 0) {
461 size_t to_copy = (length < buff_len) ? length : buff_len;
462 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
463 if (error) {
464 return error;
465 }
466 }
467
468 return 0;
469 }
470
471 static int
472 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
473 {
474 mbuf_t curr_mbuf = packet;
475
476 if (g_crypto_funcs == NULL || group->token_key == NULL) {
477 return ENOPROTOOPT;
478 }
479
480 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
481 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
482
483 while (curr_mbuf != NULL) {
484 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
485 curr_mbuf = mbuf_next(curr_mbuf);
486 }
487
488 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
489
490 return 0;
491 }
492
493 static int
494 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
495 {
496 int error = 0;
497 struct flow_divert_group *group = NULL;
498 int hmac_offset;
499 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
500 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
501 mbuf_t tail;
502
503 lck_rw_lock_shared(&g_flow_divert_group_lck);
504
505 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
506 group = g_flow_divert_groups[ctl_unit];
507 }
508
509 if (group == NULL) {
510 lck_rw_done(&g_flow_divert_group_lck);
511 return ENOPROTOOPT;
512 }
513
514 lck_rw_lock_shared(&group->lck);
515
516 if (group->token_key == NULL) {
517 error = ENOPROTOOPT;
518 goto done;
519 }
520
521 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
522 if (hmac_offset < 0) {
523 goto done;
524 }
525
526 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
527 if (error) {
528 goto done;
529 }
530
531 /* Chop off the HMAC TLV */
532 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
533 if (error) {
534 goto done;
535 }
536
537 mbuf_free(tail);
538
539 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
540 if (error) {
541 goto done;
542 }
543
544 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
545 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
546 error = EINVAL;
547 goto done;
548 }
549
550 done:
551 lck_rw_done(&group->lck);
552 lck_rw_done(&g_flow_divert_group_lck);
553 return error;
554 }
555
556 static void
557 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
558 {
559 struct inpcb *inp = NULL;
560 struct ifnet *ifp = NULL;
561 Boolean cell = FALSE;
562 Boolean wifi = FALSE;
563 Boolean wired = FALSE;
564
565 inp = sotoinpcb(fd_cb->so);
566 if (inp == NULL) {
567 return;
568 }
569
570 ifp = inp->inp_last_outifp;
571 if (ifp != NULL) {
572 cell = IFNET_IS_CELLULAR(ifp);
573 wifi = (!cell && IFNET_IS_WIFI(ifp));
574 wired = (!wifi && IFNET_IS_WIRED(ifp));
575 }
576
577 if (send) {
578 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
579 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
580 } else {
581 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
582 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
583 }
584 }
585
586 static errno_t
587 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
588 {
589 struct inpcb *inp = NULL;
590
591 inp = sotoinpcb(fd_cb->so);
592 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
593 IFNET_IS_CELLULAR(inp->inp_last_outifp))
594 return EHOSTUNREACH;
595
596 return 0;
597 }
598
599 static errno_t
600 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
601 {
602 struct inpcb *inp = NULL;
603
604 inp = sotoinpcb(fd_cb->so);
605 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
606 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
607 return EHOSTUNREACH;
608
609 return 0;
610 }
611
612 static void
613 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
614 {
615 if (how != SHUT_RD) {
616 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
617 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
618 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
619 /* If the tunnel is not accepting writes any more, then flush the send buffer */
620 sbflush(&fd_cb->so->so_snd);
621 }
622 }
623 if (how != SHUT_WR) {
624 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
625 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
626 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
627 }
628 }
629 }
630
631 static uint16_t
632 trie_node_alloc(struct flow_divert_trie *trie)
633 {
634 if (trie->nodes_free_next < trie->nodes_count) {
635 uint16_t node_idx = trie->nodes_free_next++;
636 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
637 return node_idx;
638 } else {
639 return NULL_TRIE_IDX;
640 }
641 }
642
643 static uint16_t
644 trie_child_map_alloc(struct flow_divert_trie *trie)
645 {
646 if (trie->child_maps_free_next < trie->child_maps_count) {
647 return trie->child_maps_free_next++;
648 } else {
649 return NULL_TRIE_IDX;
650 }
651 }
652
653 static uint16_t
654 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
655 {
656 uint16_t start = trie->bytes_free_next;
657 if (start + bytes_size <= trie->bytes_count) {
658 if (start != bytes_idx) {
659 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
660 }
661 trie->bytes_free_next += bytes_size;
662 return start;
663 } else {
664 return NULL_TRIE_IDX;
665 }
666 }
667
668 static uint16_t
669 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
670 {
671 uint16_t current = trie->root;
672 uint16_t child = trie->root;
673 uint16_t string_end = string_start + string_len;
674 uint16_t string_idx = string_start;
675 uint16_t string_remainder = string_len;
676
677 while (child != NULL_TRIE_IDX) {
678 uint16_t parent = current;
679 uint16_t node_idx;
680 uint16_t current_end;
681
682 current = child;
683 child = NULL_TRIE_IDX;
684
685 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
686
687 for (node_idx = TRIE_NODE(trie, current).start;
688 node_idx < current_end &&
689 string_idx < string_end &&
690 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
691 node_idx++, string_idx++);
692
693 string_remainder = string_end - string_idx;
694
695 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
696 /*
697 * We did not reach the end of the current node's string.
698 * We need to split the current node into two:
699 * 1. A new node that contains the prefix of the node that matches
700 * the prefix of the string being inserted.
701 * 2. The current node modified to point to the remainder
702 * of the current node's string.
703 */
704 uint16_t prefix = trie_node_alloc(trie);
705 if (prefix == NULL_TRIE_IDX) {
706 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
707 return NULL_TRIE_IDX;
708 }
709
710 /*
711 * Prefix points to the portion of the current nodes's string that has matched
712 * the input string thus far.
713 */
714 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
715 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
716
717 /*
718 * Prefix has the current node as the child corresponding to the first byte
719 * after the split.
720 */
721 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
722 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
723 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
724 return NULL_TRIE_IDX;
725 }
726 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
727
728 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
729 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
730
731 /* Current node is adjusted to point to the remainder */
732 TRIE_NODE(trie, current).start = node_idx;
733 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
734
735 /* We want to insert the new leaf (if any) as a child of the prefix */
736 current = prefix;
737 }
738
739 if (string_remainder > 0) {
740 /*
741 * We still have bytes in the string that have not been matched yet.
742 * If the current node has children, iterate to the child corresponding
743 * to the next byte in the string.
744 */
745 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
746 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
747 }
748 }
749 } /* while (child != NULL_TRIE_IDX) */
750
751 if (string_remainder > 0) {
752 /* Add a new leaf containing the remainder of the string */
753 uint16_t leaf = trie_node_alloc(trie);
754 if (leaf == NULL_TRIE_IDX) {
755 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
756 return NULL_TRIE_IDX;
757 }
758
759 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
760 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
761 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
762 return NULL_TRIE_IDX;
763 }
764 TRIE_NODE(trie, leaf).length = string_remainder;
765
766 /* Set the new leaf as the child of the current node */
767 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
768 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
769 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
770 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
771 return NULL_TRIE_IDX;
772 }
773 }
774 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
775 current = leaf;
776 } /* else duplicate or this string is a prefix of one of the existing strings */
777
778 return current;
779 }
780
781 static uint16_t
782 flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
783 {
784 uint16_t current = trie->root;
785 uint16_t string_idx = 0;
786
787 while (current != NULL_TRIE_IDX) {
788 uint16_t next = NULL_TRIE_IDX;
789 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
790 uint16_t node_idx;
791
792 for (node_idx = TRIE_NODE(trie, current).start;
793 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
794 node_idx++, string_idx++);
795
796 if (node_idx == node_end) {
797 if (string_bytes[string_idx] == '\0') {
798 return current; /* Got an exact match */
799 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
800 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
801 }
802 }
803 current = next;
804 }
805
806 return NULL_TRIE_IDX;
807 }
808
809 static int
810 flow_divert_get_src_proc(struct socket *so, proc_t *proc, boolean_t match_delegate)
811 {
812 int release = 0;
813
814 if (!match_delegate &&
815 (so->so_flags & SOF_DELEGATED) &&
816 (*proc == PROC_NULL || (*proc)->p_pid != so->e_pid))
817 {
818 *proc = proc_find(so->e_pid);
819 release = 1;
820 } else if (*proc == PROC_NULL) {
821 *proc = current_proc();
822 }
823
824 if (*proc != PROC_NULL) {
825 if ((*proc)->p_pid == 0) {
826 if (release) {
827 proc_rele(*proc);
828 }
829 release = 0;
830 *proc = PROC_NULL;
831 }
832 }
833
834 return release;
835 }
836
837 static int
838 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
839 {
840 int error;
841
842 if (fd_cb->group == NULL) {
843 fd_cb->so->so_error = ECONNABORTED;
844 soisdisconnected(fd_cb->so);
845 return ECONNABORTED;
846 }
847
848 lck_rw_lock_shared(&fd_cb->group->lck);
849
850 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
851 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
852 } else {
853 error = ENOBUFS;
854 }
855
856 if (error == ENOBUFS) {
857 if (enqueue) {
858 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
859 lck_rw_lock_exclusive(&fd_cb->group->lck);
860 }
861 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
862 error = 0;
863 }
864 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
865 }
866
867 lck_rw_done(&fd_cb->group->lck);
868
869 return error;
870 }
871
872 static int
873 flow_divert_send_connect(struct flow_divert_pcb *fd_cb, struct sockaddr *to, mbuf_t connect_packet)
874 {
875 int error = 0;
876
877 error = flow_divert_packet_append_tlv(connect_packet,
878 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
879 sizeof(fd_cb->so->so_traffic_class),
880 &fd_cb->so->so_traffic_class);
881 if (error) {
882 goto done;
883 }
884
885 if (fd_cb->so->so_flags & SOF_DELEGATED) {
886 error = flow_divert_packet_append_tlv(connect_packet,
887 FLOW_DIVERT_TLV_PID,
888 sizeof(fd_cb->so->e_pid),
889 &fd_cb->so->e_pid);
890 if (error) {
891 goto done;
892 }
893
894 error = flow_divert_packet_append_tlv(connect_packet,
895 FLOW_DIVERT_TLV_UUID,
896 sizeof(fd_cb->so->e_uuid),
897 &fd_cb->so->e_uuid);
898 if (error) {
899 goto done;
900 }
901 } else {
902 error = flow_divert_packet_append_tlv(connect_packet,
903 FLOW_DIVERT_TLV_PID,
904 sizeof(fd_cb->so->e_pid),
905 &fd_cb->so->last_pid);
906 if (error) {
907 goto done;
908 }
909
910 error = flow_divert_packet_append_tlv(connect_packet,
911 FLOW_DIVERT_TLV_UUID,
912 sizeof(fd_cb->so->e_uuid),
913 &fd_cb->so->last_uuid);
914 if (error) {
915 goto done;
916 }
917 }
918
919 if (fd_cb->connect_token != NULL) {
920 unsigned int token_len = m_length(fd_cb->connect_token);
921 mbuf_concatenate(connect_packet, fd_cb->connect_token);
922 mbuf_pkthdr_adjustlen(connect_packet, token_len);
923 fd_cb->connect_token = NULL;
924 } else {
925 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
926 int port;
927
928 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
929 if (error) {
930 goto done;
931 }
932
933 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, to->sa_len, to);
934 if (error) {
935 goto done;
936 }
937
938 if (to->sa_family == AF_INET) {
939 port = ntohs((satosin(to))->sin_port);
940 }
941 #if INET6
942 else {
943 port = ntohs((satosin6(to))->sin6_port);
944 }
945 #endif
946
947 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
948 if (error) {
949 goto done;
950 }
951 }
952
953 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
954 if (error) {
955 goto done;
956 }
957
958 done:
959 return error;
960 }
961
962 static int
963 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
964 {
965 int error = 0;
966 mbuf_t packet = NULL;
967 int rbuff_space = 0;
968
969 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
970 if (error) {
971 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
972 goto done;
973 }
974
975 rbuff_space = sbspace(&fd_cb->so->so_rcv);
976 if (rbuff_space < 0) {
977 rbuff_space = 0;
978 }
979 rbuff_space = htonl(rbuff_space);
980 error = flow_divert_packet_append_tlv(packet,
981 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
982 sizeof(rbuff_space),
983 &rbuff_space);
984 if (error) {
985 goto done;
986 }
987
988 error = flow_divert_send_packet(fd_cb, packet, TRUE);
989 if (error) {
990 goto done;
991 }
992
993 done:
994 if (error && packet != NULL) {
995 mbuf_free(packet);
996 }
997
998 return error;
999 }
1000
1001 static int
1002 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1003 {
1004 int error = 0;
1005 mbuf_t packet = NULL;
1006 uint32_t zero = 0;
1007
1008 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1009 if (error) {
1010 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1011 goto done;
1012 }
1013
1014 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1015 if (error) {
1016 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1017 goto done;
1018 }
1019
1020 how = htonl(how);
1021 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1022 if (error) {
1023 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1024 goto done;
1025 }
1026
1027 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1028 if (error) {
1029 goto done;
1030 }
1031
1032 done:
1033 if (error && packet != NULL) {
1034 mbuf_free(packet);
1035 }
1036
1037 return error;
1038 }
1039
1040 static int
1041 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1042 {
1043 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1044 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1045 {
1046 return SHUT_RDWR;
1047 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1048 return SHUT_RD;
1049 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1050 return SHUT_WR;
1051 }
1052
1053 return -1;
1054 }
1055
1056 /*
1057 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1058 * writes. Returns FALSE otherwise.
1059 */
1060 static void
1061 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1062 {
1063 int how = -1;
1064
1065 /* Do not send any close messages if there is still data in the send buffer */
1066 if (fd_cb->so->so_snd.sb_cc == 0) {
1067 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1068 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1069 how = SHUT_RD;
1070 }
1071 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1072 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1073 if (how == SHUT_RD) {
1074 how = SHUT_RDWR;
1075 } else {
1076 how = SHUT_WR;
1077 }
1078 }
1079 }
1080
1081 if (how != -1) {
1082 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1083 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1084 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1085 if (how != SHUT_RD) {
1086 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1087 }
1088 if (how != SHUT_WR) {
1089 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1090 }
1091 }
1092 }
1093
1094 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1095 soisdisconnected(fd_cb->so);
1096 }
1097 }
1098
1099 static errno_t
1100 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, Boolean force)
1101 {
1102 mbuf_t packet;
1103 mbuf_t last;
1104 int error = 0;
1105
1106 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1107 if (error) {
1108 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1109 return error;
1110 }
1111
1112 last = m_last(packet);
1113 mbuf_setnext(last, data);
1114 mbuf_pkthdr_adjustlen(packet, data_len);
1115
1116 error = flow_divert_send_packet(fd_cb, packet, force);
1117
1118 if (error) {
1119 mbuf_setnext(last, NULL);
1120 mbuf_free(packet);
1121 } else {
1122 fd_cb->bytes_sent += data_len;
1123 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1124 }
1125
1126 return error;
1127 }
1128
1129 static void
1130 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1131 {
1132 size_t to_send;
1133 size_t sent = 0;
1134 int error = 0;
1135 mbuf_t buffer;
1136
1137 to_send = fd_cb->so->so_snd.sb_cc;
1138 buffer = fd_cb->so->so_snd.sb_mb;
1139
1140 if (buffer == NULL && to_send > 0) {
1141 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1142 return;
1143 }
1144
1145 /* Ignore the send window if force is enabled */
1146 if (!force && (to_send > fd_cb->send_window)) {
1147 to_send = fd_cb->send_window;
1148 }
1149
1150 while (sent < to_send) {
1151 mbuf_t data;
1152 size_t data_len;
1153
1154 data_len = to_send - sent;
1155 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1156 data_len = FLOW_DIVERT_CHUNK_SIZE;
1157 }
1158
1159 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1160 if (error) {
1161 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1162 break;
1163 }
1164
1165 error = flow_divert_send_data_packet(fd_cb, data, data_len, force);
1166 if (error) {
1167 mbuf_free(data);
1168 break;
1169 }
1170
1171 sent += data_len;
1172 }
1173
1174 if (sent > 0) {
1175 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1176 if (fd_cb->send_window >= sent) {
1177 fd_cb->send_window -= sent;
1178 } else {
1179 fd_cb->send_window = 0;
1180 }
1181 sbdrop(&fd_cb->so->so_snd, sent);
1182 sowwakeup(fd_cb->so);
1183 }
1184 }
1185
1186 static int
1187 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data)
1188 {
1189 size_t to_send = mbuf_pkthdr_len(data);
1190 size_t sent = 0;
1191 int error = 0;
1192 mbuf_t remaining_data = data;
1193 mbuf_t pkt_data = NULL;
1194
1195 if (to_send > fd_cb->send_window) {
1196 to_send = fd_cb->send_window;
1197 }
1198
1199 if (fd_cb->so->so_snd.sb_cc > 0) {
1200 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1201 }
1202
1203 while (sent < to_send) {
1204 size_t pkt_data_len;
1205
1206 pkt_data = remaining_data;
1207
1208 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1209 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1210 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1211 if (error) {
1212 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1213 pkt_data = NULL;
1214 break;
1215 }
1216 } else {
1217 pkt_data_len = to_send - sent;
1218 remaining_data = NULL;
1219 }
1220
1221 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, FALSE);
1222
1223 if (error) {
1224 break;
1225 }
1226
1227 pkt_data = NULL;
1228 sent += pkt_data_len;
1229 }
1230
1231 fd_cb->send_window -= sent;
1232
1233 error = 0;
1234
1235 if (pkt_data != NULL) {
1236 if (sbspace(&fd_cb->so->so_snd) > 0) {
1237 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1238 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1239 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1240 }
1241 } else {
1242 error = ENOBUFS;
1243 }
1244 }
1245
1246 if (remaining_data != NULL) {
1247 if (sbspace(&fd_cb->so->so_snd) > 0) {
1248 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1249 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1250 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1251 }
1252 } else {
1253 error = ENOBUFS;
1254 }
1255 }
1256
1257 return error;
1258 }
1259
1260 static int
1261 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1262 {
1263 int error = 0;
1264 mbuf_t packet = NULL;
1265 uint32_t net_read_count = htonl(read_count);
1266
1267 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1268 if (error) {
1269 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1270 goto done;
1271 }
1272
1273 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1274 if (error) {
1275 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1276 goto done;
1277 }
1278
1279 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1280 if (error) {
1281 goto done;
1282 }
1283
1284 done:
1285 if (error && packet != NULL) {
1286 mbuf_free(packet);
1287 }
1288
1289 return error;
1290 }
1291
1292 static int
1293 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1294 {
1295 int error = 0;
1296 mbuf_t packet = NULL;
1297
1298 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1299 if (error) {
1300 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1301 goto done;
1302 }
1303
1304 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1305 if (error) {
1306 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1307 goto done;
1308 }
1309
1310 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1311 if (error) {
1312 goto done;
1313 }
1314
1315 done:
1316 if (error && packet != NULL) {
1317 mbuf_free(packet);
1318 }
1319
1320 return error;
1321 }
1322
1323 static void
1324 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1325 {
1326 uint32_t connect_error;
1327 uint32_t ctl_unit = 0;
1328 int error = 0;
1329 struct flow_divert_group *grp = NULL;
1330 struct sockaddr_storage local_address;
1331 int out_if_index = 0;
1332 struct sockaddr_storage remote_address;
1333 uint32_t send_window;
1334
1335 memset(&local_address, 0, sizeof(local_address));
1336 memset(&remote_address, 0, sizeof(remote_address));
1337
1338 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1339 if (error) {
1340 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1341 return;
1342 }
1343
1344 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1345
1346 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1347 if (error) {
1348 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1349 return;
1350 }
1351
1352 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1353 if (error) {
1354 FDLOG(LOG_ERR, fd_cb, "failed to get the control unit: %d", error);
1355 return;
1356 }
1357
1358 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1359 if (error) {
1360 FDLOG0(LOG_NOTICE, fd_cb, "No local address provided");
1361 }
1362
1363 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1364 if (error) {
1365 FDLOG0(LOG_NOTICE, fd_cb, "No remote address provided");
1366 }
1367
1368 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1369 if (error) {
1370 FDLOG0(LOG_NOTICE, fd_cb, "No output if index provided");
1371 }
1372
1373 connect_error = ntohl(connect_error);
1374 ctl_unit = ntohl(ctl_unit);
1375
1376 lck_rw_lock_shared(&g_flow_divert_group_lck);
1377
1378 if (connect_error == 0) {
1379 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
1380 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1381 error = EINVAL;
1382 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1383 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1384 error = EINVAL;
1385 } else {
1386 grp = g_flow_divert_groups[ctl_unit];
1387 if (grp == NULL) {
1388 error = ECONNRESET;
1389 }
1390 }
1391 }
1392
1393 FDLOCK(fd_cb);
1394 if (fd_cb->so != NULL) {
1395 struct inpcb *inp = NULL;
1396 struct ifnet *ifp = NULL;
1397 struct flow_divert_group *old_group;
1398
1399 socket_lock(fd_cb->so, 0);
1400
1401 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1402 goto done;
1403 }
1404
1405 inp = sotoinpcb(fd_cb->so);
1406
1407 if (connect_error || error) {
1408 goto set_socket_state;
1409 }
1410
1411 if (local_address.ss_family != 0) {
1412 if (local_address.ss_len > sizeof(local_address)) {
1413 local_address.ss_len = sizeof(local_address);
1414 }
1415 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1416 } else {
1417 error = EINVAL;
1418 goto set_socket_state;
1419 }
1420
1421 if (remote_address.ss_family != 0) {
1422 if (remote_address.ss_len > sizeof(remote_address)) {
1423 remote_address.ss_len = sizeof(remote_address);
1424 }
1425 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1426 } else {
1427 error = EINVAL;
1428 goto set_socket_state;
1429 }
1430
1431 ifnet_head_lock_shared();
1432 if (out_if_index > 0 && out_if_index <= if_index) {
1433 ifp = ifindex2ifnet[out_if_index];
1434 }
1435
1436 if (ifp != NULL) {
1437 inp->inp_last_outifp = ifp;
1438 } else {
1439 error = EINVAL;
1440 }
1441 ifnet_head_done();
1442
1443 if (error) {
1444 goto set_socket_state;
1445 }
1446
1447 if (fd_cb->group == NULL) {
1448 error = EINVAL;
1449 goto set_socket_state;
1450 }
1451
1452 old_group = fd_cb->group;
1453
1454 lck_rw_lock_exclusive(&old_group->lck);
1455 lck_rw_lock_exclusive(&grp->lck);
1456
1457 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1458 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1459 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1460 }
1461
1462 fd_cb->group = grp;
1463
1464 lck_rw_done(&grp->lck);
1465 lck_rw_done(&old_group->lck);
1466
1467 fd_cb->send_window = ntohl(send_window);
1468 flow_divert_send_buffered_data(fd_cb, FALSE);
1469
1470 set_socket_state:
1471 if (!connect_error && !error) {
1472 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1473 error = flow_divert_send_connect_result(fd_cb);
1474 }
1475
1476 if (connect_error || error) {
1477 if (!connect_error) {
1478 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1479 fd_cb->so->so_error = error;
1480 flow_divert_send_close_if_needed(fd_cb);
1481 } else {
1482 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1483 fd_cb->so->so_error = connect_error;
1484 }
1485 soisdisconnected(fd_cb->so);
1486 } else {
1487 soisconnected(fd_cb->so);
1488 }
1489
1490 done:
1491 socket_unlock(fd_cb->so, 0);
1492 }
1493 FDUNLOCK(fd_cb);
1494
1495 lck_rw_done(&g_flow_divert_group_lck);
1496 }
1497
1498 static void
1499 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1500 {
1501 uint32_t close_error;
1502 int error = 0;
1503 int how;
1504
1505 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1506 if (error) {
1507 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1508 return;
1509 }
1510
1511 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1512 if (error) {
1513 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1514 return;
1515 }
1516
1517 how = ntohl(how);
1518
1519 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1520
1521 FDLOCK(fd_cb);
1522 if (fd_cb->so != NULL) {
1523 socket_lock(fd_cb->so, 0);
1524
1525 fd_cb->so->so_error = ntohl(close_error);
1526
1527 flow_divert_update_closed_state(fd_cb, how, TRUE);
1528
1529 how = flow_divert_tunnel_how_closed(fd_cb);
1530 if (how == SHUT_RDWR) {
1531 soisdisconnected(fd_cb->so);
1532 } else if (how == SHUT_RD) {
1533 socantrcvmore(fd_cb->so);
1534 } else if (how == SHUT_WR) {
1535 socantsendmore(fd_cb->so);
1536 }
1537
1538 socket_unlock(fd_cb->so, 0);
1539 }
1540 FDUNLOCK(fd_cb);
1541 }
1542
1543 static void
1544 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1545 {
1546 int error = 0;
1547 mbuf_t data = NULL;
1548 size_t data_size;
1549
1550 data_size = (mbuf_pkthdr_len(packet) - offset);
1551
1552 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1553
1554 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1555 if (error || data == NULL) {
1556 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1557 return;
1558 }
1559
1560 FDLOCK(fd_cb);
1561 if (fd_cb->so != NULL) {
1562 socket_lock(fd_cb->so, 0);
1563 if (flow_divert_check_no_cellular(fd_cb) ||
1564 flow_divert_check_no_expensive(fd_cb)) {
1565 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1566 flow_divert_send_close(fd_cb, SHUT_RDWR);
1567 soisdisconnected(fd_cb->so);
1568 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1569 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1570 fd_cb->bytes_received += data_size;
1571 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1572 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1573 sorwakeup(fd_cb->so);
1574 data = NULL;
1575 } else {
1576 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1577 }
1578 }
1579 socket_unlock(fd_cb->so, 0);
1580 }
1581 FDUNLOCK(fd_cb);
1582
1583 if (data != NULL) {
1584 mbuf_free(data);
1585 }
1586 }
1587
1588 static void
1589 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1590 {
1591 uint32_t read_count;
1592 int error = 0;
1593
1594 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
1595 if (error) {
1596 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
1597 return;
1598 }
1599
1600 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", read_count);
1601
1602 FDLOCK(fd_cb);
1603 if (fd_cb->so != NULL) {
1604 socket_lock(fd_cb->so, 0);
1605 fd_cb->send_window += ntohl(read_count);
1606 flow_divert_send_buffered_data(fd_cb, FALSE);
1607 socket_unlock(fd_cb->so, 0);
1608 }
1609 FDUNLOCK(fd_cb);
1610 }
1611
1612 static void
1613 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
1614 {
1615 int error = 0;
1616 size_t key_size = 0;
1617 int log_level;
1618
1619 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
1620 if (error) {
1621 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
1622 return;
1623 }
1624
1625 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
1626 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %lu", key_size);
1627 return;
1628 }
1629
1630 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
1631 if (!error) {
1632 nil_pcb.log_level = log_level;
1633 }
1634
1635 lck_rw_lock_exclusive(&group->lck);
1636
1637 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
1638 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
1639 if (error) {
1640 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
1641 FREE(group->token_key, M_TEMP);
1642 group->token_key = NULL;
1643 lck_rw_done(&group->lck);
1644 return;
1645 }
1646
1647 group->token_key_size = key_size;
1648
1649 lck_rw_done(&group->lck);
1650 }
1651
1652 static void
1653 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1654 {
1655 int error = 0;
1656 struct sockaddr_storage local_address;
1657 int out_if_index = 0;
1658 struct sockaddr_storage remote_address;
1659
1660 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
1661
1662 memset(&local_address, 0, sizeof(local_address));
1663 memset(&remote_address, 0, sizeof(remote_address));
1664
1665 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1666 if (error) {
1667 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1668 }
1669
1670 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1671 if (error) {
1672 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1673 }
1674
1675 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1676 if (error) {
1677 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1678 }
1679
1680 FDLOCK(fd_cb);
1681 if (fd_cb->so != NULL) {
1682 struct inpcb *inp = NULL;
1683 struct ifnet *ifp = NULL;
1684
1685 socket_lock(fd_cb->so, 0);
1686
1687 inp = sotoinpcb(fd_cb->so);
1688
1689 if (local_address.ss_family != 0) {
1690 if (local_address.ss_len > sizeof(local_address)) {
1691 local_address.ss_len = sizeof(local_address);
1692 }
1693 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1694 }
1695
1696 if (remote_address.ss_family != 0) {
1697 if (remote_address.ss_len > sizeof(remote_address)) {
1698 remote_address.ss_len = sizeof(remote_address);
1699 }
1700 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1701 }
1702
1703 ifnet_head_lock_shared();
1704 if (out_if_index > 0 && out_if_index <= if_index) {
1705 ifp = ifindex2ifnet[out_if_index];
1706 }
1707
1708 if (ifp != NULL) {
1709 inp->inp_last_outifp = ifp;
1710 }
1711 ifnet_head_done();
1712
1713 socket_unlock(fd_cb->so, 0);
1714 }
1715 FDUNLOCK(fd_cb);
1716 }
1717
1718 static void
1719 flow_divert_handle_app_map_create(mbuf_t packet, int offset)
1720 {
1721 size_t bytes_mem_size;
1722 size_t child_maps_mem_size;
1723 int cursor;
1724 int error = 0;
1725 struct flow_divert_trie new_trie;
1726 int insert_error = 0;
1727 size_t nodes_mem_size;
1728 int prefix_count = 0;
1729 int signing_id_count = 0;
1730
1731 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
1732
1733 /* Re-set the current trie */
1734 if (g_signing_id_trie.memory != NULL) {
1735 FREE(g_signing_id_trie.memory, M_TEMP);
1736 }
1737 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
1738 g_signing_id_trie.root = NULL_TRIE_IDX;
1739
1740 memset(&new_trie, 0, sizeof(new_trie));
1741
1742 /* Get the number of shared prefixes in the new set of signing ID strings */
1743 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
1744
1745 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
1746 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1747 cursor >= 0;
1748 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1749 {
1750 size_t sid_size = 0;
1751 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1752 new_trie.bytes_count += sid_size;
1753 signing_id_count++;
1754 }
1755
1756 if (signing_id_count == 0) {
1757 lck_rw_done(&g_flow_divert_group_lck);
1758 return;
1759 }
1760
1761 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
1762 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
1763
1764 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
1765 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
1766
1767 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
1768 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
1769 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
1770
1771 MALLOC(new_trie.memory, void *, nodes_mem_size + child_maps_mem_size + bytes_mem_size, M_TEMP, M_WAITOK);
1772 if (new_trie.memory == NULL) {
1773 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
1774 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
1775 return;
1776 }
1777
1778 /* Initialize the free lists */
1779 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
1780 new_trie.nodes_free_next = 0;
1781 memset(new_trie.nodes, 0, nodes_mem_size);
1782
1783 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
1784 new_trie.child_maps_free_next = 0;
1785 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
1786
1787 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
1788 new_trie.bytes_free_next = 0;
1789
1790 /* The root is an empty node */
1791 new_trie.root = trie_node_alloc(&new_trie);
1792
1793 /* Add each signing ID to the trie */
1794 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1795 cursor >= 0;
1796 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1797 {
1798 size_t sid_size = 0;
1799 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1800 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
1801 boolean_t is_dns;
1802 uint16_t new_node_idx;
1803 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
1804 is_dns = (sid_size == sizeof(FLOW_DIVERT_DNS_SERVICE_SIGNING_ID) - 1 &&
1805 !memcmp(&TRIE_BYTE(&new_trie, new_trie.bytes_free_next),
1806 FLOW_DIVERT_DNS_SERVICE_SIGNING_ID,
1807 sid_size));
1808 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
1809 if (new_node_idx != NULL_TRIE_IDX) {
1810 if (is_dns) {
1811 FDLOG(LOG_INFO, &nil_pcb, "Setting group unit for %s to %d", FLOW_DIVERT_DNS_SERVICE_SIGNING_ID, DNS_SERVICE_GROUP_UNIT);
1812 TRIE_NODE(&new_trie, new_node_idx).group_unit = DNS_SERVICE_GROUP_UNIT;
1813 }
1814 } else {
1815 insert_error = EINVAL;
1816 break;
1817 }
1818 } else {
1819 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
1820 insert_error = ENOBUFS;
1821 break;
1822 }
1823 }
1824
1825 if (!insert_error) {
1826 g_signing_id_trie = new_trie;
1827 } else {
1828 FREE(new_trie.memory, M_TEMP);
1829 }
1830
1831 lck_rw_done(&g_flow_divert_group_lck);
1832 }
1833
1834 static void
1835 flow_divert_handle_app_map_update(struct flow_divert_group *group, mbuf_t packet, int offset)
1836 {
1837 int error = 0;
1838 int cursor;
1839 size_t max_size = 0;
1840 uint8_t *signing_id;
1841 uint32_t ctl_unit;
1842
1843 lck_rw_lock_shared(&group->lck);
1844 ctl_unit = group->ctl_unit;
1845 lck_rw_done(&group->lck);
1846
1847 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1848 cursor >= 0;
1849 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1850 {
1851 size_t sid_size = 0;
1852 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1853 if (sid_size > max_size) {
1854 max_size = sid_size;
1855 }
1856 }
1857
1858 MALLOC(signing_id, uint8_t *, max_size + 1, M_TEMP, M_WAITOK);
1859 if (signing_id == NULL) {
1860 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate a string to hold the signing ID (size %lu)", max_size);
1861 return;
1862 }
1863
1864 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1865 cursor >= 0;
1866 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1867 {
1868 size_t signing_id_len = 0;
1869 uint16_t node;
1870
1871 flow_divert_packet_get_tlv(packet,
1872 cursor, FLOW_DIVERT_TLV_SIGNING_ID, max_size, signing_id, &signing_id_len);
1873
1874 signing_id[signing_id_len] = '\0';
1875
1876 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
1877
1878 node = flow_divert_trie_search(&g_signing_id_trie, signing_id);
1879 if (node != NULL_TRIE_IDX) {
1880 if (TRIE_NODE(&g_signing_id_trie, node).group_unit != DNS_SERVICE_GROUP_UNIT) {
1881 FDLOG(LOG_INFO, &nil_pcb, "Setting %s to ctl unit %u", signing_id, group->ctl_unit);
1882 TRIE_NODE(&g_signing_id_trie, node).group_unit = ctl_unit;
1883 }
1884 } else {
1885 FDLOG(LOG_ERR, &nil_pcb, "Failed to find signing ID %s", signing_id);
1886 }
1887
1888 lck_rw_done(&g_flow_divert_group_lck);
1889 }
1890
1891 FREE(signing_id, M_TEMP);
1892 }
1893
1894 static int
1895 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
1896 {
1897 struct flow_divert_packet_header hdr;
1898 int error = 0;
1899 struct flow_divert_pcb *fd_cb;
1900
1901 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
1902 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
1903 error = EINVAL;
1904 goto done;
1905 }
1906
1907 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
1908 if (error) {
1909 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
1910 error = ENOBUFS;
1911 goto done;
1912 }
1913
1914 hdr.conn_id = ntohl(hdr.conn_id);
1915
1916 if (hdr.conn_id == 0) {
1917 switch (hdr.packet_type) {
1918 case FLOW_DIVERT_PKT_GROUP_INIT:
1919 flow_divert_handle_group_init(group, packet, sizeof(hdr));
1920 break;
1921 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
1922 flow_divert_handle_app_map_create(packet, sizeof(hdr));
1923 break;
1924 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
1925 flow_divert_handle_app_map_update(group, packet, sizeof(hdr));
1926 break;
1927 default:
1928 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
1929 break;
1930 }
1931 goto done;
1932 }
1933
1934 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
1935 if (fd_cb == NULL) {
1936 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
1937 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
1938 }
1939 goto done;
1940 }
1941
1942 switch (hdr.packet_type) {
1943 case FLOW_DIVERT_PKT_CONNECT_RESULT:
1944 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
1945 break;
1946 case FLOW_DIVERT_PKT_CLOSE:
1947 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
1948 break;
1949 case FLOW_DIVERT_PKT_DATA:
1950 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
1951 break;
1952 case FLOW_DIVERT_PKT_READ_NOTIFY:
1953 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
1954 break;
1955 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
1956 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
1957 break;
1958 default:
1959 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
1960 break;
1961 }
1962
1963 FDRELEASE(fd_cb);
1964
1965 done:
1966 mbuf_free(packet);
1967 return error;
1968 }
1969
1970 static void
1971 flow_divert_close_all(struct flow_divert_group *group)
1972 {
1973 struct flow_divert_pcb *fd_cb;
1974 SLIST_HEAD(, flow_divert_pcb) tmp_list;
1975
1976 SLIST_INIT(&tmp_list);
1977
1978 lck_rw_lock_exclusive(&group->lck);
1979
1980 MBUFQ_DRAIN(&group->send_queue);
1981
1982 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
1983 FDRETAIN(fd_cb);
1984 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
1985 }
1986
1987 lck_rw_done(&group->lck);
1988
1989 while (!SLIST_EMPTY(&tmp_list)) {
1990 fd_cb = SLIST_FIRST(&tmp_list);
1991 FDLOCK(fd_cb);
1992 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
1993 if (fd_cb->so != NULL) {
1994 socket_lock(fd_cb->so, 0);
1995 flow_divert_pcb_remove(fd_cb);
1996 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1997 fd_cb->so->so_error = ECONNABORTED;
1998 socket_unlock(fd_cb->so, 0);
1999 }
2000 FDUNLOCK(fd_cb);
2001 FDRELEASE(fd_cb);
2002 }
2003 }
2004
2005 void
2006 flow_divert_detach(struct socket *so)
2007 {
2008 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2009
2010 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2011
2012 so->so_flags &= ~SOF_FLOW_DIVERT;
2013 so->so_fd_pcb = NULL;
2014
2015 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2016
2017 if (fd_cb->group != NULL) {
2018 /* Last-ditch effort to send any buffered data */
2019 flow_divert_send_buffered_data(fd_cb, TRUE);
2020
2021 /* Remove from the group */
2022 flow_divert_pcb_remove(fd_cb);
2023 }
2024
2025 socket_unlock(so, 0);
2026 FDLOCK(fd_cb);
2027 fd_cb->so = NULL;
2028 FDUNLOCK(fd_cb);
2029 socket_lock(so, 0);
2030
2031 FDRELEASE(fd_cb); /* Release the socket's reference */
2032 }
2033
2034 static int
2035 flow_divert_close(struct socket *so)
2036 {
2037 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2038
2039 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2040
2041 FDLOG0(LOG_INFO, fd_cb, "Closing");
2042
2043 soisdisconnecting(so);
2044 sbflush(&so->so_rcv);
2045
2046 flow_divert_send_buffered_data(fd_cb, TRUE);
2047 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2048 flow_divert_send_close_if_needed(fd_cb);
2049
2050 /* Remove from the group */
2051 flow_divert_pcb_remove(fd_cb);
2052
2053 return 0;
2054 }
2055
2056 static int
2057 flow_divert_disconnectx(struct socket *so, associd_t aid, connid_t cid __unused)
2058 {
2059 if (aid != ASSOCID_ANY && aid != ASSOCID_ALL) {
2060 return (EINVAL);
2061 }
2062
2063 return (flow_divert_close(so));
2064 }
2065
2066 static int
2067 flow_divert_shutdown(struct socket *so)
2068 {
2069 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2070
2071 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2072
2073 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2074
2075 socantsendmore(so);
2076
2077 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2078 flow_divert_send_close_if_needed(fd_cb);
2079
2080 return 0;
2081 }
2082
2083 static int
2084 flow_divert_rcvd(struct socket *so, int flags __unused)
2085 {
2086 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2087 uint32_t latest_sb_size;
2088 uint32_t read_count;
2089
2090 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2091
2092 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2093
2094 if (fd_cb->sb_size < latest_sb_size) {
2095 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2096 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2097 }
2098
2099 read_count = fd_cb->sb_size - latest_sb_size;
2100
2101 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2102
2103 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2104 fd_cb->bytes_read_by_app += read_count;
2105 fd_cb->sb_size = latest_sb_size;
2106 }
2107
2108 return 0;
2109 }
2110
2111 static errno_t
2112 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2113 struct sockaddr **dup)
2114 {
2115 int error = 0;
2116 struct sockaddr *result;
2117 struct sockaddr_storage ss;
2118
2119 if (addr != NULL) {
2120 result = addr;
2121 } else {
2122 memset(&ss, 0, sizeof(ss));
2123 ss.ss_family = family;
2124 if (ss.ss_family == AF_INET) {
2125 ss.ss_len = sizeof(struct sockaddr_in);
2126 }
2127 #if INET6
2128 else if (ss.ss_family == AF_INET6) {
2129 ss.ss_len = sizeof(struct sockaddr_in6);
2130 }
2131 #endif /* INET6 */
2132 else {
2133 error = EINVAL;
2134 }
2135 result = (struct sockaddr *)&ss;
2136 }
2137
2138 if (!error) {
2139 *dup = dup_sockaddr(result, 1);
2140 if (*dup == NULL) {
2141 error = ENOBUFS;
2142 }
2143 }
2144
2145 return error;
2146 }
2147
2148 static errno_t
2149 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2150 {
2151 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2152
2153 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2154
2155 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2156 fd_cb->remote_address,
2157 sa);
2158 }
2159
2160 static errno_t
2161 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2162 {
2163 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2164
2165 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2166
2167 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2168 fd_cb->local_address,
2169 sa);
2170 }
2171
2172 static errno_t
2173 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2174 {
2175 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2176
2177 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2178
2179 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2180 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2181 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2182 }
2183 }
2184
2185 if (SOCK_DOM(so) == PF_INET) {
2186 return g_tcp_protosw->pr_ctloutput(so, sopt);
2187 }
2188 #if INET6
2189 else if (SOCK_DOM(so) == PF_INET6) {
2190 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2191 }
2192 #endif
2193 return 0;
2194 }
2195
2196 errno_t
2197 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2198 {
2199 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2200 int error = 0;
2201 struct inpcb *inp = sotoinpcb(so);
2202 struct sockaddr_in *sinp;
2203 mbuf_t connect_packet = NULL;
2204 char *signing_id = NULL;
2205 int free_signing_id = 0;
2206
2207 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2208
2209 if (fd_cb->group == NULL) {
2210 error = ENETUNREACH;
2211 goto done;
2212 }
2213
2214 if (inp == NULL) {
2215 error = EINVAL;
2216 goto done;
2217 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2218 if (so->so_error) {
2219 error = so->so_error;
2220 so->so_error = 0;
2221 } else {
2222 error = EINVAL;
2223 }
2224 goto done;
2225 }
2226
2227 sinp = (struct sockaddr_in *)(void *)to;
2228 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2229 error = EAFNOSUPPORT;
2230 goto done;
2231 }
2232
2233 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2234 error = EALREADY;
2235 goto done;
2236 }
2237
2238 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2239 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2240 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2241 if (fd_cb->remote_address != NULL) {
2242 soisconnected(fd_cb->so);
2243 goto done;
2244 }
2245 }
2246
2247 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
2248 if (error) {
2249 goto done;
2250 }
2251
2252 error = EPERM;
2253
2254 if (fd_cb->connect_token != NULL) {
2255 size_t sid_size = 0;
2256 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2257 if (find_error == 0 && sid_size > 0) {
2258 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
2259 if (signing_id != NULL) {
2260 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
2261 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
2262 free_signing_id = 1;
2263 }
2264 }
2265 }
2266
2267 socket_unlock(so, 0);
2268 if (g_signing_id_trie.root != NULL_TRIE_IDX) {
2269 proc_t src_proc = p;
2270 int release_proc = 0;
2271
2272 if (signing_id == NULL) {
2273 release_proc = flow_divert_get_src_proc(so, &src_proc, FALSE);
2274 if (src_proc != PROC_NULL) {
2275 proc_lock(src_proc);
2276 if (src_proc->p_csflags & CS_VALID) {
2277 signing_id = (char *)cs_identity_get(src_proc);
2278 } else {
2279 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
2280 }
2281 } else {
2282 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
2283 }
2284 } else {
2285 src_proc = PROC_NULL;
2286 }
2287
2288 if (signing_id != NULL) {
2289 uint16_t result = NULL_TRIE_IDX;
2290 lck_rw_lock_shared(&g_flow_divert_group_lck);
2291 result = flow_divert_trie_search(&g_signing_id_trie, (const uint8_t *)signing_id);
2292 lck_rw_done(&g_flow_divert_group_lck);
2293 if (result != NULL_TRIE_IDX) {
2294 error = 0;
2295 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
2296
2297 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
2298 if (error == 0) {
2299 if (src_proc != PROC_NULL) {
2300 unsigned char cdhash[SHA1_RESULTLEN];
2301 error = proc_getcdhash(src_proc, cdhash);
2302 if (error == 0) {
2303 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
2304 if (error) {
2305 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
2306 }
2307 } else {
2308 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
2309 }
2310 }
2311 } else {
2312 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
2313 }
2314 } else {
2315 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
2316 }
2317 } else {
2318 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
2319 }
2320
2321 if (src_proc != PROC_NULL) {
2322 proc_unlock(src_proc);
2323 if (release_proc) {
2324 proc_rele(src_proc);
2325 }
2326 }
2327 } else {
2328 FDLOG0(LOG_WARNING, fd_cb, "The signing ID trie is empty");
2329 }
2330 socket_lock(so, 0);
2331
2332 if (free_signing_id) {
2333 FREE(signing_id, M_TEMP);
2334 }
2335
2336 if (error) {
2337 goto done;
2338 }
2339
2340 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2341
2342 error = flow_divert_send_connect(fd_cb, to, connect_packet);
2343 if (error) {
2344 goto done;
2345 }
2346
2347 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2348
2349 soisconnecting(so);
2350
2351 done:
2352 if (error && connect_packet != NULL) {
2353 mbuf_free(connect_packet);
2354 }
2355 return error;
2356 }
2357
2358 static int
2359 flow_divert_connectx_out_common(struct socket *so, int af,
2360 struct sockaddr_list **src_sl, struct sockaddr_list **dst_sl,
2361 struct proc *p, uint32_t ifscope __unused, associd_t aid __unused,
2362 connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2363 uint32_t arglen __unused)
2364 {
2365 struct sockaddr_entry *src_se = NULL, *dst_se = NULL;
2366 struct inpcb *inp = sotoinpcb(so);
2367 int error;
2368
2369 if (inp == NULL) {
2370 return (EINVAL);
2371 }
2372
2373 VERIFY(dst_sl != NULL);
2374
2375 /* select source (if specified) and destination addresses */
2376 error = in_selectaddrs(af, src_sl, &src_se, dst_sl, &dst_se);
2377 if (error != 0) {
2378 return (error);
2379 }
2380
2381 VERIFY(*dst_sl != NULL && dst_se != NULL);
2382 VERIFY(src_se == NULL || *src_sl != NULL);
2383 VERIFY(dst_se->se_addr->sa_family == af);
2384 VERIFY(src_se == NULL || src_se->se_addr->sa_family == af);
2385
2386 error = flow_divert_connect_out(so, dst_se->se_addr, p);
2387
2388 if (error == 0 && pcid != NULL) {
2389 *pcid = 1; /* there is only 1 connection for a TCP */
2390 }
2391
2392 return (error);
2393 }
2394
2395 static int
2396 flow_divert_connectx_out(struct socket *so, struct sockaddr_list **src_sl,
2397 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2398 associd_t aid, connid_t *pcid, uint32_t flags, void *arg,
2399 uint32_t arglen)
2400 {
2401 return (flow_divert_connectx_out_common(so, AF_INET, src_sl, dst_sl,
2402 p, ifscope, aid, pcid, flags, arg, arglen));
2403 }
2404
2405 #if INET6
2406 static int
2407 flow_divert_connectx6_out(struct socket *so, struct sockaddr_list **src_sl,
2408 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2409 associd_t aid, connid_t *pcid, uint32_t flags, void *arg,
2410 uint32_t arglen)
2411 {
2412 return (flow_divert_connectx_out_common(so, AF_INET6, src_sl, dst_sl,
2413 p, ifscope, aid, pcid, flags, arg, arglen));
2414 }
2415 #endif /* INET6 */
2416
2417 static int
2418 flow_divert_getconninfo(struct socket *so, connid_t cid, uint32_t *flags,
2419 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2420 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2421 user_addr_t aux_data __unused, uint32_t *aux_len)
2422 {
2423 int error = 0;
2424 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2425 struct ifnet *ifp = NULL;
2426 struct inpcb *inp = sotoinpcb(so);
2427
2428 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2429
2430 if (so->so_fd_pcb == NULL || inp == NULL) {
2431 error = EINVAL;
2432 goto out;
2433 }
2434
2435 if (cid != CONNID_ANY && cid != CONNID_ALL && cid != 1) {
2436 error = EINVAL;
2437 goto out;
2438 }
2439
2440 ifp = inp->inp_last_outifp;
2441 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2442 *soerror = so->so_error;
2443 *flags = 0;
2444
2445 if (so->so_state & SS_ISCONNECTED) {
2446 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2447 }
2448
2449 if (fd_cb->local_address == NULL) {
2450 struct sockaddr_in sin;
2451 bzero(&sin, sizeof(sin));
2452 sin.sin_len = sizeof(sin);
2453 sin.sin_family = AF_INET;
2454 *src_len = sin.sin_len;
2455 if (src != USER_ADDR_NULL) {
2456 error = copyout(&sin, src, sin.sin_len);
2457 if (error != 0) {
2458 goto out;
2459 }
2460 }
2461 } else {
2462 *src_len = fd_cb->local_address->sa_len;
2463 if (src != USER_ADDR_NULL) {
2464 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2465 if (error != 0) {
2466 goto out;
2467 }
2468 }
2469 }
2470
2471 if (fd_cb->remote_address == NULL) {
2472 struct sockaddr_in sin;
2473 bzero(&sin, sizeof(sin));
2474 sin.sin_len = sizeof(sin);
2475 sin.sin_family = AF_INET;
2476 *dst_len = sin.sin_len;
2477 if (dst != USER_ADDR_NULL) {
2478 error = copyout(&sin, dst, sin.sin_len);
2479 if (error != 0) {
2480 goto out;
2481 }
2482 }
2483 } else {
2484 *dst_len = fd_cb->remote_address->sa_len;
2485 if (dst != USER_ADDR_NULL) {
2486 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2487 if (error != 0) {
2488 goto out;
2489 }
2490 }
2491 }
2492
2493 *aux_type = 0;
2494 *aux_len = 0;
2495
2496 out:
2497 return error;
2498 }
2499
2500 static int
2501 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
2502 {
2503 int error = 0;
2504
2505 switch (cmd) {
2506 case SIOCGCONNINFO32: {
2507 struct so_cinforeq32 cifr;
2508 bcopy(data, &cifr, sizeof (cifr));
2509 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
2510 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
2511 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
2512 &cifr.scir_aux_type, cifr.scir_aux_data,
2513 &cifr.scir_aux_len);
2514 if (error == 0) {
2515 bcopy(&cifr, data, sizeof (cifr));
2516 }
2517 break;
2518 }
2519
2520 case SIOCGCONNINFO64: {
2521 struct so_cinforeq64 cifr;
2522 bcopy(data, &cifr, sizeof (cifr));
2523 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
2524 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
2525 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
2526 &cifr.scir_aux_type, cifr.scir_aux_data,
2527 &cifr.scir_aux_len);
2528 if (error == 0) {
2529 bcopy(&cifr, data, sizeof (cifr));
2530 }
2531 break;
2532 }
2533
2534 default:
2535 error = EOPNOTSUPP;
2536 }
2537
2538 return error;
2539 }
2540
2541 static int
2542 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
2543 {
2544 int error = flow_divert_control(so, cmd, data, ifp, p);
2545
2546 if (error == EOPNOTSUPP) {
2547 error = in_control(so, cmd, data, ifp, p);
2548 }
2549
2550 return error;
2551 }
2552
2553 static int
2554 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
2555 {
2556 int error = flow_divert_control(so, cmd, data, ifp, p);
2557
2558 if (error == EOPNOTSUPP) {
2559 error = in6_control(so, cmd, data, ifp, p);
2560 }
2561
2562 return error;
2563 }
2564
2565 static errno_t
2566 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p __unused)
2567 {
2568 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2569 int error = 0;
2570 struct inpcb *inp;
2571
2572 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2573
2574 inp = sotoinpcb(so);
2575 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
2576 error = ECONNRESET;
2577 goto done;
2578 }
2579
2580 if (control && mbuf_len(control) > 0) {
2581 error = EINVAL;
2582 goto done;
2583 }
2584
2585 if (flags & MSG_OOB) {
2586 error = EINVAL;
2587 goto done; /* We don't support OOB data */
2588 }
2589
2590 error = flow_divert_check_no_cellular(fd_cb) ||
2591 flow_divert_check_no_expensive(fd_cb);
2592 if (error) {
2593 goto done;
2594 }
2595
2596 /* Implicit connect */
2597 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
2598 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
2599 error = flow_divert_connect_out(so, to, NULL);
2600 if (error) {
2601 goto done;
2602 }
2603 }
2604
2605 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
2606
2607 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
2608 error = flow_divert_send_app_data(fd_cb, data);
2609 if (error) {
2610 goto done;
2611 }
2612
2613 data = NULL;
2614
2615 if (flags & PRUS_EOF) {
2616 flow_divert_shutdown(so);
2617 }
2618
2619 done:
2620 if (data) {
2621 mbuf_free(data);
2622 }
2623 if (control) {
2624 mbuf_free(control);
2625 }
2626 return error;
2627 }
2628
2629 static void
2630 flow_divert_set_protosw(struct socket *so)
2631 {
2632 so->so_flags |= SOF_FLOW_DIVERT;
2633 if (SOCK_DOM(so) == PF_INET) {
2634 so->so_proto = &g_flow_divert_in_protosw;
2635 }
2636 #if INET6
2637 else {
2638 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
2639 }
2640 #endif /* INET6 */
2641 }
2642
2643 static errno_t
2644 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
2645 {
2646 int error = 0;
2647 struct flow_divert_pcb *fd_cb = NULL;
2648 struct ifnet *ifp = NULL;
2649 struct inpcb *inp = NULL;
2650 struct socket *old_so;
2651 mbuf_t recv_data = NULL;
2652
2653 socket_unlock(so, 0);
2654
2655 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
2656
2657 /* Find the flow divert control block */
2658 lck_rw_lock_shared(&g_flow_divert_group_lck);
2659 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
2660 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
2661 if (group != NULL) {
2662 fd_cb = flow_divert_pcb_lookup(flow_id, group);
2663 }
2664 }
2665 lck_rw_done(&g_flow_divert_group_lck);
2666
2667 if (fd_cb == NULL) {
2668 error = ENOENT;
2669 goto done;
2670 }
2671
2672 FDLOCK(fd_cb);
2673
2674 /* Dis-associate the flow divert control block from its current socket */
2675 old_so = fd_cb->so;
2676
2677 inp = sotoinpcb(old_so);
2678
2679 VERIFY(inp != NULL);
2680
2681 socket_lock(old_so, 0);
2682 soisdisconnected(old_so);
2683 old_so->so_flags &= ~SOF_FLOW_DIVERT;
2684 old_so->so_fd_pcb = NULL;
2685 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
2686 fd_cb->so = NULL;
2687 /* Save the output interface */
2688 ifp = inp->inp_last_outifp;
2689 if (old_so->so_rcv.sb_cc > 0) {
2690 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
2691 sbflush(&old_so->so_rcv);
2692 }
2693 socket_unlock(old_so, 0);
2694
2695 /* Associate the new socket with the flow divert control block */
2696 socket_lock(so, 0);
2697 so->so_fd_pcb = fd_cb;
2698 inp = sotoinpcb(so);
2699 inp->inp_last_outifp = ifp;
2700 if (recv_data != NULL) {
2701 if (sbappendstream(&so->so_rcv, recv_data)) {
2702 sorwakeup(so);
2703 }
2704 }
2705 flow_divert_set_protosw(so);
2706 socket_unlock(so, 0);
2707
2708 fd_cb->so = so;
2709 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
2710
2711 FDUNLOCK(fd_cb);
2712
2713 done:
2714 socket_lock(so, 0);
2715
2716 if (fd_cb != NULL) {
2717 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
2718 }
2719
2720 return error;
2721 }
2722
2723 errno_t
2724 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
2725 {
2726 errno_t error = 0;
2727 struct flow_divert_pcb *fd_cb;
2728
2729 if (so->so_flags & SOF_FLOW_DIVERT) {
2730 return EALREADY;
2731 }
2732
2733 fd_cb = flow_divert_pcb_create(so);
2734 if (fd_cb != NULL) {
2735 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
2736 if (error) {
2737 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
2738 FDRELEASE(fd_cb);
2739 } else {
2740 fd_cb->log_level = LOG_NOTICE;
2741 fd_cb->control_group_unit = ctl_unit;
2742 so->so_fd_pcb = fd_cb;
2743
2744 flow_divert_set_protosw(so);
2745
2746 FDLOG0(LOG_INFO, fd_cb, "Created");
2747 }
2748 } else {
2749 error = ENOMEM;
2750 }
2751
2752 return error;
2753 }
2754
2755 errno_t
2756 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
2757 {
2758 uint32_t ctl_unit = 0;
2759 uint32_t key_unit = 0;
2760 uint32_t flow_id = 0;
2761 int error = 0;
2762 mbuf_t token = NULL;
2763
2764 if (so->so_flags & SOF_FLOW_DIVERT) {
2765 error = EALREADY;
2766 goto done;
2767 }
2768
2769 if (g_init_result) {
2770 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
2771 error = ENOPROTOOPT;
2772 goto done;
2773 }
2774
2775 if (SOCK_TYPE(so) != SOCK_STREAM ||
2776 SOCK_PROTO(so) != IPPROTO_TCP ||
2777 (SOCK_DOM(so) != PF_INET
2778 #if INET6
2779 && SOCK_DOM(so) != PF_INET6
2780 #endif
2781 ))
2782 {
2783 error = EINVAL;
2784 goto done;
2785 } else {
2786 struct tcpcb *tp = sototcpcb(so);
2787 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
2788 error = EINVAL;
2789 goto done;
2790 }
2791 }
2792
2793 error = soopt_getm(sopt, &token);
2794 if (error) {
2795 goto done;
2796 }
2797
2798 error = soopt_mcopyin(sopt, token);
2799 if (error) {
2800 goto done;
2801 }
2802
2803 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
2804 if (!error) {
2805 key_unit = ntohl(key_unit);
2806 } else if (error != ENOENT) {
2807 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
2808 goto done;
2809 } else {
2810 key_unit = 0;
2811 }
2812
2813 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
2814 if (error) {
2815 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
2816 goto done;
2817 }
2818
2819 /* A valid kernel control unit is required */
2820 ctl_unit = ntohl(ctl_unit);
2821 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
2822 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
2823 error = EINVAL;
2824 goto done;
2825 }
2826
2827 socket_unlock(so, 0);
2828 error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
2829 socket_lock(so, 0);
2830
2831 if (error) {
2832 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", error);
2833 goto done;
2834 }
2835
2836 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
2837 if (error && error != ENOENT) {
2838 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
2839 goto done;
2840 }
2841
2842 if (flow_id == 0) {
2843 error = flow_divert_pcb_init(so, ctl_unit);
2844 if (error == 0) {
2845 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2846 int log_level = LOG_NOTICE;
2847
2848 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
2849 sizeof(log_level), &log_level, NULL);
2850 if (error == 0) {
2851 fd_cb->log_level = log_level;
2852 }
2853 error = 0;
2854
2855 fd_cb->connect_token = token;
2856 token = NULL;
2857 }
2858 } else {
2859 error = flow_divert_attach(so, flow_id, ctl_unit);
2860 }
2861
2862 done:
2863 if (token != NULL) {
2864 mbuf_freem(token);
2865 }
2866
2867 return error;
2868 }
2869
2870 errno_t
2871 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
2872 {
2873 uint32_t ctl_unit;
2874 int error = 0;
2875 uint8_t hmac[SHA_DIGEST_LENGTH];
2876 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2877 mbuf_t token = NULL;
2878 struct flow_divert_group *control_group = NULL;
2879
2880 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
2881 error = EINVAL;
2882 goto done;
2883 }
2884
2885 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2886
2887 if (fd_cb->group == NULL) {
2888 error = EINVAL;
2889 goto done;
2890 }
2891
2892 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
2893 if (error) {
2894 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
2895 goto done;
2896 }
2897
2898 ctl_unit = htonl(fd_cb->group->ctl_unit);
2899
2900 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
2901 if (error) {
2902 goto done;
2903 }
2904
2905 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
2906 if (error) {
2907 goto done;
2908 }
2909
2910 socket_unlock(so, 0);
2911 lck_rw_lock_shared(&g_flow_divert_group_lck);
2912
2913 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
2914 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
2915 {
2916 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
2917 }
2918
2919 if (control_group != NULL) {
2920 lck_rw_lock_shared(&control_group->lck);
2921 ctl_unit = htonl(control_group->ctl_unit);
2922 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
2923 if (!error) {
2924 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
2925 }
2926 lck_rw_done(&control_group->lck);
2927 } else {
2928 error = ENOPROTOOPT;
2929 }
2930
2931 lck_rw_done(&g_flow_divert_group_lck);
2932 socket_lock(so, 0);
2933
2934 if (error) {
2935 goto done;
2936 }
2937
2938 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
2939 if (error) {
2940 goto done;
2941 }
2942
2943 error = soopt_mcopyout(sopt, token);
2944 if (error) {
2945 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
2946 goto done;
2947 }
2948
2949 done:
2950 if (token != NULL) {
2951 mbuf_freem(token);
2952 }
2953
2954 return error;
2955 }
2956
2957 static errno_t
2958 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
2959 {
2960 struct flow_divert_group *new_group;
2961 int error = 0;
2962
2963 if (sac->sc_unit >= GROUP_COUNT_MAX) {
2964 error = EINVAL;
2965 goto done;
2966 }
2967
2968 *unitinfo = NULL;
2969
2970 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
2971 if (new_group == NULL) {
2972 error = ENOBUFS;
2973 goto done;
2974 }
2975
2976 memset(new_group, 0, sizeof(*new_group));
2977
2978 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
2979 RB_INIT(&new_group->pcb_tree);
2980 new_group->ctl_unit = sac->sc_unit;
2981 MBUFQ_INIT(&new_group->send_queue);
2982
2983 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
2984
2985 if (g_flow_divert_groups == NULL) {
2986 MALLOC(g_flow_divert_groups,
2987 struct flow_divert_group **,
2988 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
2989 M_TEMP,
2990 M_WAITOK | M_ZERO);
2991 }
2992
2993 if (g_flow_divert_groups == NULL) {
2994 error = ENOBUFS;
2995 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
2996 error = EALREADY;
2997 } else {
2998 g_flow_divert_groups[sac->sc_unit] = new_group;
2999 g_active_group_count++;
3000 }
3001
3002 lck_rw_done(&g_flow_divert_group_lck);
3003
3004 *unitinfo = new_group;
3005
3006 done:
3007 if (error != 0 && new_group != NULL) {
3008 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3009 }
3010 return error;
3011 }
3012
3013 static errno_t
3014 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3015 {
3016 struct flow_divert_group *group = NULL;
3017 errno_t error = 0;
3018 uint16_t node = 0;
3019
3020 if (unit >= GROUP_COUNT_MAX) {
3021 return EINVAL;
3022 }
3023
3024 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3025
3026 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3027
3028 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3029 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3030 g_flow_divert_groups, g_active_group_count);
3031 }
3032
3033 group = g_flow_divert_groups[unit];
3034
3035 if (group != (struct flow_divert_group *)unitinfo) {
3036 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3037 }
3038
3039 if (group != NULL) {
3040 flow_divert_close_all(group);
3041 if (group->token_key != NULL) {
3042 memset(group->token_key, 0, group->token_key_size);
3043 FREE(group->token_key, M_TEMP);
3044 group->token_key = NULL;
3045 group->token_key_size = 0;
3046 }
3047 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3048 g_flow_divert_groups[unit] = NULL;
3049 g_active_group_count--;
3050 } else {
3051 error = EINVAL;
3052 }
3053
3054 if (g_active_group_count == 0) {
3055 FREE(g_flow_divert_groups, M_TEMP);
3056 g_flow_divert_groups = NULL;
3057 }
3058
3059 /* Remove all signing IDs that point to this unit */
3060 for (node = 0; node < g_signing_id_trie.nodes_count; node++) {
3061 if (TRIE_NODE(&g_signing_id_trie, node).group_unit == unit) {
3062 TRIE_NODE(&g_signing_id_trie, node).group_unit = 0;
3063 }
3064 }
3065
3066 lck_rw_done(&g_flow_divert_group_lck);
3067
3068 return error;
3069 }
3070
3071 static errno_t
3072 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3073 {
3074 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3075 }
3076
3077 static void
3078 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3079 {
3080 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3081
3082 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3083 struct flow_divert_pcb *fd_cb;
3084 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3085
3086 lck_rw_lock_shared(&g_flow_divert_group_lck);
3087 lck_rw_lock_exclusive(&group->lck);
3088
3089 while (!MBUFQ_EMPTY(&group->send_queue)) {
3090 mbuf_t next_packet;
3091 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3092 next_packet = MBUFQ_FIRST(&group->send_queue);
3093 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3094 if (error) {
3095 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3096 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3097 lck_rw_done(&group->lck);
3098 lck_rw_done(&g_flow_divert_group_lck);
3099 return;
3100 }
3101 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3102 }
3103
3104 SLIST_INIT(&tmp_list);
3105
3106 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3107 FDRETAIN(fd_cb);
3108 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3109 }
3110
3111 lck_rw_done(&group->lck);
3112
3113 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3114 FDLOCK(fd_cb);
3115 if (fd_cb->so != NULL) {
3116 socket_lock(fd_cb->so, 0);
3117 if (fd_cb->group != NULL) {
3118 flow_divert_send_buffered_data(fd_cb, FALSE);
3119 }
3120 socket_unlock(fd_cb->so, 0);
3121 }
3122 FDUNLOCK(fd_cb);
3123 FDRELEASE(fd_cb);
3124 }
3125
3126 lck_rw_done(&g_flow_divert_group_lck);
3127 }
3128 }
3129
3130 static int
3131 flow_divert_kctl_init(void)
3132 {
3133 struct kern_ctl_reg ctl_reg;
3134 int result;
3135
3136 memset(&ctl_reg, 0, sizeof(ctl_reg));
3137
3138 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3139 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3140 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3141 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3142 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3143
3144 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3145 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3146 ctl_reg.ctl_send = flow_divert_kctl_send;
3147 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3148
3149 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3150
3151 if (result) {
3152 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3153 return result;
3154 }
3155
3156 return 0;
3157 }
3158
3159 void
3160 flow_divert_init(void)
3161 {
3162 memset(&nil_pcb, 0, sizeof(nil_pcb));
3163 nil_pcb.log_level = LOG_NOTICE;
3164
3165 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3166
3167 VERIFY(g_tcp_protosw != NULL);
3168
3169 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3170 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3171
3172 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3173 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3174 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3175 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3176 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3177 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3178 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3179 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3180 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3181 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3182
3183 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3184 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3185
3186 /*
3187 * Socket filters shouldn't attach/detach to/from this protosw
3188 * since pr_protosw is to be used instead, which points to the
3189 * real protocol; if they do, it is a bug and we should panic.
3190 */
3191 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3192 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3193 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3194 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3195
3196 #if INET6
3197 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3198
3199 VERIFY(g_tcp6_protosw != NULL);
3200
3201 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3202 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3203
3204 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3205 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3206 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3207 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3208 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3209 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3210 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3211 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3212 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3213 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3214
3215 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3216 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3217 /*
3218 * Socket filters shouldn't attach/detach to/from this protosw
3219 * since pr_protosw is to be used instead, which points to the
3220 * real protocol; if they do, it is a bug and we should panic.
3221 */
3222 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3223 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3224 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3225 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3226 #endif /* INET6 */
3227
3228 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3229 if (flow_divert_grp_attr == NULL) {
3230 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3231 g_init_result = ENOMEM;
3232 goto done;
3233 }
3234
3235 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3236 if (flow_divert_mtx_grp == NULL) {
3237 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3238 g_init_result = ENOMEM;
3239 goto done;
3240 }
3241
3242 flow_divert_mtx_attr = lck_attr_alloc_init();
3243 if (flow_divert_mtx_attr == NULL) {
3244 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3245 g_init_result = ENOMEM;
3246 goto done;
3247 }
3248
3249 g_init_result = flow_divert_kctl_init();
3250 if (g_init_result) {
3251 goto done;
3252 }
3253
3254 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3255
3256 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
3257 g_signing_id_trie.root = NULL_TRIE_IDX;
3258
3259 done:
3260 if (g_init_result != 0) {
3261 if (flow_divert_mtx_attr != NULL) {
3262 lck_attr_free(flow_divert_mtx_attr);
3263 flow_divert_mtx_attr = NULL;
3264 }
3265 if (flow_divert_mtx_grp != NULL) {
3266 lck_grp_free(flow_divert_mtx_grp);
3267 flow_divert_mtx_grp = NULL;
3268 }
3269 if (flow_divert_grp_attr != NULL) {
3270 lck_grp_attr_free(flow_divert_grp_attr);
3271 flow_divert_grp_attr = NULL;
3272 }
3273
3274 if (g_flow_divert_kctl_ref != NULL) {
3275 ctl_deregister(g_flow_divert_kctl_ref);
3276 g_flow_divert_kctl_ref = NULL;
3277 }
3278 }
3279 }
mirror of XNU