]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3248.30.4.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
59 #if INET6
60 #include <netinet6/in6_pcb.h>
61 #include <netinet6/ip6protosw.h>
62 #endif /* INET6 */
63 #include <dev/random/randomdev.h>
64 #include <libkern/crypto/sha1.h>
65 #include <libkern/crypto/crypto_internal.h>
66
67 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
68 #define FLOW_DIVERT_READ_CLOSED 0x00000002
69 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
70 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
71 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
72 #define FLOW_DIVERT_TRANSFERRED 0x00000020
73
74 #define FDLOG(level, pcb, format, ...) do { \
75 if (level <= (pcb)->log_level) { \
76 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): " format "\n", __FUNCTION__, (pcb)->hash, __VA_ARGS__); \
77 } \
78 } while (0)
79
80 #define FDLOG0(level, pcb, msg) do { \
81 if (level <= (pcb)->log_level) { \
82 log((level > LOG_NOTICE ? LOG_NOTICE : level), "%s (%u): %s\n", __FUNCTION__, (pcb)->hash, msg); \
83 } \
84 } while (0)
85
86 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
87 #define FDRELEASE(pcb) \
88 do { \
89 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
90 flow_divert_pcb_destroy(pcb); \
91 } \
92 } while (0)
93
94 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
95 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
96
97 #define FD_CTL_SENDBUFF_SIZE (2 * FLOW_DIVERT_CHUNK_SIZE)
98 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
99
100 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
101
102 #define GROUP_COUNT_MAX 32
103 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
104 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
105
106 #define DNS_SERVICE_GROUP_UNIT (GROUP_COUNT_MAX + 1)
107
108 struct flow_divert_trie_node
109 {
110 uint16_t start;
111 uint16_t length;
112 uint16_t child_map;
113 uint32_t group_unit;
114 };
115
116 struct flow_divert_trie
117 {
118 struct flow_divert_trie_node *nodes;
119 uint16_t *child_maps;
120 uint8_t *bytes;
121 void *memory;
122 size_t nodes_count;
123 size_t child_maps_count;
124 size_t bytes_count;
125 size_t nodes_free_next;
126 size_t child_maps_free_next;
127 size_t bytes_free_next;
128 uint16_t root;
129 };
130
131 #define CHILD_MAP_SIZE 256
132 #define NULL_TRIE_IDX 0xffff
133 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
134 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
135 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
136
137 static struct flow_divert_pcb nil_pcb;
138
139 decl_lck_rw_data(static, g_flow_divert_group_lck);
140 static struct flow_divert_group **g_flow_divert_groups = NULL;
141 static uint32_t g_active_group_count = 0;
142 static struct flow_divert_trie g_signing_id_trie;
143
144 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
145 static lck_attr_t *flow_divert_mtx_attr = NULL;
146 static lck_grp_t *flow_divert_mtx_grp = NULL;
147 static errno_t g_init_result = 0;
148
149 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
150
151 static struct protosw g_flow_divert_in_protosw;
152 static struct pr_usrreqs g_flow_divert_in_usrreqs;
153 static struct protosw g_flow_divert_in_udp_protosw;
154 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
155 #if INET6
156 static struct ip6protosw g_flow_divert_in6_protosw;
157 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
158 static struct ip6protosw g_flow_divert_in6_udp_protosw;
159 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
160 #endif /* INET6 */
161
162 static struct protosw *g_tcp_protosw = NULL;
163 static struct ip6protosw *g_tcp6_protosw = NULL;
164 static struct protosw *g_udp_protosw = NULL;
165 static struct ip6protosw *g_udp6_protosw = NULL;
166
167 static errno_t
168 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
169
170 static errno_t
171 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
172
173 static boolean_t
174 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
175
176 static int
177 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
178
179 struct sockaddr *
180 flow_divert_get_buffered_target_address(mbuf_t buffer);
181
182 static boolean_t
183 flow_divert_has_pcb_local_address(const struct inpcb *inp);
184
185 static void
186 flow_divert_disconnect_socket(struct socket *so);
187
188 static inline int
189 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
190 {
191 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
192 }
193
194 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
195 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
196
197 static const char *
198 flow_divert_packet_type2str(uint8_t packet_type)
199 {
200 switch (packet_type) {
201 case FLOW_DIVERT_PKT_CONNECT:
202 return "connect";
203 case FLOW_DIVERT_PKT_CONNECT_RESULT:
204 return "connect result";
205 case FLOW_DIVERT_PKT_DATA:
206 return "data";
207 case FLOW_DIVERT_PKT_CLOSE:
208 return "close";
209 case FLOW_DIVERT_PKT_READ_NOTIFY:
210 return "read notification";
211 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
212 return "properties update";
213 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
214 return "app map update";
215 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
216 return "app map create";
217 default:
218 return "unknown";
219 }
220 }
221
222 static struct flow_divert_pcb *
223 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
224 {
225 struct flow_divert_pcb key_item;
226 struct flow_divert_pcb *fd_cb = NULL;
227
228 key_item.hash = hash;
229
230 lck_rw_lock_shared(&group->lck);
231 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
232 FDRETAIN(fd_cb);
233 lck_rw_done(&group->lck);
234
235 return fd_cb;
236 }
237
238 static errno_t
239 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
240 {
241 errno_t error = 0;
242 struct flow_divert_pcb *exist = NULL;
243 struct flow_divert_group *group;
244 static uint32_t g_nextkey = 1;
245 static uint32_t g_hash_seed = 0;
246 int try_count = 0;
247
248 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
249 return EINVAL;
250 }
251
252 socket_unlock(fd_cb->so, 0);
253 lck_rw_lock_shared(&g_flow_divert_group_lck);
254
255 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
256 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
257 error = ENETUNREACH;
258 goto done;
259 }
260
261 group = g_flow_divert_groups[ctl_unit];
262 if (group == NULL) {
263 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
264 error = ENETUNREACH;
265 goto done;
266 }
267
268 socket_lock(fd_cb->so, 0);
269
270 do {
271 uint32_t key[2];
272 uint32_t idx;
273
274 key[0] = g_nextkey++;
275 key[1] = RandomULong();
276
277 if (g_hash_seed == 0) {
278 g_hash_seed = RandomULong();
279 }
280
281 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
282
283 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
284 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
285 if (curr_group != NULL && curr_group != group) {
286 lck_rw_lock_shared(&curr_group->lck);
287 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
288 lck_rw_done(&curr_group->lck);
289 if (exist != NULL) {
290 break;
291 }
292 }
293 }
294
295 if (exist == NULL) {
296 lck_rw_lock_exclusive(&group->lck);
297 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
298 lck_rw_done(&group->lck);
299 }
300 } while (exist != NULL && try_count++ < 3);
301
302 if (exist == NULL) {
303 fd_cb->group = group;
304 FDRETAIN(fd_cb); /* The group now has a reference */
305 } else {
306 fd_cb->hash = 0;
307 error = EEXIST;
308 }
309
310 socket_unlock(fd_cb->so, 0);
311
312 done:
313 lck_rw_done(&g_flow_divert_group_lck);
314 socket_lock(fd_cb->so, 0);
315
316 return error;
317 }
318
319 static struct flow_divert_pcb *
320 flow_divert_pcb_create(socket_t so)
321 {
322 struct flow_divert_pcb *new_pcb = NULL;
323
324 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
325 if (new_pcb == NULL) {
326 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
327 return NULL;
328 }
329
330 memset(new_pcb, 0, sizeof(*new_pcb));
331
332 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
333 new_pcb->so = so;
334 new_pcb->log_level = nil_pcb.log_level;
335
336 FDRETAIN(new_pcb); /* Represents the socket's reference */
337
338 return new_pcb;
339 }
340
341 static void
342 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
343 {
344 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
345 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
346
347 if (fd_cb->local_address != NULL) {
348 FREE(fd_cb->local_address, M_SONAME);
349 }
350 if (fd_cb->remote_address != NULL) {
351 FREE(fd_cb->remote_address, M_SONAME);
352 }
353 if (fd_cb->connect_token != NULL) {
354 mbuf_freem(fd_cb->connect_token);
355 }
356 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
357 }
358
359 static void
360 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
361 {
362 if (fd_cb->group != NULL) {
363 struct flow_divert_group *group = fd_cb->group;
364 lck_rw_lock_exclusive(&group->lck);
365 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
366 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
367 fd_cb->group = NULL;
368 FDRELEASE(fd_cb); /* Release the group's reference */
369 lck_rw_done(&group->lck);
370 }
371 }
372
373 static int
374 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
375 {
376 struct flow_divert_packet_header hdr;
377 int error = 0;
378
379 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
380 if (error) {
381 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
382 return error;
383 }
384
385 hdr.packet_type = packet_type;
386 hdr.conn_id = htonl(fd_cb->hash);
387
388 /* Lay down the header */
389 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
390 if (error) {
391 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
392 mbuf_freem(*packet);
393 *packet = NULL;
394 return error;
395 }
396
397 return 0;
398 }
399
400 static int
401 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
402 {
403 uint32_t net_length = htonl(length);
404 int error = 0;
405
406 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
407 if (error) {
408 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
409 return error;
410 }
411
412 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
413 if (error) {
414 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%lu)", length);
415 return error;
416 }
417
418 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
419 if (error) {
420 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
421 return error;
422 }
423
424 return error;
425 }
426
427 static int
428 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
429 {
430 size_t cursor = offset;
431 int error = 0;
432 uint32_t curr_length;
433 uint8_t curr_type;
434
435 *err = 0;
436
437 do {
438 if (!next) {
439 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
440 if (error) {
441 *err = ENOENT;
442 return -1;
443 }
444 } else {
445 next = 0;
446 curr_type = FLOW_DIVERT_TLV_NIL;
447 }
448
449 if (curr_type != type) {
450 cursor += sizeof(curr_type);
451 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
452 if (error) {
453 *err = error;
454 return -1;
455 }
456
457 cursor += (sizeof(curr_length) + ntohl(curr_length));
458 }
459 } while (curr_type != type);
460
461 return cursor;
462 }
463
464 static int
465 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
466 {
467 int error = 0;
468 uint32_t length;
469 int tlv_offset;
470
471 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
472 if (tlv_offset < 0) {
473 return error;
474 }
475
476 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
477 if (error) {
478 return error;
479 }
480
481 length = ntohl(length);
482
483 if (val_size != NULL) {
484 *val_size = length;
485 }
486
487 if (buff != NULL && buff_len > 0) {
488 size_t to_copy = (length < buff_len) ? length : buff_len;
489 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
490 if (error) {
491 return error;
492 }
493 }
494
495 return 0;
496 }
497
498 static int
499 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
500 {
501 mbuf_t curr_mbuf = packet;
502
503 if (g_crypto_funcs == NULL || group->token_key == NULL) {
504 return ENOPROTOOPT;
505 }
506
507 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
508 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
509
510 while (curr_mbuf != NULL) {
511 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
512 curr_mbuf = mbuf_next(curr_mbuf);
513 }
514
515 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
516
517 return 0;
518 }
519
520 static int
521 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
522 {
523 int error = 0;
524 struct flow_divert_group *group = NULL;
525 int hmac_offset;
526 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
527 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
528 mbuf_t tail;
529
530 lck_rw_lock_shared(&g_flow_divert_group_lck);
531
532 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
533 group = g_flow_divert_groups[ctl_unit];
534 }
535
536 if (group == NULL) {
537 lck_rw_done(&g_flow_divert_group_lck);
538 return ENOPROTOOPT;
539 }
540
541 lck_rw_lock_shared(&group->lck);
542
543 if (group->token_key == NULL) {
544 error = ENOPROTOOPT;
545 goto done;
546 }
547
548 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
549 if (hmac_offset < 0) {
550 goto done;
551 }
552
553 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
554 if (error) {
555 goto done;
556 }
557
558 /* Chop off the HMAC TLV */
559 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
560 if (error) {
561 goto done;
562 }
563
564 mbuf_free(tail);
565
566 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
567 if (error) {
568 goto done;
569 }
570
571 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
572 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
573 error = EINVAL;
574 goto done;
575 }
576
577 done:
578 lck_rw_done(&group->lck);
579 lck_rw_done(&g_flow_divert_group_lck);
580 return error;
581 }
582
583 static void
584 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
585 {
586 struct inpcb *inp = NULL;
587 struct ifnet *ifp = NULL;
588 Boolean cell = FALSE;
589 Boolean wifi = FALSE;
590 Boolean wired = FALSE;
591
592 inp = sotoinpcb(fd_cb->so);
593 if (inp == NULL) {
594 return;
595 }
596
597 ifp = inp->inp_last_outifp;
598 if (ifp != NULL) {
599 cell = IFNET_IS_CELLULAR(ifp);
600 wifi = (!cell && IFNET_IS_WIFI(ifp));
601 wired = (!wifi && IFNET_IS_WIRED(ifp));
602 }
603
604 if (send) {
605 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
606 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
607 } else {
608 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
609 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
610 }
611 }
612
613 static errno_t
614 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
615 {
616 struct inpcb *inp = NULL;
617
618 inp = sotoinpcb(fd_cb->so);
619 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
620 IFNET_IS_CELLULAR(inp->inp_last_outifp))
621 return EHOSTUNREACH;
622
623 return 0;
624 }
625
626 static errno_t
627 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
628 {
629 struct inpcb *inp = NULL;
630
631 inp = sotoinpcb(fd_cb->so);
632 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
633 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
634 return EHOSTUNREACH;
635
636 return 0;
637 }
638
639 static void
640 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
641 {
642 if (how != SHUT_RD) {
643 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
644 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
645 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
646 /* If the tunnel is not accepting writes any more, then flush the send buffer */
647 sbflush(&fd_cb->so->so_snd);
648 }
649 }
650 if (how != SHUT_WR) {
651 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
652 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
653 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
654 }
655 }
656 }
657
658 static uint16_t
659 trie_node_alloc(struct flow_divert_trie *trie)
660 {
661 if (trie->nodes_free_next < trie->nodes_count) {
662 uint16_t node_idx = trie->nodes_free_next++;
663 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
664 return node_idx;
665 } else {
666 return NULL_TRIE_IDX;
667 }
668 }
669
670 static uint16_t
671 trie_child_map_alloc(struct flow_divert_trie *trie)
672 {
673 if (trie->child_maps_free_next < trie->child_maps_count) {
674 return trie->child_maps_free_next++;
675 } else {
676 return NULL_TRIE_IDX;
677 }
678 }
679
680 static uint16_t
681 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
682 {
683 uint16_t start = trie->bytes_free_next;
684 if (start + bytes_size <= trie->bytes_count) {
685 if (start != bytes_idx) {
686 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
687 }
688 trie->bytes_free_next += bytes_size;
689 return start;
690 } else {
691 return NULL_TRIE_IDX;
692 }
693 }
694
695 static uint16_t
696 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
697 {
698 uint16_t current = trie->root;
699 uint16_t child = trie->root;
700 uint16_t string_end = string_start + string_len;
701 uint16_t string_idx = string_start;
702 uint16_t string_remainder = string_len;
703
704 while (child != NULL_TRIE_IDX) {
705 uint16_t parent = current;
706 uint16_t node_idx;
707 uint16_t current_end;
708
709 current = child;
710 child = NULL_TRIE_IDX;
711
712 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
713
714 for (node_idx = TRIE_NODE(trie, current).start;
715 node_idx < current_end &&
716 string_idx < string_end &&
717 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
718 node_idx++, string_idx++);
719
720 string_remainder = string_end - string_idx;
721
722 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
723 /*
724 * We did not reach the end of the current node's string.
725 * We need to split the current node into two:
726 * 1. A new node that contains the prefix of the node that matches
727 * the prefix of the string being inserted.
728 * 2. The current node modified to point to the remainder
729 * of the current node's string.
730 */
731 uint16_t prefix = trie_node_alloc(trie);
732 if (prefix == NULL_TRIE_IDX) {
733 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
734 return NULL_TRIE_IDX;
735 }
736
737 /*
738 * Prefix points to the portion of the current nodes's string that has matched
739 * the input string thus far.
740 */
741 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
742 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
743
744 /*
745 * Prefix has the current node as the child corresponding to the first byte
746 * after the split.
747 */
748 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
749 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
750 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
751 return NULL_TRIE_IDX;
752 }
753 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
754
755 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
756 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
757
758 /* Current node is adjusted to point to the remainder */
759 TRIE_NODE(trie, current).start = node_idx;
760 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
761
762 /* We want to insert the new leaf (if any) as a child of the prefix */
763 current = prefix;
764 }
765
766 if (string_remainder > 0) {
767 /*
768 * We still have bytes in the string that have not been matched yet.
769 * If the current node has children, iterate to the child corresponding
770 * to the next byte in the string.
771 */
772 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
773 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
774 }
775 }
776 } /* while (child != NULL_TRIE_IDX) */
777
778 if (string_remainder > 0) {
779 /* Add a new leaf containing the remainder of the string */
780 uint16_t leaf = trie_node_alloc(trie);
781 if (leaf == NULL_TRIE_IDX) {
782 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
783 return NULL_TRIE_IDX;
784 }
785
786 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
787 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
788 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
789 return NULL_TRIE_IDX;
790 }
791 TRIE_NODE(trie, leaf).length = string_remainder;
792
793 /* Set the new leaf as the child of the current node */
794 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
795 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
796 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
797 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
798 return NULL_TRIE_IDX;
799 }
800 }
801 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
802 current = leaf;
803 } /* else duplicate or this string is a prefix of one of the existing strings */
804
805 return current;
806 }
807
808 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
809 static uint16_t
810 flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
811 {
812 uint16_t current = trie->root;
813 uint16_t string_idx = 0;
814
815 while (current != NULL_TRIE_IDX) {
816 uint16_t next = NULL_TRIE_IDX;
817 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
818 uint16_t node_idx;
819
820 for (node_idx = TRIE_NODE(trie, current).start;
821 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
822 node_idx++, string_idx++);
823
824 if (node_idx == node_end) {
825 if (string_bytes[string_idx] == '\0') {
826 return current; /* Got an exact match */
827 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
828 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
829 string_bytes[string_idx] = '\0';
830 return current; /* Got an apple webclip id prefix match */
831 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
832 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
833 }
834 }
835 current = next;
836 }
837
838 return NULL_TRIE_IDX;
839 }
840
841 static int
842 flow_divert_get_src_proc(struct socket *so, proc_t *proc, boolean_t match_delegate)
843 {
844 int release = 0;
845
846 if (!match_delegate &&
847 (so->so_flags & SOF_DELEGATED) &&
848 (*proc == PROC_NULL || (*proc)->p_pid != so->e_pid))
849 {
850 *proc = proc_find(so->e_pid);
851 release = 1;
852 } else if (*proc == PROC_NULL) {
853 *proc = current_proc();
854 }
855
856 if (*proc != PROC_NULL) {
857 if ((*proc)->p_pid == 0) {
858 if (release) {
859 proc_rele(*proc);
860 }
861 release = 0;
862 *proc = PROC_NULL;
863 }
864 }
865
866 return release;
867 }
868
869 static int
870 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
871 {
872 int error;
873
874 if (fd_cb->group == NULL) {
875 fd_cb->so->so_error = ECONNABORTED;
876 flow_divert_disconnect_socket(fd_cb->so);
877 return ECONNABORTED;
878 }
879
880 lck_rw_lock_shared(&fd_cb->group->lck);
881
882 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
883 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
884 } else {
885 error = ENOBUFS;
886 }
887
888 if (error == ENOBUFS) {
889 if (enqueue) {
890 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
891 lck_rw_lock_exclusive(&fd_cb->group->lck);
892 }
893 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
894 error = 0;
895 }
896 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
897 }
898
899 lck_rw_done(&fd_cb->group->lck);
900
901 return error;
902 }
903
904 static int
905 flow_divert_send_connect(struct flow_divert_pcb *fd_cb, struct sockaddr *to, mbuf_t connect_packet)
906 {
907 int error = 0;
908 int flow_type = 0;
909
910 error = flow_divert_packet_append_tlv(connect_packet,
911 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
912 sizeof(fd_cb->so->so_traffic_class),
913 &fd_cb->so->so_traffic_class);
914 if (error) {
915 goto done;
916 }
917
918 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
919 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
920 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
921 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
922 } else {
923 error = EINVAL;
924 goto done;
925 }
926 error = flow_divert_packet_append_tlv(connect_packet,
927 FLOW_DIVERT_TLV_FLOW_TYPE,
928 sizeof(flow_type),
929 &flow_type);
930
931 if (error) {
932 goto done;
933 }
934
935 if (fd_cb->so->so_flags & SOF_DELEGATED) {
936 error = flow_divert_packet_append_tlv(connect_packet,
937 FLOW_DIVERT_TLV_PID,
938 sizeof(fd_cb->so->e_pid),
939 &fd_cb->so->e_pid);
940 if (error) {
941 goto done;
942 }
943
944 error = flow_divert_packet_append_tlv(connect_packet,
945 FLOW_DIVERT_TLV_UUID,
946 sizeof(fd_cb->so->e_uuid),
947 &fd_cb->so->e_uuid);
948 if (error) {
949 goto done;
950 }
951 } else {
952 error = flow_divert_packet_append_tlv(connect_packet,
953 FLOW_DIVERT_TLV_PID,
954 sizeof(fd_cb->so->e_pid),
955 &fd_cb->so->last_pid);
956 if (error) {
957 goto done;
958 }
959
960 error = flow_divert_packet_append_tlv(connect_packet,
961 FLOW_DIVERT_TLV_UUID,
962 sizeof(fd_cb->so->e_uuid),
963 &fd_cb->so->last_uuid);
964 if (error) {
965 goto done;
966 }
967 }
968
969 if (fd_cb->connect_token != NULL) {
970 unsigned int token_len = m_length(fd_cb->connect_token);
971 mbuf_concatenate(connect_packet, fd_cb->connect_token);
972 mbuf_pkthdr_adjustlen(connect_packet, token_len);
973 fd_cb->connect_token = NULL;
974 } else {
975 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
976
977 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
978 if (error) {
979 goto done;
980 }
981
982 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
983 if (error) {
984 goto done;
985 }
986 }
987
988 if (fd_cb->local_address != NULL) {
989 /* socket is bound. */
990 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
991 sizeof(struct sockaddr_storage), fd_cb->local_address);
992 if (error) {
993 goto done;
994 }
995 }
996
997 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
998 if (error) {
999 goto done;
1000 }
1001
1002 done:
1003 return error;
1004 }
1005
1006 static int
1007 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1008 {
1009 int error = 0;
1010 mbuf_t packet = NULL;
1011 int rbuff_space = 0;
1012
1013 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1014 if (error) {
1015 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1016 goto done;
1017 }
1018
1019 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1020 if (rbuff_space < 0) {
1021 rbuff_space = 0;
1022 }
1023 rbuff_space = htonl(rbuff_space);
1024 error = flow_divert_packet_append_tlv(packet,
1025 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1026 sizeof(rbuff_space),
1027 &rbuff_space);
1028 if (error) {
1029 goto done;
1030 }
1031
1032 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1033 if (error) {
1034 goto done;
1035 }
1036
1037 done:
1038 if (error && packet != NULL) {
1039 mbuf_freem(packet);
1040 }
1041
1042 return error;
1043 }
1044
1045 static int
1046 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1047 {
1048 int error = 0;
1049 mbuf_t packet = NULL;
1050 uint32_t zero = 0;
1051
1052 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1053 if (error) {
1054 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1055 goto done;
1056 }
1057
1058 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1059 if (error) {
1060 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1061 goto done;
1062 }
1063
1064 how = htonl(how);
1065 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1066 if (error) {
1067 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1068 goto done;
1069 }
1070
1071 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1072 if (error) {
1073 goto done;
1074 }
1075
1076 done:
1077 if (error && packet != NULL) {
1078 mbuf_free(packet);
1079 }
1080
1081 return error;
1082 }
1083
1084 static int
1085 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1086 {
1087 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1088 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1089 {
1090 return SHUT_RDWR;
1091 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1092 return SHUT_RD;
1093 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1094 return SHUT_WR;
1095 }
1096
1097 return -1;
1098 }
1099
1100 /*
1101 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1102 * writes. Returns FALSE otherwise.
1103 */
1104 static void
1105 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1106 {
1107 int how = -1;
1108
1109 /* Do not send any close messages if there is still data in the send buffer */
1110 if (fd_cb->so->so_snd.sb_cc == 0) {
1111 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1112 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1113 how = SHUT_RD;
1114 }
1115 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1116 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1117 if (how == SHUT_RD) {
1118 how = SHUT_RDWR;
1119 } else {
1120 how = SHUT_WR;
1121 }
1122 }
1123 }
1124
1125 if (how != -1) {
1126 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1127 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1128 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1129 if (how != SHUT_RD) {
1130 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1131 }
1132 if (how != SHUT_WR) {
1133 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1134 }
1135 }
1136 }
1137
1138 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1139 flow_divert_disconnect_socket(fd_cb->so);
1140 }
1141 }
1142
1143 static errno_t
1144 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1145 {
1146 mbuf_t packet;
1147 mbuf_t last;
1148 int error = 0;
1149
1150 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1151 if (error) {
1152 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1153 return error;
1154 }
1155
1156 if (toaddr != NULL) {
1157 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1158 if (error) {
1159 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1160 return error;
1161 }
1162 }
1163
1164 last = m_last(packet);
1165 mbuf_setnext(last, data);
1166 mbuf_pkthdr_adjustlen(packet, data_len);
1167 error = flow_divert_send_packet(fd_cb, packet, force);
1168
1169 if (error) {
1170 mbuf_setnext(last, NULL);
1171 mbuf_freem(packet);
1172 } else {
1173 fd_cb->bytes_sent += data_len;
1174 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1175 }
1176
1177 return error;
1178 }
1179
1180 static void
1181 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1182 {
1183 size_t to_send;
1184 size_t sent = 0;
1185 int error = 0;
1186 mbuf_t buffer;
1187
1188 to_send = fd_cb->so->so_snd.sb_cc;
1189 buffer = fd_cb->so->so_snd.sb_mb;
1190
1191 if (buffer == NULL && to_send > 0) {
1192 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1193 return;
1194 }
1195
1196 /* Ignore the send window if force is enabled */
1197 if (!force && (to_send > fd_cb->send_window)) {
1198 to_send = fd_cb->send_window;
1199 }
1200
1201 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1202 while (sent < to_send) {
1203 mbuf_t data;
1204 size_t data_len;
1205
1206 data_len = to_send - sent;
1207 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1208 data_len = FLOW_DIVERT_CHUNK_SIZE;
1209 }
1210
1211 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1212 if (error) {
1213 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1214 break;
1215 }
1216
1217 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1218 if (error) {
1219 mbuf_freem(data);
1220 break;
1221 }
1222
1223 sent += data_len;
1224 }
1225 sbdrop(&fd_cb->so->so_snd, sent);
1226 sowwakeup(fd_cb->so);
1227 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1228 mbuf_t data;
1229 mbuf_t m;
1230 size_t data_len;
1231
1232 while(buffer) {
1233 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1234
1235 m = buffer;
1236 if (toaddr != NULL) {
1237 /* look for data in the chain */
1238 do {
1239 m = m->m_next;
1240 if (m != NULL && m->m_type == MT_DATA) {
1241 break;
1242 }
1243 } while(m);
1244 if (m == NULL) {
1245 /* unexpected */
1246 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1247 goto move_on;
1248 }
1249 }
1250 data_len = mbuf_pkthdr_len(m);
1251 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %u", data_len);
1252 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1253 if (error) {
1254 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1255 break;
1256 }
1257 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1258 if (error) {
1259 mbuf_freem(data);
1260 break;
1261 }
1262 sent += data_len;
1263 move_on:
1264 buffer = buffer->m_nextpkt;
1265 (void) sbdroprecord(&(fd_cb->so->so_snd));
1266 }
1267 }
1268
1269 if (sent > 0) {
1270 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1271 if (fd_cb->send_window >= sent) {
1272 fd_cb->send_window -= sent;
1273 } else {
1274 fd_cb->send_window = 0;
1275 }
1276 }
1277 }
1278
1279 static int
1280 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1281 {
1282 size_t to_send = mbuf_pkthdr_len(data);
1283 int error = 0;
1284
1285 if (to_send > fd_cb->send_window) {
1286 to_send = fd_cb->send_window;
1287 }
1288
1289 if (fd_cb->so->so_snd.sb_cc > 0) {
1290 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1291 }
1292
1293 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1294 size_t sent = 0;
1295 mbuf_t remaining_data = data;
1296 mbuf_t pkt_data = NULL;
1297 while (sent < to_send) {
1298 size_t pkt_data_len;
1299
1300 pkt_data = remaining_data;
1301
1302 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1303 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1304 } else {
1305 pkt_data_len = to_send - sent;
1306 }
1307
1308 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1309 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1310 if (error) {
1311 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1312 pkt_data = NULL;
1313 break;
1314 }
1315 } else {
1316 remaining_data = NULL;
1317 }
1318
1319 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1320
1321 if (error) {
1322 break;
1323 }
1324
1325 pkt_data = NULL;
1326 sent += pkt_data_len;
1327 }
1328
1329 fd_cb->send_window -= sent;
1330
1331 error = 0;
1332
1333 if (pkt_data != NULL) {
1334 if (sbspace(&fd_cb->so->so_snd) > 0) {
1335 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1336 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1337 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1338 }
1339 } else {
1340 error = ENOBUFS;
1341 }
1342 }
1343
1344 if (remaining_data != NULL) {
1345 if (sbspace(&fd_cb->so->so_snd) > 0) {
1346 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1347 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1348 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1349 }
1350 } else {
1351 error = ENOBUFS;
1352 }
1353 }
1354 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1355 if (to_send) {
1356 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1357 if (error) {
1358 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %u", to_send);
1359 } else {
1360 fd_cb->send_window -= to_send;
1361 }
1362 } else {
1363 /* buffer it */
1364 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1365 if (toaddr != NULL) {
1366 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1367 FDLOG(LOG_ERR, fd_cb,
1368 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1369 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1370 }
1371 } else {
1372 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1373 FDLOG(LOG_ERR, fd_cb,
1374 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1375 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1376 }
1377 }
1378 } else {
1379 error = ENOBUFS;
1380 }
1381 }
1382 }
1383
1384 return error;
1385 }
1386
1387 static int
1388 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1389 {
1390 int error = 0;
1391 mbuf_t packet = NULL;
1392 uint32_t net_read_count = htonl(read_count);
1393
1394 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1395 if (error) {
1396 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1397 goto done;
1398 }
1399
1400 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1401 if (error) {
1402 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1403 goto done;
1404 }
1405
1406 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1407 if (error) {
1408 goto done;
1409 }
1410
1411 done:
1412 if (error && packet != NULL) {
1413 mbuf_free(packet);
1414 }
1415
1416 return error;
1417 }
1418
1419 static int
1420 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1421 {
1422 int error = 0;
1423 mbuf_t packet = NULL;
1424
1425 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1426 if (error) {
1427 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1428 goto done;
1429 }
1430
1431 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1432 if (error) {
1433 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1434 goto done;
1435 }
1436
1437 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1438 if (error) {
1439 goto done;
1440 }
1441
1442 done:
1443 if (error && packet != NULL) {
1444 mbuf_free(packet);
1445 }
1446
1447 return error;
1448 }
1449
1450 static void
1451 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1452 {
1453 uint32_t connect_error;
1454 uint32_t ctl_unit = 0;
1455 int error = 0;
1456 struct flow_divert_group *grp = NULL;
1457 struct sockaddr_storage local_address;
1458 int out_if_index = 0;
1459 struct sockaddr_storage remote_address;
1460 uint32_t send_window;
1461
1462 memset(&local_address, 0, sizeof(local_address));
1463 memset(&remote_address, 0, sizeof(remote_address));
1464
1465 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1466 if (error) {
1467 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1468 return;
1469 }
1470
1471 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1472
1473 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1474 if (error) {
1475 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1476 return;
1477 }
1478
1479 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1480 if (error) {
1481 FDLOG(LOG_ERR, fd_cb, "failed to get the control unit: %d", error);
1482 return;
1483 }
1484
1485 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1486 if (error) {
1487 FDLOG0(LOG_NOTICE, fd_cb, "No local address provided");
1488 }
1489
1490 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1491 if (error) {
1492 FDLOG0(LOG_NOTICE, fd_cb, "No remote address provided");
1493 }
1494
1495 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1496 if (error) {
1497 FDLOG0(LOG_NOTICE, fd_cb, "No output if index provided");
1498 }
1499
1500 connect_error = ntohl(connect_error);
1501 ctl_unit = ntohl(ctl_unit);
1502
1503 lck_rw_lock_shared(&g_flow_divert_group_lck);
1504
1505 if (connect_error == 0) {
1506 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
1507 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1508 error = EINVAL;
1509 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1510 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1511 error = EINVAL;
1512 } else {
1513 grp = g_flow_divert_groups[ctl_unit];
1514 if (grp == NULL) {
1515 error = ECONNRESET;
1516 }
1517 }
1518 }
1519
1520 FDLOCK(fd_cb);
1521 if (fd_cb->so != NULL) {
1522 struct inpcb *inp = NULL;
1523 struct ifnet *ifp = NULL;
1524 struct flow_divert_group *old_group;
1525
1526 socket_lock(fd_cb->so, 0);
1527
1528 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1529 goto done;
1530 }
1531
1532 inp = sotoinpcb(fd_cb->so);
1533
1534 if (connect_error || error) {
1535 goto set_socket_state;
1536 }
1537
1538 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1539 error = EINVAL;
1540 goto set_socket_state;
1541 }
1542 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1543 if (local_address.ss_len > sizeof(local_address)) {
1544 local_address.ss_len = sizeof(local_address);
1545 }
1546 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1547 }
1548
1549 if (remote_address.ss_family != 0) {
1550 if (remote_address.ss_len > sizeof(remote_address)) {
1551 remote_address.ss_len = sizeof(remote_address);
1552 }
1553 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1554 } else {
1555 error = EINVAL;
1556 goto set_socket_state;
1557 }
1558
1559 ifnet_head_lock_shared();
1560 if (out_if_index > 0 && out_if_index <= if_index) {
1561 ifp = ifindex2ifnet[out_if_index];
1562 }
1563
1564 if (ifp != NULL) {
1565 inp->inp_last_outifp = ifp;
1566 } else {
1567 error = EINVAL;
1568 }
1569 ifnet_head_done();
1570
1571 if (error) {
1572 goto set_socket_state;
1573 }
1574
1575 if (fd_cb->group == NULL) {
1576 error = EINVAL;
1577 goto set_socket_state;
1578 }
1579
1580 old_group = fd_cb->group;
1581
1582 lck_rw_lock_exclusive(&old_group->lck);
1583 lck_rw_lock_exclusive(&grp->lck);
1584
1585 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1586 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1587 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1588 }
1589
1590 fd_cb->group = grp;
1591
1592 lck_rw_done(&grp->lck);
1593 lck_rw_done(&old_group->lck);
1594
1595 fd_cb->send_window = ntohl(send_window);
1596 flow_divert_send_buffered_data(fd_cb, FALSE);
1597
1598 set_socket_state:
1599 if (!connect_error && !error) {
1600 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1601 error = flow_divert_send_connect_result(fd_cb);
1602 }
1603
1604 if (connect_error || error) {
1605 if (!connect_error) {
1606 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1607 fd_cb->so->so_error = error;
1608 flow_divert_send_close_if_needed(fd_cb);
1609 } else {
1610 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1611 fd_cb->so->so_error = connect_error;
1612 }
1613 flow_divert_disconnect_socket(fd_cb->so);
1614 } else {
1615 soisconnected(fd_cb->so);
1616 }
1617
1618 done:
1619 socket_unlock(fd_cb->so, 0);
1620 }
1621 FDUNLOCK(fd_cb);
1622
1623 lck_rw_done(&g_flow_divert_group_lck);
1624 }
1625
1626 static void
1627 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1628 {
1629 uint32_t close_error;
1630 int error = 0;
1631 int how;
1632
1633 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1634 if (error) {
1635 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1636 return;
1637 }
1638
1639 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1640 if (error) {
1641 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1642 return;
1643 }
1644
1645 how = ntohl(how);
1646
1647 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1648
1649 FDLOCK(fd_cb);
1650 if (fd_cb->so != NULL) {
1651 socket_lock(fd_cb->so, 0);
1652
1653 fd_cb->so->so_error = ntohl(close_error);
1654
1655 flow_divert_update_closed_state(fd_cb, how, TRUE);
1656
1657 how = flow_divert_tunnel_how_closed(fd_cb);
1658 if (how == SHUT_RDWR) {
1659 flow_divert_disconnect_socket(fd_cb->so);
1660 } else if (how == SHUT_RD) {
1661 socantrcvmore(fd_cb->so);
1662 } else if (how == SHUT_WR) {
1663 socantsendmore(fd_cb->so);
1664 }
1665
1666 socket_unlock(fd_cb->so, 0);
1667 }
1668 FDUNLOCK(fd_cb);
1669 }
1670
1671 static mbuf_t
1672 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
1673 {
1674 struct inpcb *inp = sotoinpcb(fd_cb->so);
1675 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1676 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
1677
1678 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1679 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1680 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1681 struct in6_pktinfo pi6;
1682
1683 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1684 pi6.ipi6_ifindex = 0;
1685 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
1686 }
1687 return (NULL);
1688 }
1689
1690 static void
1691 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1692 {
1693 FDLOCK(fd_cb);
1694 if (fd_cb->so != NULL) {
1695 int error = 0;
1696 mbuf_t data = NULL;
1697 size_t data_size;
1698 struct sockaddr_storage remote_address;
1699 boolean_t got_remote_sa = FALSE;
1700
1701 socket_lock(fd_cb->so, 0);
1702
1703 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1704 uint32_t val_size = 0;
1705
1706 /* check if we got remote address with data */
1707 memset(&remote_address, 0, sizeof(remote_address));
1708 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1709 if (error || val_size > sizeof(remote_address)) {
1710 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1711 error = 0;
1712 } else {
1713 /* validate the address */
1714 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1715 got_remote_sa = TRUE;
1716 }
1717 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1718 }
1719 }
1720
1721 data_size = (mbuf_pkthdr_len(packet) - offset);
1722
1723 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1724
1725 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1726 if (error || data == NULL) {
1727 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1728 } else {
1729 if (flow_divert_check_no_cellular(fd_cb) ||
1730 flow_divert_check_no_expensive(fd_cb))
1731 {
1732 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1733 flow_divert_send_close(fd_cb, SHUT_RDWR);
1734 flow_divert_disconnect_socket(fd_cb->so);
1735 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1736 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1737 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1738 fd_cb->bytes_received += data_size;
1739 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1740 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1741 sorwakeup(fd_cb->so);
1742 data = NULL;
1743 } else {
1744 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1745 }
1746 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1747 struct sockaddr *append_sa;
1748 mbuf_t mctl;
1749
1750 if (got_remote_sa == TRUE) {
1751 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1752 (struct sockaddr *)&remote_address, &append_sa);
1753 } else {
1754 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1755 fd_cb->remote_address, &append_sa);
1756 }
1757 if (error) {
1758 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
1759 }
1760
1761 mctl = flow_divert_get_control_mbuf(fd_cb);
1762 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
1763 fd_cb->bytes_received += data_size;
1764 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1765 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1766 sorwakeup(fd_cb->so);
1767 data = NULL;
1768 } else {
1769 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
1770 }
1771 if (!error) {
1772 FREE(append_sa, M_TEMP);
1773 }
1774 }
1775 }
1776 }
1777 socket_unlock(fd_cb->so, 0);
1778
1779 if (data != NULL) {
1780 mbuf_freem(data);
1781 }
1782 }
1783 FDUNLOCK(fd_cb);
1784 }
1785
1786 static void
1787 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1788 {
1789 uint32_t read_count;
1790 int error = 0;
1791
1792 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
1793 if (error) {
1794 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
1795 return;
1796 }
1797
1798 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
1799
1800 FDLOCK(fd_cb);
1801 if (fd_cb->so != NULL) {
1802 socket_lock(fd_cb->so, 0);
1803 fd_cb->send_window += ntohl(read_count);
1804 flow_divert_send_buffered_data(fd_cb, FALSE);
1805 socket_unlock(fd_cb->so, 0);
1806 }
1807 FDUNLOCK(fd_cb);
1808 }
1809
1810 static void
1811 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
1812 {
1813 int error = 0;
1814 uint32_t key_size = 0;
1815 int log_level;
1816
1817 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
1818 if (error) {
1819 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
1820 return;
1821 }
1822
1823 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
1824 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %lu", key_size);
1825 return;
1826 }
1827
1828 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
1829 if (!error) {
1830 nil_pcb.log_level = log_level;
1831 }
1832
1833 lck_rw_lock_exclusive(&group->lck);
1834
1835 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
1836 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
1837 if (error) {
1838 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
1839 FREE(group->token_key, M_TEMP);
1840 group->token_key = NULL;
1841 lck_rw_done(&group->lck);
1842 return;
1843 }
1844
1845 group->token_key_size = key_size;
1846
1847 lck_rw_done(&group->lck);
1848 }
1849
1850 static void
1851 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1852 {
1853 int error = 0;
1854 struct sockaddr_storage local_address;
1855 int out_if_index = 0;
1856 struct sockaddr_storage remote_address;
1857
1858 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
1859
1860 memset(&local_address, 0, sizeof(local_address));
1861 memset(&remote_address, 0, sizeof(remote_address));
1862
1863 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1864 if (error) {
1865 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1866 }
1867
1868 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1869 if (error) {
1870 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1871 }
1872
1873 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1874 if (error) {
1875 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1876 }
1877
1878 FDLOCK(fd_cb);
1879 if (fd_cb->so != NULL) {
1880 struct inpcb *inp = NULL;
1881 struct ifnet *ifp = NULL;
1882
1883 socket_lock(fd_cb->so, 0);
1884
1885 inp = sotoinpcb(fd_cb->so);
1886
1887 if (local_address.ss_family != 0) {
1888 if (local_address.ss_len > sizeof(local_address)) {
1889 local_address.ss_len = sizeof(local_address);
1890 }
1891 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1892 }
1893
1894 if (remote_address.ss_family != 0) {
1895 if (remote_address.ss_len > sizeof(remote_address)) {
1896 remote_address.ss_len = sizeof(remote_address);
1897 }
1898 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1899 }
1900
1901 ifnet_head_lock_shared();
1902 if (out_if_index > 0 && out_if_index <= if_index) {
1903 ifp = ifindex2ifnet[out_if_index];
1904 }
1905
1906 if (ifp != NULL) {
1907 inp->inp_last_outifp = ifp;
1908 }
1909 ifnet_head_done();
1910
1911 socket_unlock(fd_cb->so, 0);
1912 }
1913 FDUNLOCK(fd_cb);
1914 }
1915
1916 static void
1917 flow_divert_handle_app_map_create(mbuf_t packet, int offset)
1918 {
1919 size_t bytes_mem_size;
1920 size_t child_maps_mem_size;
1921 int cursor;
1922 int error = 0;
1923 struct flow_divert_trie new_trie;
1924 int insert_error = 0;
1925 size_t nodes_mem_size;
1926 int prefix_count = 0;
1927 int signing_id_count = 0;
1928
1929 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
1930
1931 /* Re-set the current trie */
1932 if (g_signing_id_trie.memory != NULL) {
1933 FREE(g_signing_id_trie.memory, M_TEMP);
1934 }
1935 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
1936 g_signing_id_trie.root = NULL_TRIE_IDX;
1937
1938 memset(&new_trie, 0, sizeof(new_trie));
1939
1940 /* Get the number of shared prefixes in the new set of signing ID strings */
1941 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
1942
1943 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
1944 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1945 cursor >= 0;
1946 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1947 {
1948 uint32_t sid_size = 0;
1949 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1950 new_trie.bytes_count += sid_size;
1951 signing_id_count++;
1952 }
1953
1954 if (signing_id_count == 0) {
1955 lck_rw_done(&g_flow_divert_group_lck);
1956 return;
1957 }
1958
1959 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
1960 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
1961
1962 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
1963 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
1964
1965 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
1966 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
1967 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
1968
1969 MALLOC(new_trie.memory, void *, nodes_mem_size + child_maps_mem_size + bytes_mem_size, M_TEMP, M_WAITOK);
1970 if (new_trie.memory == NULL) {
1971 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
1972 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
1973 return;
1974 }
1975
1976 /* Initialize the free lists */
1977 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
1978 new_trie.nodes_free_next = 0;
1979 memset(new_trie.nodes, 0, nodes_mem_size);
1980
1981 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
1982 new_trie.child_maps_free_next = 0;
1983 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
1984
1985 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
1986 new_trie.bytes_free_next = 0;
1987
1988 /* The root is an empty node */
1989 new_trie.root = trie_node_alloc(&new_trie);
1990
1991 /* Add each signing ID to the trie */
1992 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
1993 cursor >= 0;
1994 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
1995 {
1996 uint32_t sid_size = 0;
1997 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1998 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
1999 boolean_t is_dns;
2000 uint16_t new_node_idx;
2001 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2002 is_dns = (sid_size == sizeof(FLOW_DIVERT_DNS_SERVICE_SIGNING_ID) - 1 &&
2003 !memcmp(&TRIE_BYTE(&new_trie, new_trie.bytes_free_next),
2004 FLOW_DIVERT_DNS_SERVICE_SIGNING_ID,
2005 sid_size));
2006 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2007 if (new_node_idx != NULL_TRIE_IDX) {
2008 if (is_dns) {
2009 FDLOG(LOG_INFO, &nil_pcb, "Setting group unit for %s to %d", FLOW_DIVERT_DNS_SERVICE_SIGNING_ID, DNS_SERVICE_GROUP_UNIT);
2010 TRIE_NODE(&new_trie, new_node_idx).group_unit = DNS_SERVICE_GROUP_UNIT;
2011 }
2012 } else {
2013 insert_error = EINVAL;
2014 break;
2015 }
2016 } else {
2017 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2018 insert_error = ENOBUFS;
2019 break;
2020 }
2021 }
2022
2023 if (!insert_error) {
2024 g_signing_id_trie = new_trie;
2025 } else {
2026 FREE(new_trie.memory, M_TEMP);
2027 }
2028
2029 lck_rw_done(&g_flow_divert_group_lck);
2030 }
2031
2032 static void
2033 flow_divert_handle_app_map_update(struct flow_divert_group *group, mbuf_t packet, int offset)
2034 {
2035 int error = 0;
2036 int cursor;
2037 size_t max_size = 0;
2038 uint8_t *signing_id;
2039 uint32_t ctl_unit;
2040
2041 lck_rw_lock_shared(&group->lck);
2042 ctl_unit = group->ctl_unit;
2043 lck_rw_done(&group->lck);
2044
2045 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2046 cursor >= 0;
2047 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2048 {
2049 uint32_t sid_size = 0;
2050 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2051 if (sid_size > max_size) {
2052 max_size = sid_size;
2053 }
2054 }
2055
2056 MALLOC(signing_id, uint8_t *, max_size + 1, M_TEMP, M_WAITOK);
2057 if (signing_id == NULL) {
2058 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate a string to hold the signing ID (size %lu)", max_size);
2059 return;
2060 }
2061
2062 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2063 cursor >= 0;
2064 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2065 {
2066 uint32_t signing_id_len = 0;
2067 uint16_t node;
2068
2069 flow_divert_packet_get_tlv(packet,
2070 cursor, FLOW_DIVERT_TLV_SIGNING_ID, max_size, signing_id, &signing_id_len);
2071
2072 signing_id[signing_id_len] = '\0';
2073
2074 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
2075
2076 node = flow_divert_trie_search(&g_signing_id_trie, signing_id);
2077 if (node != NULL_TRIE_IDX) {
2078 if (TRIE_NODE(&g_signing_id_trie, node).group_unit != DNS_SERVICE_GROUP_UNIT) {
2079 FDLOG(LOG_INFO, &nil_pcb, "Setting %s to ctl unit %u", signing_id, group->ctl_unit);
2080 TRIE_NODE(&g_signing_id_trie, node).group_unit = ctl_unit;
2081 }
2082 } else {
2083 FDLOG(LOG_ERR, &nil_pcb, "Failed to find signing ID %s", signing_id);
2084 }
2085
2086 lck_rw_done(&g_flow_divert_group_lck);
2087 }
2088
2089 FREE(signing_id, M_TEMP);
2090 }
2091
2092 static int
2093 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2094 {
2095 struct flow_divert_packet_header hdr;
2096 int error = 0;
2097 struct flow_divert_pcb *fd_cb;
2098
2099 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2100 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2101 error = EINVAL;
2102 goto done;
2103 }
2104
2105 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2106 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %lu", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2107 error = EINVAL;
2108 goto done;
2109 }
2110
2111 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2112 if (error) {
2113 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2114 error = ENOBUFS;
2115 goto done;
2116 }
2117
2118 hdr.conn_id = ntohl(hdr.conn_id);
2119
2120 if (hdr.conn_id == 0) {
2121 switch (hdr.packet_type) {
2122 case FLOW_DIVERT_PKT_GROUP_INIT:
2123 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2124 break;
2125 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2126 flow_divert_handle_app_map_create(packet, sizeof(hdr));
2127 break;
2128 case FLOW_DIVERT_PKT_APP_MAP_UPDATE:
2129 flow_divert_handle_app_map_update(group, packet, sizeof(hdr));
2130 break;
2131 default:
2132 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2133 break;
2134 }
2135 goto done;
2136 }
2137
2138 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2139 if (fd_cb == NULL) {
2140 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2141 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2142 }
2143 goto done;
2144 }
2145
2146 switch (hdr.packet_type) {
2147 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2148 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2149 break;
2150 case FLOW_DIVERT_PKT_CLOSE:
2151 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2152 break;
2153 case FLOW_DIVERT_PKT_DATA:
2154 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2155 break;
2156 case FLOW_DIVERT_PKT_READ_NOTIFY:
2157 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2158 break;
2159 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2160 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2161 break;
2162 default:
2163 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2164 break;
2165 }
2166
2167 FDRELEASE(fd_cb);
2168
2169 done:
2170 mbuf_freem(packet);
2171 return error;
2172 }
2173
2174 static void
2175 flow_divert_close_all(struct flow_divert_group *group)
2176 {
2177 struct flow_divert_pcb *fd_cb;
2178 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2179
2180 SLIST_INIT(&tmp_list);
2181
2182 lck_rw_lock_exclusive(&group->lck);
2183
2184 MBUFQ_DRAIN(&group->send_queue);
2185
2186 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2187 FDRETAIN(fd_cb);
2188 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2189 }
2190
2191 lck_rw_done(&group->lck);
2192
2193 while (!SLIST_EMPTY(&tmp_list)) {
2194 fd_cb = SLIST_FIRST(&tmp_list);
2195 FDLOCK(fd_cb);
2196 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2197 if (fd_cb->so != NULL) {
2198 socket_lock(fd_cb->so, 0);
2199 flow_divert_pcb_remove(fd_cb);
2200 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2201 fd_cb->so->so_error = ECONNABORTED;
2202 socket_unlock(fd_cb->so, 0);
2203 }
2204 FDUNLOCK(fd_cb);
2205 FDRELEASE(fd_cb);
2206 }
2207 }
2208
2209 void
2210 flow_divert_detach(struct socket *so)
2211 {
2212 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2213
2214 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2215
2216 so->so_flags &= ~SOF_FLOW_DIVERT;
2217 so->so_fd_pcb = NULL;
2218
2219 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2220
2221 if (fd_cb->group != NULL) {
2222 /* Last-ditch effort to send any buffered data */
2223 flow_divert_send_buffered_data(fd_cb, TRUE);
2224
2225 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2226 flow_divert_send_close_if_needed(fd_cb);
2227 /* Remove from the group */
2228 flow_divert_pcb_remove(fd_cb);
2229 }
2230
2231 socket_unlock(so, 0);
2232 FDLOCK(fd_cb);
2233 fd_cb->so = NULL;
2234 FDUNLOCK(fd_cb);
2235 socket_lock(so, 0);
2236
2237 FDRELEASE(fd_cb); /* Release the socket's reference */
2238 }
2239
2240 static int
2241 flow_divert_close(struct socket *so)
2242 {
2243 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2244
2245 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2246
2247 FDLOG0(LOG_INFO, fd_cb, "Closing");
2248
2249 if (SOCK_TYPE(so) == SOCK_STREAM) {
2250 soisdisconnecting(so);
2251 sbflush(&so->so_rcv);
2252 }
2253
2254 flow_divert_send_buffered_data(fd_cb, TRUE);
2255 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2256 flow_divert_send_close_if_needed(fd_cb);
2257
2258 /* Remove from the group */
2259 flow_divert_pcb_remove(fd_cb);
2260
2261 return 0;
2262 }
2263
2264 static int
2265 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2266 sae_connid_t cid __unused)
2267 {
2268 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2269 return (EINVAL);
2270 }
2271
2272 return (flow_divert_close(so));
2273 }
2274
2275 static int
2276 flow_divert_shutdown(struct socket *so)
2277 {
2278 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2279
2280 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2281
2282 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2283
2284 socantsendmore(so);
2285
2286 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2287 flow_divert_send_close_if_needed(fd_cb);
2288
2289 return 0;
2290 }
2291
2292 static int
2293 flow_divert_rcvd(struct socket *so, int flags __unused)
2294 {
2295 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2296 uint32_t latest_sb_size;
2297 uint32_t read_count;
2298
2299 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2300
2301 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2302
2303 if (fd_cb->sb_size < latest_sb_size) {
2304 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2305 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2306 }
2307
2308 read_count = fd_cb->sb_size - latest_sb_size;
2309
2310 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2311
2312 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2313 fd_cb->bytes_read_by_app += read_count;
2314 fd_cb->sb_size = latest_sb_size;
2315 }
2316
2317 return 0;
2318 }
2319
2320 static int
2321 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2322 {
2323 int error = 0;
2324 int port = 0;
2325
2326 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2327 if (error) {
2328 goto done;
2329 }
2330
2331 if (toaddr->sa_family == AF_INET) {
2332 port = ntohs((satosin(toaddr))->sin_port);
2333 }
2334 #if INET6
2335 else {
2336 port = ntohs((satosin6(toaddr))->sin6_port);
2337 }
2338 #endif
2339
2340 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2341 if (error) {
2342 goto done;
2343 }
2344
2345 done:
2346 return error;
2347 }
2348
2349 struct sockaddr *
2350 flow_divert_get_buffered_target_address(mbuf_t buffer)
2351 {
2352 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2353 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2354 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2355 return toaddr;
2356 }
2357 }
2358 return NULL;
2359 }
2360
2361 static boolean_t
2362 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2363 {
2364 switch(addr->sa_family)
2365 {
2366 case AF_INET:
2367 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2368 return FALSE;
2369 }
2370 break;
2371 #if INET6
2372 case AF_INET6:
2373 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2374 return FALSE;
2375 }
2376 break;
2377 #endif /* INET6 */
2378 default:
2379 return FALSE;
2380 }
2381 return TRUE;
2382 }
2383
2384 static errno_t
2385 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2386 {
2387 int error = 0;
2388 union sockaddr_in_4_6 sin46;
2389
2390 bzero(&sin46, sizeof(sin46));
2391 if (inp->inp_vflag & INP_IPV4) {
2392 struct sockaddr_in *sin = &sin46.sin;
2393
2394 sin->sin_family = AF_INET;
2395 sin->sin_len = sizeof(*sin);
2396 sin->sin_port = inp->inp_lport;
2397 sin->sin_addr = inp->inp_laddr;
2398 } else if (inp->inp_vflag & INP_IPV6) {
2399 struct sockaddr_in6 *sin6 = &sin46.sin6;
2400
2401 sin6->sin6_len = sizeof(*sin6);
2402 sin6->sin6_family = AF_INET6;
2403 sin6->sin6_port = inp->inp_lport;
2404 sin6->sin6_addr = inp->in6p_laddr;
2405 }
2406 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2407 if (*local_socket == NULL) {
2408 error = ENOBUFS;
2409 }
2410 return (error);
2411 }
2412
2413 static boolean_t
2414 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2415 {
2416 return (inp->inp_lport != 0
2417 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2418 }
2419
2420 static errno_t
2421 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2422 struct sockaddr **dup)
2423 {
2424 int error = 0;
2425 struct sockaddr *result;
2426 struct sockaddr_storage ss;
2427
2428 if (addr != NULL) {
2429 result = addr;
2430 } else {
2431 memset(&ss, 0, sizeof(ss));
2432 ss.ss_family = family;
2433 if (ss.ss_family == AF_INET) {
2434 ss.ss_len = sizeof(struct sockaddr_in);
2435 }
2436 #if INET6
2437 else if (ss.ss_family == AF_INET6) {
2438 ss.ss_len = sizeof(struct sockaddr_in6);
2439 }
2440 #endif /* INET6 */
2441 else {
2442 error = EINVAL;
2443 }
2444 result = (struct sockaddr *)&ss;
2445 }
2446
2447 if (!error) {
2448 *dup = dup_sockaddr(result, 1);
2449 if (*dup == NULL) {
2450 error = ENOBUFS;
2451 }
2452 }
2453
2454 return error;
2455 }
2456
2457 static void
2458 flow_divert_disconnect_socket(struct socket *so)
2459 {
2460 soisdisconnected(so);
2461 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2462 struct inpcb *inp = NULL;
2463
2464 inp = sotoinpcb(so);
2465 if (inp != NULL) {
2466 #if INET6
2467 if (SOCK_CHECK_DOM(so, PF_INET6))
2468 in6_pcbdetach(inp);
2469 else
2470 #endif /* INET6 */
2471 in_pcbdetach(inp);
2472 }
2473 }
2474 }
2475
2476 static errno_t
2477 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2478 {
2479 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2480
2481 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2482
2483 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2484 fd_cb->remote_address,
2485 sa);
2486 }
2487
2488 static errno_t
2489 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2490 {
2491 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2492
2493 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2494
2495 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2496 fd_cb->local_address,
2497 sa);
2498 }
2499
2500 static errno_t
2501 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2502 {
2503 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2504
2505 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2506
2507 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2508 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2509 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2510 }
2511 }
2512
2513 if (SOCK_DOM(so) == PF_INET) {
2514 return g_tcp_protosw->pr_ctloutput(so, sopt);
2515 }
2516 #if INET6
2517 else if (SOCK_DOM(so) == PF_INET6) {
2518 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2519 }
2520 #endif
2521 return 0;
2522 }
2523
2524 errno_t
2525 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2526 {
2527 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2528 int error = 0;
2529 struct inpcb *inp = sotoinpcb(so);
2530 struct sockaddr_in *sinp;
2531 mbuf_t connect_packet = NULL;
2532 char *signing_id = NULL;
2533 int free_signing_id = 0;
2534
2535 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2536
2537 if (fd_cb->group == NULL) {
2538 error = ENETUNREACH;
2539 goto done;
2540 }
2541
2542 if (inp == NULL) {
2543 error = EINVAL;
2544 goto done;
2545 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2546 if (so->so_error) {
2547 error = so->so_error;
2548 so->so_error = 0;
2549 } else {
2550 error = EINVAL;
2551 }
2552 goto done;
2553 }
2554
2555 sinp = (struct sockaddr_in *)(void *)to;
2556 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2557 error = EAFNOSUPPORT;
2558 goto done;
2559 }
2560
2561 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2562 error = EALREADY;
2563 goto done;
2564 }
2565
2566 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2567 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2568 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2569 if (fd_cb->remote_address != NULL) {
2570 soisconnected(fd_cb->so);
2571 goto done;
2572 }
2573 }
2574
2575 if (fd_cb->local_address != NULL) {
2576 error = EALREADY;
2577 goto done;
2578 } else {
2579 if (flow_divert_has_pcb_local_address(inp)) {
2580 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
2581 if (error) {
2582 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
2583 goto done;
2584 }
2585 }
2586 }
2587
2588
2589 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
2590 if (error) {
2591 goto done;
2592 }
2593
2594 error = EPERM;
2595
2596 if (fd_cb->connect_token != NULL) {
2597 uint32_t sid_size = 0;
2598 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2599 if (find_error == 0 && sid_size > 0) {
2600 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
2601 if (signing_id != NULL) {
2602 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
2603 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
2604 free_signing_id = 1;
2605 }
2606 }
2607 }
2608
2609 socket_unlock(so, 0);
2610 if (g_signing_id_trie.root != NULL_TRIE_IDX) {
2611 proc_t src_proc = p;
2612 int release_proc = 0;
2613
2614 if (signing_id == NULL) {
2615 release_proc = flow_divert_get_src_proc(so, &src_proc, FALSE);
2616 if (src_proc != PROC_NULL) {
2617 proc_lock(src_proc);
2618 if (src_proc->p_csflags & CS_VALID) {
2619 const char * cs_id;
2620 cs_id = cs_identity_get(src_proc);
2621 signing_id = __DECONST(char *, cs_id);
2622 } else {
2623 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
2624 }
2625 } else {
2626 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
2627 }
2628 } else {
2629 src_proc = PROC_NULL;
2630 }
2631
2632 if (signing_id != NULL) {
2633 uint16_t result = NULL_TRIE_IDX;
2634 lck_rw_lock_shared(&g_flow_divert_group_lck);
2635 result = flow_divert_trie_search(&g_signing_id_trie, (uint8_t *)signing_id);
2636 lck_rw_done(&g_flow_divert_group_lck);
2637 if (result != NULL_TRIE_IDX) {
2638 error = 0;
2639 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
2640
2641 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
2642 if (error == 0) {
2643 if (src_proc != PROC_NULL) {
2644 unsigned char cdhash[SHA1_RESULTLEN];
2645 error = proc_getcdhash(src_proc, cdhash);
2646 if (error == 0) {
2647 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
2648 if (error) {
2649 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
2650 }
2651 } else {
2652 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
2653 }
2654 }
2655 } else {
2656 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
2657 }
2658 } else {
2659 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
2660 }
2661 } else {
2662 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
2663 }
2664
2665 if (src_proc != PROC_NULL) {
2666 proc_unlock(src_proc);
2667 if (release_proc) {
2668 proc_rele(src_proc);
2669 }
2670 }
2671 } else {
2672 FDLOG0(LOG_WARNING, fd_cb, "The signing ID trie is empty");
2673 }
2674 socket_lock(so, 0);
2675
2676 if (free_signing_id) {
2677 FREE(signing_id, M_TEMP);
2678 }
2679
2680 if (error) {
2681 goto done;
2682 }
2683
2684 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2685
2686 error = flow_divert_send_connect(fd_cb, to, connect_packet);
2687 if (error) {
2688 goto done;
2689 }
2690
2691 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2692
2693 soisconnecting(so);
2694
2695 done:
2696 if (error && connect_packet != NULL) {
2697 mbuf_freem(connect_packet);
2698 }
2699 return error;
2700 }
2701
2702 static int
2703 flow_divert_connectx_out_common(struct socket *so, int af,
2704 struct sockaddr_list **src_sl, struct sockaddr_list **dst_sl,
2705 struct proc *p, uint32_t ifscope __unused, sae_associd_t aid __unused,
2706 sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2707 uint32_t arglen __unused)
2708 {
2709 struct sockaddr_entry *src_se = NULL, *dst_se = NULL;
2710 struct inpcb *inp = sotoinpcb(so);
2711 int error;
2712
2713 if (inp == NULL) {
2714 return (EINVAL);
2715 }
2716
2717 VERIFY(dst_sl != NULL);
2718
2719 /* select source (if specified) and destination addresses */
2720 error = in_selectaddrs(af, src_sl, &src_se, dst_sl, &dst_se);
2721 if (error != 0) {
2722 return (error);
2723 }
2724
2725 VERIFY(*dst_sl != NULL && dst_se != NULL);
2726 VERIFY(src_se == NULL || *src_sl != NULL);
2727 VERIFY(dst_se->se_addr->sa_family == af);
2728 VERIFY(src_se == NULL || src_se->se_addr->sa_family == af);
2729
2730 error = flow_divert_connect_out(so, dst_se->se_addr, p);
2731
2732 if (error == 0 && pcid != NULL) {
2733 *pcid = 1; /* there is only 1 connection for a TCP */
2734 }
2735
2736 return (error);
2737 }
2738
2739 static int
2740 flow_divert_connectx_out(struct socket *so, struct sockaddr_list **src_sl,
2741 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2742 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2743 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
2744 {
2745 #pragma unused(uio, bytes_written)
2746 return (flow_divert_connectx_out_common(so, AF_INET, src_sl, dst_sl,
2747 p, ifscope, aid, pcid, flags, arg, arglen));
2748 }
2749
2750 #if INET6
2751 static int
2752 flow_divert_connectx6_out(struct socket *so, struct sockaddr_list **src_sl,
2753 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2754 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2755 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
2756 {
2757 #pragma unused(uio, bytes_written)
2758 return (flow_divert_connectx_out_common(so, AF_INET6, src_sl, dst_sl,
2759 p, ifscope, aid, pcid, flags, arg, arglen));
2760 }
2761 #endif /* INET6 */
2762
2763 static int
2764 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
2765 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2766 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2767 user_addr_t aux_data __unused, uint32_t *aux_len)
2768 {
2769 int error = 0;
2770 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2771 struct ifnet *ifp = NULL;
2772 struct inpcb *inp = sotoinpcb(so);
2773
2774 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2775
2776 if (so->so_fd_pcb == NULL || inp == NULL) {
2777 error = EINVAL;
2778 goto out;
2779 }
2780
2781 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
2782 error = EINVAL;
2783 goto out;
2784 }
2785
2786 ifp = inp->inp_last_outifp;
2787 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2788 *soerror = so->so_error;
2789 *flags = 0;
2790
2791 if (so->so_state & SS_ISCONNECTED) {
2792 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2793 }
2794
2795 if (fd_cb->local_address == NULL) {
2796 struct sockaddr_in sin;
2797 bzero(&sin, sizeof(sin));
2798 sin.sin_len = sizeof(sin);
2799 sin.sin_family = AF_INET;
2800 *src_len = sin.sin_len;
2801 if (src != USER_ADDR_NULL) {
2802 error = copyout(&sin, src, sin.sin_len);
2803 if (error != 0) {
2804 goto out;
2805 }
2806 }
2807 } else {
2808 *src_len = fd_cb->local_address->sa_len;
2809 if (src != USER_ADDR_NULL) {
2810 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2811 if (error != 0) {
2812 goto out;
2813 }
2814 }
2815 }
2816
2817 if (fd_cb->remote_address == NULL) {
2818 struct sockaddr_in sin;
2819 bzero(&sin, sizeof(sin));
2820 sin.sin_len = sizeof(sin);
2821 sin.sin_family = AF_INET;
2822 *dst_len = sin.sin_len;
2823 if (dst != USER_ADDR_NULL) {
2824 error = copyout(&sin, dst, sin.sin_len);
2825 if (error != 0) {
2826 goto out;
2827 }
2828 }
2829 } else {
2830 *dst_len = fd_cb->remote_address->sa_len;
2831 if (dst != USER_ADDR_NULL) {
2832 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2833 if (error != 0) {
2834 goto out;
2835 }
2836 }
2837 }
2838
2839 *aux_type = 0;
2840 *aux_len = 0;
2841
2842 out:
2843 return error;
2844 }
2845
2846 static int
2847 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
2848 {
2849 int error = 0;
2850
2851 switch (cmd) {
2852 case SIOCGCONNINFO32: {
2853 struct so_cinforeq32 cifr;
2854 bcopy(data, &cifr, sizeof (cifr));
2855 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
2856 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
2857 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
2858 &cifr.scir_aux_type, cifr.scir_aux_data,
2859 &cifr.scir_aux_len);
2860 if (error == 0) {
2861 bcopy(&cifr, data, sizeof (cifr));
2862 }
2863 break;
2864 }
2865
2866 case SIOCGCONNINFO64: {
2867 struct so_cinforeq64 cifr;
2868 bcopy(data, &cifr, sizeof (cifr));
2869 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
2870 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
2871 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
2872 &cifr.scir_aux_type, cifr.scir_aux_data,
2873 &cifr.scir_aux_len);
2874 if (error == 0) {
2875 bcopy(&cifr, data, sizeof (cifr));
2876 }
2877 break;
2878 }
2879
2880 default:
2881 error = EOPNOTSUPP;
2882 }
2883
2884 return error;
2885 }
2886
2887 static int
2888 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
2889 {
2890 int error = flow_divert_control(so, cmd, data, ifp, p);
2891
2892 if (error == EOPNOTSUPP) {
2893 error = in_control(so, cmd, data, ifp, p);
2894 }
2895
2896 return error;
2897 }
2898
2899 static int
2900 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
2901 {
2902 int error = flow_divert_control(so, cmd, data, ifp, p);
2903
2904 if (error == EOPNOTSUPP) {
2905 error = in6_control(so, cmd, data, ifp, p);
2906 }
2907
2908 return error;
2909 }
2910
2911 static errno_t
2912 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p __unused)
2913 {
2914 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2915 int error = 0;
2916 struct inpcb *inp;
2917
2918 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2919
2920 inp = sotoinpcb(so);
2921 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
2922 error = ECONNRESET;
2923 goto done;
2924 }
2925
2926 if (control && mbuf_len(control) > 0) {
2927 error = EINVAL;
2928 goto done;
2929 }
2930
2931 if (flags & MSG_OOB) {
2932 error = EINVAL;
2933 goto done; /* We don't support OOB data */
2934 }
2935
2936 error = flow_divert_check_no_cellular(fd_cb) ||
2937 flow_divert_check_no_expensive(fd_cb);
2938 if (error) {
2939 goto done;
2940 }
2941
2942 /* Implicit connect */
2943 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
2944 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
2945 error = flow_divert_connect_out(so, to, NULL);
2946 if (error) {
2947 goto done;
2948 }
2949 }
2950
2951 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
2952
2953 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
2954 error = flow_divert_send_app_data(fd_cb, data, to);
2955 if (error) {
2956 goto done;
2957 }
2958
2959 data = NULL;
2960
2961 if (flags & PRUS_EOF) {
2962 flow_divert_shutdown(so);
2963 }
2964
2965 done:
2966 if (data) {
2967 mbuf_freem(data);
2968 }
2969 if (control) {
2970 mbuf_free(control);
2971 }
2972 return error;
2973 }
2974
2975 static void
2976 flow_divert_set_protosw(struct socket *so)
2977 {
2978 so->so_flags |= SOF_FLOW_DIVERT;
2979 if (SOCK_DOM(so) == PF_INET) {
2980 so->so_proto = &g_flow_divert_in_protosw;
2981 }
2982 #if INET6
2983 else {
2984 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
2985 }
2986 #endif /* INET6 */
2987 }
2988
2989 static void
2990 flow_divert_set_udp_protosw(struct socket *so)
2991 {
2992 so->so_flags |= SOF_FLOW_DIVERT;
2993 if (SOCK_DOM(so) == PF_INET) {
2994 so->so_proto = &g_flow_divert_in_udp_protosw;
2995 }
2996 #if INET6
2997 else {
2998 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
2999 }
3000 #endif /* INET6 */
3001 }
3002
3003 static errno_t
3004 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3005 {
3006 int error = 0;
3007 struct flow_divert_pcb *fd_cb = NULL;
3008 struct ifnet *ifp = NULL;
3009 struct inpcb *inp = NULL;
3010 struct socket *old_so;
3011 mbuf_t recv_data = NULL;
3012
3013 socket_unlock(so, 0);
3014
3015 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3016
3017 /* Find the flow divert control block */
3018 lck_rw_lock_shared(&g_flow_divert_group_lck);
3019 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3020 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3021 if (group != NULL) {
3022 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3023 }
3024 }
3025 lck_rw_done(&g_flow_divert_group_lck);
3026
3027 if (fd_cb == NULL) {
3028 error = ENOENT;
3029 goto done;
3030 }
3031
3032 FDLOCK(fd_cb);
3033
3034 /* Dis-associate the flow divert control block from its current socket */
3035 old_so = fd_cb->so;
3036
3037 inp = sotoinpcb(old_so);
3038
3039 VERIFY(inp != NULL);
3040
3041 socket_lock(old_so, 0);
3042 flow_divert_disconnect_socket(old_so);
3043 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3044 old_so->so_fd_pcb = NULL;
3045 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3046 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3047 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3048 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3049 }
3050 fd_cb->so = NULL;
3051 /* Save the output interface */
3052 ifp = inp->inp_last_outifp;
3053 if (old_so->so_rcv.sb_cc > 0) {
3054 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3055 sbflush(&old_so->so_rcv);
3056 }
3057 socket_unlock(old_so, 0);
3058
3059 /* Associate the new socket with the flow divert control block */
3060 socket_lock(so, 0);
3061 so->so_fd_pcb = fd_cb;
3062 inp = sotoinpcb(so);
3063 inp->inp_last_outifp = ifp;
3064 if (recv_data != NULL) {
3065 if (sbappendstream(&so->so_rcv, recv_data)) {
3066 sorwakeup(so);
3067 }
3068 }
3069 flow_divert_set_protosw(so);
3070 socket_unlock(so, 0);
3071
3072 fd_cb->so = so;
3073 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3074
3075 FDUNLOCK(fd_cb);
3076
3077 done:
3078 socket_lock(so, 0);
3079
3080 if (fd_cb != NULL) {
3081 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3082 }
3083
3084 return error;
3085 }
3086
3087 errno_t
3088 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3089 {
3090 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3091 struct inpcb *inp;
3092 int error = 0;
3093
3094 inp = sotoinpcb(so);
3095 if (inp == NULL) {
3096 return (EINVAL);
3097 }
3098
3099 if (fd_cb == NULL) {
3100 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3101 if (fd_ctl_unit > 0) {
3102 error = flow_divert_pcb_init(so, fd_ctl_unit);
3103 fd_cb = so->so_fd_pcb;
3104 if (error != 0 || fd_cb == NULL) {
3105 goto done;
3106 }
3107 } else {
3108 error = ENETDOWN;
3109 goto done;
3110 }
3111 }
3112 return flow_divert_data_out(so, flags, data, to, control, p);
3113
3114 done:
3115 if (data) {
3116 mbuf_freem(data);
3117 }
3118 if (control) {
3119 mbuf_free(control);
3120 }
3121
3122 return error;
3123 }
3124
3125 errno_t
3126 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3127 {
3128 errno_t error = 0;
3129 struct flow_divert_pcb *fd_cb;
3130
3131 if (so->so_flags & SOF_FLOW_DIVERT) {
3132 return EALREADY;
3133 }
3134
3135 fd_cb = flow_divert_pcb_create(so);
3136 if (fd_cb != NULL) {
3137 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3138 if (error) {
3139 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3140 FDRELEASE(fd_cb);
3141 } else {
3142 fd_cb->control_group_unit = ctl_unit;
3143 so->so_fd_pcb = fd_cb;
3144
3145 if (SOCK_TYPE(so) == SOCK_STREAM) {
3146 flow_divert_set_protosw(so);
3147 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3148 flow_divert_set_udp_protosw(so);
3149 }
3150
3151 FDLOG0(LOG_INFO, fd_cb, "Created");
3152 }
3153 } else {
3154 error = ENOMEM;
3155 }
3156
3157 return error;
3158 }
3159
3160 errno_t
3161 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3162 {
3163 uint32_t ctl_unit = 0;
3164 uint32_t key_unit = 0;
3165 uint32_t flow_id = 0;
3166 int error = 0;
3167 mbuf_t token = NULL;
3168
3169 if (so->so_flags & SOF_FLOW_DIVERT) {
3170 error = EALREADY;
3171 goto done;
3172 }
3173
3174 if (g_init_result) {
3175 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3176 error = ENOPROTOOPT;
3177 goto done;
3178 }
3179
3180 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3181 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3182 (SOCK_DOM(so) != PF_INET
3183 #if INET6
3184 && SOCK_DOM(so) != PF_INET6
3185 #endif
3186 ))
3187 {
3188 error = EINVAL;
3189 goto done;
3190 } else {
3191 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3192 struct tcpcb *tp = sototcpcb(so);
3193 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3194 error = EINVAL;
3195 goto done;
3196 }
3197 }
3198 }
3199
3200 error = soopt_getm(sopt, &token);
3201 if (error) {
3202 goto done;
3203 }
3204
3205 error = soopt_mcopyin(sopt, token);
3206 if (error) {
3207 goto done;
3208 }
3209
3210 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3211 if (!error) {
3212 key_unit = ntohl(key_unit);
3213 } else if (error != ENOENT) {
3214 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3215 goto done;
3216 } else {
3217 key_unit = 0;
3218 }
3219
3220 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3221 if (error) {
3222 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3223 goto done;
3224 }
3225
3226 /* A valid kernel control unit is required */
3227 ctl_unit = ntohl(ctl_unit);
3228 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3229 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3230 error = EINVAL;
3231 goto done;
3232 }
3233
3234 socket_unlock(so, 0);
3235 error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3236 socket_lock(so, 0);
3237
3238 if (error) {
3239 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", error);
3240 goto done;
3241 }
3242
3243 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3244 if (error && error != ENOENT) {
3245 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3246 goto done;
3247 }
3248
3249 if (flow_id == 0) {
3250 error = flow_divert_pcb_init(so, ctl_unit);
3251 if (error == 0) {
3252 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3253 int log_level = LOG_NOTICE;
3254
3255 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3256 sizeof(log_level), &log_level, NULL);
3257 if (error == 0) {
3258 fd_cb->log_level = log_level;
3259 }
3260 error = 0;
3261
3262 fd_cb->connect_token = token;
3263 token = NULL;
3264 }
3265 } else {
3266 error = flow_divert_attach(so, flow_id, ctl_unit);
3267 }
3268
3269 done:
3270 if (token != NULL) {
3271 mbuf_freem(token);
3272 }
3273
3274 return error;
3275 }
3276
3277 errno_t
3278 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3279 {
3280 uint32_t ctl_unit;
3281 int error = 0;
3282 uint8_t hmac[SHA_DIGEST_LENGTH];
3283 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3284 mbuf_t token = NULL;
3285 struct flow_divert_group *control_group = NULL;
3286
3287 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3288 error = EINVAL;
3289 goto done;
3290 }
3291
3292 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3293
3294 if (fd_cb->group == NULL) {
3295 error = EINVAL;
3296 goto done;
3297 }
3298
3299 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3300 if (error) {
3301 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3302 goto done;
3303 }
3304
3305 ctl_unit = htonl(fd_cb->group->ctl_unit);
3306
3307 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3308 if (error) {
3309 goto done;
3310 }
3311
3312 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3313 if (error) {
3314 goto done;
3315 }
3316
3317 socket_unlock(so, 0);
3318 lck_rw_lock_shared(&g_flow_divert_group_lck);
3319
3320 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3321 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3322 {
3323 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3324 }
3325
3326 if (control_group != NULL) {
3327 lck_rw_lock_shared(&control_group->lck);
3328 ctl_unit = htonl(control_group->ctl_unit);
3329 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3330 if (!error) {
3331 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3332 }
3333 lck_rw_done(&control_group->lck);
3334 } else {
3335 error = ENOPROTOOPT;
3336 }
3337
3338 lck_rw_done(&g_flow_divert_group_lck);
3339 socket_lock(so, 0);
3340
3341 if (error) {
3342 goto done;
3343 }
3344
3345 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3346 if (error) {
3347 goto done;
3348 }
3349
3350 error = soopt_mcopyout(sopt, token);
3351 if (error) {
3352 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3353 goto done;
3354 }
3355
3356 done:
3357 if (token != NULL) {
3358 mbuf_freem(token);
3359 }
3360
3361 return error;
3362 }
3363
3364 static errno_t
3365 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3366 {
3367 struct flow_divert_group *new_group = NULL;
3368 int error = 0;
3369
3370 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3371 error = EINVAL;
3372 goto done;
3373 }
3374
3375 *unitinfo = NULL;
3376
3377 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3378 if (new_group == NULL) {
3379 error = ENOBUFS;
3380 goto done;
3381 }
3382
3383 memset(new_group, 0, sizeof(*new_group));
3384
3385 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3386 RB_INIT(&new_group->pcb_tree);
3387 new_group->ctl_unit = sac->sc_unit;
3388 MBUFQ_INIT(&new_group->send_queue);
3389
3390 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3391
3392 if (g_flow_divert_groups == NULL) {
3393 MALLOC(g_flow_divert_groups,
3394 struct flow_divert_group **,
3395 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3396 M_TEMP,
3397 M_WAITOK | M_ZERO);
3398 }
3399
3400 if (g_flow_divert_groups == NULL) {
3401 error = ENOBUFS;
3402 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3403 error = EALREADY;
3404 } else {
3405 g_flow_divert_groups[sac->sc_unit] = new_group;
3406 g_active_group_count++;
3407 }
3408
3409 lck_rw_done(&g_flow_divert_group_lck);
3410
3411 *unitinfo = new_group;
3412
3413 done:
3414 if (error != 0 && new_group != NULL) {
3415 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3416 }
3417 return error;
3418 }
3419
3420 static errno_t
3421 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3422 {
3423 struct flow_divert_group *group = NULL;
3424 errno_t error = 0;
3425 uint16_t node = 0;
3426
3427 if (unit >= GROUP_COUNT_MAX) {
3428 return EINVAL;
3429 }
3430
3431 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3432
3433 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3434
3435 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3436 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3437 g_flow_divert_groups, g_active_group_count);
3438 }
3439
3440 group = g_flow_divert_groups[unit];
3441
3442 if (group != (struct flow_divert_group *)unitinfo) {
3443 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3444 }
3445
3446 if (group != NULL) {
3447 flow_divert_close_all(group);
3448 if (group->token_key != NULL) {
3449 memset(group->token_key, 0, group->token_key_size);
3450 FREE(group->token_key, M_TEMP);
3451 group->token_key = NULL;
3452 group->token_key_size = 0;
3453 }
3454 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3455 g_flow_divert_groups[unit] = NULL;
3456 g_active_group_count--;
3457 } else {
3458 error = EINVAL;
3459 }
3460
3461 if (g_active_group_count == 0) {
3462 FREE(g_flow_divert_groups, M_TEMP);
3463 g_flow_divert_groups = NULL;
3464 }
3465
3466 /* Remove all signing IDs that point to this unit */
3467 for (node = 0; node < g_signing_id_trie.nodes_count; node++) {
3468 if (TRIE_NODE(&g_signing_id_trie, node).group_unit == unit) {
3469 TRIE_NODE(&g_signing_id_trie, node).group_unit = 0;
3470 }
3471 }
3472
3473 lck_rw_done(&g_flow_divert_group_lck);
3474
3475 return error;
3476 }
3477
3478 static errno_t
3479 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3480 {
3481 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3482 }
3483
3484 static void
3485 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3486 {
3487 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3488
3489 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3490 struct flow_divert_pcb *fd_cb;
3491 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3492
3493 lck_rw_lock_shared(&g_flow_divert_group_lck);
3494 lck_rw_lock_exclusive(&group->lck);
3495
3496 while (!MBUFQ_EMPTY(&group->send_queue)) {
3497 mbuf_t next_packet;
3498 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3499 next_packet = MBUFQ_FIRST(&group->send_queue);
3500 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3501 if (error) {
3502 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3503 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3504 lck_rw_done(&group->lck);
3505 lck_rw_done(&g_flow_divert_group_lck);
3506 return;
3507 }
3508 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3509 }
3510
3511 SLIST_INIT(&tmp_list);
3512
3513 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3514 FDRETAIN(fd_cb);
3515 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3516 }
3517
3518 lck_rw_done(&group->lck);
3519
3520 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3521 FDLOCK(fd_cb);
3522 if (fd_cb->so != NULL) {
3523 socket_lock(fd_cb->so, 0);
3524 if (fd_cb->group != NULL) {
3525 flow_divert_send_buffered_data(fd_cb, FALSE);
3526 }
3527 socket_unlock(fd_cb->so, 0);
3528 }
3529 FDUNLOCK(fd_cb);
3530 FDRELEASE(fd_cb);
3531 }
3532
3533 lck_rw_done(&g_flow_divert_group_lck);
3534 }
3535 }
3536
3537 static int
3538 flow_divert_kctl_init(void)
3539 {
3540 struct kern_ctl_reg ctl_reg;
3541 int result;
3542
3543 memset(&ctl_reg, 0, sizeof(ctl_reg));
3544
3545 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3546 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3547 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3548 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3549 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3550
3551 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3552 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3553 ctl_reg.ctl_send = flow_divert_kctl_send;
3554 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3555
3556 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3557
3558 if (result) {
3559 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3560 return result;
3561 }
3562
3563 return 0;
3564 }
3565
3566 void
3567 flow_divert_init(void)
3568 {
3569 memset(&nil_pcb, 0, sizeof(nil_pcb));
3570 nil_pcb.log_level = LOG_NOTICE;
3571
3572 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3573
3574 VERIFY(g_tcp_protosw != NULL);
3575
3576 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3577 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3578
3579 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3580 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3581 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3582 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3583 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3584 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3585 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3586 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3587 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3588 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3589
3590 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3591 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3592
3593 /*
3594 * Socket filters shouldn't attach/detach to/from this protosw
3595 * since pr_protosw is to be used instead, which points to the
3596 * real protocol; if they do, it is a bug and we should panic.
3597 */
3598 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3599 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3600 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3601 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3602
3603 /* UDP */
3604 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3605 VERIFY(g_udp_protosw != NULL);
3606
3607 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3608 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3609
3610 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3611 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3612 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3613 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3614 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3615 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3616 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3617 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3618 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3619 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3620 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3621 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3622
3623 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3624 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3625
3626 /*
3627 * Socket filters shouldn't attach/detach to/from this protosw
3628 * since pr_protosw is to be used instead, which points to the
3629 * real protocol; if they do, it is a bug and we should panic.
3630 */
3631 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3632 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3633 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3634 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3635
3636 #if INET6
3637 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3638
3639 VERIFY(g_tcp6_protosw != NULL);
3640
3641 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3642 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3643
3644 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3645 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3646 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3647 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3648 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3649 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3650 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3651 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3652 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3653 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3654
3655 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3656 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3657 /*
3658 * Socket filters shouldn't attach/detach to/from this protosw
3659 * since pr_protosw is to be used instead, which points to the
3660 * real protocol; if they do, it is a bug and we should panic.
3661 */
3662 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3663 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3664 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3665 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3666
3667 /* UDP6 */
3668 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3669
3670 VERIFY(g_udp6_protosw != NULL);
3671
3672 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3673 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3674
3675 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3676 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3677 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3678 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3679 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3680 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3681 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3682 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3683 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3684 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3685 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3686 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3687
3688 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3689 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3690 /*
3691 * Socket filters shouldn't attach/detach to/from this protosw
3692 * since pr_protosw is to be used instead, which points to the
3693 * real protocol; if they do, it is a bug and we should panic.
3694 */
3695 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3696 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3697 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3698 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3699 #endif /* INET6 */
3700
3701 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3702 if (flow_divert_grp_attr == NULL) {
3703 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3704 g_init_result = ENOMEM;
3705 goto done;
3706 }
3707
3708 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3709 if (flow_divert_mtx_grp == NULL) {
3710 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3711 g_init_result = ENOMEM;
3712 goto done;
3713 }
3714
3715 flow_divert_mtx_attr = lck_attr_alloc_init();
3716 if (flow_divert_mtx_attr == NULL) {
3717 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3718 g_init_result = ENOMEM;
3719 goto done;
3720 }
3721
3722 g_init_result = flow_divert_kctl_init();
3723 if (g_init_result) {
3724 goto done;
3725 }
3726
3727 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3728
3729 memset(&g_signing_id_trie, 0, sizeof(g_signing_id_trie));
3730 g_signing_id_trie.root = NULL_TRIE_IDX;
3731
3732 done:
3733 if (g_init_result != 0) {
3734 if (flow_divert_mtx_attr != NULL) {
3735 lck_attr_free(flow_divert_mtx_attr);
3736 flow_divert_mtx_attr = NULL;
3737 }
3738 if (flow_divert_mtx_grp != NULL) {
3739 lck_grp_free(flow_divert_mtx_grp);
3740 flow_divert_mtx_grp = NULL;
3741 }
3742 if (flow_divert_grp_attr != NULL) {
3743 lck_grp_attr_free(flow_divert_grp_attr);
3744 flow_divert_grp_attr = NULL;
3745 }
3746
3747 if (g_flow_divert_kctl_ref != NULL) {
3748 ctl_deregister(g_flow_divert_kctl_ref);
3749 g_flow_divert_kctl_ref = NULL;
3750 }
3751 }
3752 }
mirror of XNU