]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:


13b9cad5d7ff1637fb7d40f064de93a33f40cc6f
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
59 #if INET6
60 #include <netinet6/in6_pcb.h>
61 #include <netinet6/ip6protosw.h>
62 #endif /* INET6 */
63 #include <dev/random/randomdev.h>
64 #include <libkern/crypto/sha1.h>
65 #include <libkern/crypto/crypto_internal.h>
66 #include <os/log.h>
67
68 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69 #define FLOW_DIVERT_READ_CLOSED 0x00000002
70 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73 #define FLOW_DIVERT_TRANSFERRED 0x00000020
74 #define FLOW_DIVERT_HAS_HMAC 0x00000040
75
76 #define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
78
79 #define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
81
82 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83 #define FDRELEASE(pcb) \
84 do { \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
87 } \
88 } while (0)
89
90 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
92
93 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
94 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
95
96 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
97
98 #define GROUP_COUNT_MAX 32
99 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
100 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
101 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
102
103 struct flow_divert_trie_node {
104 uint16_t start;
105 uint16_t length;
106 uint16_t child_map;
107 };
108
109 #define CHILD_MAP_SIZE 256
110 #define NULL_TRIE_IDX 0xffff
111 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
112 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
113 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
114
115 static struct flow_divert_pcb nil_pcb;
116
117 decl_lck_rw_data(static, g_flow_divert_group_lck);
118 static struct flow_divert_group **g_flow_divert_groups = NULL;
119 static uint32_t g_active_group_count = 0;
120
121 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
122 static lck_attr_t *flow_divert_mtx_attr = NULL;
123 static lck_grp_t *flow_divert_mtx_grp = NULL;
124 static errno_t g_init_result = 0;
125
126 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
127
128 static struct protosw g_flow_divert_in_protosw;
129 static struct pr_usrreqs g_flow_divert_in_usrreqs;
130 static struct protosw g_flow_divert_in_udp_protosw;
131 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
132 #if INET6
133 static struct ip6protosw g_flow_divert_in6_protosw;
134 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
135 static struct ip6protosw g_flow_divert_in6_udp_protosw;
136 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
137 #endif /* INET6 */
138
139 static struct protosw *g_tcp_protosw = NULL;
140 static struct ip6protosw *g_tcp6_protosw = NULL;
141 static struct protosw *g_udp_protosw = NULL;
142 static struct ip6protosw *g_udp6_protosw = NULL;
143
144 static errno_t
145 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
146
147 static errno_t
148 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
149
150 static boolean_t
151 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
152
153 static int
154 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
155
156 struct sockaddr *
157 flow_divert_get_buffered_target_address(mbuf_t buffer);
158
159 static boolean_t
160 flow_divert_has_pcb_local_address(const struct inpcb *inp);
161
162 static void
163 flow_divert_disconnect_socket(struct socket *so);
164
165 static inline uint8_t
166 flow_divert_syslog_type_to_oslog_type(int syslog_type)
167 {
168 switch (syslog_type) {
169 case LOG_ERR: return OS_LOG_TYPE_ERROR;
170 case LOG_INFO: return OS_LOG_TYPE_INFO;
171 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
172 default: return OS_LOG_TYPE_DEFAULT;
173 }
174 }
175
176 static inline int
177 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
178 {
179 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
180 }
181
182 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
183 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
184
185 static const char *
186 flow_divert_packet_type2str(uint8_t packet_type)
187 {
188 switch (packet_type) {
189 case FLOW_DIVERT_PKT_CONNECT:
190 return "connect";
191 case FLOW_DIVERT_PKT_CONNECT_RESULT:
192 return "connect result";
193 case FLOW_DIVERT_PKT_DATA:
194 return "data";
195 case FLOW_DIVERT_PKT_CLOSE:
196 return "close";
197 case FLOW_DIVERT_PKT_READ_NOTIFY:
198 return "read notification";
199 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
200 return "properties update";
201 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
202 return "app map create";
203 default:
204 return "unknown";
205 }
206 }
207
208 static struct flow_divert_pcb *
209 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
210 {
211 struct flow_divert_pcb key_item;
212 struct flow_divert_pcb *fd_cb = NULL;
213
214 key_item.hash = hash;
215
216 lck_rw_lock_shared(&group->lck);
217 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
218 FDRETAIN(fd_cb);
219 lck_rw_done(&group->lck);
220
221 return fd_cb;
222 }
223
224 static errno_t
225 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
226 {
227 errno_t error = 0;
228 struct flow_divert_pcb *exist = NULL;
229 struct flow_divert_group *group;
230 static uint32_t g_nextkey = 1;
231 static uint32_t g_hash_seed = 0;
232 int try_count = 0;
233
234 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
235 return EINVAL;
236 }
237
238 socket_unlock(fd_cb->so, 0);
239 lck_rw_lock_shared(&g_flow_divert_group_lck);
240
241 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
242 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
243 error = ENETUNREACH;
244 goto done;
245 }
246
247 group = g_flow_divert_groups[ctl_unit];
248 if (group == NULL) {
249 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
250 error = ENETUNREACH;
251 goto done;
252 }
253
254 socket_lock(fd_cb->so, 0);
255
256 do {
257 uint32_t key[2];
258 uint32_t idx;
259
260 key[0] = g_nextkey++;
261 key[1] = RandomULong();
262
263 if (g_hash_seed == 0) {
264 g_hash_seed = RandomULong();
265 }
266
267 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
268
269 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
270 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
271 if (curr_group != NULL && curr_group != group) {
272 lck_rw_lock_shared(&curr_group->lck);
273 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
274 lck_rw_done(&curr_group->lck);
275 if (exist != NULL) {
276 break;
277 }
278 }
279 }
280
281 if (exist == NULL) {
282 lck_rw_lock_exclusive(&group->lck);
283 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
284 lck_rw_done(&group->lck);
285 }
286 } while (exist != NULL && try_count++ < 3);
287
288 if (exist == NULL) {
289 fd_cb->group = group;
290 FDRETAIN(fd_cb); /* The group now has a reference */
291 } else {
292 fd_cb->hash = 0;
293 error = EEXIST;
294 }
295
296 socket_unlock(fd_cb->so, 0);
297
298 done:
299 lck_rw_done(&g_flow_divert_group_lck);
300 socket_lock(fd_cb->so, 0);
301
302 return error;
303 }
304
305 static struct flow_divert_pcb *
306 flow_divert_pcb_create(socket_t so)
307 {
308 struct flow_divert_pcb *new_pcb = NULL;
309
310 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
311 if (new_pcb == NULL) {
312 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
313 return NULL;
314 }
315
316 memset(new_pcb, 0, sizeof(*new_pcb));
317
318 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
319 new_pcb->so = so;
320 new_pcb->log_level = nil_pcb.log_level;
321
322 FDRETAIN(new_pcb); /* Represents the socket's reference */
323
324 return new_pcb;
325 }
326
327 static void
328 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
329 {
330 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
331 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
332
333 if (fd_cb->local_address != NULL) {
334 FREE(fd_cb->local_address, M_SONAME);
335 }
336 if (fd_cb->remote_address != NULL) {
337 FREE(fd_cb->remote_address, M_SONAME);
338 }
339 if (fd_cb->connect_token != NULL) {
340 mbuf_freem(fd_cb->connect_token);
341 }
342 if (fd_cb->connect_packet != NULL) {
343 mbuf_freem(fd_cb->connect_packet);
344 }
345 if (fd_cb->app_data != NULL) {
346 FREE(fd_cb->app_data, M_TEMP);
347 }
348 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
349 }
350
351 static void
352 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
353 {
354 if (fd_cb->group != NULL) {
355 struct flow_divert_group *group = fd_cb->group;
356 lck_rw_lock_exclusive(&group->lck);
357 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
358 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
359 fd_cb->group = NULL;
360 FDRELEASE(fd_cb); /* Release the group's reference */
361 lck_rw_done(&group->lck);
362 }
363 }
364
365 static int
366 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
367 {
368 struct flow_divert_packet_header hdr;
369 int error = 0;
370
371 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
372 if (error) {
373 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
374 return error;
375 }
376
377 hdr.packet_type = packet_type;
378 hdr.conn_id = htonl(fd_cb->hash);
379
380 /* Lay down the header */
381 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
382 if (error) {
383 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
384 mbuf_freem(*packet);
385 *packet = NULL;
386 return error;
387 }
388
389 return 0;
390 }
391
392 static int
393 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
394 {
395 uint32_t net_length = htonl(length);
396 int error = 0;
397
398 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
399 if (error) {
400 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
401 return error;
402 }
403
404 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
405 if (error) {
406 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
407 return error;
408 }
409
410 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
411 if (error) {
412 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
413 return error;
414 }
415
416 return error;
417 }
418
419 static int
420 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
421 {
422 size_t cursor = offset;
423 int error = 0;
424 uint32_t curr_length;
425 uint8_t curr_type;
426
427 *err = 0;
428
429 do {
430 if (!next) {
431 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
432 if (error) {
433 *err = ENOENT;
434 return -1;
435 }
436 } else {
437 next = 0;
438 curr_type = FLOW_DIVERT_TLV_NIL;
439 }
440
441 if (curr_type != type) {
442 cursor += sizeof(curr_type);
443 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
444 if (error) {
445 *err = error;
446 return -1;
447 }
448
449 cursor += (sizeof(curr_length) + ntohl(curr_length));
450 }
451 } while (curr_type != type);
452
453 return cursor;
454 }
455
456 static int
457 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
458 {
459 int error = 0;
460 uint32_t length;
461 int tlv_offset;
462
463 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
464 if (tlv_offset < 0) {
465 return error;
466 }
467
468 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
469 if (error) {
470 return error;
471 }
472
473 length = ntohl(length);
474
475 if (val_size != NULL) {
476 *val_size = length;
477 }
478
479 if (buff != NULL && buff_len > 0) {
480 size_t to_copy = (length < buff_len) ? length : buff_len;
481 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
482 if (error) {
483 return error;
484 }
485 }
486
487 return 0;
488 }
489
490 static int
491 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
492 {
493 mbuf_t curr_mbuf = packet;
494
495 if (g_crypto_funcs == NULL || group->token_key == NULL) {
496 return ENOPROTOOPT;
497 }
498
499 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
500 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
501
502 while (curr_mbuf != NULL) {
503 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
504 curr_mbuf = mbuf_next(curr_mbuf);
505 }
506
507 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
508
509 return 0;
510 }
511
512 static int
513 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
514 {
515 int error = 0;
516 struct flow_divert_group *group = NULL;
517 int hmac_offset;
518 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
519 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
520 mbuf_t tail;
521
522 lck_rw_lock_shared(&g_flow_divert_group_lck);
523
524 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
525 group = g_flow_divert_groups[ctl_unit];
526 }
527
528 if (group == NULL) {
529 lck_rw_done(&g_flow_divert_group_lck);
530 return ENOPROTOOPT;
531 }
532
533 lck_rw_lock_shared(&group->lck);
534
535 if (group->token_key == NULL) {
536 error = ENOPROTOOPT;
537 goto done;
538 }
539
540 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
541 if (hmac_offset < 0) {
542 goto done;
543 }
544
545 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
546 if (error) {
547 goto done;
548 }
549
550 /* Chop off the HMAC TLV */
551 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
552 if (error) {
553 goto done;
554 }
555
556 mbuf_free(tail);
557
558 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
559 if (error) {
560 goto done;
561 }
562
563 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
564 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
565 error = EINVAL;
566 goto done;
567 }
568
569 done:
570 lck_rw_done(&group->lck);
571 lck_rw_done(&g_flow_divert_group_lck);
572 return error;
573 }
574
575 static void
576 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
577 {
578 struct inpcb *inp = NULL;
579 struct ifnet *ifp = NULL;
580 Boolean cell = FALSE;
581 Boolean wifi = FALSE;
582 Boolean wired = FALSE;
583
584 inp = sotoinpcb(fd_cb->so);
585 if (inp == NULL) {
586 return;
587 }
588
589 ifp = inp->inp_last_outifp;
590 if (ifp != NULL) {
591 cell = IFNET_IS_CELLULAR(ifp);
592 wifi = (!cell && IFNET_IS_WIFI(ifp));
593 wired = (!wifi && IFNET_IS_WIRED(ifp));
594 }
595
596 if (send) {
597 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
598 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
599 } else {
600 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
601 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
602 }
603 inp_set_activity_bitmap(inp);
604 }
605
606 static errno_t
607 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
608 {
609 struct inpcb *inp = NULL;
610
611 inp = sotoinpcb(fd_cb->so);
612 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
613 IFNET_IS_CELLULAR(inp->inp_last_outifp)) {
614 return EHOSTUNREACH;
615 }
616
617 return 0;
618 }
619
620 static errno_t
621 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
622 {
623 struct inpcb *inp = NULL;
624
625 inp = sotoinpcb(fd_cb->so);
626 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
627 IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) {
628 return EHOSTUNREACH;
629 }
630
631 return 0;
632 }
633
634 static void
635 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
636 {
637 if (how != SHUT_RD) {
638 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
639 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
640 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
641 /* If the tunnel is not accepting writes any more, then flush the send buffer */
642 sbflush(&fd_cb->so->so_snd);
643 }
644 }
645 if (how != SHUT_WR) {
646 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
647 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
648 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
649 }
650 }
651 }
652
653 static uint16_t
654 trie_node_alloc(struct flow_divert_trie *trie)
655 {
656 if (trie->nodes_free_next < trie->nodes_count) {
657 uint16_t node_idx = trie->nodes_free_next++;
658 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
659 return node_idx;
660 } else {
661 return NULL_TRIE_IDX;
662 }
663 }
664
665 static uint16_t
666 trie_child_map_alloc(struct flow_divert_trie *trie)
667 {
668 if (trie->child_maps_free_next < trie->child_maps_count) {
669 return trie->child_maps_free_next++;
670 } else {
671 return NULL_TRIE_IDX;
672 }
673 }
674
675 static uint16_t
676 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
677 {
678 uint16_t start = trie->bytes_free_next;
679 if (start + bytes_size <= trie->bytes_count) {
680 if (start != bytes_idx) {
681 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
682 }
683 trie->bytes_free_next += bytes_size;
684 return start;
685 } else {
686 return NULL_TRIE_IDX;
687 }
688 }
689
690 static uint16_t
691 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
692 {
693 uint16_t current = trie->root;
694 uint16_t child = trie->root;
695 uint16_t string_end = string_start + string_len;
696 uint16_t string_idx = string_start;
697 uint16_t string_remainder = string_len;
698
699 while (child != NULL_TRIE_IDX) {
700 uint16_t parent = current;
701 uint16_t node_idx;
702 uint16_t current_end;
703
704 current = child;
705 child = NULL_TRIE_IDX;
706
707 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
708
709 for (node_idx = TRIE_NODE(trie, current).start;
710 node_idx < current_end &&
711 string_idx < string_end &&
712 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
713 node_idx++, string_idx++) {
714 ;
715 }
716
717 string_remainder = string_end - string_idx;
718
719 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
720 /*
721 * We did not reach the end of the current node's string.
722 * We need to split the current node into two:
723 * 1. A new node that contains the prefix of the node that matches
724 * the prefix of the string being inserted.
725 * 2. The current node modified to point to the remainder
726 * of the current node's string.
727 */
728 uint16_t prefix = trie_node_alloc(trie);
729 if (prefix == NULL_TRIE_IDX) {
730 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
731 return NULL_TRIE_IDX;
732 }
733
734 /*
735 * Prefix points to the portion of the current nodes's string that has matched
736 * the input string thus far.
737 */
738 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
739 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
740
741 /*
742 * Prefix has the current node as the child corresponding to the first byte
743 * after the split.
744 */
745 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
746 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
747 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
748 return NULL_TRIE_IDX;
749 }
750 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
751
752 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
753 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
754
755 /* Current node is adjusted to point to the remainder */
756 TRIE_NODE(trie, current).start = node_idx;
757 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
758
759 /* We want to insert the new leaf (if any) as a child of the prefix */
760 current = prefix;
761 }
762
763 if (string_remainder > 0) {
764 /*
765 * We still have bytes in the string that have not been matched yet.
766 * If the current node has children, iterate to the child corresponding
767 * to the next byte in the string.
768 */
769 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
770 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
771 }
772 }
773 } /* while (child != NULL_TRIE_IDX) */
774
775 if (string_remainder > 0) {
776 /* Add a new leaf containing the remainder of the string */
777 uint16_t leaf = trie_node_alloc(trie);
778 if (leaf == NULL_TRIE_IDX) {
779 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
780 return NULL_TRIE_IDX;
781 }
782
783 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
784 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
785 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
786 return NULL_TRIE_IDX;
787 }
788 TRIE_NODE(trie, leaf).length = string_remainder;
789
790 /* Set the new leaf as the child of the current node */
791 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
792 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
793 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
794 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
795 return NULL_TRIE_IDX;
796 }
797 }
798 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
799 current = leaf;
800 } /* else duplicate or this string is a prefix of one of the existing strings */
801
802 return current;
803 }
804
805 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
806 static uint16_t
807 flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
808 {
809 uint16_t current = trie->root;
810 uint16_t string_idx = 0;
811
812 while (current != NULL_TRIE_IDX) {
813 uint16_t next = NULL_TRIE_IDX;
814 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
815 uint16_t node_idx;
816
817 for (node_idx = TRIE_NODE(trie, current).start;
818 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
819 node_idx++, string_idx++) {
820 ;
821 }
822
823 if (node_idx == node_end) {
824 if (string_bytes[string_idx] == '\0') {
825 return current; /* Got an exact match */
826 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
827 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
828 string_bytes[string_idx] = '\0';
829 return current; /* Got an apple webclip id prefix match */
830 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
831 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
832 }
833 }
834 current = next;
835 }
836
837 return NULL_TRIE_IDX;
838 }
839
840 struct uuid_search_info {
841 uuid_t target_uuid;
842 char *found_signing_id;
843 boolean_t found_multiple_signing_ids;
844 proc_t found_proc;
845 };
846
847 static int
848 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
849 {
850 struct uuid_search_info *info = (struct uuid_search_info *)arg;
851 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
852
853 if (info->found_signing_id != NULL) {
854 if (!info->found_multiple_signing_ids) {
855 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
856 info->found_proc = p;
857 result = PROC_CLAIMED_DONE;
858 } else {
859 uuid_string_t uuid_str;
860 uuid_unparse(info->target_uuid, uuid_str);
861 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
862 }
863 FREE(info->found_signing_id, M_TEMP);
864 info->found_signing_id = NULL;
865 }
866
867 if (result == PROC_RETURNED_DONE) {
868 uuid_string_t uuid_str;
869 uuid_unparse(info->target_uuid, uuid_str);
870 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
871 }
872
873 return result;
874 }
875
876 static int
877 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
878 {
879 struct uuid_search_info *info = (struct uuid_search_info *)arg;
880 int include = 0;
881
882 if (info->found_multiple_signing_ids) {
883 return include;
884 }
885
886 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
887 if (include) {
888 const char *signing_id = cs_identity_get(p);
889 if (signing_id != NULL) {
890 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
891 size_t signing_id_size = strlen(signing_id) + 1;
892 if (info->found_signing_id == NULL) {
893 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
894 memcpy(info->found_signing_id, signing_id, signing_id_size);
895 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
896 info->found_multiple_signing_ids = TRUE;
897 }
898 } else {
899 info->found_multiple_signing_ids = TRUE;
900 }
901 include = !info->found_multiple_signing_ids;
902 }
903
904 return include;
905 }
906
907 static proc_t
908 flow_divert_find_proc_by_uuid(uuid_t uuid)
909 {
910 struct uuid_search_info info;
911
912 if (LOG_INFO <= nil_pcb.log_level) {
913 uuid_string_t uuid_str;
914 uuid_unparse(uuid, uuid_str);
915 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
916 }
917
918 memset(&info, 0, sizeof(info));
919 info.found_proc = PROC_NULL;
920 uuid_copy(info.target_uuid, uuid);
921
922 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
923
924 return info.found_proc;
925 }
926
927 static int
928 flow_divert_get_src_proc(struct socket *so, proc_t *proc)
929 {
930 int release = 0;
931
932 if (so->so_flags & SOF_DELEGATED) {
933 if ((*proc)->p_pid != so->e_pid) {
934 *proc = proc_find(so->e_pid);
935 release = 1;
936 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
937 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
938 release = 1;
939 }
940 } else if (*proc == PROC_NULL) {
941 *proc = current_proc();
942 }
943
944 if (*proc != PROC_NULL) {
945 if ((*proc)->p_pid == 0) {
946 if (release) {
947 proc_rele(*proc);
948 }
949 release = 0;
950 *proc = PROC_NULL;
951 }
952 }
953
954 return release;
955 }
956
957 static int
958 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
959 {
960 int error;
961
962 if (fd_cb->group == NULL) {
963 fd_cb->so->so_error = ECONNABORTED;
964 flow_divert_disconnect_socket(fd_cb->so);
965 return ECONNABORTED;
966 }
967
968 lck_rw_lock_shared(&fd_cb->group->lck);
969
970 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
971 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
972 } else {
973 error = ENOBUFS;
974 }
975
976 if (error == ENOBUFS) {
977 if (enqueue) {
978 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
979 lck_rw_lock_exclusive(&fd_cb->group->lck);
980 }
981 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
982 error = 0;
983 }
984 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
985 }
986
987 lck_rw_done(&fd_cb->group->lck);
988
989 return error;
990 }
991
992 static int
993 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
994 {
995 int error = 0;
996 int flow_type = 0;
997 char *signing_id = NULL;
998 int free_signing_id = 0;
999 mbuf_t connect_packet = NULL;
1000 proc_t src_proc = p;
1001 int release_proc = 0;
1002
1003 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1004 if (error) {
1005 goto done;
1006 }
1007
1008 error = EPERM;
1009
1010 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1011 uint32_t sid_size = 0;
1012 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1013 if (find_error == 0 && sid_size > 0) {
1014 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1015 if (signing_id != NULL) {
1016 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1017 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1018 free_signing_id = 1;
1019 }
1020 }
1021 }
1022
1023 socket_unlock(so, 0);
1024
1025 if (signing_id == NULL) {
1026 release_proc = flow_divert_get_src_proc(so, &src_proc);
1027 if (src_proc != PROC_NULL) {
1028 proc_lock(src_proc);
1029 if (src_proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
1030 const char * cs_id;
1031 cs_id = cs_identity_get(src_proc);
1032 signing_id = __DECONST(char *, cs_id);
1033 } else {
1034 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
1035 }
1036 } else {
1037 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
1038 }
1039 } else {
1040 src_proc = PROC_NULL;
1041 }
1042
1043 if (signing_id != NULL) {
1044 uint16_t result = NULL_TRIE_IDX;
1045 lck_rw_lock_shared(&fd_cb->group->lck);
1046 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1047 result = 1;
1048 } else {
1049 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1050 }
1051 lck_rw_done(&fd_cb->group->lck);
1052 if (result != NULL_TRIE_IDX) {
1053 error = 0;
1054 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
1055
1056 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1057 if (error == 0) {
1058 if (src_proc != PROC_NULL) {
1059 unsigned char cdhash[SHA1_RESULTLEN];
1060 error = proc_getcdhash(src_proc, cdhash);
1061 if (error == 0) {
1062 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1063 if (error) {
1064 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1065 }
1066 } else {
1067 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
1068 }
1069 }
1070 } else {
1071 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1072 }
1073 } else {
1074 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
1075 }
1076 } else {
1077 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1078 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1079 error = 0;
1080 }
1081 }
1082
1083 if (src_proc != PROC_NULL) {
1084 proc_unlock(src_proc);
1085 if (release_proc) {
1086 proc_rele(src_proc);
1087 }
1088 }
1089 socket_lock(so, 0);
1090
1091 if (free_signing_id) {
1092 FREE(signing_id, M_TEMP);
1093 }
1094
1095 if (error) {
1096 goto done;
1097 }
1098
1099 error = flow_divert_packet_append_tlv(connect_packet,
1100 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1101 sizeof(fd_cb->so->so_traffic_class),
1102 &fd_cb->so->so_traffic_class);
1103 if (error) {
1104 goto done;
1105 }
1106
1107 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1108 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1109 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1110 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1111 } else {
1112 error = EINVAL;
1113 goto done;
1114 }
1115 error = flow_divert_packet_append_tlv(connect_packet,
1116 FLOW_DIVERT_TLV_FLOW_TYPE,
1117 sizeof(flow_type),
1118 &flow_type);
1119
1120 if (error) {
1121 goto done;
1122 }
1123
1124 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1125 error = flow_divert_packet_append_tlv(connect_packet,
1126 FLOW_DIVERT_TLV_PID,
1127 sizeof(fd_cb->so->e_pid),
1128 &fd_cb->so->e_pid);
1129 if (error) {
1130 goto done;
1131 }
1132
1133 error = flow_divert_packet_append_tlv(connect_packet,
1134 FLOW_DIVERT_TLV_UUID,
1135 sizeof(fd_cb->so->e_uuid),
1136 &fd_cb->so->e_uuid);
1137 if (error) {
1138 goto done;
1139 }
1140 } else {
1141 error = flow_divert_packet_append_tlv(connect_packet,
1142 FLOW_DIVERT_TLV_PID,
1143 sizeof(fd_cb->so->e_pid),
1144 &fd_cb->so->last_pid);
1145 if (error) {
1146 goto done;
1147 }
1148
1149 error = flow_divert_packet_append_tlv(connect_packet,
1150 FLOW_DIVERT_TLV_UUID,
1151 sizeof(fd_cb->so->e_uuid),
1152 &fd_cb->so->last_uuid);
1153 if (error) {
1154 goto done;
1155 }
1156 }
1157
1158 if (fd_cb->connect_token != NULL) {
1159 unsigned int token_len = m_length(fd_cb->connect_token);
1160 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1161 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1162 fd_cb->connect_token = NULL;
1163 } else {
1164 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
1165
1166 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1167 if (error) {
1168 goto done;
1169 }
1170
1171 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1172 if (error) {
1173 goto done;
1174 }
1175 }
1176
1177 if (fd_cb->local_address != NULL) {
1178 error = EALREADY;
1179 goto done;
1180 } else {
1181 struct inpcb *inp = sotoinpcb(so);
1182 if (flow_divert_has_pcb_local_address(inp)) {
1183 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1184 if (error) {
1185 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1186 goto done;
1187 }
1188 }
1189 }
1190
1191 if (fd_cb->local_address != NULL) {
1192 /* socket is bound. */
1193 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1194 fd_cb->local_address->sa_len, fd_cb->local_address);
1195 if (error) {
1196 goto done;
1197 }
1198 }
1199
1200 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1201 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1202 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1203 if (error) {
1204 goto done;
1205 }
1206 }
1207
1208 done:
1209 if (!error) {
1210 *out_connect_packet = connect_packet;
1211 } else if (connect_packet != NULL) {
1212 mbuf_freem(connect_packet);
1213 }
1214
1215 return error;
1216 }
1217
1218 static int
1219 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1220 {
1221 int error = 0;
1222 mbuf_t packet = NULL;
1223 int rbuff_space = 0;
1224
1225 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1226 if (error) {
1227 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1228 goto done;
1229 }
1230
1231 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1232 if (rbuff_space < 0) {
1233 rbuff_space = 0;
1234 }
1235 rbuff_space = htonl(rbuff_space);
1236 error = flow_divert_packet_append_tlv(packet,
1237 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1238 sizeof(rbuff_space),
1239 &rbuff_space);
1240 if (error) {
1241 goto done;
1242 }
1243
1244 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1245 if (error) {
1246 goto done;
1247 }
1248
1249 done:
1250 if (error && packet != NULL) {
1251 mbuf_freem(packet);
1252 }
1253
1254 return error;
1255 }
1256
1257 static int
1258 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1259 {
1260 int error = 0;
1261 mbuf_t packet = NULL;
1262 uint32_t zero = 0;
1263
1264 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1265 if (error) {
1266 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1267 goto done;
1268 }
1269
1270 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1271 if (error) {
1272 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1273 goto done;
1274 }
1275
1276 how = htonl(how);
1277 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1278 if (error) {
1279 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1280 goto done;
1281 }
1282
1283 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1284 if (error) {
1285 goto done;
1286 }
1287
1288 done:
1289 if (error && packet != NULL) {
1290 mbuf_free(packet);
1291 }
1292
1293 return error;
1294 }
1295
1296 static int
1297 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1298 {
1299 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1300 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1301 return SHUT_RDWR;
1302 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1303 return SHUT_RD;
1304 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1305 return SHUT_WR;
1306 }
1307
1308 return -1;
1309 }
1310
1311 /*
1312 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1313 * writes. Returns FALSE otherwise.
1314 */
1315 static void
1316 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1317 {
1318 int how = -1;
1319
1320 /* Do not send any close messages if there is still data in the send buffer */
1321 if (fd_cb->so->so_snd.sb_cc == 0) {
1322 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1323 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1324 how = SHUT_RD;
1325 }
1326 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1327 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1328 if (how == SHUT_RD) {
1329 how = SHUT_RDWR;
1330 } else {
1331 how = SHUT_WR;
1332 }
1333 }
1334 }
1335
1336 if (how != -1) {
1337 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1338 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1339 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1340 if (how != SHUT_RD) {
1341 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1342 }
1343 if (how != SHUT_WR) {
1344 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1345 }
1346 }
1347 }
1348
1349 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1350 flow_divert_disconnect_socket(fd_cb->so);
1351 }
1352 }
1353
1354 static errno_t
1355 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1356 {
1357 mbuf_t packet;
1358 mbuf_t last;
1359 int error = 0;
1360
1361 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1362 if (error) {
1363 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1364 return error;
1365 }
1366
1367 if (toaddr != NULL) {
1368 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1369 if (error) {
1370 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1371 return error;
1372 }
1373 }
1374
1375 if (data_len > 0 && data != NULL) {
1376 last = m_last(packet);
1377 mbuf_setnext(last, data);
1378 mbuf_pkthdr_adjustlen(packet, data_len);
1379 }
1380 error = flow_divert_send_packet(fd_cb, packet, force);
1381
1382 if (error) {
1383 mbuf_setnext(last, NULL);
1384 mbuf_freem(packet);
1385 } else {
1386 fd_cb->bytes_sent += data_len;
1387 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1388 }
1389
1390 return error;
1391 }
1392
1393 static void
1394 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1395 {
1396 size_t to_send;
1397 size_t sent = 0;
1398 int error = 0;
1399 mbuf_t buffer;
1400
1401 to_send = fd_cb->so->so_snd.sb_cc;
1402 buffer = fd_cb->so->so_snd.sb_mb;
1403
1404 if (buffer == NULL && to_send > 0) {
1405 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1406 return;
1407 }
1408
1409 /* Ignore the send window if force is enabled */
1410 if (!force && (to_send > fd_cb->send_window)) {
1411 to_send = fd_cb->send_window;
1412 }
1413
1414 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1415 while (sent < to_send) {
1416 mbuf_t data;
1417 size_t data_len;
1418
1419 data_len = to_send - sent;
1420 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1421 data_len = FLOW_DIVERT_CHUNK_SIZE;
1422 }
1423
1424 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1425 if (error) {
1426 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1427 break;
1428 }
1429
1430 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1431 if (error) {
1432 mbuf_freem(data);
1433 break;
1434 }
1435
1436 sent += data_len;
1437 }
1438 sbdrop(&fd_cb->so->so_snd, sent);
1439 sowwakeup(fd_cb->so);
1440 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1441 mbuf_t data;
1442 mbuf_t m;
1443 size_t data_len;
1444
1445 while (buffer) {
1446 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1447
1448 m = buffer;
1449 if (toaddr != NULL) {
1450 /* look for data in the chain */
1451 do {
1452 m = m->m_next;
1453 if (m != NULL && m->m_type == MT_DATA) {
1454 break;
1455 }
1456 } while (m);
1457 if (m == NULL) {
1458 /* unexpected */
1459 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1460 goto move_on;
1461 }
1462 }
1463 data_len = mbuf_pkthdr_len(m);
1464 if (data_len > 0) {
1465 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1466 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1467 if (error) {
1468 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1469 break;
1470 }
1471 } else {
1472 data = NULL;
1473 }
1474 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1475 if (error) {
1476 mbuf_freem(data);
1477 break;
1478 }
1479 sent += data_len;
1480 move_on:
1481 buffer = buffer->m_nextpkt;
1482 (void) sbdroprecord(&(fd_cb->so->so_snd));
1483 }
1484 }
1485
1486 if (sent > 0) {
1487 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1488 if (fd_cb->send_window >= sent) {
1489 fd_cb->send_window -= sent;
1490 } else {
1491 fd_cb->send_window = 0;
1492 }
1493 }
1494 }
1495
1496 static int
1497 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1498 {
1499 size_t to_send = mbuf_pkthdr_len(data);
1500 int error = 0;
1501
1502 if (to_send > fd_cb->send_window) {
1503 to_send = fd_cb->send_window;
1504 }
1505
1506 if (fd_cb->so->so_snd.sb_cc > 0) {
1507 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1508 }
1509
1510 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1511 size_t sent = 0;
1512 mbuf_t remaining_data = data;
1513 mbuf_t pkt_data = NULL;
1514 while (sent < to_send && remaining_data != NULL) {
1515 size_t pkt_data_len;
1516
1517 pkt_data = remaining_data;
1518
1519 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1520 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1521 } else {
1522 pkt_data_len = to_send - sent;
1523 }
1524
1525 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1526 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1527 if (error) {
1528 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1529 pkt_data = NULL;
1530 break;
1531 }
1532 } else {
1533 remaining_data = NULL;
1534 }
1535
1536 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1537
1538 if (error) {
1539 break;
1540 }
1541
1542 pkt_data = NULL;
1543 sent += pkt_data_len;
1544 }
1545
1546 fd_cb->send_window -= sent;
1547
1548 error = 0;
1549
1550 if (pkt_data != NULL) {
1551 if (sbspace(&fd_cb->so->so_snd) > 0) {
1552 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1553 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1554 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1555 }
1556 } else {
1557 error = ENOBUFS;
1558 }
1559 }
1560
1561 if (remaining_data != NULL) {
1562 if (sbspace(&fd_cb->so->so_snd) > 0) {
1563 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1564 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1565 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1566 }
1567 } else {
1568 error = ENOBUFS;
1569 }
1570 }
1571 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1572 if (to_send || mbuf_pkthdr_len(data) == 0) {
1573 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1574 if (error) {
1575 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1576 } else {
1577 fd_cb->send_window -= to_send;
1578 }
1579 } else {
1580 /* buffer it */
1581 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1582 if (toaddr != NULL) {
1583 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1584 FDLOG(LOG_ERR, fd_cb,
1585 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1586 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1587 }
1588 } else {
1589 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1590 FDLOG(LOG_ERR, fd_cb,
1591 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1592 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1593 }
1594 }
1595 } else {
1596 error = ENOBUFS;
1597 }
1598 }
1599 }
1600
1601 return error;
1602 }
1603
1604 static int
1605 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1606 {
1607 int error = 0;
1608 mbuf_t packet = NULL;
1609 uint32_t net_read_count = htonl(read_count);
1610
1611 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1612 if (error) {
1613 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1614 goto done;
1615 }
1616
1617 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1618 if (error) {
1619 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1620 goto done;
1621 }
1622
1623 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1624 if (error) {
1625 goto done;
1626 }
1627
1628 done:
1629 if (error && packet != NULL) {
1630 mbuf_free(packet);
1631 }
1632
1633 return error;
1634 }
1635
1636 static int
1637 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1638 {
1639 int error = 0;
1640 mbuf_t packet = NULL;
1641
1642 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1643 if (error) {
1644 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1645 goto done;
1646 }
1647
1648 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1649 if (error) {
1650 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1651 goto done;
1652 }
1653
1654 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1655 if (error) {
1656 goto done;
1657 }
1658
1659 done:
1660 if (error && packet != NULL) {
1661 mbuf_free(packet);
1662 }
1663
1664 return error;
1665 }
1666
1667 static void
1668 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1669 {
1670 uint32_t connect_error;
1671 uint32_t ctl_unit = 0;
1672 int error = 0;
1673 struct flow_divert_group *grp = NULL;
1674 struct sockaddr_storage local_address;
1675 int out_if_index = 0;
1676 struct sockaddr_storage remote_address;
1677 uint32_t send_window;
1678 uint32_t app_data_length = 0;
1679
1680 memset(&local_address, 0, sizeof(local_address));
1681 memset(&remote_address, 0, sizeof(remote_address));
1682
1683 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1684 if (error) {
1685 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1686 return;
1687 }
1688
1689 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1690
1691 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1692 if (error) {
1693 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1694 return;
1695 }
1696
1697 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1698 if (error) {
1699 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
1700 }
1701
1702 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1703 if (error) {
1704 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1705 }
1706
1707 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1708 if (error) {
1709 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1710 }
1711
1712 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1713 if (error) {
1714 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1715 }
1716
1717 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1718 if (error) {
1719 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
1720 }
1721
1722 error = 0;
1723 connect_error = ntohl(connect_error);
1724 ctl_unit = ntohl(ctl_unit);
1725
1726 lck_rw_lock_shared(&g_flow_divert_group_lck);
1727
1728 if (connect_error == 0 && ctl_unit > 0) {
1729 if (ctl_unit >= GROUP_COUNT_MAX) {
1730 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1731 error = EINVAL;
1732 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1733 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1734 error = EINVAL;
1735 } else {
1736 grp = g_flow_divert_groups[ctl_unit];
1737 if (grp == NULL) {
1738 error = ECONNRESET;
1739 }
1740 }
1741 }
1742
1743 FDLOCK(fd_cb);
1744 if (fd_cb->so != NULL) {
1745 struct inpcb *inp = NULL;
1746 struct ifnet *ifp = NULL;
1747 struct flow_divert_group *old_group;
1748
1749 socket_lock(fd_cb->so, 0);
1750
1751 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1752 goto done;
1753 }
1754
1755 inp = sotoinpcb(fd_cb->so);
1756
1757 if (connect_error || error) {
1758 goto set_socket_state;
1759 }
1760
1761 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1762 error = EINVAL;
1763 goto set_socket_state;
1764 }
1765 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1766 if (local_address.ss_len > sizeof(local_address)) {
1767 local_address.ss_len = sizeof(local_address);
1768 }
1769 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1770 }
1771
1772 if (remote_address.ss_family != 0) {
1773 if (remote_address.ss_len > sizeof(remote_address)) {
1774 remote_address.ss_len = sizeof(remote_address);
1775 }
1776 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1777 } else {
1778 error = EINVAL;
1779 goto set_socket_state;
1780 }
1781
1782 if (app_data_length > 0) {
1783 uint8_t *app_data = NULL;
1784 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1785 if (app_data != NULL) {
1786 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1787 if (error == 0) {
1788 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1789 if (fd_cb->app_data != NULL) {
1790 FREE(fd_cb->app_data, M_TEMP);
1791 }
1792 fd_cb->app_data = app_data;
1793 fd_cb->app_data_length = app_data_length;
1794 } else {
1795 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1796 FREE(app_data, M_TEMP);
1797 }
1798 } else {
1799 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1800 }
1801 }
1802
1803 ifnet_head_lock_shared();
1804 if (out_if_index > 0 && out_if_index <= if_index) {
1805 ifp = ifindex2ifnet[out_if_index];
1806 }
1807
1808 if (ifp != NULL) {
1809 inp->inp_last_outifp = ifp;
1810 } else {
1811 error = EINVAL;
1812 }
1813 ifnet_head_done();
1814
1815 if (error) {
1816 goto set_socket_state;
1817 }
1818
1819 if (fd_cb->group == NULL) {
1820 error = EINVAL;
1821 goto set_socket_state;
1822 }
1823
1824 if (grp != NULL) {
1825 old_group = fd_cb->group;
1826
1827 lck_rw_lock_exclusive(&old_group->lck);
1828 lck_rw_lock_exclusive(&grp->lck);
1829
1830 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1831 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1832 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1833 }
1834
1835 fd_cb->group = grp;
1836
1837 lck_rw_done(&grp->lck);
1838 lck_rw_done(&old_group->lck);
1839 }
1840
1841 fd_cb->send_window = ntohl(send_window);
1842
1843 set_socket_state:
1844 if (!connect_error && !error) {
1845 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1846 error = flow_divert_send_connect_result(fd_cb);
1847 }
1848
1849 if (connect_error || error) {
1850 if (!connect_error) {
1851 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1852 fd_cb->so->so_error = error;
1853 flow_divert_send_close_if_needed(fd_cb);
1854 } else {
1855 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1856 fd_cb->so->so_error = connect_error;
1857 }
1858 flow_divert_disconnect_socket(fd_cb->so);
1859 } else {
1860 flow_divert_send_buffered_data(fd_cb, FALSE);
1861 soisconnected(fd_cb->so);
1862 }
1863
1864 done:
1865 socket_unlock(fd_cb->so, 0);
1866 }
1867 FDUNLOCK(fd_cb);
1868
1869 lck_rw_done(&g_flow_divert_group_lck);
1870 }
1871
1872 static void
1873 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1874 {
1875 uint32_t close_error;
1876 int error = 0;
1877 int how;
1878
1879 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1880 if (error) {
1881 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1882 return;
1883 }
1884
1885 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1886 if (error) {
1887 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1888 return;
1889 }
1890
1891 how = ntohl(how);
1892
1893 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1894
1895 FDLOCK(fd_cb);
1896 if (fd_cb->so != NULL) {
1897 socket_lock(fd_cb->so, 0);
1898
1899 fd_cb->so->so_error = ntohl(close_error);
1900
1901 flow_divert_update_closed_state(fd_cb, how, TRUE);
1902
1903 how = flow_divert_tunnel_how_closed(fd_cb);
1904 if (how == SHUT_RDWR) {
1905 flow_divert_disconnect_socket(fd_cb->so);
1906 } else if (how == SHUT_RD) {
1907 socantrcvmore(fd_cb->so);
1908 } else if (how == SHUT_WR) {
1909 socantsendmore(fd_cb->so);
1910 }
1911
1912 socket_unlock(fd_cb->so, 0);
1913 }
1914 FDUNLOCK(fd_cb);
1915 }
1916
1917 static mbuf_t
1918 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
1919 {
1920 if (fd_cb->local_address != NULL) {
1921 struct inpcb *inp = sotoinpcb(fd_cb->so);
1922 if ((inp->inp_vflag & INP_IPV4) &&
1923 (inp->inp_flags & INP_RECVDSTADDR) &&
1924 fd_cb->local_address->sa_family == AF_INET &&
1925 fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in)) {
1926 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
1927
1928 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1929 } else if ((inp->inp_vflag & INP_IPV6) &&
1930 (inp->inp_flags & IN6P_PKTINFO) &&
1931 fd_cb->local_address->sa_family == AF_INET6 &&
1932 fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in6)) {
1933 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1934 struct in6_pktinfo pi6;
1935
1936 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof(struct in6_addr));
1937 pi6.ipi6_ifindex = 0;
1938 return sbcreatecontrol((caddr_t)&pi6, sizeof(struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
1939 }
1940 }
1941 return NULL;
1942 }
1943
1944 static void
1945 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1946 {
1947 FDLOCK(fd_cb);
1948 if (fd_cb->so != NULL) {
1949 int error = 0;
1950 mbuf_t data = NULL;
1951 size_t data_size;
1952 struct sockaddr_storage remote_address;
1953 boolean_t got_remote_sa = FALSE;
1954
1955 socket_lock(fd_cb->so, 0);
1956
1957 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1958 uint32_t val_size = 0;
1959
1960 /* check if we got remote address with data */
1961 memset(&remote_address, 0, sizeof(remote_address));
1962 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1963 if (error || val_size > sizeof(remote_address)) {
1964 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1965 error = 0;
1966 } else {
1967 /* validate the address */
1968 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1969 got_remote_sa = TRUE;
1970 }
1971 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1972 }
1973 }
1974
1975 data_size = (mbuf_pkthdr_len(packet) - offset);
1976
1977 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1978
1979 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1980 if (error || data == NULL) {
1981 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1982 } else {
1983 if (flow_divert_check_no_cellular(fd_cb) ||
1984 flow_divert_check_no_expensive(fd_cb)) {
1985 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1986 flow_divert_send_close(fd_cb, SHUT_RDWR);
1987 flow_divert_disconnect_socket(fd_cb->so);
1988 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1989 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1990 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1991 fd_cb->bytes_received += data_size;
1992 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1993 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1994 sorwakeup(fd_cb->so);
1995 data = NULL;
1996 } else {
1997 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1998 }
1999 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2000 struct sockaddr *append_sa;
2001 mbuf_t mctl;
2002
2003 if (got_remote_sa == TRUE) {
2004 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2005 (struct sockaddr *)&remote_address, &append_sa);
2006 } else {
2007 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2008 fd_cb->remote_address, &append_sa);
2009 }
2010 if (error) {
2011 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2012 }
2013
2014 mctl = flow_divert_get_control_mbuf(fd_cb);
2015 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
2016 fd_cb->bytes_received += data_size;
2017 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2018 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2019 sorwakeup(fd_cb->so);
2020 data = NULL;
2021 } else {
2022 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
2023 }
2024 if (!error) {
2025 FREE(append_sa, M_TEMP);
2026 }
2027 }
2028 }
2029 }
2030 socket_unlock(fd_cb->so, 0);
2031 }
2032 FDUNLOCK(fd_cb);
2033 }
2034
2035 static void
2036 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2037 {
2038 uint32_t read_count;
2039 int error = 0;
2040
2041 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2042 if (error) {
2043 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2044 return;
2045 }
2046
2047 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2048
2049 FDLOCK(fd_cb);
2050 if (fd_cb->so != NULL) {
2051 socket_lock(fd_cb->so, 0);
2052 fd_cb->send_window += ntohl(read_count);
2053 flow_divert_send_buffered_data(fd_cb, FALSE);
2054 socket_unlock(fd_cb->so, 0);
2055 }
2056 FDUNLOCK(fd_cb);
2057 }
2058
2059 static void
2060 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2061 {
2062 int error = 0;
2063 uint32_t key_size = 0;
2064 int log_level;
2065 uint32_t flags = 0;
2066
2067 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2068 if (error) {
2069 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2070 return;
2071 }
2072
2073 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2074 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2075 return;
2076 }
2077
2078 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2079 if (!error) {
2080 nil_pcb.log_level = log_level;
2081 }
2082
2083 lck_rw_lock_exclusive(&group->lck);
2084
2085 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2086 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2087 if (error) {
2088 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2089 FREE(group->token_key, M_TEMP);
2090 group->token_key = NULL;
2091 lck_rw_done(&group->lck);
2092 return;
2093 }
2094
2095 group->token_key_size = key_size;
2096
2097 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2098 if (!error) {
2099 group->flags = flags;
2100 }
2101
2102 lck_rw_done(&group->lck);
2103 }
2104
2105 static void
2106 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2107 {
2108 int error = 0;
2109 struct sockaddr_storage local_address;
2110 int out_if_index = 0;
2111 struct sockaddr_storage remote_address;
2112 uint32_t app_data_length = 0;
2113
2114 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2115
2116 memset(&local_address, 0, sizeof(local_address));
2117 memset(&remote_address, 0, sizeof(remote_address));
2118
2119 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2120 if (error) {
2121 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
2122 }
2123
2124 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2125 if (error) {
2126 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
2127 }
2128
2129 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2130 if (error) {
2131 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2132 }
2133
2134 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2135 if (error) {
2136 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2137 }
2138
2139 FDLOCK(fd_cb);
2140 if (fd_cb->so != NULL) {
2141 socket_lock(fd_cb->so, 0);
2142
2143 if (local_address.ss_family != 0) {
2144 if (local_address.ss_len > sizeof(local_address)) {
2145 local_address.ss_len = sizeof(local_address);
2146 }
2147 if (fd_cb->local_address != NULL) {
2148 FREE(fd_cb->local_address, M_SONAME);
2149 fd_cb->local_address = NULL;
2150 }
2151 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2152 }
2153
2154 if (remote_address.ss_family != 0) {
2155 if (remote_address.ss_len > sizeof(remote_address)) {
2156 remote_address.ss_len = sizeof(remote_address);
2157 }
2158 if (fd_cb->remote_address != NULL) {
2159 FREE(fd_cb->remote_address, M_SONAME);
2160 fd_cb->remote_address = NULL;
2161 }
2162 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2163 }
2164
2165 if (out_if_index > 0) {
2166 struct inpcb *inp = NULL;
2167 struct ifnet *ifp = NULL;
2168
2169 inp = sotoinpcb(fd_cb->so);
2170
2171 ifnet_head_lock_shared();
2172 if (out_if_index <= if_index) {
2173 ifp = ifindex2ifnet[out_if_index];
2174 }
2175
2176 if (ifp != NULL) {
2177 inp->inp_last_outifp = ifp;
2178 }
2179 ifnet_head_done();
2180 }
2181
2182 if (app_data_length > 0) {
2183 uint8_t *app_data = NULL;
2184 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2185 if (app_data != NULL) {
2186 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2187 if (error == 0) {
2188 if (fd_cb->app_data != NULL) {
2189 FREE(fd_cb->app_data, M_TEMP);
2190 }
2191 fd_cb->app_data = app_data;
2192 fd_cb->app_data_length = app_data_length;
2193 } else {
2194 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2195 FREE(app_data, M_TEMP);
2196 }
2197 } else {
2198 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2199 }
2200 }
2201
2202 socket_unlock(fd_cb->so, 0);
2203 }
2204 FDUNLOCK(fd_cb);
2205 }
2206
2207 static void
2208 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2209 {
2210 size_t bytes_mem_size;
2211 size_t child_maps_mem_size;
2212 int cursor;
2213 int error = 0;
2214 struct flow_divert_trie new_trie;
2215 int insert_error = 0;
2216 size_t nodes_mem_size;
2217 int prefix_count = 0;
2218 int signing_id_count = 0;
2219 size_t trie_memory_size = 0;
2220
2221 lck_rw_lock_exclusive(&group->lck);
2222
2223 /* Re-set the current trie */
2224 if (group->signing_id_trie.memory != NULL) {
2225 FREE(group->signing_id_trie.memory, M_TEMP);
2226 }
2227 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2228 group->signing_id_trie.root = NULL_TRIE_IDX;
2229
2230 memset(&new_trie, 0, sizeof(new_trie));
2231
2232 /* Get the number of shared prefixes in the new set of signing ID strings */
2233 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2234
2235 if (prefix_count < 0) {
2236 lck_rw_done(&group->lck);
2237 return;
2238 }
2239
2240 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2241 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2242 cursor >= 0;
2243 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2244 uint32_t sid_size = 0;
2245 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2246 new_trie.bytes_count += sid_size;
2247 signing_id_count++;
2248 }
2249
2250 if (signing_id_count == 0) {
2251 lck_rw_done(&group->lck);
2252 return;
2253 }
2254
2255 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2256 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2257
2258 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2259 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2260
2261 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2262 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2263 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2264
2265 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2266 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2267 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2268 lck_rw_done(&group->lck);
2269 return;
2270 }
2271
2272 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2273 if (new_trie.memory == NULL) {
2274 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2275 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2276 lck_rw_done(&group->lck);
2277 return;
2278 }
2279
2280 /* Initialize the free lists */
2281 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2282 new_trie.nodes_free_next = 0;
2283 memset(new_trie.nodes, 0, nodes_mem_size);
2284
2285 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2286 new_trie.child_maps_free_next = 0;
2287 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2288
2289 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2290 new_trie.bytes_free_next = 0;
2291
2292 /* The root is an empty node */
2293 new_trie.root = trie_node_alloc(&new_trie);
2294
2295 /* Add each signing ID to the trie */
2296 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2297 cursor >= 0;
2298 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2299 uint32_t sid_size = 0;
2300 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2301 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2302 uint16_t new_node_idx;
2303 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2304 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2305 if (new_node_idx == NULL_TRIE_IDX) {
2306 insert_error = EINVAL;
2307 break;
2308 }
2309 } else {
2310 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2311 insert_error = ENOBUFS;
2312 break;
2313 }
2314 }
2315
2316 if (!insert_error) {
2317 group->signing_id_trie = new_trie;
2318 } else {
2319 FREE(new_trie.memory, M_TEMP);
2320 }
2321
2322 lck_rw_done(&group->lck);
2323 }
2324
2325 static int
2326 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2327 {
2328 struct flow_divert_packet_header hdr;
2329 int error = 0;
2330 struct flow_divert_pcb *fd_cb;
2331
2332 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2333 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2334 error = EINVAL;
2335 goto done;
2336 }
2337
2338 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2339 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2340 error = EINVAL;
2341 goto done;
2342 }
2343
2344 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2345 if (error) {
2346 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2347 error = ENOBUFS;
2348 goto done;
2349 }
2350
2351 hdr.conn_id = ntohl(hdr.conn_id);
2352
2353 if (hdr.conn_id == 0) {
2354 switch (hdr.packet_type) {
2355 case FLOW_DIVERT_PKT_GROUP_INIT:
2356 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2357 break;
2358 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2359 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2360 break;
2361 default:
2362 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2363 break;
2364 }
2365 goto done;
2366 }
2367
2368 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2369 if (fd_cb == NULL) {
2370 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2371 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2372 }
2373 goto done;
2374 }
2375
2376 switch (hdr.packet_type) {
2377 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2378 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2379 break;
2380 case FLOW_DIVERT_PKT_CLOSE:
2381 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2382 break;
2383 case FLOW_DIVERT_PKT_DATA:
2384 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2385 break;
2386 case FLOW_DIVERT_PKT_READ_NOTIFY:
2387 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2388 break;
2389 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2390 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2391 break;
2392 default:
2393 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2394 break;
2395 }
2396
2397 FDRELEASE(fd_cb);
2398
2399 done:
2400 mbuf_freem(packet);
2401 return error;
2402 }
2403
2404 static void
2405 flow_divert_close_all(struct flow_divert_group *group)
2406 {
2407 struct flow_divert_pcb *fd_cb;
2408 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2409
2410 SLIST_INIT(&tmp_list);
2411
2412 lck_rw_lock_exclusive(&group->lck);
2413
2414 MBUFQ_DRAIN(&group->send_queue);
2415
2416 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2417 FDRETAIN(fd_cb);
2418 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2419 }
2420
2421 lck_rw_done(&group->lck);
2422
2423 while (!SLIST_EMPTY(&tmp_list)) {
2424 fd_cb = SLIST_FIRST(&tmp_list);
2425 FDLOCK(fd_cb);
2426 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2427 if (fd_cb->so != NULL) {
2428 socket_lock(fd_cb->so, 0);
2429 flow_divert_pcb_remove(fd_cb);
2430 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2431 fd_cb->so->so_error = ECONNABORTED;
2432 flow_divert_disconnect_socket(fd_cb->so);
2433 socket_unlock(fd_cb->so, 0);
2434 }
2435 FDUNLOCK(fd_cb);
2436 FDRELEASE(fd_cb);
2437 }
2438 }
2439
2440 void
2441 flow_divert_detach(struct socket *so)
2442 {
2443 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2444
2445 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2446
2447 so->so_flags &= ~SOF_FLOW_DIVERT;
2448 so->so_fd_pcb = NULL;
2449
2450 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2451
2452 if (fd_cb->group != NULL) {
2453 /* Last-ditch effort to send any buffered data */
2454 flow_divert_send_buffered_data(fd_cb, TRUE);
2455
2456 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2457 flow_divert_send_close_if_needed(fd_cb);
2458 /* Remove from the group */
2459 flow_divert_pcb_remove(fd_cb);
2460 }
2461
2462 socket_unlock(so, 0);
2463 FDLOCK(fd_cb);
2464 fd_cb->so = NULL;
2465 FDUNLOCK(fd_cb);
2466 socket_lock(so, 0);
2467
2468 FDRELEASE(fd_cb); /* Release the socket's reference */
2469 }
2470
2471 static int
2472 flow_divert_close(struct socket *so)
2473 {
2474 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2475
2476 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2477
2478 FDLOG0(LOG_INFO, fd_cb, "Closing");
2479
2480 if (SOCK_TYPE(so) == SOCK_STREAM) {
2481 soisdisconnecting(so);
2482 sbflush(&so->so_rcv);
2483 }
2484
2485 flow_divert_send_buffered_data(fd_cb, TRUE);
2486 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2487 flow_divert_send_close_if_needed(fd_cb);
2488
2489 /* Remove from the group */
2490 flow_divert_pcb_remove(fd_cb);
2491
2492 return 0;
2493 }
2494
2495 static int
2496 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2497 sae_connid_t cid __unused)
2498 {
2499 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2500 return EINVAL;
2501 }
2502
2503 return flow_divert_close(so);
2504 }
2505
2506 static int
2507 flow_divert_shutdown(struct socket *so)
2508 {
2509 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2510
2511 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2512
2513 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2514
2515 socantsendmore(so);
2516
2517 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2518 flow_divert_send_close_if_needed(fd_cb);
2519
2520 return 0;
2521 }
2522
2523 static int
2524 flow_divert_rcvd(struct socket *so, int flags __unused)
2525 {
2526 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2527 uint32_t latest_sb_size;
2528 uint32_t read_count;
2529
2530 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2531
2532 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2533
2534 if (fd_cb->sb_size < latest_sb_size) {
2535 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2536 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2537 }
2538
2539 read_count = fd_cb->sb_size - latest_sb_size;
2540
2541 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2542
2543 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2544 fd_cb->bytes_read_by_app += read_count;
2545 fd_cb->sb_size = latest_sb_size;
2546 }
2547
2548 return 0;
2549 }
2550
2551 static int
2552 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2553 {
2554 int error = 0;
2555 int port = 0;
2556
2557 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2558 if (error) {
2559 goto done;
2560 }
2561
2562 if (toaddr->sa_family == AF_INET) {
2563 port = ntohs((satosin(toaddr))->sin_port);
2564 }
2565 #if INET6
2566 else {
2567 port = ntohs((satosin6(toaddr))->sin6_port);
2568 }
2569 #endif
2570
2571 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2572 if (error) {
2573 goto done;
2574 }
2575
2576 done:
2577 return error;
2578 }
2579
2580 struct sockaddr *
2581 flow_divert_get_buffered_target_address(mbuf_t buffer)
2582 {
2583 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2584 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2585 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2586 return toaddr;
2587 }
2588 }
2589 return NULL;
2590 }
2591
2592 static boolean_t
2593 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2594 {
2595 switch (addr->sa_family) {
2596 case AF_INET:
2597 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2598 return FALSE;
2599 }
2600 break;
2601 #if INET6
2602 case AF_INET6:
2603 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2604 return FALSE;
2605 }
2606 break;
2607 #endif /* INET6 */
2608 default:
2609 return FALSE;
2610 }
2611 return TRUE;
2612 }
2613
2614 static errno_t
2615 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2616 {
2617 int error = 0;
2618 union sockaddr_in_4_6 sin46;
2619
2620 bzero(&sin46, sizeof(sin46));
2621 if (inp->inp_vflag & INP_IPV4) {
2622 struct sockaddr_in *sin = &sin46.sin;
2623
2624 sin->sin_family = AF_INET;
2625 sin->sin_len = sizeof(*sin);
2626 sin->sin_port = inp->inp_lport;
2627 sin->sin_addr = inp->inp_laddr;
2628 } else if (inp->inp_vflag & INP_IPV6) {
2629 struct sockaddr_in6 *sin6 = &sin46.sin6;
2630
2631 sin6->sin6_len = sizeof(*sin6);
2632 sin6->sin6_family = AF_INET6;
2633 sin6->sin6_port = inp->inp_lport;
2634 sin6->sin6_addr = inp->in6p_laddr;
2635 }
2636 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2637 if (*local_socket == NULL) {
2638 error = ENOBUFS;
2639 }
2640 return error;
2641 }
2642
2643 static boolean_t
2644 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2645 {
2646 return inp->inp_lport != 0
2647 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr));
2648 }
2649
2650 static errno_t
2651 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2652 struct sockaddr **dup)
2653 {
2654 int error = 0;
2655 struct sockaddr *result;
2656 struct sockaddr_storage ss;
2657
2658 if (addr != NULL) {
2659 result = addr;
2660 } else {
2661 memset(&ss, 0, sizeof(ss));
2662 ss.ss_family = family;
2663 if (ss.ss_family == AF_INET) {
2664 ss.ss_len = sizeof(struct sockaddr_in);
2665 }
2666 #if INET6
2667 else if (ss.ss_family == AF_INET6) {
2668 ss.ss_len = sizeof(struct sockaddr_in6);
2669 }
2670 #endif /* INET6 */
2671 else {
2672 error = EINVAL;
2673 }
2674 result = (struct sockaddr *)&ss;
2675 }
2676
2677 if (!error) {
2678 *dup = dup_sockaddr(result, 1);
2679 if (*dup == NULL) {
2680 error = ENOBUFS;
2681 }
2682 }
2683
2684 return error;
2685 }
2686
2687 static void
2688 flow_divert_disconnect_socket(struct socket *so)
2689 {
2690 soisdisconnected(so);
2691 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2692 struct inpcb *inp = NULL;
2693
2694 inp = sotoinpcb(so);
2695 if (inp != NULL) {
2696 #if INET6
2697 if (SOCK_CHECK_DOM(so, PF_INET6)) {
2698 in6_pcbdetach(inp);
2699 } else
2700 #endif /* INET6 */
2701 in_pcbdetach(inp);
2702 }
2703 }
2704 }
2705
2706 static errno_t
2707 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2708 {
2709 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2710
2711 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2712
2713 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2714 fd_cb->remote_address,
2715 sa);
2716 }
2717
2718 static errno_t
2719 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2720 {
2721 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2722
2723 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2724
2725 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2726 fd_cb->local_address,
2727 sa);
2728 }
2729
2730 static errno_t
2731 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2732 {
2733 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2734
2735 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2736
2737 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2738 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2739 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2740 }
2741 }
2742
2743 if (SOCK_DOM(so) == PF_INET) {
2744 return g_tcp_protosw->pr_ctloutput(so, sopt);
2745 }
2746 #if INET6
2747 else if (SOCK_DOM(so) == PF_INET6) {
2748 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2749 }
2750 #endif
2751 return 0;
2752 }
2753
2754 errno_t
2755 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2756 {
2757 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2758 int error = 0;
2759 struct inpcb *inp = sotoinpcb(so);
2760 struct sockaddr_in *sinp;
2761 mbuf_t connect_packet = NULL;
2762 int do_send = 1;
2763
2764 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2765
2766 if (fd_cb->group == NULL) {
2767 error = ENETUNREACH;
2768 goto done;
2769 }
2770
2771 if (inp == NULL) {
2772 error = EINVAL;
2773 goto done;
2774 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2775 if (so->so_error) {
2776 error = so->so_error;
2777 so->so_error = 0;
2778 } else {
2779 error = EINVAL;
2780 }
2781 goto done;
2782 }
2783
2784 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2785 error = EALREADY;
2786 goto done;
2787 }
2788
2789 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2790 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2791 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2792 if (fd_cb->remote_address != NULL) {
2793 soisconnected(fd_cb->so);
2794 goto done;
2795 }
2796 }
2797
2798 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2799
2800 if (fd_cb->connect_packet == NULL) {
2801 if (to == NULL) {
2802 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2803 error = EINVAL;
2804 goto done;
2805 }
2806
2807 sinp = (struct sockaddr_in *)(void *)to;
2808 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2809 error = EAFNOSUPPORT;
2810 goto done;
2811 }
2812
2813 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2814 if (error) {
2815 goto done;
2816 }
2817
2818 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2819 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2820 do_send = 0;
2821 }
2822 } else {
2823 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2824 connect_packet = fd_cb->connect_packet;
2825 fd_cb->connect_packet = NULL;
2826 }
2827
2828 if (do_send) {
2829 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2830 if (error) {
2831 goto done;
2832 }
2833
2834 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2835 } else {
2836 fd_cb->connect_packet = connect_packet;
2837 connect_packet = NULL;
2838 }
2839
2840 soisconnecting(so);
2841
2842 done:
2843 if (error && connect_packet != NULL) {
2844 mbuf_freem(connect_packet);
2845 }
2846 return error;
2847 }
2848
2849 static int
2850 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2851 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
2852 {
2853 struct inpcb *inp = sotoinpcb(so);
2854 int error;
2855
2856 if (inp == NULL) {
2857 return EINVAL;
2858 }
2859
2860 VERIFY(dst != NULL);
2861
2862 error = flow_divert_connect_out(so, dst, p);
2863
2864 if (error != 0) {
2865 return error;
2866 }
2867
2868 /* if there is data, send it */
2869 if (auio != NULL) {
2870 user_ssize_t datalen = 0;
2871
2872 socket_unlock(so, 0);
2873
2874 VERIFY(bytes_written != NULL);
2875
2876 datalen = uio_resid(auio);
2877 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2878 socket_lock(so, 0);
2879
2880 if (error == 0 || error == EWOULDBLOCK) {
2881 *bytes_written = datalen - uio_resid(auio);
2882 }
2883
2884 /*
2885 * sosend returns EWOULDBLOCK if it's a non-blocking
2886 * socket or a timeout occured (this allows to return
2887 * the amount of queued data through sendit()).
2888 *
2889 * However, connectx() returns EINPROGRESS in case of a
2890 * blocking socket. So we change the return value here.
2891 */
2892 if (error == EWOULDBLOCK) {
2893 error = EINPROGRESS;
2894 }
2895 }
2896
2897 if (error == 0 && pcid != NULL) {
2898 *pcid = 1; /* there is only 1 connection for a TCP */
2899 }
2900
2901 return error;
2902 }
2903
2904 static int
2905 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
2906 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2907 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2908 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2909 {
2910 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
2911 }
2912
2913 #if INET6
2914 static int
2915 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
2916 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2917 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2918 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2919 {
2920 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
2921 }
2922 #endif /* INET6 */
2923
2924 static int
2925 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
2926 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2927 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2928 user_addr_t aux_data __unused, uint32_t *aux_len)
2929 {
2930 int error = 0;
2931 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2932 struct ifnet *ifp = NULL;
2933 struct inpcb *inp = sotoinpcb(so);
2934
2935 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2936
2937 if (so->so_fd_pcb == NULL || inp == NULL) {
2938 error = EINVAL;
2939 goto out;
2940 }
2941
2942 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
2943 error = EINVAL;
2944 goto out;
2945 }
2946
2947 ifp = inp->inp_last_outifp;
2948 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2949 *soerror = so->so_error;
2950 *flags = 0;
2951
2952 if (so->so_state & SS_ISCONNECTED) {
2953 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2954 }
2955
2956 if (fd_cb->local_address == NULL) {
2957 struct sockaddr_in sin;
2958 bzero(&sin, sizeof(sin));
2959 sin.sin_len = sizeof(sin);
2960 sin.sin_family = AF_INET;
2961 *src_len = sin.sin_len;
2962 if (src != USER_ADDR_NULL) {
2963 error = copyout(&sin, src, sin.sin_len);
2964 if (error != 0) {
2965 goto out;
2966 }
2967 }
2968 } else {
2969 *src_len = fd_cb->local_address->sa_len;
2970 if (src != USER_ADDR_NULL) {
2971 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2972 if (error != 0) {
2973 goto out;
2974 }
2975 }
2976 }
2977
2978 if (fd_cb->remote_address == NULL) {
2979 struct sockaddr_in sin;
2980 bzero(&sin, sizeof(sin));
2981 sin.sin_len = sizeof(sin);
2982 sin.sin_family = AF_INET;
2983 *dst_len = sin.sin_len;
2984 if (dst != USER_ADDR_NULL) {
2985 error = copyout(&sin, dst, sin.sin_len);
2986 if (error != 0) {
2987 goto out;
2988 }
2989 }
2990 } else {
2991 *dst_len = fd_cb->remote_address->sa_len;
2992 if (dst != USER_ADDR_NULL) {
2993 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2994 if (error != 0) {
2995 goto out;
2996 }
2997 }
2998 }
2999
3000 *aux_type = 0;
3001 *aux_len = 0;
3002
3003 out:
3004 return error;
3005 }
3006
3007 static int
3008 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3009 {
3010 int error = 0;
3011
3012 switch (cmd) {
3013 case SIOCGCONNINFO32: {
3014 struct so_cinforeq32 cifr;
3015 bcopy(data, &cifr, sizeof(cifr));
3016 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3017 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3018 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3019 &cifr.scir_aux_type, cifr.scir_aux_data,
3020 &cifr.scir_aux_len);
3021 if (error == 0) {
3022 bcopy(&cifr, data, sizeof(cifr));
3023 }
3024 break;
3025 }
3026
3027 case SIOCGCONNINFO64: {
3028 struct so_cinforeq64 cifr;
3029 bcopy(data, &cifr, sizeof(cifr));
3030 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3031 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3032 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3033 &cifr.scir_aux_type, cifr.scir_aux_data,
3034 &cifr.scir_aux_len);
3035 if (error == 0) {
3036 bcopy(&cifr, data, sizeof(cifr));
3037 }
3038 break;
3039 }
3040
3041 default:
3042 error = EOPNOTSUPP;
3043 }
3044
3045 return error;
3046 }
3047
3048 static int
3049 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3050 {
3051 int error = flow_divert_control(so, cmd, data, ifp, p);
3052
3053 if (error == EOPNOTSUPP) {
3054 error = in_control(so, cmd, data, ifp, p);
3055 }
3056
3057 return error;
3058 }
3059
3060 static int
3061 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3062 {
3063 int error = flow_divert_control(so, cmd, data, ifp, p);
3064
3065 if (error == EOPNOTSUPP) {
3066 error = in6_control(so, cmd, data, ifp, p);
3067 }
3068
3069 return error;
3070 }
3071
3072 static errno_t
3073 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3074 {
3075 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3076 int error = 0;
3077 struct inpcb *inp;
3078
3079 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3080
3081 inp = sotoinpcb(so);
3082 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3083 error = ECONNRESET;
3084 goto done;
3085 }
3086
3087 if (control && mbuf_len(control) > 0) {
3088 error = EINVAL;
3089 goto done;
3090 }
3091
3092 if (flags & MSG_OOB) {
3093 error = EINVAL;
3094 goto done; /* We don't support OOB data */
3095 }
3096
3097 error = flow_divert_check_no_cellular(fd_cb) ||
3098 flow_divert_check_no_expensive(fd_cb);
3099 if (error) {
3100 goto done;
3101 }
3102
3103 /* Implicit connect */
3104 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3105 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3106 error = flow_divert_connect_out(so, to, p);
3107 if (error) {
3108 goto done;
3109 }
3110
3111 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3112 /* Open up the send window so that the data will get sent right away */
3113 fd_cb->send_window = mbuf_pkthdr_len(data);
3114 }
3115 }
3116
3117 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3118
3119 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3120 error = flow_divert_send_app_data(fd_cb, data, to);
3121 if (error) {
3122 goto done;
3123 }
3124
3125 data = NULL;
3126
3127 if (flags & PRUS_EOF) {
3128 flow_divert_shutdown(so);
3129 }
3130
3131 done:
3132 if (data) {
3133 mbuf_freem(data);
3134 }
3135 if (control) {
3136 mbuf_free(control);
3137 }
3138 return error;
3139 }
3140
3141 static int
3142 flow_divert_preconnect(struct socket *so)
3143 {
3144 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3145 int error = 0;
3146
3147 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3148 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3149 mbuf_t connect_packet = fd_cb->connect_packet;
3150 fd_cb->connect_packet = NULL;
3151
3152 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3153 if (error) {
3154 mbuf_freem(connect_packet);
3155 }
3156
3157 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3158 }
3159
3160 soclearfastopen(so);
3161
3162 return error;
3163 }
3164
3165 static void
3166 flow_divert_set_protosw(struct socket *so)
3167 {
3168 so->so_flags |= SOF_FLOW_DIVERT;
3169 if (SOCK_DOM(so) == PF_INET) {
3170 so->so_proto = &g_flow_divert_in_protosw;
3171 }
3172 #if INET6
3173 else {
3174 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3175 }
3176 #endif /* INET6 */
3177 }
3178
3179 static void
3180 flow_divert_set_udp_protosw(struct socket *so)
3181 {
3182 so->so_flags |= SOF_FLOW_DIVERT;
3183 if (SOCK_DOM(so) == PF_INET) {
3184 so->so_proto = &g_flow_divert_in_udp_protosw;
3185 }
3186 #if INET6
3187 else {
3188 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3189 }
3190 #endif /* INET6 */
3191 }
3192
3193 static errno_t
3194 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3195 {
3196 int error = 0;
3197 struct flow_divert_pcb *fd_cb = NULL;
3198 struct ifnet *ifp = NULL;
3199 struct inpcb *inp = NULL;
3200 struct socket *old_so;
3201 mbuf_t recv_data = NULL;
3202
3203 socket_unlock(so, 0);
3204
3205 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3206
3207 /* Find the flow divert control block */
3208 lck_rw_lock_shared(&g_flow_divert_group_lck);
3209 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3210 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3211 if (group != NULL) {
3212 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3213 }
3214 }
3215 lck_rw_done(&g_flow_divert_group_lck);
3216
3217 if (fd_cb == NULL) {
3218 error = ENOENT;
3219 goto done;
3220 }
3221
3222 FDLOCK(fd_cb);
3223
3224 /* Dis-associate the flow divert control block from its current socket */
3225 old_so = fd_cb->so;
3226
3227 inp = sotoinpcb(old_so);
3228
3229 VERIFY(inp != NULL);
3230
3231 socket_lock(old_so, 0);
3232 flow_divert_disconnect_socket(old_so);
3233 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3234 old_so->so_fd_pcb = NULL;
3235 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3236 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3237 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3238 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3239 }
3240 fd_cb->so = NULL;
3241 /* Save the output interface */
3242 ifp = inp->inp_last_outifp;
3243 if (old_so->so_rcv.sb_cc > 0) {
3244 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3245 sbflush(&old_so->so_rcv);
3246 }
3247 socket_unlock(old_so, 0);
3248
3249 /* Associate the new socket with the flow divert control block */
3250 socket_lock(so, 0);
3251 so->so_fd_pcb = fd_cb;
3252 inp = sotoinpcb(so);
3253 inp->inp_last_outifp = ifp;
3254 if (recv_data != NULL) {
3255 if (sbappendstream(&so->so_rcv, recv_data)) {
3256 sorwakeup(so);
3257 }
3258 }
3259 flow_divert_set_protosw(so);
3260 socket_unlock(so, 0);
3261
3262 fd_cb->so = so;
3263 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3264
3265 FDUNLOCK(fd_cb);
3266
3267 done:
3268 socket_lock(so, 0);
3269
3270 if (fd_cb != NULL) {
3271 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3272 }
3273
3274 return error;
3275 }
3276
3277 errno_t
3278 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3279 {
3280 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3281 struct inpcb *inp;
3282 int error = 0;
3283
3284 inp = sotoinpcb(so);
3285 if (inp == NULL) {
3286 return EINVAL;
3287 }
3288
3289 if (fd_cb == NULL) {
3290 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3291 if (fd_ctl_unit > 0) {
3292 error = flow_divert_pcb_init(so, fd_ctl_unit);
3293 fd_cb = so->so_fd_pcb;
3294 if (error != 0 || fd_cb == NULL) {
3295 goto done;
3296 }
3297 } else {
3298 error = ENETDOWN;
3299 goto done;
3300 }
3301 }
3302 return flow_divert_data_out(so, flags, data, to, control, p);
3303
3304 done:
3305 if (data) {
3306 mbuf_freem(data);
3307 }
3308 if (control) {
3309 mbuf_free(control);
3310 }
3311
3312 return error;
3313 }
3314
3315 errno_t
3316 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3317 {
3318 errno_t error = 0;
3319 struct flow_divert_pcb *fd_cb;
3320
3321 if (so->so_flags & SOF_FLOW_DIVERT) {
3322 return EALREADY;
3323 }
3324
3325 fd_cb = flow_divert_pcb_create(so);
3326 if (fd_cb != NULL) {
3327 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3328 if (error) {
3329 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3330 FDRELEASE(fd_cb);
3331 } else {
3332 fd_cb->control_group_unit = ctl_unit;
3333 so->so_fd_pcb = fd_cb;
3334
3335 if (SOCK_TYPE(so) == SOCK_STREAM) {
3336 flow_divert_set_protosw(so);
3337 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3338 flow_divert_set_udp_protosw(so);
3339 }
3340
3341 FDLOG0(LOG_INFO, fd_cb, "Created");
3342 }
3343 } else {
3344 error = ENOMEM;
3345 }
3346
3347 return error;
3348 }
3349
3350 errno_t
3351 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3352 {
3353 uint32_t ctl_unit = 0;
3354 uint32_t key_unit = 0;
3355 uint32_t flow_id = 0;
3356 int error = 0;
3357 int hmac_error = 0;
3358 mbuf_t token = NULL;
3359
3360 if (so->so_flags & SOF_FLOW_DIVERT) {
3361 error = EALREADY;
3362 goto done;
3363 }
3364
3365 if (g_init_result) {
3366 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3367 error = ENOPROTOOPT;
3368 goto done;
3369 }
3370
3371 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3372 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3373 (SOCK_DOM(so) != PF_INET
3374 #if INET6
3375 && SOCK_DOM(so) != PF_INET6
3376 #endif
3377 )) {
3378 error = EINVAL;
3379 goto done;
3380 } else {
3381 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3382 struct tcpcb *tp = sototcpcb(so);
3383 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3384 error = EINVAL;
3385 goto done;
3386 }
3387 }
3388 }
3389
3390 error = soopt_getm(sopt, &token);
3391 if (error) {
3392 token = NULL;
3393 goto done;
3394 }
3395
3396 error = soopt_mcopyin(sopt, token);
3397 if (error) {
3398 token = NULL;
3399 goto done;
3400 }
3401
3402 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3403 if (!error) {
3404 key_unit = ntohl(key_unit);
3405 if (key_unit >= GROUP_COUNT_MAX) {
3406 key_unit = 0;
3407 }
3408 } else if (error != ENOENT) {
3409 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3410 goto done;
3411 } else {
3412 key_unit = 0;
3413 }
3414
3415 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3416 if (error) {
3417 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3418 goto done;
3419 }
3420
3421 /* A valid kernel control unit is required */
3422 ctl_unit = ntohl(ctl_unit);
3423 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3424 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3425 error = EINVAL;
3426 goto done;
3427 }
3428
3429 socket_unlock(so, 0);
3430 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3431 socket_lock(so, 0);
3432
3433 if (hmac_error && hmac_error != ENOENT) {
3434 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3435 error = hmac_error;
3436 goto done;
3437 }
3438
3439 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3440 if (error && error != ENOENT) {
3441 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3442 goto done;
3443 }
3444
3445 if (flow_id == 0) {
3446 error = flow_divert_pcb_init(so, ctl_unit);
3447 if (error == 0) {
3448 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3449 int log_level = LOG_NOTICE;
3450
3451 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3452 sizeof(log_level), &log_level, NULL);
3453 if (error == 0) {
3454 fd_cb->log_level = log_level;
3455 }
3456 error = 0;
3457
3458 fd_cb->connect_token = token;
3459 token = NULL;
3460 }
3461 } else {
3462 error = flow_divert_attach(so, flow_id, ctl_unit);
3463 }
3464
3465 if (hmac_error == 0) {
3466 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3467 if (fd_cb != NULL) {
3468 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3469 }
3470 }
3471
3472 done:
3473 if (token != NULL) {
3474 mbuf_freem(token);
3475 }
3476
3477 return error;
3478 }
3479
3480 errno_t
3481 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3482 {
3483 uint32_t ctl_unit;
3484 int error = 0;
3485 uint8_t hmac[SHA_DIGEST_LENGTH];
3486 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3487 mbuf_t token = NULL;
3488 struct flow_divert_group *control_group = NULL;
3489
3490 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3491 error = EINVAL;
3492 goto done;
3493 }
3494
3495 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3496
3497 if (fd_cb->group == NULL) {
3498 error = EINVAL;
3499 goto done;
3500 }
3501
3502 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3503 if (error) {
3504 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3505 goto done;
3506 }
3507
3508 ctl_unit = htonl(fd_cb->group->ctl_unit);
3509
3510 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3511 if (error) {
3512 goto done;
3513 }
3514
3515 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3516 if (error) {
3517 goto done;
3518 }
3519
3520 if (fd_cb->app_data != NULL) {
3521 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3522 if (error) {
3523 goto done;
3524 }
3525 }
3526
3527 socket_unlock(so, 0);
3528 lck_rw_lock_shared(&g_flow_divert_group_lck);
3529
3530 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3531 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
3532 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3533 }
3534
3535 if (control_group != NULL) {
3536 lck_rw_lock_shared(&control_group->lck);
3537 ctl_unit = htonl(control_group->ctl_unit);
3538 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3539 if (!error) {
3540 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3541 }
3542 lck_rw_done(&control_group->lck);
3543 } else {
3544 error = ENOPROTOOPT;
3545 }
3546
3547 lck_rw_done(&g_flow_divert_group_lck);
3548 socket_lock(so, 0);
3549
3550 if (error) {
3551 goto done;
3552 }
3553
3554 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3555 if (error) {
3556 goto done;
3557 }
3558
3559 if (sopt->sopt_val == USER_ADDR_NULL) {
3560 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3561 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3562 goto done;
3563 }
3564
3565 error = soopt_mcopyout(sopt, token);
3566 if (error) {
3567 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3568 goto done;
3569 }
3570
3571 done:
3572 if (token != NULL) {
3573 mbuf_freem(token);
3574 }
3575
3576 return error;
3577 }
3578
3579 static errno_t
3580 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3581 {
3582 struct flow_divert_group *new_group = NULL;
3583 int error = 0;
3584
3585 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3586 error = EINVAL;
3587 goto done;
3588 }
3589
3590 *unitinfo = NULL;
3591
3592 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3593 if (new_group == NULL) {
3594 error = ENOBUFS;
3595 goto done;
3596 }
3597
3598 memset(new_group, 0, sizeof(*new_group));
3599
3600 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3601 RB_INIT(&new_group->pcb_tree);
3602 new_group->ctl_unit = sac->sc_unit;
3603 MBUFQ_INIT(&new_group->send_queue);
3604 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3605
3606 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3607
3608 if (g_flow_divert_groups == NULL) {
3609 MALLOC(g_flow_divert_groups,
3610 struct flow_divert_group **,
3611 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3612 M_TEMP,
3613 M_WAITOK | M_ZERO);
3614 }
3615
3616 if (g_flow_divert_groups == NULL) {
3617 error = ENOBUFS;
3618 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3619 error = EALREADY;
3620 } else {
3621 g_flow_divert_groups[sac->sc_unit] = new_group;
3622 g_active_group_count++;
3623 }
3624
3625 lck_rw_done(&g_flow_divert_group_lck);
3626
3627 *unitinfo = new_group;
3628
3629 done:
3630 if (error != 0 && new_group != NULL) {
3631 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3632 }
3633 return error;
3634 }
3635
3636 static errno_t
3637 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3638 {
3639 struct flow_divert_group *group = NULL;
3640 errno_t error = 0;
3641
3642 if (unit >= GROUP_COUNT_MAX) {
3643 return EINVAL;
3644 }
3645
3646 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3647
3648 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3649
3650 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3651 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3652 g_flow_divert_groups, g_active_group_count);
3653 }
3654
3655 group = g_flow_divert_groups[unit];
3656
3657 if (group != (struct flow_divert_group *)unitinfo) {
3658 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3659 }
3660
3661 if (group != NULL) {
3662 flow_divert_close_all(group);
3663 if (group->token_key != NULL) {
3664 memset(group->token_key, 0, group->token_key_size);
3665 FREE(group->token_key, M_TEMP);
3666 group->token_key = NULL;
3667 group->token_key_size = 0;
3668 }
3669
3670 /* Re-set the current trie */
3671 if (group->signing_id_trie.memory != NULL) {
3672 FREE(group->signing_id_trie.memory, M_TEMP);
3673 }
3674 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3675 group->signing_id_trie.root = NULL_TRIE_IDX;
3676
3677 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3678 g_flow_divert_groups[unit] = NULL;
3679 g_active_group_count--;
3680 } else {
3681 error = EINVAL;
3682 }
3683
3684 if (g_active_group_count == 0) {
3685 FREE(g_flow_divert_groups, M_TEMP);
3686 g_flow_divert_groups = NULL;
3687 }
3688
3689 lck_rw_done(&g_flow_divert_group_lck);
3690
3691 return error;
3692 }
3693
3694 static errno_t
3695 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3696 {
3697 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3698 }
3699
3700 static void
3701 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3702 {
3703 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3704
3705 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3706 struct flow_divert_pcb *fd_cb;
3707 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3708
3709 lck_rw_lock_shared(&g_flow_divert_group_lck);
3710 lck_rw_lock_exclusive(&group->lck);
3711
3712 while (!MBUFQ_EMPTY(&group->send_queue)) {
3713 mbuf_t next_packet;
3714 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3715 next_packet = MBUFQ_FIRST(&group->send_queue);
3716 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3717 if (error) {
3718 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3719 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3720 lck_rw_done(&group->lck);
3721 lck_rw_done(&g_flow_divert_group_lck);
3722 return;
3723 }
3724 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3725 }
3726
3727 SLIST_INIT(&tmp_list);
3728
3729 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3730 FDRETAIN(fd_cb);
3731 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3732 }
3733
3734 lck_rw_done(&group->lck);
3735
3736 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3737 FDLOCK(fd_cb);
3738 if (fd_cb->so != NULL) {
3739 socket_lock(fd_cb->so, 0);
3740 if (fd_cb->group != NULL) {
3741 flow_divert_send_buffered_data(fd_cb, FALSE);
3742 }
3743 socket_unlock(fd_cb->so, 0);
3744 }
3745 FDUNLOCK(fd_cb);
3746 FDRELEASE(fd_cb);
3747 }
3748
3749 lck_rw_done(&g_flow_divert_group_lck);
3750 }
3751 }
3752
3753 static int
3754 flow_divert_kctl_init(void)
3755 {
3756 struct kern_ctl_reg ctl_reg;
3757 int result;
3758
3759 memset(&ctl_reg, 0, sizeof(ctl_reg));
3760
3761 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3762 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
3763 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3764 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3765 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3766
3767 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3768 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3769 ctl_reg.ctl_send = flow_divert_kctl_send;
3770 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3771
3772 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3773
3774 if (result) {
3775 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3776 return result;
3777 }
3778
3779 return 0;
3780 }
3781
3782 void
3783 flow_divert_init(void)
3784 {
3785 memset(&nil_pcb, 0, sizeof(nil_pcb));
3786 nil_pcb.log_level = LOG_NOTICE;
3787
3788 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3789
3790 VERIFY(g_tcp_protosw != NULL);
3791
3792 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3793 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3794
3795 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3796 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3797 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3798 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3799 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3800 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3801 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3802 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3803 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3804 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3805 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
3806
3807 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3808 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3809
3810 /*
3811 * Socket filters shouldn't attach/detach to/from this protosw
3812 * since pr_protosw is to be used instead, which points to the
3813 * real protocol; if they do, it is a bug and we should panic.
3814 */
3815 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3816 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3817 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3818 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3819
3820 /* UDP */
3821 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3822 VERIFY(g_udp_protosw != NULL);
3823
3824 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3825 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3826
3827 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3828 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3829 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3830 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3831 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3832 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3833 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3834 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3835 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3836 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3837 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3838 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3839 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3840
3841 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3842 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3843
3844 /*
3845 * Socket filters shouldn't attach/detach to/from this protosw
3846 * since pr_protosw is to be used instead, which points to the
3847 * real protocol; if they do, it is a bug and we should panic.
3848 */
3849 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3850 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3851 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3852 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3853
3854 #if INET6
3855 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3856
3857 VERIFY(g_tcp6_protosw != NULL);
3858
3859 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3860 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3861
3862 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3863 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3864 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3865 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3866 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3867 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3868 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3869 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3870 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3871 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3872 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
3873
3874 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3875 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3876 /*
3877 * Socket filters shouldn't attach/detach to/from this protosw
3878 * since pr_protosw is to be used instead, which points to the
3879 * real protocol; if they do, it is a bug and we should panic.
3880 */
3881 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3882 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3883 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3884 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3885
3886 /* UDP6 */
3887 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3888
3889 VERIFY(g_udp6_protosw != NULL);
3890
3891 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3892 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3893
3894 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3895 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3896 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3897 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3898 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3899 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3900 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3901 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3902 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3903 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3904 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3905 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3906 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3907
3908 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3909 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3910 /*
3911 * Socket filters shouldn't attach/detach to/from this protosw
3912 * since pr_protosw is to be used instead, which points to the
3913 * real protocol; if they do, it is a bug and we should panic.
3914 */
3915 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3916 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3917 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3918 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3919 #endif /* INET6 */
3920
3921 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3922 if (flow_divert_grp_attr == NULL) {
3923 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3924 g_init_result = ENOMEM;
3925 goto done;
3926 }
3927
3928 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3929 if (flow_divert_mtx_grp == NULL) {
3930 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3931 g_init_result = ENOMEM;
3932 goto done;
3933 }
3934
3935 flow_divert_mtx_attr = lck_attr_alloc_init();
3936 if (flow_divert_mtx_attr == NULL) {
3937 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3938 g_init_result = ENOMEM;
3939 goto done;
3940 }
3941
3942 g_init_result = flow_divert_kctl_init();
3943 if (g_init_result) {
3944 goto done;
3945 }
3946
3947 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3948
3949 done:
3950 if (g_init_result != 0) {
3951 if (flow_divert_mtx_attr != NULL) {
3952 lck_attr_free(flow_divert_mtx_attr);
3953 flow_divert_mtx_attr = NULL;
3954 }
3955 if (flow_divert_mtx_grp != NULL) {
3956 lck_grp_free(flow_divert_mtx_grp);
3957 flow_divert_mtx_grp = NULL;
3958 }
3959 if (flow_divert_grp_attr != NULL) {
3960 lck_grp_attr_free(flow_divert_grp_attr);
3961 flow_divert_grp_attr = NULL;
3962 }
3963
3964 if (g_flow_divert_kctl_ref != NULL) {
3965 ctl_deregister(g_flow_divert_kctl_ref);
3966 g_flow_divert_kctl_ref = NULL;
3967 }
3968 }
3969 }
mirror of XNU