]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
b531933cc506f0b5e7072c74d15da991cba29164
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
59 #if INET6
60 #include <netinet6/in6_pcb.h>
61 #include <netinet6/ip6protosw.h>
62 #endif /* INET6 */
63 #include <dev/random/randomdev.h>
64 #include <libkern/crypto/sha1.h>
65 #include <libkern/crypto/crypto_internal.h>
66 #include <os/log.h>
67
68 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69 #define FLOW_DIVERT_READ_CLOSED 0x00000002
70 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73 #define FLOW_DIVERT_TRANSFERRED 0x00000020
74 #define FLOW_DIVERT_HAS_HMAC 0x00000040
75
76 #define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
78
79 #define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
81
82 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83 #define FDRELEASE(pcb) \
84 do { \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
87 } \
88 } while (0)
89
90 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
92
93 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
94 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
95
96 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
97
98 #define GROUP_COUNT_MAX 32
99 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
100 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
101 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
102
103 struct flow_divert_trie_node
104 {
105 uint16_t start;
106 uint16_t length;
107 uint16_t child_map;
108 };
109
110 #define CHILD_MAP_SIZE 256
111 #define NULL_TRIE_IDX 0xffff
112 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
113 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
114 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
115
116 static struct flow_divert_pcb nil_pcb;
117
118 decl_lck_rw_data(static, g_flow_divert_group_lck);
119 static struct flow_divert_group **g_flow_divert_groups = NULL;
120 static uint32_t g_active_group_count = 0;
121
122 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
123 static lck_attr_t *flow_divert_mtx_attr = NULL;
124 static lck_grp_t *flow_divert_mtx_grp = NULL;
125 static errno_t g_init_result = 0;
126
127 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
128
129 static struct protosw g_flow_divert_in_protosw;
130 static struct pr_usrreqs g_flow_divert_in_usrreqs;
131 static struct protosw g_flow_divert_in_udp_protosw;
132 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
133 #if INET6
134 static struct ip6protosw g_flow_divert_in6_protosw;
135 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
136 static struct ip6protosw g_flow_divert_in6_udp_protosw;
137 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
138 #endif /* INET6 */
139
140 static struct protosw *g_tcp_protosw = NULL;
141 static struct ip6protosw *g_tcp6_protosw = NULL;
142 static struct protosw *g_udp_protosw = NULL;
143 static struct ip6protosw *g_udp6_protosw = NULL;
144
145 static errno_t
146 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
147
148 static errno_t
149 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
150
151 static boolean_t
152 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
153
154 static int
155 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
156
157 struct sockaddr *
158 flow_divert_get_buffered_target_address(mbuf_t buffer);
159
160 static boolean_t
161 flow_divert_has_pcb_local_address(const struct inpcb *inp);
162
163 static void
164 flow_divert_disconnect_socket(struct socket *so);
165
166 static inline uint8_t
167 flow_divert_syslog_type_to_oslog_type(int syslog_type)
168 {
169 switch (syslog_type) {
170 case LOG_ERR: return OS_LOG_TYPE_ERROR;
171 case LOG_INFO: return OS_LOG_TYPE_INFO;
172 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
173 default: return OS_LOG_TYPE_DEFAULT;
174 }
175 }
176
177 static inline int
178 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
179 {
180 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
181 }
182
183 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
184 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
185
186 static const char *
187 flow_divert_packet_type2str(uint8_t packet_type)
188 {
189 switch (packet_type) {
190 case FLOW_DIVERT_PKT_CONNECT:
191 return "connect";
192 case FLOW_DIVERT_PKT_CONNECT_RESULT:
193 return "connect result";
194 case FLOW_DIVERT_PKT_DATA:
195 return "data";
196 case FLOW_DIVERT_PKT_CLOSE:
197 return "close";
198 case FLOW_DIVERT_PKT_READ_NOTIFY:
199 return "read notification";
200 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
201 return "properties update";
202 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
203 return "app map create";
204 default:
205 return "unknown";
206 }
207 }
208
209 static struct flow_divert_pcb *
210 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
211 {
212 struct flow_divert_pcb key_item;
213 struct flow_divert_pcb *fd_cb = NULL;
214
215 key_item.hash = hash;
216
217 lck_rw_lock_shared(&group->lck);
218 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
219 FDRETAIN(fd_cb);
220 lck_rw_done(&group->lck);
221
222 return fd_cb;
223 }
224
225 static errno_t
226 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
227 {
228 errno_t error = 0;
229 struct flow_divert_pcb *exist = NULL;
230 struct flow_divert_group *group;
231 static uint32_t g_nextkey = 1;
232 static uint32_t g_hash_seed = 0;
233 int try_count = 0;
234
235 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
236 return EINVAL;
237 }
238
239 socket_unlock(fd_cb->so, 0);
240 lck_rw_lock_shared(&g_flow_divert_group_lck);
241
242 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
243 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
244 error = ENETUNREACH;
245 goto done;
246 }
247
248 group = g_flow_divert_groups[ctl_unit];
249 if (group == NULL) {
250 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
251 error = ENETUNREACH;
252 goto done;
253 }
254
255 socket_lock(fd_cb->so, 0);
256
257 do {
258 uint32_t key[2];
259 uint32_t idx;
260
261 key[0] = g_nextkey++;
262 key[1] = RandomULong();
263
264 if (g_hash_seed == 0) {
265 g_hash_seed = RandomULong();
266 }
267
268 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
269
270 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
271 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
272 if (curr_group != NULL && curr_group != group) {
273 lck_rw_lock_shared(&curr_group->lck);
274 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
275 lck_rw_done(&curr_group->lck);
276 if (exist != NULL) {
277 break;
278 }
279 }
280 }
281
282 if (exist == NULL) {
283 lck_rw_lock_exclusive(&group->lck);
284 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
285 lck_rw_done(&group->lck);
286 }
287 } while (exist != NULL && try_count++ < 3);
288
289 if (exist == NULL) {
290 fd_cb->group = group;
291 FDRETAIN(fd_cb); /* The group now has a reference */
292 } else {
293 fd_cb->hash = 0;
294 error = EEXIST;
295 }
296
297 socket_unlock(fd_cb->so, 0);
298
299 done:
300 lck_rw_done(&g_flow_divert_group_lck);
301 socket_lock(fd_cb->so, 0);
302
303 return error;
304 }
305
306 static struct flow_divert_pcb *
307 flow_divert_pcb_create(socket_t so)
308 {
309 struct flow_divert_pcb *new_pcb = NULL;
310
311 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
312 if (new_pcb == NULL) {
313 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
314 return NULL;
315 }
316
317 memset(new_pcb, 0, sizeof(*new_pcb));
318
319 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
320 new_pcb->so = so;
321 new_pcb->log_level = nil_pcb.log_level;
322
323 FDRETAIN(new_pcb); /* Represents the socket's reference */
324
325 return new_pcb;
326 }
327
328 static void
329 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
330 {
331 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
332 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
333
334 if (fd_cb->local_address != NULL) {
335 FREE(fd_cb->local_address, M_SONAME);
336 }
337 if (fd_cb->remote_address != NULL) {
338 FREE(fd_cb->remote_address, M_SONAME);
339 }
340 if (fd_cb->connect_token != NULL) {
341 mbuf_freem(fd_cb->connect_token);
342 }
343 if (fd_cb->connect_packet != NULL) {
344 mbuf_freem(fd_cb->connect_packet);
345 }
346 if (fd_cb->app_data != NULL) {
347 FREE(fd_cb->app_data, M_TEMP);
348 }
349 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
350 }
351
352 static void
353 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
354 {
355 if (fd_cb->group != NULL) {
356 struct flow_divert_group *group = fd_cb->group;
357 lck_rw_lock_exclusive(&group->lck);
358 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
359 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
360 fd_cb->group = NULL;
361 FDRELEASE(fd_cb); /* Release the group's reference */
362 lck_rw_done(&group->lck);
363 }
364 }
365
366 static int
367 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
368 {
369 struct flow_divert_packet_header hdr;
370 int error = 0;
371
372 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
373 if (error) {
374 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
375 return error;
376 }
377
378 hdr.packet_type = packet_type;
379 hdr.conn_id = htonl(fd_cb->hash);
380
381 /* Lay down the header */
382 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
383 if (error) {
384 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
385 mbuf_freem(*packet);
386 *packet = NULL;
387 return error;
388 }
389
390 return 0;
391 }
392
393 static int
394 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
395 {
396 uint32_t net_length = htonl(length);
397 int error = 0;
398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
400 if (error) {
401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
406 if (error) {
407 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
408 return error;
409 }
410
411 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
412 if (error) {
413 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
414 return error;
415 }
416
417 return error;
418 }
419
420 static int
421 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
422 {
423 size_t cursor = offset;
424 int error = 0;
425 uint32_t curr_length;
426 uint8_t curr_type;
427
428 *err = 0;
429
430 do {
431 if (!next) {
432 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
433 if (error) {
434 *err = ENOENT;
435 return -1;
436 }
437 } else {
438 next = 0;
439 curr_type = FLOW_DIVERT_TLV_NIL;
440 }
441
442 if (curr_type != type) {
443 cursor += sizeof(curr_type);
444 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
445 if (error) {
446 *err = error;
447 return -1;
448 }
449
450 cursor += (sizeof(curr_length) + ntohl(curr_length));
451 }
452 } while (curr_type != type);
453
454 return cursor;
455 }
456
457 static int
458 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
459 {
460 int error = 0;
461 uint32_t length;
462 int tlv_offset;
463
464 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
465 if (tlv_offset < 0) {
466 return error;
467 }
468
469 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
470 if (error) {
471 return error;
472 }
473
474 length = ntohl(length);
475
476 if (val_size != NULL) {
477 *val_size = length;
478 }
479
480 if (buff != NULL && buff_len > 0) {
481 size_t to_copy = (length < buff_len) ? length : buff_len;
482 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
483 if (error) {
484 return error;
485 }
486 }
487
488 return 0;
489 }
490
491 static int
492 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
493 {
494 mbuf_t curr_mbuf = packet;
495
496 if (g_crypto_funcs == NULL || group->token_key == NULL) {
497 return ENOPROTOOPT;
498 }
499
500 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
501 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
502
503 while (curr_mbuf != NULL) {
504 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
505 curr_mbuf = mbuf_next(curr_mbuf);
506 }
507
508 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
509
510 return 0;
511 }
512
513 static int
514 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
515 {
516 int error = 0;
517 struct flow_divert_group *group = NULL;
518 int hmac_offset;
519 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
520 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
521 mbuf_t tail;
522
523 lck_rw_lock_shared(&g_flow_divert_group_lck);
524
525 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
526 group = g_flow_divert_groups[ctl_unit];
527 }
528
529 if (group == NULL) {
530 lck_rw_done(&g_flow_divert_group_lck);
531 return ENOPROTOOPT;
532 }
533
534 lck_rw_lock_shared(&group->lck);
535
536 if (group->token_key == NULL) {
537 error = ENOPROTOOPT;
538 goto done;
539 }
540
541 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
542 if (hmac_offset < 0) {
543 goto done;
544 }
545
546 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
547 if (error) {
548 goto done;
549 }
550
551 /* Chop off the HMAC TLV */
552 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
553 if (error) {
554 goto done;
555 }
556
557 mbuf_free(tail);
558
559 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
560 if (error) {
561 goto done;
562 }
563
564 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
565 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
566 error = EINVAL;
567 goto done;
568 }
569
570 done:
571 lck_rw_done(&group->lck);
572 lck_rw_done(&g_flow_divert_group_lck);
573 return error;
574 }
575
576 static void
577 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
578 {
579 struct inpcb *inp = NULL;
580 struct ifnet *ifp = NULL;
581 Boolean cell = FALSE;
582 Boolean wifi = FALSE;
583 Boolean wired = FALSE;
584
585 inp = sotoinpcb(fd_cb->so);
586 if (inp == NULL) {
587 return;
588 }
589
590 ifp = inp->inp_last_outifp;
591 if (ifp != NULL) {
592 cell = IFNET_IS_CELLULAR(ifp);
593 wifi = (!cell && IFNET_IS_WIFI(ifp));
594 wired = (!wifi && IFNET_IS_WIRED(ifp));
595 }
596
597 if (send) {
598 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
599 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
600 } else {
601 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
602 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
603 }
604 }
605
606 static errno_t
607 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
608 {
609 struct inpcb *inp = NULL;
610
611 inp = sotoinpcb(fd_cb->so);
612 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
613 IFNET_IS_CELLULAR(inp->inp_last_outifp))
614 return EHOSTUNREACH;
615
616 return 0;
617 }
618
619 static errno_t
620 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
621 {
622 struct inpcb *inp = NULL;
623
624 inp = sotoinpcb(fd_cb->so);
625 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
626 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
627 return EHOSTUNREACH;
628
629 return 0;
630 }
631
632 static void
633 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
634 {
635 if (how != SHUT_RD) {
636 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
637 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
638 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
639 /* If the tunnel is not accepting writes any more, then flush the send buffer */
640 sbflush(&fd_cb->so->so_snd);
641 }
642 }
643 if (how != SHUT_WR) {
644 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
645 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
646 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
647 }
648 }
649 }
650
651 static uint16_t
652 trie_node_alloc(struct flow_divert_trie *trie)
653 {
654 if (trie->nodes_free_next < trie->nodes_count) {
655 uint16_t node_idx = trie->nodes_free_next++;
656 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
657 return node_idx;
658 } else {
659 return NULL_TRIE_IDX;
660 }
661 }
662
663 static uint16_t
664 trie_child_map_alloc(struct flow_divert_trie *trie)
665 {
666 if (trie->child_maps_free_next < trie->child_maps_count) {
667 return trie->child_maps_free_next++;
668 } else {
669 return NULL_TRIE_IDX;
670 }
671 }
672
673 static uint16_t
674 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
675 {
676 uint16_t start = trie->bytes_free_next;
677 if (start + bytes_size <= trie->bytes_count) {
678 if (start != bytes_idx) {
679 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
680 }
681 trie->bytes_free_next += bytes_size;
682 return start;
683 } else {
684 return NULL_TRIE_IDX;
685 }
686 }
687
688 static uint16_t
689 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
690 {
691 uint16_t current = trie->root;
692 uint16_t child = trie->root;
693 uint16_t string_end = string_start + string_len;
694 uint16_t string_idx = string_start;
695 uint16_t string_remainder = string_len;
696
697 while (child != NULL_TRIE_IDX) {
698 uint16_t parent = current;
699 uint16_t node_idx;
700 uint16_t current_end;
701
702 current = child;
703 child = NULL_TRIE_IDX;
704
705 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
706
707 for (node_idx = TRIE_NODE(trie, current).start;
708 node_idx < current_end &&
709 string_idx < string_end &&
710 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
711 node_idx++, string_idx++);
712
713 string_remainder = string_end - string_idx;
714
715 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
716 /*
717 * We did not reach the end of the current node's string.
718 * We need to split the current node into two:
719 * 1. A new node that contains the prefix of the node that matches
720 * the prefix of the string being inserted.
721 * 2. The current node modified to point to the remainder
722 * of the current node's string.
723 */
724 uint16_t prefix = trie_node_alloc(trie);
725 if (prefix == NULL_TRIE_IDX) {
726 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
727 return NULL_TRIE_IDX;
728 }
729
730 /*
731 * Prefix points to the portion of the current nodes's string that has matched
732 * the input string thus far.
733 */
734 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
735 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
736
737 /*
738 * Prefix has the current node as the child corresponding to the first byte
739 * after the split.
740 */
741 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
742 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
743 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
744 return NULL_TRIE_IDX;
745 }
746 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
747
748 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
749 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
750
751 /* Current node is adjusted to point to the remainder */
752 TRIE_NODE(trie, current).start = node_idx;
753 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
754
755 /* We want to insert the new leaf (if any) as a child of the prefix */
756 current = prefix;
757 }
758
759 if (string_remainder > 0) {
760 /*
761 * We still have bytes in the string that have not been matched yet.
762 * If the current node has children, iterate to the child corresponding
763 * to the next byte in the string.
764 */
765 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
766 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
767 }
768 }
769 } /* while (child != NULL_TRIE_IDX) */
770
771 if (string_remainder > 0) {
772 /* Add a new leaf containing the remainder of the string */
773 uint16_t leaf = trie_node_alloc(trie);
774 if (leaf == NULL_TRIE_IDX) {
775 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
776 return NULL_TRIE_IDX;
777 }
778
779 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
780 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
781 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
782 return NULL_TRIE_IDX;
783 }
784 TRIE_NODE(trie, leaf).length = string_remainder;
785
786 /* Set the new leaf as the child of the current node */
787 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
788 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
789 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
790 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
791 return NULL_TRIE_IDX;
792 }
793 }
794 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
795 current = leaf;
796 } /* else duplicate or this string is a prefix of one of the existing strings */
797
798 return current;
799 }
800
801 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
802 static uint16_t
803 flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
804 {
805 uint16_t current = trie->root;
806 uint16_t string_idx = 0;
807
808 while (current != NULL_TRIE_IDX) {
809 uint16_t next = NULL_TRIE_IDX;
810 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
811 uint16_t node_idx;
812
813 for (node_idx = TRIE_NODE(trie, current).start;
814 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
815 node_idx++, string_idx++);
816
817 if (node_idx == node_end) {
818 if (string_bytes[string_idx] == '\0') {
819 return current; /* Got an exact match */
820 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
821 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
822 string_bytes[string_idx] = '\0';
823 return current; /* Got an apple webclip id prefix match */
824 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
825 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
826 }
827 }
828 current = next;
829 }
830
831 return NULL_TRIE_IDX;
832 }
833
834 struct uuid_search_info {
835 uuid_t target_uuid;
836 char *found_signing_id;
837 boolean_t found_multiple_signing_ids;
838 proc_t found_proc;
839 };
840
841 static int
842 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
843 {
844 struct uuid_search_info *info = (struct uuid_search_info *)arg;
845 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
846
847 if (info->found_signing_id != NULL) {
848 if (!info->found_multiple_signing_ids) {
849 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
850 info->found_proc = p;
851 result = PROC_CLAIMED_DONE;
852 } else {
853 uuid_string_t uuid_str;
854 uuid_unparse(info->target_uuid, uuid_str);
855 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
856 }
857 FREE(info->found_signing_id, M_TEMP);
858 info->found_signing_id = NULL;
859 }
860
861 if (result == PROC_RETURNED_DONE) {
862 uuid_string_t uuid_str;
863 uuid_unparse(info->target_uuid, uuid_str);
864 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
865 }
866
867 return result;
868 }
869
870 static int
871 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
872 {
873 struct uuid_search_info *info = (struct uuid_search_info *)arg;
874 int include = 0;
875
876 if (info->found_multiple_signing_ids) {
877 return include;
878 }
879
880 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
881 if (include) {
882 const char *signing_id = cs_identity_get(p);
883 if (signing_id != NULL) {
884 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
885 size_t signing_id_size = strlen(signing_id) + 1;
886 if (info->found_signing_id == NULL) {
887 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
888 memcpy(info->found_signing_id, signing_id, signing_id_size);
889 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
890 info->found_multiple_signing_ids = TRUE;
891 }
892 } else {
893 info->found_multiple_signing_ids = TRUE;
894 }
895 include = !info->found_multiple_signing_ids;
896 }
897
898 return include;
899 }
900
901 static proc_t
902 flow_divert_find_proc_by_uuid(uuid_t uuid)
903 {
904 struct uuid_search_info info;
905
906 if (LOG_INFO <= nil_pcb.log_level) {
907 uuid_string_t uuid_str;
908 uuid_unparse(uuid, uuid_str);
909 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
910 }
911
912 memset(&info, 0, sizeof(info));
913 info.found_proc = PROC_NULL;
914 uuid_copy(info.target_uuid, uuid);
915
916 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
917
918 return info.found_proc;
919 }
920
921 static int
922 flow_divert_get_src_proc(struct socket *so, proc_t *proc)
923 {
924 int release = 0;
925
926 if (so->so_flags & SOF_DELEGATED) {
927 if ((*proc)->p_pid != so->e_pid) {
928 *proc = proc_find(so->e_pid);
929 release = 1;
930 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
931 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
932 release = 1;
933 }
934 } else if (*proc == PROC_NULL) {
935 *proc = current_proc();
936 }
937
938 if (*proc != PROC_NULL) {
939 if ((*proc)->p_pid == 0) {
940 if (release) {
941 proc_rele(*proc);
942 }
943 release = 0;
944 *proc = PROC_NULL;
945 }
946 }
947
948 return release;
949 }
950
951 static int
952 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
953 {
954 int error;
955
956 if (fd_cb->group == NULL) {
957 fd_cb->so->so_error = ECONNABORTED;
958 flow_divert_disconnect_socket(fd_cb->so);
959 return ECONNABORTED;
960 }
961
962 lck_rw_lock_shared(&fd_cb->group->lck);
963
964 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
965 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
966 } else {
967 error = ENOBUFS;
968 }
969
970 if (error == ENOBUFS) {
971 if (enqueue) {
972 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
973 lck_rw_lock_exclusive(&fd_cb->group->lck);
974 }
975 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
976 error = 0;
977 }
978 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
979 }
980
981 lck_rw_done(&fd_cb->group->lck);
982
983 return error;
984 }
985
986 static int
987 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
988 {
989 int error = 0;
990 int flow_type = 0;
991 char *signing_id = NULL;
992 int free_signing_id = 0;
993 mbuf_t connect_packet = NULL;
994 proc_t src_proc = p;
995 int release_proc = 0;
996
997 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
998 if (error) {
999 goto done;
1000 }
1001
1002 error = EPERM;
1003
1004 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1005 uint32_t sid_size = 0;
1006 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1007 if (find_error == 0 && sid_size > 0) {
1008 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1009 if (signing_id != NULL) {
1010 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1011 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1012 free_signing_id = 1;
1013 }
1014 }
1015 }
1016
1017 socket_unlock(so, 0);
1018
1019 if (signing_id == NULL) {
1020 release_proc = flow_divert_get_src_proc(so, &src_proc);
1021 if (src_proc != PROC_NULL) {
1022 proc_lock(src_proc);
1023 if (src_proc->p_csflags & (CS_VALID|CS_DEBUGGED)) {
1024 const char * cs_id;
1025 cs_id = cs_identity_get(src_proc);
1026 signing_id = __DECONST(char *, cs_id);
1027 } else {
1028 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
1029 }
1030 } else {
1031 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
1032 }
1033 } else {
1034 src_proc = PROC_NULL;
1035 }
1036
1037 if (signing_id != NULL) {
1038 uint16_t result = NULL_TRIE_IDX;
1039 lck_rw_lock_shared(&fd_cb->group->lck);
1040 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1041 lck_rw_done(&fd_cb->group->lck);
1042 if (result != NULL_TRIE_IDX) {
1043 error = 0;
1044 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
1045
1046 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1047 if (error == 0) {
1048 if (src_proc != PROC_NULL) {
1049 unsigned char cdhash[SHA1_RESULTLEN];
1050 error = proc_getcdhash(src_proc, cdhash);
1051 if (error == 0) {
1052 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1053 if (error) {
1054 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1055 }
1056 } else {
1057 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
1058 }
1059 }
1060 } else {
1061 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1062 }
1063 } else {
1064 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
1065 }
1066 } else {
1067 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1068 }
1069
1070 if (src_proc != PROC_NULL) {
1071 proc_unlock(src_proc);
1072 if (release_proc) {
1073 proc_rele(src_proc);
1074 }
1075 }
1076 socket_lock(so, 0);
1077
1078 if (free_signing_id) {
1079 FREE(signing_id, M_TEMP);
1080 }
1081
1082 if (error) {
1083 goto done;
1084 }
1085
1086 error = flow_divert_packet_append_tlv(connect_packet,
1087 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1088 sizeof(fd_cb->so->so_traffic_class),
1089 &fd_cb->so->so_traffic_class);
1090 if (error) {
1091 goto done;
1092 }
1093
1094 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1095 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1096 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1097 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1098 } else {
1099 error = EINVAL;
1100 goto done;
1101 }
1102 error = flow_divert_packet_append_tlv(connect_packet,
1103 FLOW_DIVERT_TLV_FLOW_TYPE,
1104 sizeof(flow_type),
1105 &flow_type);
1106
1107 if (error) {
1108 goto done;
1109 }
1110
1111 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1112 error = flow_divert_packet_append_tlv(connect_packet,
1113 FLOW_DIVERT_TLV_PID,
1114 sizeof(fd_cb->so->e_pid),
1115 &fd_cb->so->e_pid);
1116 if (error) {
1117 goto done;
1118 }
1119
1120 error = flow_divert_packet_append_tlv(connect_packet,
1121 FLOW_DIVERT_TLV_UUID,
1122 sizeof(fd_cb->so->e_uuid),
1123 &fd_cb->so->e_uuid);
1124 if (error) {
1125 goto done;
1126 }
1127 } else {
1128 error = flow_divert_packet_append_tlv(connect_packet,
1129 FLOW_DIVERT_TLV_PID,
1130 sizeof(fd_cb->so->e_pid),
1131 &fd_cb->so->last_pid);
1132 if (error) {
1133 goto done;
1134 }
1135
1136 error = flow_divert_packet_append_tlv(connect_packet,
1137 FLOW_DIVERT_TLV_UUID,
1138 sizeof(fd_cb->so->e_uuid),
1139 &fd_cb->so->last_uuid);
1140 if (error) {
1141 goto done;
1142 }
1143 }
1144
1145 if (fd_cb->connect_token != NULL) {
1146 unsigned int token_len = m_length(fd_cb->connect_token);
1147 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1148 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1149 fd_cb->connect_token = NULL;
1150 } else {
1151 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
1152
1153 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1154 if (error) {
1155 goto done;
1156 }
1157
1158 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1159 if (error) {
1160 goto done;
1161 }
1162 }
1163
1164 if (fd_cb->local_address != NULL) {
1165 error = EALREADY;
1166 goto done;
1167 } else {
1168 struct inpcb *inp = sotoinpcb(so);
1169 if (flow_divert_has_pcb_local_address(inp)) {
1170 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1171 if (error) {
1172 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1173 goto done;
1174 }
1175 }
1176 }
1177
1178 if (fd_cb->local_address != NULL) {
1179 /* socket is bound. */
1180 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1181 sizeof(struct sockaddr_storage), fd_cb->local_address);
1182 if (error) {
1183 goto done;
1184 }
1185 }
1186
1187 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1188 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1189 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1190 if (error) {
1191 goto done;
1192 }
1193 }
1194
1195 done:
1196 if (!error) {
1197 *out_connect_packet = connect_packet;
1198 } else if (connect_packet != NULL) {
1199 mbuf_freem(connect_packet);
1200 }
1201
1202 return error;
1203 }
1204
1205 static int
1206 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1207 {
1208 int error = 0;
1209 mbuf_t packet = NULL;
1210 int rbuff_space = 0;
1211
1212 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1213 if (error) {
1214 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1215 goto done;
1216 }
1217
1218 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1219 if (rbuff_space < 0) {
1220 rbuff_space = 0;
1221 }
1222 rbuff_space = htonl(rbuff_space);
1223 error = flow_divert_packet_append_tlv(packet,
1224 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1225 sizeof(rbuff_space),
1226 &rbuff_space);
1227 if (error) {
1228 goto done;
1229 }
1230
1231 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1232 if (error) {
1233 goto done;
1234 }
1235
1236 done:
1237 if (error && packet != NULL) {
1238 mbuf_freem(packet);
1239 }
1240
1241 return error;
1242 }
1243
1244 static int
1245 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1246 {
1247 int error = 0;
1248 mbuf_t packet = NULL;
1249 uint32_t zero = 0;
1250
1251 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1252 if (error) {
1253 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1254 goto done;
1255 }
1256
1257 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1258 if (error) {
1259 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1260 goto done;
1261 }
1262
1263 how = htonl(how);
1264 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1265 if (error) {
1266 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1267 goto done;
1268 }
1269
1270 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1271 if (error) {
1272 goto done;
1273 }
1274
1275 done:
1276 if (error && packet != NULL) {
1277 mbuf_free(packet);
1278 }
1279
1280 return error;
1281 }
1282
1283 static int
1284 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1285 {
1286 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1287 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1288 {
1289 return SHUT_RDWR;
1290 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1291 return SHUT_RD;
1292 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1293 return SHUT_WR;
1294 }
1295
1296 return -1;
1297 }
1298
1299 /*
1300 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1301 * writes. Returns FALSE otherwise.
1302 */
1303 static void
1304 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1305 {
1306 int how = -1;
1307
1308 /* Do not send any close messages if there is still data in the send buffer */
1309 if (fd_cb->so->so_snd.sb_cc == 0) {
1310 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1311 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1312 how = SHUT_RD;
1313 }
1314 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1315 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1316 if (how == SHUT_RD) {
1317 how = SHUT_RDWR;
1318 } else {
1319 how = SHUT_WR;
1320 }
1321 }
1322 }
1323
1324 if (how != -1) {
1325 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1326 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1327 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1328 if (how != SHUT_RD) {
1329 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1330 }
1331 if (how != SHUT_WR) {
1332 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1333 }
1334 }
1335 }
1336
1337 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1338 flow_divert_disconnect_socket(fd_cb->so);
1339 }
1340 }
1341
1342 static errno_t
1343 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1344 {
1345 mbuf_t packet;
1346 mbuf_t last;
1347 int error = 0;
1348
1349 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1350 if (error) {
1351 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1352 return error;
1353 }
1354
1355 if (toaddr != NULL) {
1356 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1357 if (error) {
1358 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1359 return error;
1360 }
1361 }
1362
1363 last = m_last(packet);
1364 mbuf_setnext(last, data);
1365 mbuf_pkthdr_adjustlen(packet, data_len);
1366 error = flow_divert_send_packet(fd_cb, packet, force);
1367
1368 if (error) {
1369 mbuf_setnext(last, NULL);
1370 mbuf_freem(packet);
1371 } else {
1372 fd_cb->bytes_sent += data_len;
1373 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1374 }
1375
1376 return error;
1377 }
1378
1379 static void
1380 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1381 {
1382 size_t to_send;
1383 size_t sent = 0;
1384 int error = 0;
1385 mbuf_t buffer;
1386
1387 to_send = fd_cb->so->so_snd.sb_cc;
1388 buffer = fd_cb->so->so_snd.sb_mb;
1389
1390 if (buffer == NULL && to_send > 0) {
1391 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1392 return;
1393 }
1394
1395 /* Ignore the send window if force is enabled */
1396 if (!force && (to_send > fd_cb->send_window)) {
1397 to_send = fd_cb->send_window;
1398 }
1399
1400 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1401 while (sent < to_send) {
1402 mbuf_t data;
1403 size_t data_len;
1404
1405 data_len = to_send - sent;
1406 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1407 data_len = FLOW_DIVERT_CHUNK_SIZE;
1408 }
1409
1410 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1411 if (error) {
1412 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1413 break;
1414 }
1415
1416 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1417 if (error) {
1418 mbuf_freem(data);
1419 break;
1420 }
1421
1422 sent += data_len;
1423 }
1424 sbdrop(&fd_cb->so->so_snd, sent);
1425 sowwakeup(fd_cb->so);
1426 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1427 mbuf_t data;
1428 mbuf_t m;
1429 size_t data_len;
1430
1431 while(buffer) {
1432 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1433
1434 m = buffer;
1435 if (toaddr != NULL) {
1436 /* look for data in the chain */
1437 do {
1438 m = m->m_next;
1439 if (m != NULL && m->m_type == MT_DATA) {
1440 break;
1441 }
1442 } while(m);
1443 if (m == NULL) {
1444 /* unexpected */
1445 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1446 goto move_on;
1447 }
1448 }
1449 data_len = mbuf_pkthdr_len(m);
1450 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1451 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1452 if (error) {
1453 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1454 break;
1455 }
1456 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1457 if (error) {
1458 mbuf_freem(data);
1459 break;
1460 }
1461 sent += data_len;
1462 move_on:
1463 buffer = buffer->m_nextpkt;
1464 (void) sbdroprecord(&(fd_cb->so->so_snd));
1465 }
1466 }
1467
1468 if (sent > 0) {
1469 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1470 if (fd_cb->send_window >= sent) {
1471 fd_cb->send_window -= sent;
1472 } else {
1473 fd_cb->send_window = 0;
1474 }
1475 }
1476 }
1477
1478 static int
1479 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1480 {
1481 size_t to_send = mbuf_pkthdr_len(data);
1482 int error = 0;
1483
1484 if (to_send > fd_cb->send_window) {
1485 to_send = fd_cb->send_window;
1486 }
1487
1488 if (fd_cb->so->so_snd.sb_cc > 0) {
1489 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1490 }
1491
1492 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1493 size_t sent = 0;
1494 mbuf_t remaining_data = data;
1495 mbuf_t pkt_data = NULL;
1496 while (sent < to_send && remaining_data != NULL) {
1497 size_t pkt_data_len;
1498
1499 pkt_data = remaining_data;
1500
1501 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1502 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1503 } else {
1504 pkt_data_len = to_send - sent;
1505 }
1506
1507 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1508 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1509 if (error) {
1510 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1511 pkt_data = NULL;
1512 break;
1513 }
1514 } else {
1515 remaining_data = NULL;
1516 }
1517
1518 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1519
1520 if (error) {
1521 break;
1522 }
1523
1524 pkt_data = NULL;
1525 sent += pkt_data_len;
1526 }
1527
1528 fd_cb->send_window -= sent;
1529
1530 error = 0;
1531
1532 if (pkt_data != NULL) {
1533 if (sbspace(&fd_cb->so->so_snd) > 0) {
1534 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1535 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1536 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1537 }
1538 } else {
1539 error = ENOBUFS;
1540 }
1541 }
1542
1543 if (remaining_data != NULL) {
1544 if (sbspace(&fd_cb->so->so_snd) > 0) {
1545 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1546 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1547 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1548 }
1549 } else {
1550 error = ENOBUFS;
1551 }
1552 }
1553 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1554 if (to_send) {
1555 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1556 if (error) {
1557 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1558 } else {
1559 fd_cb->send_window -= to_send;
1560 }
1561 } else {
1562 /* buffer it */
1563 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1564 if (toaddr != NULL) {
1565 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1566 FDLOG(LOG_ERR, fd_cb,
1567 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1568 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1569 }
1570 } else {
1571 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1572 FDLOG(LOG_ERR, fd_cb,
1573 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1574 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1575 }
1576 }
1577 } else {
1578 error = ENOBUFS;
1579 }
1580 }
1581 }
1582
1583 return error;
1584 }
1585
1586 static int
1587 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1588 {
1589 int error = 0;
1590 mbuf_t packet = NULL;
1591 uint32_t net_read_count = htonl(read_count);
1592
1593 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1594 if (error) {
1595 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1596 goto done;
1597 }
1598
1599 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1600 if (error) {
1601 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1602 goto done;
1603 }
1604
1605 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1606 if (error) {
1607 goto done;
1608 }
1609
1610 done:
1611 if (error && packet != NULL) {
1612 mbuf_free(packet);
1613 }
1614
1615 return error;
1616 }
1617
1618 static int
1619 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1620 {
1621 int error = 0;
1622 mbuf_t packet = NULL;
1623
1624 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1625 if (error) {
1626 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1627 goto done;
1628 }
1629
1630 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1631 if (error) {
1632 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1633 goto done;
1634 }
1635
1636 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1637 if (error) {
1638 goto done;
1639 }
1640
1641 done:
1642 if (error && packet != NULL) {
1643 mbuf_free(packet);
1644 }
1645
1646 return error;
1647 }
1648
1649 static void
1650 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1651 {
1652 uint32_t connect_error;
1653 uint32_t ctl_unit = 0;
1654 int error = 0;
1655 struct flow_divert_group *grp = NULL;
1656 struct sockaddr_storage local_address;
1657 int out_if_index = 0;
1658 struct sockaddr_storage remote_address;
1659 uint32_t send_window;
1660 uint32_t app_data_length = 0;
1661
1662 memset(&local_address, 0, sizeof(local_address));
1663 memset(&remote_address, 0, sizeof(remote_address));
1664
1665 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1666 if (error) {
1667 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1668 return;
1669 }
1670
1671 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1672
1673 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1674 if (error) {
1675 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1676 return;
1677 }
1678
1679 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1680 if (error) {
1681 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
1682 }
1683
1684 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1685 if (error) {
1686 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1687 }
1688
1689 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1690 if (error) {
1691 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1692 }
1693
1694 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1695 if (error) {
1696 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1697 }
1698
1699 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1700 if (error) {
1701 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
1702 }
1703
1704 error = 0;
1705 connect_error = ntohl(connect_error);
1706 ctl_unit = ntohl(ctl_unit);
1707
1708 lck_rw_lock_shared(&g_flow_divert_group_lck);
1709
1710 if (connect_error == 0 && ctl_unit > 0) {
1711 if (ctl_unit >= GROUP_COUNT_MAX) {
1712 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1713 error = EINVAL;
1714 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1715 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1716 error = EINVAL;
1717 } else {
1718 grp = g_flow_divert_groups[ctl_unit];
1719 if (grp == NULL) {
1720 error = ECONNRESET;
1721 }
1722 }
1723 }
1724
1725 FDLOCK(fd_cb);
1726 if (fd_cb->so != NULL) {
1727 struct inpcb *inp = NULL;
1728 struct ifnet *ifp = NULL;
1729 struct flow_divert_group *old_group;
1730
1731 socket_lock(fd_cb->so, 0);
1732
1733 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1734 goto done;
1735 }
1736
1737 inp = sotoinpcb(fd_cb->so);
1738
1739 if (connect_error || error) {
1740 goto set_socket_state;
1741 }
1742
1743 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1744 error = EINVAL;
1745 goto set_socket_state;
1746 }
1747 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1748 if (local_address.ss_len > sizeof(local_address)) {
1749 local_address.ss_len = sizeof(local_address);
1750 }
1751 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1752 }
1753
1754 if (remote_address.ss_family != 0) {
1755 if (remote_address.ss_len > sizeof(remote_address)) {
1756 remote_address.ss_len = sizeof(remote_address);
1757 }
1758 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1759 } else {
1760 error = EINVAL;
1761 goto set_socket_state;
1762 }
1763
1764 if (app_data_length > 0) {
1765 uint8_t *app_data = NULL;
1766 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1767 if (app_data != NULL) {
1768 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1769 if (error == 0) {
1770 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1771 if (fd_cb->app_data != NULL) {
1772 FREE(fd_cb->app_data, M_TEMP);
1773 }
1774 fd_cb->app_data = app_data;
1775 fd_cb->app_data_length = app_data_length;
1776 } else {
1777 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1778 FREE(app_data, M_TEMP);
1779 }
1780 } else {
1781 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1782 }
1783 }
1784
1785 ifnet_head_lock_shared();
1786 if (out_if_index > 0 && out_if_index <= if_index) {
1787 ifp = ifindex2ifnet[out_if_index];
1788 }
1789
1790 if (ifp != NULL) {
1791 inp->inp_last_outifp = ifp;
1792 } else {
1793 error = EINVAL;
1794 }
1795 ifnet_head_done();
1796
1797 if (error) {
1798 goto set_socket_state;
1799 }
1800
1801 if (fd_cb->group == NULL) {
1802 error = EINVAL;
1803 goto set_socket_state;
1804 }
1805
1806 if (grp != NULL) {
1807 old_group = fd_cb->group;
1808
1809 lck_rw_lock_exclusive(&old_group->lck);
1810 lck_rw_lock_exclusive(&grp->lck);
1811
1812 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1813 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1814 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1815 }
1816
1817 fd_cb->group = grp;
1818
1819 lck_rw_done(&grp->lck);
1820 lck_rw_done(&old_group->lck);
1821 }
1822
1823 fd_cb->send_window = ntohl(send_window);
1824
1825 set_socket_state:
1826 if (!connect_error && !error) {
1827 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1828 error = flow_divert_send_connect_result(fd_cb);
1829 }
1830
1831 if (connect_error || error) {
1832 if (!connect_error) {
1833 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1834 fd_cb->so->so_error = error;
1835 flow_divert_send_close_if_needed(fd_cb);
1836 } else {
1837 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1838 fd_cb->so->so_error = connect_error;
1839 }
1840 flow_divert_disconnect_socket(fd_cb->so);
1841 } else {
1842 flow_divert_send_buffered_data(fd_cb, FALSE);
1843 soisconnected(fd_cb->so);
1844 }
1845
1846 done:
1847 socket_unlock(fd_cb->so, 0);
1848 }
1849 FDUNLOCK(fd_cb);
1850
1851 lck_rw_done(&g_flow_divert_group_lck);
1852 }
1853
1854 static void
1855 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1856 {
1857 uint32_t close_error;
1858 int error = 0;
1859 int how;
1860
1861 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1862 if (error) {
1863 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1864 return;
1865 }
1866
1867 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1868 if (error) {
1869 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1870 return;
1871 }
1872
1873 how = ntohl(how);
1874
1875 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1876
1877 FDLOCK(fd_cb);
1878 if (fd_cb->so != NULL) {
1879 socket_lock(fd_cb->so, 0);
1880
1881 fd_cb->so->so_error = ntohl(close_error);
1882
1883 flow_divert_update_closed_state(fd_cb, how, TRUE);
1884
1885 how = flow_divert_tunnel_how_closed(fd_cb);
1886 if (how == SHUT_RDWR) {
1887 flow_divert_disconnect_socket(fd_cb->so);
1888 } else if (how == SHUT_RD) {
1889 socantrcvmore(fd_cb->so);
1890 } else if (how == SHUT_WR) {
1891 socantsendmore(fd_cb->so);
1892 }
1893
1894 socket_unlock(fd_cb->so, 0);
1895 }
1896 FDUNLOCK(fd_cb);
1897 }
1898
1899 static mbuf_t
1900 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
1901 {
1902 struct inpcb *inp = sotoinpcb(fd_cb->so);
1903 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1904 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
1905
1906 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1907 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1908 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1909 struct in6_pktinfo pi6;
1910
1911 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1912 pi6.ipi6_ifindex = 0;
1913 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
1914 }
1915 return (NULL);
1916 }
1917
1918 static void
1919 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1920 {
1921 FDLOCK(fd_cb);
1922 if (fd_cb->so != NULL) {
1923 int error = 0;
1924 mbuf_t data = NULL;
1925 size_t data_size;
1926 struct sockaddr_storage remote_address;
1927 boolean_t got_remote_sa = FALSE;
1928
1929 socket_lock(fd_cb->so, 0);
1930
1931 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1932 uint32_t val_size = 0;
1933
1934 /* check if we got remote address with data */
1935 memset(&remote_address, 0, sizeof(remote_address));
1936 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1937 if (error || val_size > sizeof(remote_address)) {
1938 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1939 error = 0;
1940 } else {
1941 /* validate the address */
1942 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1943 got_remote_sa = TRUE;
1944 }
1945 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1946 }
1947 }
1948
1949 data_size = (mbuf_pkthdr_len(packet) - offset);
1950
1951 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1952
1953 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1954 if (error || data == NULL) {
1955 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1956 } else {
1957 if (flow_divert_check_no_cellular(fd_cb) ||
1958 flow_divert_check_no_expensive(fd_cb))
1959 {
1960 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1961 flow_divert_send_close(fd_cb, SHUT_RDWR);
1962 flow_divert_disconnect_socket(fd_cb->so);
1963 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1964 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1965 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1966 fd_cb->bytes_received += data_size;
1967 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1968 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1969 sorwakeup(fd_cb->so);
1970 data = NULL;
1971 } else {
1972 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1973 }
1974 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1975 struct sockaddr *append_sa;
1976 mbuf_t mctl;
1977
1978 if (got_remote_sa == TRUE) {
1979 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1980 (struct sockaddr *)&remote_address, &append_sa);
1981 } else {
1982 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1983 fd_cb->remote_address, &append_sa);
1984 }
1985 if (error) {
1986 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
1987 }
1988
1989 mctl = flow_divert_get_control_mbuf(fd_cb);
1990 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
1991 fd_cb->bytes_received += data_size;
1992 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1993 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1994 sorwakeup(fd_cb->so);
1995 data = NULL;
1996 } else {
1997 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
1998 }
1999 if (!error) {
2000 FREE(append_sa, M_TEMP);
2001 }
2002 }
2003 }
2004 }
2005 socket_unlock(fd_cb->so, 0);
2006
2007 if (data != NULL) {
2008 mbuf_freem(data);
2009 }
2010 }
2011 FDUNLOCK(fd_cb);
2012 }
2013
2014 static void
2015 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2016 {
2017 uint32_t read_count;
2018 int error = 0;
2019
2020 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2021 if (error) {
2022 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2023 return;
2024 }
2025
2026 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2027
2028 FDLOCK(fd_cb);
2029 if (fd_cb->so != NULL) {
2030 socket_lock(fd_cb->so, 0);
2031 fd_cb->send_window += ntohl(read_count);
2032 flow_divert_send_buffered_data(fd_cb, FALSE);
2033 socket_unlock(fd_cb->so, 0);
2034 }
2035 FDUNLOCK(fd_cb);
2036 }
2037
2038 static void
2039 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2040 {
2041 int error = 0;
2042 uint32_t key_size = 0;
2043 int log_level;
2044
2045 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2046 if (error) {
2047 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2048 return;
2049 }
2050
2051 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2052 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2053 return;
2054 }
2055
2056 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2057 if (!error) {
2058 nil_pcb.log_level = log_level;
2059 }
2060
2061 lck_rw_lock_exclusive(&group->lck);
2062
2063 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2064 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2065 if (error) {
2066 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2067 FREE(group->token_key, M_TEMP);
2068 group->token_key = NULL;
2069 lck_rw_done(&group->lck);
2070 return;
2071 }
2072
2073 group->token_key_size = key_size;
2074
2075 lck_rw_done(&group->lck);
2076 }
2077
2078 static void
2079 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2080 {
2081 int error = 0;
2082 struct sockaddr_storage local_address;
2083 int out_if_index = 0;
2084 struct sockaddr_storage remote_address;
2085 uint32_t app_data_length = 0;
2086
2087 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2088
2089 memset(&local_address, 0, sizeof(local_address));
2090 memset(&remote_address, 0, sizeof(remote_address));
2091
2092 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2093 if (error) {
2094 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
2095 }
2096
2097 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2098 if (error) {
2099 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
2100 }
2101
2102 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2103 if (error) {
2104 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2105 }
2106
2107 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2108 if (error) {
2109 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2110 }
2111
2112 FDLOCK(fd_cb);
2113 if (fd_cb->so != NULL) {
2114 socket_lock(fd_cb->so, 0);
2115
2116 if (local_address.ss_family != 0) {
2117 if (local_address.ss_len > sizeof(local_address)) {
2118 local_address.ss_len = sizeof(local_address);
2119 }
2120 if (fd_cb->local_address != NULL) {
2121 FREE(fd_cb->local_address, M_SONAME);
2122 fd_cb->local_address = NULL;
2123 }
2124 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2125 }
2126
2127 if (remote_address.ss_family != 0) {
2128 if (remote_address.ss_len > sizeof(remote_address)) {
2129 remote_address.ss_len = sizeof(remote_address);
2130 }
2131 if (fd_cb->remote_address != NULL) {
2132 FREE(fd_cb->remote_address, M_SONAME);
2133 fd_cb->remote_address = NULL;
2134 }
2135 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2136 }
2137
2138 if (out_if_index > 0) {
2139 struct inpcb *inp = NULL;
2140 struct ifnet *ifp = NULL;
2141
2142 inp = sotoinpcb(fd_cb->so);
2143
2144 ifnet_head_lock_shared();
2145 if (out_if_index <= if_index) {
2146 ifp = ifindex2ifnet[out_if_index];
2147 }
2148
2149 if (ifp != NULL) {
2150 inp->inp_last_outifp = ifp;
2151 }
2152 ifnet_head_done();
2153 }
2154
2155 if (app_data_length > 0) {
2156 uint8_t *app_data = NULL;
2157 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2158 if (app_data != NULL) {
2159 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2160 if (error == 0) {
2161 if (fd_cb->app_data != NULL) {
2162 FREE(fd_cb->app_data, M_TEMP);
2163 }
2164 fd_cb->app_data = app_data;
2165 fd_cb->app_data_length = app_data_length;
2166 } else {
2167 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2168 FREE(app_data, M_TEMP);
2169 }
2170 } else {
2171 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2172 }
2173 }
2174
2175 socket_unlock(fd_cb->so, 0);
2176 }
2177 FDUNLOCK(fd_cb);
2178 }
2179
2180 static void
2181 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2182 {
2183 size_t bytes_mem_size;
2184 size_t child_maps_mem_size;
2185 int cursor;
2186 int error = 0;
2187 struct flow_divert_trie new_trie;
2188 int insert_error = 0;
2189 size_t nodes_mem_size;
2190 int prefix_count = 0;
2191 int signing_id_count = 0;
2192 size_t trie_memory_size = 0;
2193
2194 lck_rw_lock_exclusive(&group->lck);
2195
2196 /* Re-set the current trie */
2197 if (group->signing_id_trie.memory != NULL) {
2198 FREE(group->signing_id_trie.memory, M_TEMP);
2199 }
2200 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2201 group->signing_id_trie.root = NULL_TRIE_IDX;
2202
2203 memset(&new_trie, 0, sizeof(new_trie));
2204
2205 /* Get the number of shared prefixes in the new set of signing ID strings */
2206 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2207
2208 if (prefix_count < 0) {
2209 lck_rw_done(&group->lck);
2210 return;
2211 }
2212
2213 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2214 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2215 cursor >= 0;
2216 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2217 {
2218 uint32_t sid_size = 0;
2219 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2220 new_trie.bytes_count += sid_size;
2221 signing_id_count++;
2222 }
2223
2224 if (signing_id_count == 0) {
2225 lck_rw_done(&group->lck);
2226 return;
2227 }
2228
2229 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2230 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2231
2232 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2233 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2234
2235 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2236 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2237 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2238
2239 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2240 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2241 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2242 lck_rw_done(&group->lck);
2243 return;
2244 }
2245
2246 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2247 if (new_trie.memory == NULL) {
2248 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2249 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2250 lck_rw_done(&group->lck);
2251 return;
2252 }
2253
2254 /* Initialize the free lists */
2255 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2256 new_trie.nodes_free_next = 0;
2257 memset(new_trie.nodes, 0, nodes_mem_size);
2258
2259 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2260 new_trie.child_maps_free_next = 0;
2261 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2262
2263 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2264 new_trie.bytes_free_next = 0;
2265
2266 /* The root is an empty node */
2267 new_trie.root = trie_node_alloc(&new_trie);
2268
2269 /* Add each signing ID to the trie */
2270 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2271 cursor >= 0;
2272 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2273 {
2274 uint32_t sid_size = 0;
2275 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2276 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2277 uint16_t new_node_idx;
2278 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2279 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2280 if (new_node_idx == NULL_TRIE_IDX) {
2281 insert_error = EINVAL;
2282 break;
2283 }
2284 } else {
2285 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2286 insert_error = ENOBUFS;
2287 break;
2288 }
2289 }
2290
2291 if (!insert_error) {
2292 group->signing_id_trie = new_trie;
2293 } else {
2294 FREE(new_trie.memory, M_TEMP);
2295 }
2296
2297 lck_rw_done(&group->lck);
2298 }
2299
2300 static int
2301 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2302 {
2303 struct flow_divert_packet_header hdr;
2304 int error = 0;
2305 struct flow_divert_pcb *fd_cb;
2306
2307 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2308 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2309 error = EINVAL;
2310 goto done;
2311 }
2312
2313 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2314 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2315 error = EINVAL;
2316 goto done;
2317 }
2318
2319 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2320 if (error) {
2321 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2322 error = ENOBUFS;
2323 goto done;
2324 }
2325
2326 hdr.conn_id = ntohl(hdr.conn_id);
2327
2328 if (hdr.conn_id == 0) {
2329 switch (hdr.packet_type) {
2330 case FLOW_DIVERT_PKT_GROUP_INIT:
2331 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2332 break;
2333 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2334 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2335 break;
2336 default:
2337 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2338 break;
2339 }
2340 goto done;
2341 }
2342
2343 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2344 if (fd_cb == NULL) {
2345 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2346 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2347 }
2348 goto done;
2349 }
2350
2351 switch (hdr.packet_type) {
2352 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2353 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2354 break;
2355 case FLOW_DIVERT_PKT_CLOSE:
2356 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2357 break;
2358 case FLOW_DIVERT_PKT_DATA:
2359 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2360 break;
2361 case FLOW_DIVERT_PKT_READ_NOTIFY:
2362 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2363 break;
2364 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2365 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2366 break;
2367 default:
2368 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2369 break;
2370 }
2371
2372 FDRELEASE(fd_cb);
2373
2374 done:
2375 mbuf_freem(packet);
2376 return error;
2377 }
2378
2379 static void
2380 flow_divert_close_all(struct flow_divert_group *group)
2381 {
2382 struct flow_divert_pcb *fd_cb;
2383 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2384
2385 SLIST_INIT(&tmp_list);
2386
2387 lck_rw_lock_exclusive(&group->lck);
2388
2389 MBUFQ_DRAIN(&group->send_queue);
2390
2391 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2392 FDRETAIN(fd_cb);
2393 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2394 }
2395
2396 lck_rw_done(&group->lck);
2397
2398 while (!SLIST_EMPTY(&tmp_list)) {
2399 fd_cb = SLIST_FIRST(&tmp_list);
2400 FDLOCK(fd_cb);
2401 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2402 if (fd_cb->so != NULL) {
2403 socket_lock(fd_cb->so, 0);
2404 flow_divert_pcb_remove(fd_cb);
2405 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2406 fd_cb->so->so_error = ECONNABORTED;
2407 flow_divert_disconnect_socket(fd_cb->so);
2408 socket_unlock(fd_cb->so, 0);
2409 }
2410 FDUNLOCK(fd_cb);
2411 FDRELEASE(fd_cb);
2412 }
2413 }
2414
2415 void
2416 flow_divert_detach(struct socket *so)
2417 {
2418 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2419
2420 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2421
2422 so->so_flags &= ~SOF_FLOW_DIVERT;
2423 so->so_fd_pcb = NULL;
2424
2425 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2426
2427 if (fd_cb->group != NULL) {
2428 /* Last-ditch effort to send any buffered data */
2429 flow_divert_send_buffered_data(fd_cb, TRUE);
2430
2431 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2432 flow_divert_send_close_if_needed(fd_cb);
2433 /* Remove from the group */
2434 flow_divert_pcb_remove(fd_cb);
2435 }
2436
2437 socket_unlock(so, 0);
2438 FDLOCK(fd_cb);
2439 fd_cb->so = NULL;
2440 FDUNLOCK(fd_cb);
2441 socket_lock(so, 0);
2442
2443 FDRELEASE(fd_cb); /* Release the socket's reference */
2444 }
2445
2446 static int
2447 flow_divert_close(struct socket *so)
2448 {
2449 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2450
2451 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2452
2453 FDLOG0(LOG_INFO, fd_cb, "Closing");
2454
2455 if (SOCK_TYPE(so) == SOCK_STREAM) {
2456 soisdisconnecting(so);
2457 sbflush(&so->so_rcv);
2458 }
2459
2460 flow_divert_send_buffered_data(fd_cb, TRUE);
2461 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2462 flow_divert_send_close_if_needed(fd_cb);
2463
2464 /* Remove from the group */
2465 flow_divert_pcb_remove(fd_cb);
2466
2467 return 0;
2468 }
2469
2470 static int
2471 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2472 sae_connid_t cid __unused)
2473 {
2474 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2475 return (EINVAL);
2476 }
2477
2478 return (flow_divert_close(so));
2479 }
2480
2481 static int
2482 flow_divert_shutdown(struct socket *so)
2483 {
2484 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2485
2486 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2487
2488 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2489
2490 socantsendmore(so);
2491
2492 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2493 flow_divert_send_close_if_needed(fd_cb);
2494
2495 return 0;
2496 }
2497
2498 static int
2499 flow_divert_rcvd(struct socket *so, int flags __unused)
2500 {
2501 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2502 uint32_t latest_sb_size;
2503 uint32_t read_count;
2504
2505 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2506
2507 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2508
2509 if (fd_cb->sb_size < latest_sb_size) {
2510 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2511 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2512 }
2513
2514 read_count = fd_cb->sb_size - latest_sb_size;
2515
2516 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2517
2518 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2519 fd_cb->bytes_read_by_app += read_count;
2520 fd_cb->sb_size = latest_sb_size;
2521 }
2522
2523 return 0;
2524 }
2525
2526 static int
2527 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2528 {
2529 int error = 0;
2530 int port = 0;
2531
2532 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2533 if (error) {
2534 goto done;
2535 }
2536
2537 if (toaddr->sa_family == AF_INET) {
2538 port = ntohs((satosin(toaddr))->sin_port);
2539 }
2540 #if INET6
2541 else {
2542 port = ntohs((satosin6(toaddr))->sin6_port);
2543 }
2544 #endif
2545
2546 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2547 if (error) {
2548 goto done;
2549 }
2550
2551 done:
2552 return error;
2553 }
2554
2555 struct sockaddr *
2556 flow_divert_get_buffered_target_address(mbuf_t buffer)
2557 {
2558 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2559 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2560 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2561 return toaddr;
2562 }
2563 }
2564 return NULL;
2565 }
2566
2567 static boolean_t
2568 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2569 {
2570 switch(addr->sa_family)
2571 {
2572 case AF_INET:
2573 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2574 return FALSE;
2575 }
2576 break;
2577 #if INET6
2578 case AF_INET6:
2579 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2580 return FALSE;
2581 }
2582 break;
2583 #endif /* INET6 */
2584 default:
2585 return FALSE;
2586 }
2587 return TRUE;
2588 }
2589
2590 static errno_t
2591 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2592 {
2593 int error = 0;
2594 union sockaddr_in_4_6 sin46;
2595
2596 bzero(&sin46, sizeof(sin46));
2597 if (inp->inp_vflag & INP_IPV4) {
2598 struct sockaddr_in *sin = &sin46.sin;
2599
2600 sin->sin_family = AF_INET;
2601 sin->sin_len = sizeof(*sin);
2602 sin->sin_port = inp->inp_lport;
2603 sin->sin_addr = inp->inp_laddr;
2604 } else if (inp->inp_vflag & INP_IPV6) {
2605 struct sockaddr_in6 *sin6 = &sin46.sin6;
2606
2607 sin6->sin6_len = sizeof(*sin6);
2608 sin6->sin6_family = AF_INET6;
2609 sin6->sin6_port = inp->inp_lport;
2610 sin6->sin6_addr = inp->in6p_laddr;
2611 }
2612 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2613 if (*local_socket == NULL) {
2614 error = ENOBUFS;
2615 }
2616 return (error);
2617 }
2618
2619 static boolean_t
2620 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2621 {
2622 return (inp->inp_lport != 0
2623 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2624 }
2625
2626 static errno_t
2627 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2628 struct sockaddr **dup)
2629 {
2630 int error = 0;
2631 struct sockaddr *result;
2632 struct sockaddr_storage ss;
2633
2634 if (addr != NULL) {
2635 result = addr;
2636 } else {
2637 memset(&ss, 0, sizeof(ss));
2638 ss.ss_family = family;
2639 if (ss.ss_family == AF_INET) {
2640 ss.ss_len = sizeof(struct sockaddr_in);
2641 }
2642 #if INET6
2643 else if (ss.ss_family == AF_INET6) {
2644 ss.ss_len = sizeof(struct sockaddr_in6);
2645 }
2646 #endif /* INET6 */
2647 else {
2648 error = EINVAL;
2649 }
2650 result = (struct sockaddr *)&ss;
2651 }
2652
2653 if (!error) {
2654 *dup = dup_sockaddr(result, 1);
2655 if (*dup == NULL) {
2656 error = ENOBUFS;
2657 }
2658 }
2659
2660 return error;
2661 }
2662
2663 static void
2664 flow_divert_disconnect_socket(struct socket *so)
2665 {
2666 soisdisconnected(so);
2667 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2668 struct inpcb *inp = NULL;
2669
2670 inp = sotoinpcb(so);
2671 if (inp != NULL) {
2672 #if INET6
2673 if (SOCK_CHECK_DOM(so, PF_INET6))
2674 in6_pcbdetach(inp);
2675 else
2676 #endif /* INET6 */
2677 in_pcbdetach(inp);
2678 }
2679 }
2680 }
2681
2682 static errno_t
2683 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2684 {
2685 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2686
2687 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2688
2689 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2690 fd_cb->remote_address,
2691 sa);
2692 }
2693
2694 static errno_t
2695 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2696 {
2697 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2698
2699 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2700
2701 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2702 fd_cb->local_address,
2703 sa);
2704 }
2705
2706 static errno_t
2707 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2708 {
2709 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2710
2711 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2712
2713 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2714 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2715 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2716 }
2717 }
2718
2719 if (SOCK_DOM(so) == PF_INET) {
2720 return g_tcp_protosw->pr_ctloutput(so, sopt);
2721 }
2722 #if INET6
2723 else if (SOCK_DOM(so) == PF_INET6) {
2724 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2725 }
2726 #endif
2727 return 0;
2728 }
2729
2730 errno_t
2731 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2732 {
2733 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2734 int error = 0;
2735 struct inpcb *inp = sotoinpcb(so);
2736 struct sockaddr_in *sinp;
2737 mbuf_t connect_packet = NULL;
2738 int do_send = 1;
2739
2740 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2741
2742 if (fd_cb->group == NULL) {
2743 error = ENETUNREACH;
2744 goto done;
2745 }
2746
2747 if (inp == NULL) {
2748 error = EINVAL;
2749 goto done;
2750 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2751 if (so->so_error) {
2752 error = so->so_error;
2753 so->so_error = 0;
2754 } else {
2755 error = EINVAL;
2756 }
2757 goto done;
2758 }
2759
2760 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2761 error = EALREADY;
2762 goto done;
2763 }
2764
2765 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2766 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2767 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2768 if (fd_cb->remote_address != NULL) {
2769 soisconnected(fd_cb->so);
2770 goto done;
2771 }
2772 }
2773
2774 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2775
2776 if (fd_cb->connect_packet == NULL) {
2777 if (to == NULL) {
2778 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2779 error = EINVAL;
2780 goto done;
2781 }
2782
2783 sinp = (struct sockaddr_in *)(void *)to;
2784 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2785 error = EAFNOSUPPORT;
2786 goto done;
2787 }
2788
2789 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2790 if (error) {
2791 goto done;
2792 }
2793
2794 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2795 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2796 do_send = 0;
2797 }
2798 } else {
2799 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2800 connect_packet = fd_cb->connect_packet;
2801 fd_cb->connect_packet = NULL;
2802 }
2803
2804 if (do_send) {
2805 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2806 if (error) {
2807 goto done;
2808 }
2809
2810 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2811 } else {
2812 fd_cb->connect_packet = connect_packet;
2813 connect_packet = NULL;
2814 }
2815
2816 soisconnecting(so);
2817
2818 done:
2819 if (error && connect_packet != NULL) {
2820 mbuf_freem(connect_packet);
2821 }
2822 return error;
2823 }
2824
2825 static int
2826 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2827 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
2828 {
2829 struct inpcb *inp = sotoinpcb(so);
2830 int error;
2831
2832 if (inp == NULL) {
2833 return (EINVAL);
2834 }
2835
2836 VERIFY(dst != NULL);
2837
2838 error = flow_divert_connect_out(so, dst, p);
2839
2840 if (error != 0) {
2841 return error;
2842 }
2843
2844 /* if there is data, send it */
2845 if (auio != NULL) {
2846 user_ssize_t datalen = 0;
2847
2848 socket_unlock(so, 0);
2849
2850 VERIFY(bytes_written != NULL);
2851
2852 datalen = uio_resid(auio);
2853 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2854 socket_lock(so, 0);
2855
2856 if (error == 0 || error == EWOULDBLOCK) {
2857 *bytes_written = datalen - uio_resid(auio);
2858 }
2859
2860 /*
2861 * sosend returns EWOULDBLOCK if it's a non-blocking
2862 * socket or a timeout occured (this allows to return
2863 * the amount of queued data through sendit()).
2864 *
2865 * However, connectx() returns EINPROGRESS in case of a
2866 * blocking socket. So we change the return value here.
2867 */
2868 if (error == EWOULDBLOCK) {
2869 error = EINPROGRESS;
2870 }
2871 }
2872
2873 if (error == 0 && pcid != NULL) {
2874 *pcid = 1; /* there is only 1 connection for a TCP */
2875 }
2876
2877 return (error);
2878 }
2879
2880 static int
2881 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
2882 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2883 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2884 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2885 {
2886 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
2887 }
2888
2889 #if INET6
2890 static int
2891 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
2892 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
2893 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2894 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
2895 {
2896 return (flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written));
2897 }
2898 #endif /* INET6 */
2899
2900 static int
2901 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
2902 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2903 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2904 user_addr_t aux_data __unused, uint32_t *aux_len)
2905 {
2906 int error = 0;
2907 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2908 struct ifnet *ifp = NULL;
2909 struct inpcb *inp = sotoinpcb(so);
2910
2911 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2912
2913 if (so->so_fd_pcb == NULL || inp == NULL) {
2914 error = EINVAL;
2915 goto out;
2916 }
2917
2918 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
2919 error = EINVAL;
2920 goto out;
2921 }
2922
2923 ifp = inp->inp_last_outifp;
2924 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2925 *soerror = so->so_error;
2926 *flags = 0;
2927
2928 if (so->so_state & SS_ISCONNECTED) {
2929 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2930 }
2931
2932 if (fd_cb->local_address == NULL) {
2933 struct sockaddr_in sin;
2934 bzero(&sin, sizeof(sin));
2935 sin.sin_len = sizeof(sin);
2936 sin.sin_family = AF_INET;
2937 *src_len = sin.sin_len;
2938 if (src != USER_ADDR_NULL) {
2939 error = copyout(&sin, src, sin.sin_len);
2940 if (error != 0) {
2941 goto out;
2942 }
2943 }
2944 } else {
2945 *src_len = fd_cb->local_address->sa_len;
2946 if (src != USER_ADDR_NULL) {
2947 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2948 if (error != 0) {
2949 goto out;
2950 }
2951 }
2952 }
2953
2954 if (fd_cb->remote_address == NULL) {
2955 struct sockaddr_in sin;
2956 bzero(&sin, sizeof(sin));
2957 sin.sin_len = sizeof(sin);
2958 sin.sin_family = AF_INET;
2959 *dst_len = sin.sin_len;
2960 if (dst != USER_ADDR_NULL) {
2961 error = copyout(&sin, dst, sin.sin_len);
2962 if (error != 0) {
2963 goto out;
2964 }
2965 }
2966 } else {
2967 *dst_len = fd_cb->remote_address->sa_len;
2968 if (dst != USER_ADDR_NULL) {
2969 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2970 if (error != 0) {
2971 goto out;
2972 }
2973 }
2974 }
2975
2976 *aux_type = 0;
2977 *aux_len = 0;
2978
2979 out:
2980 return error;
2981 }
2982
2983 static int
2984 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
2985 {
2986 int error = 0;
2987
2988 switch (cmd) {
2989 case SIOCGCONNINFO32: {
2990 struct so_cinforeq32 cifr;
2991 bcopy(data, &cifr, sizeof (cifr));
2992 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
2993 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
2994 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
2995 &cifr.scir_aux_type, cifr.scir_aux_data,
2996 &cifr.scir_aux_len);
2997 if (error == 0) {
2998 bcopy(&cifr, data, sizeof (cifr));
2999 }
3000 break;
3001 }
3002
3003 case SIOCGCONNINFO64: {
3004 struct so_cinforeq64 cifr;
3005 bcopy(data, &cifr, sizeof (cifr));
3006 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3007 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3008 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3009 &cifr.scir_aux_type, cifr.scir_aux_data,
3010 &cifr.scir_aux_len);
3011 if (error == 0) {
3012 bcopy(&cifr, data, sizeof (cifr));
3013 }
3014 break;
3015 }
3016
3017 default:
3018 error = EOPNOTSUPP;
3019 }
3020
3021 return error;
3022 }
3023
3024 static int
3025 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3026 {
3027 int error = flow_divert_control(so, cmd, data, ifp, p);
3028
3029 if (error == EOPNOTSUPP) {
3030 error = in_control(so, cmd, data, ifp, p);
3031 }
3032
3033 return error;
3034 }
3035
3036 static int
3037 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3038 {
3039 int error = flow_divert_control(so, cmd, data, ifp, p);
3040
3041 if (error == EOPNOTSUPP) {
3042 error = in6_control(so, cmd, data, ifp, p);
3043 }
3044
3045 return error;
3046 }
3047
3048 static errno_t
3049 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3050 {
3051 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3052 int error = 0;
3053 struct inpcb *inp;
3054
3055 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3056
3057 inp = sotoinpcb(so);
3058 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3059 error = ECONNRESET;
3060 goto done;
3061 }
3062
3063 if (control && mbuf_len(control) > 0) {
3064 error = EINVAL;
3065 goto done;
3066 }
3067
3068 if (flags & MSG_OOB) {
3069 error = EINVAL;
3070 goto done; /* We don't support OOB data */
3071 }
3072
3073 error = flow_divert_check_no_cellular(fd_cb) ||
3074 flow_divert_check_no_expensive(fd_cb);
3075 if (error) {
3076 goto done;
3077 }
3078
3079 /* Implicit connect */
3080 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3081 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3082 error = flow_divert_connect_out(so, to, p);
3083 if (error) {
3084 goto done;
3085 }
3086
3087 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3088 /* Open up the send window so that the data will get sent right away */
3089 fd_cb->send_window = mbuf_pkthdr_len(data);
3090 }
3091 }
3092
3093 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3094
3095 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3096 error = flow_divert_send_app_data(fd_cb, data, to);
3097 if (error) {
3098 goto done;
3099 }
3100
3101 data = NULL;
3102
3103 if (flags & PRUS_EOF) {
3104 flow_divert_shutdown(so);
3105 }
3106
3107 done:
3108 if (data) {
3109 mbuf_freem(data);
3110 }
3111 if (control) {
3112 mbuf_free(control);
3113 }
3114 return error;
3115 }
3116
3117 static int
3118 flow_divert_preconnect(struct socket *so)
3119 {
3120 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3121 int error = 0;
3122
3123 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3124 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3125 mbuf_t connect_packet = fd_cb->connect_packet;
3126 fd_cb->connect_packet = NULL;
3127
3128 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3129 if (error) {
3130 mbuf_freem(connect_packet);
3131 }
3132
3133 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3134 }
3135
3136 so->so_flags1 &= ~SOF1_PRECONNECT_DATA;
3137
3138 return error;
3139 }
3140
3141 static void
3142 flow_divert_set_protosw(struct socket *so)
3143 {
3144 so->so_flags |= SOF_FLOW_DIVERT;
3145 if (SOCK_DOM(so) == PF_INET) {
3146 so->so_proto = &g_flow_divert_in_protosw;
3147 }
3148 #if INET6
3149 else {
3150 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3151 }
3152 #endif /* INET6 */
3153 }
3154
3155 static void
3156 flow_divert_set_udp_protosw(struct socket *so)
3157 {
3158 so->so_flags |= SOF_FLOW_DIVERT;
3159 if (SOCK_DOM(so) == PF_INET) {
3160 so->so_proto = &g_flow_divert_in_udp_protosw;
3161 }
3162 #if INET6
3163 else {
3164 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3165 }
3166 #endif /* INET6 */
3167 }
3168
3169 static errno_t
3170 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3171 {
3172 int error = 0;
3173 struct flow_divert_pcb *fd_cb = NULL;
3174 struct ifnet *ifp = NULL;
3175 struct inpcb *inp = NULL;
3176 struct socket *old_so;
3177 mbuf_t recv_data = NULL;
3178
3179 socket_unlock(so, 0);
3180
3181 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3182
3183 /* Find the flow divert control block */
3184 lck_rw_lock_shared(&g_flow_divert_group_lck);
3185 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3186 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3187 if (group != NULL) {
3188 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3189 }
3190 }
3191 lck_rw_done(&g_flow_divert_group_lck);
3192
3193 if (fd_cb == NULL) {
3194 error = ENOENT;
3195 goto done;
3196 }
3197
3198 FDLOCK(fd_cb);
3199
3200 /* Dis-associate the flow divert control block from its current socket */
3201 old_so = fd_cb->so;
3202
3203 inp = sotoinpcb(old_so);
3204
3205 VERIFY(inp != NULL);
3206
3207 socket_lock(old_so, 0);
3208 flow_divert_disconnect_socket(old_so);
3209 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3210 old_so->so_fd_pcb = NULL;
3211 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3212 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3213 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3214 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3215 }
3216 fd_cb->so = NULL;
3217 /* Save the output interface */
3218 ifp = inp->inp_last_outifp;
3219 if (old_so->so_rcv.sb_cc > 0) {
3220 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3221 sbflush(&old_so->so_rcv);
3222 }
3223 socket_unlock(old_so, 0);
3224
3225 /* Associate the new socket with the flow divert control block */
3226 socket_lock(so, 0);
3227 so->so_fd_pcb = fd_cb;
3228 inp = sotoinpcb(so);
3229 inp->inp_last_outifp = ifp;
3230 if (recv_data != NULL) {
3231 if (sbappendstream(&so->so_rcv, recv_data)) {
3232 sorwakeup(so);
3233 }
3234 }
3235 flow_divert_set_protosw(so);
3236 socket_unlock(so, 0);
3237
3238 fd_cb->so = so;
3239 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3240
3241 FDUNLOCK(fd_cb);
3242
3243 done:
3244 socket_lock(so, 0);
3245
3246 if (fd_cb != NULL) {
3247 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3248 }
3249
3250 return error;
3251 }
3252
3253 errno_t
3254 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3255 {
3256 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3257 struct inpcb *inp;
3258 int error = 0;
3259
3260 inp = sotoinpcb(so);
3261 if (inp == NULL) {
3262 return (EINVAL);
3263 }
3264
3265 if (fd_cb == NULL) {
3266 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3267 if (fd_ctl_unit > 0) {
3268 error = flow_divert_pcb_init(so, fd_ctl_unit);
3269 fd_cb = so->so_fd_pcb;
3270 if (error != 0 || fd_cb == NULL) {
3271 goto done;
3272 }
3273 } else {
3274 error = ENETDOWN;
3275 goto done;
3276 }
3277 }
3278 return flow_divert_data_out(so, flags, data, to, control, p);
3279
3280 done:
3281 if (data) {
3282 mbuf_freem(data);
3283 }
3284 if (control) {
3285 mbuf_free(control);
3286 }
3287
3288 return error;
3289 }
3290
3291 errno_t
3292 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3293 {
3294 errno_t error = 0;
3295 struct flow_divert_pcb *fd_cb;
3296
3297 if (so->so_flags & SOF_FLOW_DIVERT) {
3298 return EALREADY;
3299 }
3300
3301 fd_cb = flow_divert_pcb_create(so);
3302 if (fd_cb != NULL) {
3303 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3304 if (error) {
3305 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3306 FDRELEASE(fd_cb);
3307 } else {
3308 fd_cb->control_group_unit = ctl_unit;
3309 so->so_fd_pcb = fd_cb;
3310
3311 if (SOCK_TYPE(so) == SOCK_STREAM) {
3312 flow_divert_set_protosw(so);
3313 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3314 flow_divert_set_udp_protosw(so);
3315 }
3316
3317 FDLOG0(LOG_INFO, fd_cb, "Created");
3318 }
3319 } else {
3320 error = ENOMEM;
3321 }
3322
3323 return error;
3324 }
3325
3326 errno_t
3327 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3328 {
3329 uint32_t ctl_unit = 0;
3330 uint32_t key_unit = 0;
3331 uint32_t flow_id = 0;
3332 int error = 0;
3333 int hmac_error = 0;
3334 mbuf_t token = NULL;
3335
3336 if (so->so_flags & SOF_FLOW_DIVERT) {
3337 error = EALREADY;
3338 goto done;
3339 }
3340
3341 if (g_init_result) {
3342 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3343 error = ENOPROTOOPT;
3344 goto done;
3345 }
3346
3347 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3348 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3349 (SOCK_DOM(so) != PF_INET
3350 #if INET6
3351 && SOCK_DOM(so) != PF_INET6
3352 #endif
3353 ))
3354 {
3355 error = EINVAL;
3356 goto done;
3357 } else {
3358 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3359 struct tcpcb *tp = sototcpcb(so);
3360 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3361 error = EINVAL;
3362 goto done;
3363 }
3364 }
3365 }
3366
3367 error = soopt_getm(sopt, &token);
3368 if (error) {
3369 goto done;
3370 }
3371
3372 error = soopt_mcopyin(sopt, token);
3373 if (error) {
3374 goto done;
3375 }
3376
3377 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3378 if (!error) {
3379 key_unit = ntohl(key_unit);
3380 } else if (error != ENOENT) {
3381 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3382 goto done;
3383 } else {
3384 key_unit = 0;
3385 }
3386
3387 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3388 if (error) {
3389 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3390 goto done;
3391 }
3392
3393 /* A valid kernel control unit is required */
3394 ctl_unit = ntohl(ctl_unit);
3395 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3396 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3397 error = EINVAL;
3398 goto done;
3399 }
3400
3401 socket_unlock(so, 0);
3402 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3403 socket_lock(so, 0);
3404
3405 if (hmac_error && hmac_error != ENOENT) {
3406 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3407 error = hmac_error;
3408 goto done;
3409 }
3410
3411 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3412 if (error && error != ENOENT) {
3413 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3414 goto done;
3415 }
3416
3417 if (flow_id == 0) {
3418 error = flow_divert_pcb_init(so, ctl_unit);
3419 if (error == 0) {
3420 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3421 int log_level = LOG_NOTICE;
3422
3423 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3424 sizeof(log_level), &log_level, NULL);
3425 if (error == 0) {
3426 fd_cb->log_level = log_level;
3427 }
3428 error = 0;
3429
3430 fd_cb->connect_token = token;
3431 token = NULL;
3432 }
3433 } else {
3434 error = flow_divert_attach(so, flow_id, ctl_unit);
3435 }
3436
3437 if (hmac_error == 0) {
3438 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3439 if (fd_cb != NULL) {
3440 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3441 }
3442 }
3443
3444 done:
3445 if (token != NULL) {
3446 mbuf_freem(token);
3447 }
3448
3449 return error;
3450 }
3451
3452 errno_t
3453 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3454 {
3455 uint32_t ctl_unit;
3456 int error = 0;
3457 uint8_t hmac[SHA_DIGEST_LENGTH];
3458 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3459 mbuf_t token = NULL;
3460 struct flow_divert_group *control_group = NULL;
3461
3462 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3463 error = EINVAL;
3464 goto done;
3465 }
3466
3467 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3468
3469 if (fd_cb->group == NULL) {
3470 error = EINVAL;
3471 goto done;
3472 }
3473
3474 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3475 if (error) {
3476 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3477 goto done;
3478 }
3479
3480 ctl_unit = htonl(fd_cb->group->ctl_unit);
3481
3482 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3483 if (error) {
3484 goto done;
3485 }
3486
3487 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3488 if (error) {
3489 goto done;
3490 }
3491
3492 if (fd_cb->app_data != NULL) {
3493 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3494 if (error) {
3495 goto done;
3496 }
3497 }
3498
3499 socket_unlock(so, 0);
3500 lck_rw_lock_shared(&g_flow_divert_group_lck);
3501
3502 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3503 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3504 {
3505 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3506 }
3507
3508 if (control_group != NULL) {
3509 lck_rw_lock_shared(&control_group->lck);
3510 ctl_unit = htonl(control_group->ctl_unit);
3511 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3512 if (!error) {
3513 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3514 }
3515 lck_rw_done(&control_group->lck);
3516 } else {
3517 error = ENOPROTOOPT;
3518 }
3519
3520 lck_rw_done(&g_flow_divert_group_lck);
3521 socket_lock(so, 0);
3522
3523 if (error) {
3524 goto done;
3525 }
3526
3527 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3528 if (error) {
3529 goto done;
3530 }
3531
3532 if (sopt->sopt_val == USER_ADDR_NULL) {
3533 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3534 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3535 goto done;
3536 }
3537
3538 error = soopt_mcopyout(sopt, token);
3539 if (error) {
3540 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3541 goto done;
3542 }
3543
3544 done:
3545 if (token != NULL) {
3546 mbuf_freem(token);
3547 }
3548
3549 return error;
3550 }
3551
3552 static errno_t
3553 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3554 {
3555 struct flow_divert_group *new_group = NULL;
3556 int error = 0;
3557
3558 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3559 error = EINVAL;
3560 goto done;
3561 }
3562
3563 *unitinfo = NULL;
3564
3565 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3566 if (new_group == NULL) {
3567 error = ENOBUFS;
3568 goto done;
3569 }
3570
3571 memset(new_group, 0, sizeof(*new_group));
3572
3573 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3574 RB_INIT(&new_group->pcb_tree);
3575 new_group->ctl_unit = sac->sc_unit;
3576 MBUFQ_INIT(&new_group->send_queue);
3577 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3578
3579 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3580
3581 if (g_flow_divert_groups == NULL) {
3582 MALLOC(g_flow_divert_groups,
3583 struct flow_divert_group **,
3584 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3585 M_TEMP,
3586 M_WAITOK | M_ZERO);
3587 }
3588
3589 if (g_flow_divert_groups == NULL) {
3590 error = ENOBUFS;
3591 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3592 error = EALREADY;
3593 } else {
3594 g_flow_divert_groups[sac->sc_unit] = new_group;
3595 g_active_group_count++;
3596 }
3597
3598 lck_rw_done(&g_flow_divert_group_lck);
3599
3600 *unitinfo = new_group;
3601
3602 done:
3603 if (error != 0 && new_group != NULL) {
3604 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3605 }
3606 return error;
3607 }
3608
3609 static errno_t
3610 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3611 {
3612 struct flow_divert_group *group = NULL;
3613 errno_t error = 0;
3614
3615 if (unit >= GROUP_COUNT_MAX) {
3616 return EINVAL;
3617 }
3618
3619 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3620
3621 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3622
3623 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3624 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3625 g_flow_divert_groups, g_active_group_count);
3626 }
3627
3628 group = g_flow_divert_groups[unit];
3629
3630 if (group != (struct flow_divert_group *)unitinfo) {
3631 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3632 }
3633
3634 if (group != NULL) {
3635 flow_divert_close_all(group);
3636 if (group->token_key != NULL) {
3637 memset(group->token_key, 0, group->token_key_size);
3638 FREE(group->token_key, M_TEMP);
3639 group->token_key = NULL;
3640 group->token_key_size = 0;
3641 }
3642
3643 /* Re-set the current trie */
3644 if (group->signing_id_trie.memory != NULL) {
3645 FREE(group->signing_id_trie.memory, M_TEMP);
3646 }
3647 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3648 group->signing_id_trie.root = NULL_TRIE_IDX;
3649
3650 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3651 g_flow_divert_groups[unit] = NULL;
3652 g_active_group_count--;
3653 } else {
3654 error = EINVAL;
3655 }
3656
3657 if (g_active_group_count == 0) {
3658 FREE(g_flow_divert_groups, M_TEMP);
3659 g_flow_divert_groups = NULL;
3660 }
3661
3662 lck_rw_done(&g_flow_divert_group_lck);
3663
3664 return error;
3665 }
3666
3667 static errno_t
3668 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3669 {
3670 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3671 }
3672
3673 static void
3674 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3675 {
3676 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3677
3678 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3679 struct flow_divert_pcb *fd_cb;
3680 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3681
3682 lck_rw_lock_shared(&g_flow_divert_group_lck);
3683 lck_rw_lock_exclusive(&group->lck);
3684
3685 while (!MBUFQ_EMPTY(&group->send_queue)) {
3686 mbuf_t next_packet;
3687 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3688 next_packet = MBUFQ_FIRST(&group->send_queue);
3689 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3690 if (error) {
3691 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3692 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3693 lck_rw_done(&group->lck);
3694 lck_rw_done(&g_flow_divert_group_lck);
3695 return;
3696 }
3697 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3698 }
3699
3700 SLIST_INIT(&tmp_list);
3701
3702 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3703 FDRETAIN(fd_cb);
3704 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3705 }
3706
3707 lck_rw_done(&group->lck);
3708
3709 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3710 FDLOCK(fd_cb);
3711 if (fd_cb->so != NULL) {
3712 socket_lock(fd_cb->so, 0);
3713 if (fd_cb->group != NULL) {
3714 flow_divert_send_buffered_data(fd_cb, FALSE);
3715 }
3716 socket_unlock(fd_cb->so, 0);
3717 }
3718 FDUNLOCK(fd_cb);
3719 FDRELEASE(fd_cb);
3720 }
3721
3722 lck_rw_done(&g_flow_divert_group_lck);
3723 }
3724 }
3725
3726 static int
3727 flow_divert_kctl_init(void)
3728 {
3729 struct kern_ctl_reg ctl_reg;
3730 int result;
3731
3732 memset(&ctl_reg, 0, sizeof(ctl_reg));
3733
3734 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3735 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3736 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3737 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3738 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3739
3740 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3741 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3742 ctl_reg.ctl_send = flow_divert_kctl_send;
3743 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3744
3745 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3746
3747 if (result) {
3748 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3749 return result;
3750 }
3751
3752 return 0;
3753 }
3754
3755 void
3756 flow_divert_init(void)
3757 {
3758 memset(&nil_pcb, 0, sizeof(nil_pcb));
3759 nil_pcb.log_level = LOG_NOTICE;
3760
3761 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3762
3763 VERIFY(g_tcp_protosw != NULL);
3764
3765 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3766 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3767
3768 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3769 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3770 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3771 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3772 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3773 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3774 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3775 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3776 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3777 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3778 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
3779
3780 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3781 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3782
3783 /*
3784 * Socket filters shouldn't attach/detach to/from this protosw
3785 * since pr_protosw is to be used instead, which points to the
3786 * real protocol; if they do, it is a bug and we should panic.
3787 */
3788 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3789 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3790 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3791 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3792
3793 /* UDP */
3794 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3795 VERIFY(g_udp_protosw != NULL);
3796
3797 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3798 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3799
3800 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3801 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3802 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3803 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3804 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3805 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3806 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3807 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3808 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3809 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3810 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3811 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3812 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3813
3814 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3815 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3816
3817 /*
3818 * Socket filters shouldn't attach/detach to/from this protosw
3819 * since pr_protosw is to be used instead, which points to the
3820 * real protocol; if they do, it is a bug and we should panic.
3821 */
3822 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3823 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3824 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3825 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3826
3827 #if INET6
3828 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3829
3830 VERIFY(g_tcp6_protosw != NULL);
3831
3832 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3833 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3834
3835 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3836 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3837 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3838 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3839 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3840 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3841 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3842 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3843 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3844 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3845 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
3846
3847 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3848 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3849 /*
3850 * Socket filters shouldn't attach/detach to/from this protosw
3851 * since pr_protosw is to be used instead, which points to the
3852 * real protocol; if they do, it is a bug and we should panic.
3853 */
3854 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3855 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3856 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3857 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3858
3859 /* UDP6 */
3860 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3861
3862 VERIFY(g_udp6_protosw != NULL);
3863
3864 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3865 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3866
3867 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3868 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3869 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3870 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3871 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3872 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3873 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3874 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3875 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3876 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3877 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3878 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3879 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3880
3881 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3882 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3883 /*
3884 * Socket filters shouldn't attach/detach to/from this protosw
3885 * since pr_protosw is to be used instead, which points to the
3886 * real protocol; if they do, it is a bug and we should panic.
3887 */
3888 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3889 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3890 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3891 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3892 #endif /* INET6 */
3893
3894 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3895 if (flow_divert_grp_attr == NULL) {
3896 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3897 g_init_result = ENOMEM;
3898 goto done;
3899 }
3900
3901 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3902 if (flow_divert_mtx_grp == NULL) {
3903 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3904 g_init_result = ENOMEM;
3905 goto done;
3906 }
3907
3908 flow_divert_mtx_attr = lck_attr_alloc_init();
3909 if (flow_divert_mtx_attr == NULL) {
3910 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3911 g_init_result = ENOMEM;
3912 goto done;
3913 }
3914
3915 g_init_result = flow_divert_kctl_init();
3916 if (g_init_result) {
3917 goto done;
3918 }
3919
3920 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3921
3922 done:
3923 if (g_init_result != 0) {
3924 if (flow_divert_mtx_attr != NULL) {
3925 lck_attr_free(flow_divert_mtx_attr);
3926 flow_divert_mtx_attr = NULL;
3927 }
3928 if (flow_divert_mtx_grp != NULL) {
3929 lck_grp_free(flow_divert_mtx_grp);
3930 flow_divert_mtx_grp = NULL;
3931 }
3932 if (flow_divert_grp_attr != NULL) {
3933 lck_grp_attr_free(flow_divert_grp_attr);
3934 flow_divert_grp_attr = NULL;
3935 }
3936
3937 if (g_flow_divert_kctl_ref != NULL) {
3938 ctl_deregister(g_flow_divert_kctl_ref);
3939 g_flow_divert_kctl_ref = NULL;
3940 }
3941 }
3942 }