]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.31.2.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
59 #if INET6
60 #include <netinet6/in6_pcb.h>
61 #include <netinet6/ip6protosw.h>
62 #endif /* INET6 */
63 #include <dev/random/randomdev.h>
64 #include <libkern/crypto/sha1.h>
65 #include <libkern/crypto/crypto_internal.h>
66 #include <os/log.h>
67
68 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69 #define FLOW_DIVERT_READ_CLOSED 0x00000002
70 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73 #define FLOW_DIVERT_TRANSFERRED 0x00000020
74 #define FLOW_DIVERT_HAS_HMAC 0x00000040
75
76 #define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
78
79 #define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
81
82 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83 #define FDRELEASE(pcb) \
84 do { \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
87 } \
88 } while (0)
89
90 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
92
93 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
94 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
95
96 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
97
98 #define GROUP_COUNT_MAX 32
99 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
100 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
101 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
102
103 struct flow_divert_trie_node
104 {
105 uint16_t start;
106 uint16_t length;
107 uint16_t child_map;
108 };
109
110 #define CHILD_MAP_SIZE 256
111 #define NULL_TRIE_IDX 0xffff
112 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
113 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
114 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
115
116 static struct flow_divert_pcb nil_pcb;
117
118 decl_lck_rw_data(static, g_flow_divert_group_lck);
119 static struct flow_divert_group **g_flow_divert_groups = NULL;
120 static uint32_t g_active_group_count = 0;
121
122 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
123 static lck_attr_t *flow_divert_mtx_attr = NULL;
124 static lck_grp_t *flow_divert_mtx_grp = NULL;
125 static errno_t g_init_result = 0;
126
127 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
128
129 static struct protosw g_flow_divert_in_protosw;
130 static struct pr_usrreqs g_flow_divert_in_usrreqs;
131 static struct protosw g_flow_divert_in_udp_protosw;
132 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
133 #if INET6
134 static struct ip6protosw g_flow_divert_in6_protosw;
135 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
136 static struct ip6protosw g_flow_divert_in6_udp_protosw;
137 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
138 #endif /* INET6 */
139
140 static struct protosw *g_tcp_protosw = NULL;
141 static struct ip6protosw *g_tcp6_protosw = NULL;
142 static struct protosw *g_udp_protosw = NULL;
143 static struct ip6protosw *g_udp6_protosw = NULL;
144
145 static errno_t
146 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
147
148 static errno_t
149 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
150
151 static boolean_t
152 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
153
154 static int
155 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
156
157 struct sockaddr *
158 flow_divert_get_buffered_target_address(mbuf_t buffer);
159
160 static boolean_t
161 flow_divert_has_pcb_local_address(const struct inpcb *inp);
162
163 static void
164 flow_divert_disconnect_socket(struct socket *so);
165
166 static inline uint8_t
167 flow_divert_syslog_type_to_oslog_type(int syslog_type)
168 {
169 switch (syslog_type) {
170 case LOG_ERR: return OS_LOG_TYPE_ERROR;
171 case LOG_INFO: return OS_LOG_TYPE_INFO;
172 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
173 default: return OS_LOG_TYPE_DEFAULT;
174 }
175 }
176
177 static inline int
178 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
179 {
180 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
181 }
182
183 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
184 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
185
186 static const char *
187 flow_divert_packet_type2str(uint8_t packet_type)
188 {
189 switch (packet_type) {
190 case FLOW_DIVERT_PKT_CONNECT:
191 return "connect";
192 case FLOW_DIVERT_PKT_CONNECT_RESULT:
193 return "connect result";
194 case FLOW_DIVERT_PKT_DATA:
195 return "data";
196 case FLOW_DIVERT_PKT_CLOSE:
197 return "close";
198 case FLOW_DIVERT_PKT_READ_NOTIFY:
199 return "read notification";
200 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
201 return "properties update";
202 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
203 return "app map create";
204 default:
205 return "unknown";
206 }
207 }
208
209 static struct flow_divert_pcb *
210 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
211 {
212 struct flow_divert_pcb key_item;
213 struct flow_divert_pcb *fd_cb = NULL;
214
215 key_item.hash = hash;
216
217 lck_rw_lock_shared(&group->lck);
218 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
219 FDRETAIN(fd_cb);
220 lck_rw_done(&group->lck);
221
222 return fd_cb;
223 }
224
225 static errno_t
226 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
227 {
228 errno_t error = 0;
229 struct flow_divert_pcb *exist = NULL;
230 struct flow_divert_group *group;
231 static uint32_t g_nextkey = 1;
232 static uint32_t g_hash_seed = 0;
233 int try_count = 0;
234
235 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
236 return EINVAL;
237 }
238
239 socket_unlock(fd_cb->so, 0);
240 lck_rw_lock_shared(&g_flow_divert_group_lck);
241
242 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
243 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
244 error = ENETUNREACH;
245 goto done;
246 }
247
248 group = g_flow_divert_groups[ctl_unit];
249 if (group == NULL) {
250 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
251 error = ENETUNREACH;
252 goto done;
253 }
254
255 socket_lock(fd_cb->so, 0);
256
257 do {
258 uint32_t key[2];
259 uint32_t idx;
260
261 key[0] = g_nextkey++;
262 key[1] = RandomULong();
263
264 if (g_hash_seed == 0) {
265 g_hash_seed = RandomULong();
266 }
267
268 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
269
270 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
271 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
272 if (curr_group != NULL && curr_group != group) {
273 lck_rw_lock_shared(&curr_group->lck);
274 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
275 lck_rw_done(&curr_group->lck);
276 if (exist != NULL) {
277 break;
278 }
279 }
280 }
281
282 if (exist == NULL) {
283 lck_rw_lock_exclusive(&group->lck);
284 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
285 lck_rw_done(&group->lck);
286 }
287 } while (exist != NULL && try_count++ < 3);
288
289 if (exist == NULL) {
290 fd_cb->group = group;
291 FDRETAIN(fd_cb); /* The group now has a reference */
292 } else {
293 fd_cb->hash = 0;
294 error = EEXIST;
295 }
296
297 socket_unlock(fd_cb->so, 0);
298
299 done:
300 lck_rw_done(&g_flow_divert_group_lck);
301 socket_lock(fd_cb->so, 0);
302
303 return error;
304 }
305
306 static struct flow_divert_pcb *
307 flow_divert_pcb_create(socket_t so)
308 {
309 struct flow_divert_pcb *new_pcb = NULL;
310
311 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
312 if (new_pcb == NULL) {
313 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
314 return NULL;
315 }
316
317 memset(new_pcb, 0, sizeof(*new_pcb));
318
319 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
320 new_pcb->so = so;
321 new_pcb->log_level = nil_pcb.log_level;
322
323 FDRETAIN(new_pcb); /* Represents the socket's reference */
324
325 return new_pcb;
326 }
327
328 static void
329 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
330 {
331 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
332 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
333
334 if (fd_cb->local_address != NULL) {
335 FREE(fd_cb->local_address, M_SONAME);
336 }
337 if (fd_cb->remote_address != NULL) {
338 FREE(fd_cb->remote_address, M_SONAME);
339 }
340 if (fd_cb->connect_token != NULL) {
341 mbuf_freem(fd_cb->connect_token);
342 }
343 if (fd_cb->connect_packet != NULL) {
344 mbuf_freem(fd_cb->connect_packet);
345 }
346 if (fd_cb->app_data != NULL) {
347 FREE(fd_cb->app_data, M_TEMP);
348 }
349 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
350 }
351
352 static void
353 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
354 {
355 if (fd_cb->group != NULL) {
356 struct flow_divert_group *group = fd_cb->group;
357 lck_rw_lock_exclusive(&group->lck);
358 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
359 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
360 fd_cb->group = NULL;
361 FDRELEASE(fd_cb); /* Release the group's reference */
362 lck_rw_done(&group->lck);
363 }
364 }
365
366 static int
367 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
368 {
369 struct flow_divert_packet_header hdr;
370 int error = 0;
371
372 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
373 if (error) {
374 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
375 return error;
376 }
377
378 hdr.packet_type = packet_type;
379 hdr.conn_id = htonl(fd_cb->hash);
380
381 /* Lay down the header */
382 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
383 if (error) {
384 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
385 mbuf_freem(*packet);
386 *packet = NULL;
387 return error;
388 }
389
390 return 0;
391 }
392
393 static int
394 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
395 {
396 uint32_t net_length = htonl(length);
397 int error = 0;
398
399 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
400 if (error) {
401 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
402 return error;
403 }
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
406 if (error) {
407 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
408 return error;
409 }
410
411 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
412 if (error) {
413 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
414 return error;
415 }
416
417 return error;
418 }
419
420 static int
421 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
422 {
423 size_t cursor = offset;
424 int error = 0;
425 uint32_t curr_length;
426 uint8_t curr_type;
427
428 *err = 0;
429
430 do {
431 if (!next) {
432 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
433 if (error) {
434 *err = ENOENT;
435 return -1;
436 }
437 } else {
438 next = 0;
439 curr_type = FLOW_DIVERT_TLV_NIL;
440 }
441
442 if (curr_type != type) {
443 cursor += sizeof(curr_type);
444 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
445 if (error) {
446 *err = error;
447 return -1;
448 }
449
450 cursor += (sizeof(curr_length) + ntohl(curr_length));
451 }
452 } while (curr_type != type);
453
454 return cursor;
455 }
456
457 static int
458 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
459 {
460 int error = 0;
461 uint32_t length;
462 int tlv_offset;
463
464 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
465 if (tlv_offset < 0) {
466 return error;
467 }
468
469 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
470 if (error) {
471 return error;
472 }
473
474 length = ntohl(length);
475
476 if (val_size != NULL) {
477 *val_size = length;
478 }
479
480 if (buff != NULL && buff_len > 0) {
481 size_t to_copy = (length < buff_len) ? length : buff_len;
482 error = mbuf_copydata(packet, tlv_offset + sizeof(type) + sizeof(length), to_copy, buff);
483 if (error) {
484 return error;
485 }
486 }
487
488 return 0;
489 }
490
491 static int
492 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
493 {
494 mbuf_t curr_mbuf = packet;
495
496 if (g_crypto_funcs == NULL || group->token_key == NULL) {
497 return ENOPROTOOPT;
498 }
499
500 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
501 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
502
503 while (curr_mbuf != NULL) {
504 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
505 curr_mbuf = mbuf_next(curr_mbuf);
506 }
507
508 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
509
510 return 0;
511 }
512
513 static int
514 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
515 {
516 int error = 0;
517 struct flow_divert_group *group = NULL;
518 int hmac_offset;
519 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
520 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
521 mbuf_t tail;
522
523 lck_rw_lock_shared(&g_flow_divert_group_lck);
524
525 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
526 group = g_flow_divert_groups[ctl_unit];
527 }
528
529 if (group == NULL) {
530 lck_rw_done(&g_flow_divert_group_lck);
531 return ENOPROTOOPT;
532 }
533
534 lck_rw_lock_shared(&group->lck);
535
536 if (group->token_key == NULL) {
537 error = ENOPROTOOPT;
538 goto done;
539 }
540
541 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
542 if (hmac_offset < 0) {
543 goto done;
544 }
545
546 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
547 if (error) {
548 goto done;
549 }
550
551 /* Chop off the HMAC TLV */
552 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
553 if (error) {
554 goto done;
555 }
556
557 mbuf_free(tail);
558
559 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
560 if (error) {
561 goto done;
562 }
563
564 if (memcmp(packet_hmac, computed_hmac, sizeof(packet_hmac))) {
565 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
566 error = EINVAL;
567 goto done;
568 }
569
570 done:
571 lck_rw_done(&group->lck);
572 lck_rw_done(&g_flow_divert_group_lck);
573 return error;
574 }
575
576 static void
577 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
578 {
579 struct inpcb *inp = NULL;
580 struct ifnet *ifp = NULL;
581 Boolean cell = FALSE;
582 Boolean wifi = FALSE;
583 Boolean wired = FALSE;
584
585 inp = sotoinpcb(fd_cb->so);
586 if (inp == NULL) {
587 return;
588 }
589
590 ifp = inp->inp_last_outifp;
591 if (ifp != NULL) {
592 cell = IFNET_IS_CELLULAR(ifp);
593 wifi = (!cell && IFNET_IS_WIFI(ifp));
594 wired = (!wifi && IFNET_IS_WIRED(ifp));
595 }
596
597 if (send) {
598 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
599 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
600 } else {
601 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
602 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
603 }
604 }
605
606 static errno_t
607 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
608 {
609 struct inpcb *inp = NULL;
610
611 inp = sotoinpcb(fd_cb->so);
612 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
613 IFNET_IS_CELLULAR(inp->inp_last_outifp))
614 return EHOSTUNREACH;
615
616 return 0;
617 }
618
619 static errno_t
620 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
621 {
622 struct inpcb *inp = NULL;
623
624 inp = sotoinpcb(fd_cb->so);
625 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
626 IFNET_IS_EXPENSIVE(inp->inp_last_outifp))
627 return EHOSTUNREACH;
628
629 return 0;
630 }
631
632 static void
633 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
634 {
635 if (how != SHUT_RD) {
636 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
637 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
638 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
639 /* If the tunnel is not accepting writes any more, then flush the send buffer */
640 sbflush(&fd_cb->so->so_snd);
641 }
642 }
643 if (how != SHUT_WR) {
644 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
645 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
646 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
647 }
648 }
649 }
650
651 static uint16_t
652 trie_node_alloc(struct flow_divert_trie *trie)
653 {
654 if (trie->nodes_free_next < trie->nodes_count) {
655 uint16_t node_idx = trie->nodes_free_next++;
656 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
657 return node_idx;
658 } else {
659 return NULL_TRIE_IDX;
660 }
661 }
662
663 static uint16_t
664 trie_child_map_alloc(struct flow_divert_trie *trie)
665 {
666 if (trie->child_maps_free_next < trie->child_maps_count) {
667 return trie->child_maps_free_next++;
668 } else {
669 return NULL_TRIE_IDX;
670 }
671 }
672
673 static uint16_t
674 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
675 {
676 uint16_t start = trie->bytes_free_next;
677 if (start + bytes_size <= trie->bytes_count) {
678 if (start != bytes_idx) {
679 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
680 }
681 trie->bytes_free_next += bytes_size;
682 return start;
683 } else {
684 return NULL_TRIE_IDX;
685 }
686 }
687
688 static uint16_t
689 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
690 {
691 uint16_t current = trie->root;
692 uint16_t child = trie->root;
693 uint16_t string_end = string_start + string_len;
694 uint16_t string_idx = string_start;
695 uint16_t string_remainder = string_len;
696
697 while (child != NULL_TRIE_IDX) {
698 uint16_t parent = current;
699 uint16_t node_idx;
700 uint16_t current_end;
701
702 current = child;
703 child = NULL_TRIE_IDX;
704
705 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
706
707 for (node_idx = TRIE_NODE(trie, current).start;
708 node_idx < current_end &&
709 string_idx < string_end &&
710 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
711 node_idx++, string_idx++);
712
713 string_remainder = string_end - string_idx;
714
715 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
716 /*
717 * We did not reach the end of the current node's string.
718 * We need to split the current node into two:
719 * 1. A new node that contains the prefix of the node that matches
720 * the prefix of the string being inserted.
721 * 2. The current node modified to point to the remainder
722 * of the current node's string.
723 */
724 uint16_t prefix = trie_node_alloc(trie);
725 if (prefix == NULL_TRIE_IDX) {
726 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
727 return NULL_TRIE_IDX;
728 }
729
730 /*
731 * Prefix points to the portion of the current nodes's string that has matched
732 * the input string thus far.
733 */
734 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
735 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
736
737 /*
738 * Prefix has the current node as the child corresponding to the first byte
739 * after the split.
740 */
741 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
742 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
743 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
744 return NULL_TRIE_IDX;
745 }
746 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
747
748 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
749 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
750
751 /* Current node is adjusted to point to the remainder */
752 TRIE_NODE(trie, current).start = node_idx;
753 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
754
755 /* We want to insert the new leaf (if any) as a child of the prefix */
756 current = prefix;
757 }
758
759 if (string_remainder > 0) {
760 /*
761 * We still have bytes in the string that have not been matched yet.
762 * If the current node has children, iterate to the child corresponding
763 * to the next byte in the string.
764 */
765 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
766 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
767 }
768 }
769 } /* while (child != NULL_TRIE_IDX) */
770
771 if (string_remainder > 0) {
772 /* Add a new leaf containing the remainder of the string */
773 uint16_t leaf = trie_node_alloc(trie);
774 if (leaf == NULL_TRIE_IDX) {
775 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
776 return NULL_TRIE_IDX;
777 }
778
779 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
780 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
781 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
782 return NULL_TRIE_IDX;
783 }
784 TRIE_NODE(trie, leaf).length = string_remainder;
785
786 /* Set the new leaf as the child of the current node */
787 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
788 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
789 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
790 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
791 return NULL_TRIE_IDX;
792 }
793 }
794 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
795 current = leaf;
796 } /* else duplicate or this string is a prefix of one of the existing strings */
797
798 return current;
799 }
800
801 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
802 static uint16_t
803 flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
804 {
805 uint16_t current = trie->root;
806 uint16_t string_idx = 0;
807
808 while (current != NULL_TRIE_IDX) {
809 uint16_t next = NULL_TRIE_IDX;
810 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
811 uint16_t node_idx;
812
813 for (node_idx = TRIE_NODE(trie, current).start;
814 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
815 node_idx++, string_idx++);
816
817 if (node_idx == node_end) {
818 if (string_bytes[string_idx] == '\0') {
819 return current; /* Got an exact match */
820 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
821 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
822 string_bytes[string_idx] = '\0';
823 return current; /* Got an apple webclip id prefix match */
824 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
825 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
826 }
827 }
828 current = next;
829 }
830
831 return NULL_TRIE_IDX;
832 }
833
834 struct uuid_search_info {
835 uuid_t target_uuid;
836 char *found_signing_id;
837 boolean_t found_multiple_signing_ids;
838 proc_t found_proc;
839 };
840
841 static int
842 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
843 {
844 struct uuid_search_info *info = (struct uuid_search_info *)arg;
845 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
846
847 if (info->found_signing_id != NULL) {
848 if (!info->found_multiple_signing_ids) {
849 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
850 info->found_proc = p;
851 result = PROC_CLAIMED_DONE;
852 } else {
853 uuid_string_t uuid_str;
854 uuid_unparse(info->target_uuid, uuid_str);
855 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
856 }
857 FREE(info->found_signing_id, M_TEMP);
858 info->found_signing_id = NULL;
859 }
860
861 if (result == PROC_RETURNED_DONE) {
862 uuid_string_t uuid_str;
863 uuid_unparse(info->target_uuid, uuid_str);
864 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
865 }
866
867 return result;
868 }
869
870 static int
871 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
872 {
873 struct uuid_search_info *info = (struct uuid_search_info *)arg;
874 int include = 0;
875
876 if (info->found_multiple_signing_ids) {
877 return include;
878 }
879
880 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
881 if (include) {
882 const char *signing_id = cs_identity_get(p);
883 if (signing_id != NULL) {
884 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
885 size_t signing_id_size = strlen(signing_id) + 1;
886 if (info->found_signing_id == NULL) {
887 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
888 memcpy(info->found_signing_id, signing_id, signing_id_size);
889 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
890 info->found_multiple_signing_ids = TRUE;
891 }
892 } else {
893 info->found_multiple_signing_ids = TRUE;
894 }
895 include = !info->found_multiple_signing_ids;
896 }
897
898 return include;
899 }
900
901 static proc_t
902 flow_divert_find_proc_by_uuid(uuid_t uuid)
903 {
904 struct uuid_search_info info;
905
906 if (LOG_INFO <= nil_pcb.log_level) {
907 uuid_string_t uuid_str;
908 uuid_unparse(uuid, uuid_str);
909 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
910 }
911
912 memset(&info, 0, sizeof(info));
913 info.found_proc = PROC_NULL;
914 uuid_copy(info.target_uuid, uuid);
915
916 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
917
918 return info.found_proc;
919 }
920
921 static int
922 flow_divert_get_src_proc(struct socket *so, proc_t *proc)
923 {
924 int release = 0;
925
926 if (so->so_flags & SOF_DELEGATED) {
927 if ((*proc)->p_pid != so->e_pid) {
928 *proc = proc_find(so->e_pid);
929 release = 1;
930 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
931 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
932 release = 1;
933 }
934 } else if (*proc == PROC_NULL) {
935 *proc = current_proc();
936 }
937
938 if (*proc != PROC_NULL) {
939 if ((*proc)->p_pid == 0) {
940 if (release) {
941 proc_rele(*proc);
942 }
943 release = 0;
944 *proc = PROC_NULL;
945 }
946 }
947
948 return release;
949 }
950
951 static int
952 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
953 {
954 int error;
955
956 if (fd_cb->group == NULL) {
957 fd_cb->so->so_error = ECONNABORTED;
958 flow_divert_disconnect_socket(fd_cb->so);
959 return ECONNABORTED;
960 }
961
962 lck_rw_lock_shared(&fd_cb->group->lck);
963
964 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
965 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
966 } else {
967 error = ENOBUFS;
968 }
969
970 if (error == ENOBUFS) {
971 if (enqueue) {
972 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
973 lck_rw_lock_exclusive(&fd_cb->group->lck);
974 }
975 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
976 error = 0;
977 }
978 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
979 }
980
981 lck_rw_done(&fd_cb->group->lck);
982
983 return error;
984 }
985
986 static int
987 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
988 {
989 int error = 0;
990 int flow_type = 0;
991 char *signing_id = NULL;
992 int free_signing_id = 0;
993 mbuf_t connect_packet = NULL;
994 proc_t src_proc = p;
995 int release_proc = 0;
996
997 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
998 if (error) {
999 goto done;
1000 }
1001
1002 error = EPERM;
1003
1004 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1005 uint32_t sid_size = 0;
1006 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1007 if (find_error == 0 && sid_size > 0) {
1008 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1009 if (signing_id != NULL) {
1010 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1011 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1012 free_signing_id = 1;
1013 }
1014 }
1015 }
1016
1017 socket_unlock(so, 0);
1018
1019 if (signing_id == NULL) {
1020 release_proc = flow_divert_get_src_proc(so, &src_proc);
1021 if (src_proc != PROC_NULL) {
1022 proc_lock(src_proc);
1023 if (src_proc->p_csflags & (CS_VALID|CS_DEBUGGED)) {
1024 const char * cs_id;
1025 cs_id = cs_identity_get(src_proc);
1026 signing_id = __DECONST(char *, cs_id);
1027 } else {
1028 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
1029 }
1030 } else {
1031 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
1032 }
1033 } else {
1034 src_proc = PROC_NULL;
1035 }
1036
1037 if (signing_id != NULL) {
1038 uint16_t result = NULL_TRIE_IDX;
1039 lck_rw_lock_shared(&fd_cb->group->lck);
1040 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1041 lck_rw_done(&fd_cb->group->lck);
1042 if (result != NULL_TRIE_IDX) {
1043 error = 0;
1044 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
1045
1046 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1047 if (error == 0) {
1048 if (src_proc != PROC_NULL) {
1049 unsigned char cdhash[SHA1_RESULTLEN];
1050 error = proc_getcdhash(src_proc, cdhash);
1051 if (error == 0) {
1052 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1053 if (error) {
1054 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1055 }
1056 } else {
1057 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
1058 }
1059 }
1060 } else {
1061 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1062 }
1063 } else {
1064 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
1065 }
1066 } else {
1067 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
1068 }
1069
1070 if (src_proc != PROC_NULL) {
1071 proc_unlock(src_proc);
1072 if (release_proc) {
1073 proc_rele(src_proc);
1074 }
1075 }
1076 socket_lock(so, 0);
1077
1078 if (free_signing_id) {
1079 FREE(signing_id, M_TEMP);
1080 }
1081
1082 if (error) {
1083 goto done;
1084 }
1085
1086 error = flow_divert_packet_append_tlv(connect_packet,
1087 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1088 sizeof(fd_cb->so->so_traffic_class),
1089 &fd_cb->so->so_traffic_class);
1090 if (error) {
1091 goto done;
1092 }
1093
1094 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1095 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1096 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1097 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1098 } else {
1099 error = EINVAL;
1100 goto done;
1101 }
1102 error = flow_divert_packet_append_tlv(connect_packet,
1103 FLOW_DIVERT_TLV_FLOW_TYPE,
1104 sizeof(flow_type),
1105 &flow_type);
1106
1107 if (error) {
1108 goto done;
1109 }
1110
1111 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1112 error = flow_divert_packet_append_tlv(connect_packet,
1113 FLOW_DIVERT_TLV_PID,
1114 sizeof(fd_cb->so->e_pid),
1115 &fd_cb->so->e_pid);
1116 if (error) {
1117 goto done;
1118 }
1119
1120 error = flow_divert_packet_append_tlv(connect_packet,
1121 FLOW_DIVERT_TLV_UUID,
1122 sizeof(fd_cb->so->e_uuid),
1123 &fd_cb->so->e_uuid);
1124 if (error) {
1125 goto done;
1126 }
1127 } else {
1128 error = flow_divert_packet_append_tlv(connect_packet,
1129 FLOW_DIVERT_TLV_PID,
1130 sizeof(fd_cb->so->e_pid),
1131 &fd_cb->so->last_pid);
1132 if (error) {
1133 goto done;
1134 }
1135
1136 error = flow_divert_packet_append_tlv(connect_packet,
1137 FLOW_DIVERT_TLV_UUID,
1138 sizeof(fd_cb->so->e_uuid),
1139 &fd_cb->so->last_uuid);
1140 if (error) {
1141 goto done;
1142 }
1143 }
1144
1145 if (fd_cb->connect_token != NULL) {
1146 unsigned int token_len = m_length(fd_cb->connect_token);
1147 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1148 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1149 fd_cb->connect_token = NULL;
1150 } else {
1151 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
1152
1153 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1154 if (error) {
1155 goto done;
1156 }
1157
1158 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1159 if (error) {
1160 goto done;
1161 }
1162 }
1163
1164 if (fd_cb->local_address != NULL) {
1165 error = EALREADY;
1166 goto done;
1167 } else {
1168 struct inpcb *inp = sotoinpcb(so);
1169 if (flow_divert_has_pcb_local_address(inp)) {
1170 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1171 if (error) {
1172 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1173 goto done;
1174 }
1175 }
1176 }
1177
1178 if (fd_cb->local_address != NULL) {
1179 /* socket is bound. */
1180 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1181 sizeof(struct sockaddr_storage), fd_cb->local_address);
1182 if (error) {
1183 goto done;
1184 }
1185 }
1186
1187 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1188 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1189 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1190 if (error) {
1191 goto done;
1192 }
1193 }
1194
1195 done:
1196 if (!error) {
1197 *out_connect_packet = connect_packet;
1198 } else if (connect_packet != NULL) {
1199 mbuf_freem(connect_packet);
1200 }
1201
1202 return error;
1203 }
1204
1205 static int
1206 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1207 {
1208 int error = 0;
1209 mbuf_t packet = NULL;
1210 int rbuff_space = 0;
1211
1212 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1213 if (error) {
1214 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1215 goto done;
1216 }
1217
1218 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1219 if (rbuff_space < 0) {
1220 rbuff_space = 0;
1221 }
1222 rbuff_space = htonl(rbuff_space);
1223 error = flow_divert_packet_append_tlv(packet,
1224 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1225 sizeof(rbuff_space),
1226 &rbuff_space);
1227 if (error) {
1228 goto done;
1229 }
1230
1231 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1232 if (error) {
1233 goto done;
1234 }
1235
1236 done:
1237 if (error && packet != NULL) {
1238 mbuf_freem(packet);
1239 }
1240
1241 return error;
1242 }
1243
1244 static int
1245 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1246 {
1247 int error = 0;
1248 mbuf_t packet = NULL;
1249 uint32_t zero = 0;
1250
1251 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1252 if (error) {
1253 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1254 goto done;
1255 }
1256
1257 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1258 if (error) {
1259 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1260 goto done;
1261 }
1262
1263 how = htonl(how);
1264 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1265 if (error) {
1266 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1267 goto done;
1268 }
1269
1270 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1271 if (error) {
1272 goto done;
1273 }
1274
1275 done:
1276 if (error && packet != NULL) {
1277 mbuf_free(packet);
1278 }
1279
1280 return error;
1281 }
1282
1283 static int
1284 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1285 {
1286 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1287 (FLOW_DIVERT_TUNNEL_RD_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED))
1288 {
1289 return SHUT_RDWR;
1290 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1291 return SHUT_RD;
1292 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1293 return SHUT_WR;
1294 }
1295
1296 return -1;
1297 }
1298
1299 /*
1300 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1301 * writes. Returns FALSE otherwise.
1302 */
1303 static void
1304 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1305 {
1306 int how = -1;
1307
1308 /* Do not send any close messages if there is still data in the send buffer */
1309 if (fd_cb->so->so_snd.sb_cc == 0) {
1310 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED|FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1311 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1312 how = SHUT_RD;
1313 }
1314 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED|FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1315 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1316 if (how == SHUT_RD) {
1317 how = SHUT_RDWR;
1318 } else {
1319 how = SHUT_WR;
1320 }
1321 }
1322 }
1323
1324 if (how != -1) {
1325 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1326 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1327 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1328 if (how != SHUT_RD) {
1329 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1330 }
1331 if (how != SHUT_WR) {
1332 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1333 }
1334 }
1335 }
1336
1337 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1338 flow_divert_disconnect_socket(fd_cb->so);
1339 }
1340 }
1341
1342 static errno_t
1343 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1344 {
1345 mbuf_t packet;
1346 mbuf_t last;
1347 int error = 0;
1348
1349 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1350 if (error) {
1351 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1352 return error;
1353 }
1354
1355 if (toaddr != NULL) {
1356 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1357 if (error) {
1358 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1359 return error;
1360 }
1361 }
1362
1363 last = m_last(packet);
1364 mbuf_setnext(last, data);
1365 mbuf_pkthdr_adjustlen(packet, data_len);
1366 error = flow_divert_send_packet(fd_cb, packet, force);
1367
1368 if (error) {
1369 mbuf_setnext(last, NULL);
1370 mbuf_freem(packet);
1371 } else {
1372 fd_cb->bytes_sent += data_len;
1373 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1374 }
1375
1376 return error;
1377 }
1378
1379 static void
1380 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1381 {
1382 size_t to_send;
1383 size_t sent = 0;
1384 int error = 0;
1385 mbuf_t buffer;
1386
1387 to_send = fd_cb->so->so_snd.sb_cc;
1388 buffer = fd_cb->so->so_snd.sb_mb;
1389
1390 if (buffer == NULL && to_send > 0) {
1391 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1392 return;
1393 }
1394
1395 /* Ignore the send window if force is enabled */
1396 if (!force && (to_send > fd_cb->send_window)) {
1397 to_send = fd_cb->send_window;
1398 }
1399
1400 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1401 while (sent < to_send) {
1402 mbuf_t data;
1403 size_t data_len;
1404
1405 data_len = to_send - sent;
1406 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1407 data_len = FLOW_DIVERT_CHUNK_SIZE;
1408 }
1409
1410 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1411 if (error) {
1412 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1413 break;
1414 }
1415
1416 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1417 if (error) {
1418 mbuf_freem(data);
1419 break;
1420 }
1421
1422 sent += data_len;
1423 }
1424 sbdrop(&fd_cb->so->so_snd, sent);
1425 sowwakeup(fd_cb->so);
1426 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1427 mbuf_t data;
1428 mbuf_t m;
1429 size_t data_len;
1430
1431 while(buffer) {
1432 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1433
1434 m = buffer;
1435 if (toaddr != NULL) {
1436 /* look for data in the chain */
1437 do {
1438 m = m->m_next;
1439 if (m != NULL && m->m_type == MT_DATA) {
1440 break;
1441 }
1442 } while(m);
1443 if (m == NULL) {
1444 /* unexpected */
1445 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1446 goto move_on;
1447 }
1448 }
1449 data_len = mbuf_pkthdr_len(m);
1450 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1451 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1452 if (error) {
1453 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1454 break;
1455 }
1456 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1457 if (error) {
1458 mbuf_freem(data);
1459 break;
1460 }
1461 sent += data_len;
1462 move_on:
1463 buffer = buffer->m_nextpkt;
1464 (void) sbdroprecord(&(fd_cb->so->so_snd));
1465 }
1466 }
1467
1468 if (sent > 0) {
1469 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1470 if (fd_cb->send_window >= sent) {
1471 fd_cb->send_window -= sent;
1472 } else {
1473 fd_cb->send_window = 0;
1474 }
1475 }
1476 }
1477
1478 static int
1479 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1480 {
1481 size_t to_send = mbuf_pkthdr_len(data);
1482 int error = 0;
1483
1484 if (to_send > fd_cb->send_window) {
1485 to_send = fd_cb->send_window;
1486 }
1487
1488 if (fd_cb->so->so_snd.sb_cc > 0) {
1489 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1490 }
1491
1492 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1493 size_t sent = 0;
1494 mbuf_t remaining_data = data;
1495 mbuf_t pkt_data = NULL;
1496 while (sent < to_send && remaining_data != NULL) {
1497 size_t pkt_data_len;
1498
1499 pkt_data = remaining_data;
1500
1501 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1502 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1503 } else {
1504 pkt_data_len = to_send - sent;
1505 }
1506
1507 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1508 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1509 if (error) {
1510 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1511 pkt_data = NULL;
1512 break;
1513 }
1514 } else {
1515 remaining_data = NULL;
1516 }
1517
1518 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1519
1520 if (error) {
1521 break;
1522 }
1523
1524 pkt_data = NULL;
1525 sent += pkt_data_len;
1526 }
1527
1528 fd_cb->send_window -= sent;
1529
1530 error = 0;
1531
1532 if (pkt_data != NULL) {
1533 if (sbspace(&fd_cb->so->so_snd) > 0) {
1534 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1535 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1536 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1537 }
1538 } else {
1539 error = ENOBUFS;
1540 }
1541 }
1542
1543 if (remaining_data != NULL) {
1544 if (sbspace(&fd_cb->so->so_snd) > 0) {
1545 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1546 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1547 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1548 }
1549 } else {
1550 error = ENOBUFS;
1551 }
1552 }
1553 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1554 if (to_send) {
1555 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1556 if (error) {
1557 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1558 } else {
1559 fd_cb->send_window -= to_send;
1560 }
1561 } else {
1562 /* buffer it */
1563 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1564 if (toaddr != NULL) {
1565 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1566 FDLOG(LOG_ERR, fd_cb,
1567 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1568 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1569 }
1570 } else {
1571 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1572 FDLOG(LOG_ERR, fd_cb,
1573 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1574 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1575 }
1576 }
1577 } else {
1578 error = ENOBUFS;
1579 }
1580 }
1581 }
1582
1583 return error;
1584 }
1585
1586 static int
1587 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1588 {
1589 int error = 0;
1590 mbuf_t packet = NULL;
1591 uint32_t net_read_count = htonl(read_count);
1592
1593 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1594 if (error) {
1595 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1596 goto done;
1597 }
1598
1599 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1600 if (error) {
1601 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1602 goto done;
1603 }
1604
1605 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1606 if (error) {
1607 goto done;
1608 }
1609
1610 done:
1611 if (error && packet != NULL) {
1612 mbuf_free(packet);
1613 }
1614
1615 return error;
1616 }
1617
1618 static int
1619 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1620 {
1621 int error = 0;
1622 mbuf_t packet = NULL;
1623
1624 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1625 if (error) {
1626 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1627 goto done;
1628 }
1629
1630 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1631 if (error) {
1632 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1633 goto done;
1634 }
1635
1636 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1637 if (error) {
1638 goto done;
1639 }
1640
1641 done:
1642 if (error && packet != NULL) {
1643 mbuf_free(packet);
1644 }
1645
1646 return error;
1647 }
1648
1649 static void
1650 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1651 {
1652 uint32_t connect_error;
1653 uint32_t ctl_unit = 0;
1654 int error = 0;
1655 struct flow_divert_group *grp = NULL;
1656 struct sockaddr_storage local_address;
1657 int out_if_index = 0;
1658 struct sockaddr_storage remote_address;
1659 uint32_t send_window;
1660 uint32_t app_data_length = 0;
1661
1662 memset(&local_address, 0, sizeof(local_address));
1663 memset(&remote_address, 0, sizeof(remote_address));
1664
1665 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1666 if (error) {
1667 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1668 return;
1669 }
1670
1671 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1672
1673 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1674 if (error) {
1675 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1676 return;
1677 }
1678
1679 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1680 if (error) {
1681 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
1682 }
1683
1684 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1685 if (error) {
1686 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1687 }
1688
1689 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1690 if (error) {
1691 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1692 }
1693
1694 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1695 if (error) {
1696 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1697 }
1698
1699 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1700 if (error) {
1701 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
1702 }
1703
1704 error = 0;
1705 connect_error = ntohl(connect_error);
1706 ctl_unit = ntohl(ctl_unit);
1707
1708 lck_rw_lock_shared(&g_flow_divert_group_lck);
1709
1710 if (connect_error == 0 && ctl_unit > 0) {
1711 if (ctl_unit >= GROUP_COUNT_MAX) {
1712 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1713 error = EINVAL;
1714 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1715 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1716 error = EINVAL;
1717 } else {
1718 grp = g_flow_divert_groups[ctl_unit];
1719 if (grp == NULL) {
1720 error = ECONNRESET;
1721 }
1722 }
1723 }
1724
1725 FDLOCK(fd_cb);
1726 if (fd_cb->so != NULL) {
1727 struct inpcb *inp = NULL;
1728 struct ifnet *ifp = NULL;
1729 struct flow_divert_group *old_group;
1730
1731 socket_lock(fd_cb->so, 0);
1732
1733 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1734 goto done;
1735 }
1736
1737 inp = sotoinpcb(fd_cb->so);
1738
1739 if (connect_error || error) {
1740 goto set_socket_state;
1741 }
1742
1743 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1744 error = EINVAL;
1745 goto set_socket_state;
1746 }
1747 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1748 if (local_address.ss_len > sizeof(local_address)) {
1749 local_address.ss_len = sizeof(local_address);
1750 }
1751 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1752 }
1753
1754 if (remote_address.ss_family != 0) {
1755 if (remote_address.ss_len > sizeof(remote_address)) {
1756 remote_address.ss_len = sizeof(remote_address);
1757 }
1758 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1759 } else {
1760 error = EINVAL;
1761 goto set_socket_state;
1762 }
1763
1764 if (app_data_length > 0) {
1765 uint8_t *app_data = NULL;
1766 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1767 if (app_data != NULL) {
1768 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1769 if (error == 0) {
1770 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1771 if (fd_cb->app_data != NULL) {
1772 FREE(fd_cb->app_data, M_TEMP);
1773 }
1774 fd_cb->app_data = app_data;
1775 fd_cb->app_data_length = app_data_length;
1776 } else {
1777 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1778 FREE(app_data, M_TEMP);
1779 }
1780 } else {
1781 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1782 }
1783 }
1784
1785 ifnet_head_lock_shared();
1786 if (out_if_index > 0 && out_if_index <= if_index) {
1787 ifp = ifindex2ifnet[out_if_index];
1788 }
1789
1790 if (ifp != NULL) {
1791 inp->inp_last_outifp = ifp;
1792 } else {
1793 error = EINVAL;
1794 }
1795 ifnet_head_done();
1796
1797 if (error) {
1798 goto set_socket_state;
1799 }
1800
1801 if (fd_cb->group == NULL) {
1802 error = EINVAL;
1803 goto set_socket_state;
1804 }
1805
1806 if (grp != NULL) {
1807 old_group = fd_cb->group;
1808
1809 lck_rw_lock_exclusive(&old_group->lck);
1810 lck_rw_lock_exclusive(&grp->lck);
1811
1812 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1813 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1814 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1815 }
1816
1817 fd_cb->group = grp;
1818
1819 lck_rw_done(&grp->lck);
1820 lck_rw_done(&old_group->lck);
1821 }
1822
1823 fd_cb->send_window = ntohl(send_window);
1824
1825 set_socket_state:
1826 if (!connect_error && !error) {
1827 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1828 error = flow_divert_send_connect_result(fd_cb);
1829 }
1830
1831 if (connect_error || error) {
1832 if (!connect_error) {
1833 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1834 fd_cb->so->so_error = error;
1835 flow_divert_send_close_if_needed(fd_cb);
1836 } else {
1837 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1838 fd_cb->so->so_error = connect_error;
1839 }
1840 flow_divert_disconnect_socket(fd_cb->so);
1841 } else {
1842 flow_divert_send_buffered_data(fd_cb, FALSE);
1843 soisconnected(fd_cb->so);
1844 }
1845
1846 done:
1847 socket_unlock(fd_cb->so, 0);
1848 }
1849 FDUNLOCK(fd_cb);
1850
1851 lck_rw_done(&g_flow_divert_group_lck);
1852 }
1853
1854 static void
1855 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1856 {
1857 uint32_t close_error;
1858 int error = 0;
1859 int how;
1860
1861 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1862 if (error) {
1863 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1864 return;
1865 }
1866
1867 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1868 if (error) {
1869 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1870 return;
1871 }
1872
1873 how = ntohl(how);
1874
1875 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1876
1877 FDLOCK(fd_cb);
1878 if (fd_cb->so != NULL) {
1879 socket_lock(fd_cb->so, 0);
1880
1881 fd_cb->so->so_error = ntohl(close_error);
1882
1883 flow_divert_update_closed_state(fd_cb, how, TRUE);
1884
1885 how = flow_divert_tunnel_how_closed(fd_cb);
1886 if (how == SHUT_RDWR) {
1887 flow_divert_disconnect_socket(fd_cb->so);
1888 } else if (how == SHUT_RD) {
1889 socantrcvmore(fd_cb->so);
1890 } else if (how == SHUT_WR) {
1891 socantsendmore(fd_cb->so);
1892 }
1893
1894 socket_unlock(fd_cb->so, 0);
1895 }
1896 FDUNLOCK(fd_cb);
1897 }
1898
1899 static mbuf_t
1900 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
1901 {
1902 struct inpcb *inp = sotoinpcb(fd_cb->so);
1903 if (inp->inp_vflag & INP_IPV4 && inp->inp_flags & INP_RECVDSTADDR) {
1904 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
1905
1906 return sbcreatecontrol((caddr_t) &sin->sin_addr, sizeof(struct in_addr), IP_RECVDSTADDR, IPPROTO_IP);
1907 } else if (inp->inp_vflag & INP_IPV6 && (inp->inp_flags & IN6P_PKTINFO) != 0) {
1908 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
1909 struct in6_pktinfo pi6;
1910
1911 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof (struct in6_addr));
1912 pi6.ipi6_ifindex = 0;
1913 return sbcreatecontrol((caddr_t)&pi6, sizeof (struct in6_pktinfo), IPV6_PKTINFO, IPPROTO_IPV6);
1914 }
1915 return (NULL);
1916 }
1917
1918 static void
1919 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
1920 {
1921 FDLOCK(fd_cb);
1922 if (fd_cb->so != NULL) {
1923 int error = 0;
1924 mbuf_t data = NULL;
1925 size_t data_size;
1926 struct sockaddr_storage remote_address;
1927 boolean_t got_remote_sa = FALSE;
1928
1929 socket_lock(fd_cb->so, 0);
1930
1931 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1932 uint32_t val_size = 0;
1933
1934 /* check if we got remote address with data */
1935 memset(&remote_address, 0, sizeof(remote_address));
1936 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
1937 if (error || val_size > sizeof(remote_address)) {
1938 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1939 error = 0;
1940 } else {
1941 /* validate the address */
1942 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1943 got_remote_sa = TRUE;
1944 }
1945 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
1946 }
1947 }
1948
1949 data_size = (mbuf_pkthdr_len(packet) - offset);
1950
1951 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
1952
1953 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
1954 if (error || data == NULL) {
1955 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1956 } else {
1957 if (flow_divert_check_no_cellular(fd_cb) ||
1958 flow_divert_check_no_expensive(fd_cb))
1959 {
1960 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1961 flow_divert_send_close(fd_cb, SHUT_RDWR);
1962 flow_divert_disconnect_socket(fd_cb->so);
1963 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
1964 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1965 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
1966 fd_cb->bytes_received += data_size;
1967 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1968 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1969 sorwakeup(fd_cb->so);
1970 data = NULL;
1971 } else {
1972 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
1973 }
1974 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1975 struct sockaddr *append_sa;
1976 mbuf_t mctl;
1977
1978 if (got_remote_sa == TRUE) {
1979 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1980 (struct sockaddr *)&remote_address, &append_sa);
1981 } else {
1982 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
1983 fd_cb->remote_address, &append_sa);
1984 }
1985 if (error) {
1986 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
1987 }
1988
1989 mctl = flow_divert_get_control_mbuf(fd_cb);
1990 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, NULL)) {
1991 fd_cb->bytes_received += data_size;
1992 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
1993 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
1994 sorwakeup(fd_cb->so);
1995 data = NULL;
1996 } else {
1997 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
1998 }
1999 if (!error) {
2000 FREE(append_sa, M_TEMP);
2001 }
2002 }
2003 }
2004 }
2005 socket_unlock(fd_cb->so, 0);
2006
2007 if (data != NULL) {
2008 mbuf_freem(data);
2009 }
2010 }
2011 FDUNLOCK(fd_cb);
2012 }
2013
2014 static void
2015 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2016 {
2017 uint32_t read_count;
2018 int error = 0;
2019
2020 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2021 if (error) {
2022 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2023 return;
2024 }
2025
2026 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2027
2028 FDLOCK(fd_cb);
2029 if (fd_cb->so != NULL) {
2030 socket_lock(fd_cb->so, 0);
2031 fd_cb->send_window += ntohl(read_count);
2032 flow_divert_send_buffered_data(fd_cb, FALSE);
2033 socket_unlock(fd_cb->so, 0);
2034 }
2035 FDUNLOCK(fd_cb);
2036 }
2037
2038 static void
2039 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2040 {
2041 int error = 0;
2042 uint32_t key_size = 0;
2043 int log_level;
2044
2045 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2046 if (error) {
2047 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2048 return;
2049 }
2050
2051 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2052 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2053 return;
2054 }
2055
2056 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2057 if (!error) {
2058 nil_pcb.log_level = log_level;
2059 }
2060
2061 lck_rw_lock_exclusive(&group->lck);
2062
2063 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2064 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2065 if (error) {
2066 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2067 FREE(group->token_key, M_TEMP);
2068 group->token_key = NULL;
2069 lck_rw_done(&group->lck);
2070 return;
2071 }
2072
2073 group->token_key_size = key_size;
2074
2075 lck_rw_done(&group->lck);
2076 }
2077
2078 static void
2079 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2080 {
2081 int error = 0;
2082 struct sockaddr_storage local_address;
2083 int out_if_index = 0;
2084 struct sockaddr_storage remote_address;
2085 uint32_t app_data_length = 0;
2086
2087 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2088
2089 memset(&local_address, 0, sizeof(local_address));
2090 memset(&remote_address, 0, sizeof(remote_address));
2091
2092 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2093 if (error) {
2094 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
2095 }
2096
2097 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2098 if (error) {
2099 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
2100 }
2101
2102 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2103 if (error) {
2104 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2105 }
2106
2107 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2108 if (error) {
2109 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2110 }
2111
2112 FDLOCK(fd_cb);
2113 if (fd_cb->so != NULL) {
2114 socket_lock(fd_cb->so, 0);
2115
2116 if (local_address.ss_family != 0) {
2117 if (local_address.ss_len > sizeof(local_address)) {
2118 local_address.ss_len = sizeof(local_address);
2119 }
2120 if (fd_cb->local_address != NULL) {
2121 FREE(fd_cb->local_address, M_SONAME);
2122 fd_cb->local_address = NULL;
2123 }
2124 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2125 }
2126
2127 if (remote_address.ss_family != 0) {
2128 if (remote_address.ss_len > sizeof(remote_address)) {
2129 remote_address.ss_len = sizeof(remote_address);
2130 }
2131 if (fd_cb->remote_address != NULL) {
2132 FREE(fd_cb->remote_address, M_SONAME);
2133 fd_cb->remote_address = NULL;
2134 }
2135 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2136 }
2137
2138 if (out_if_index > 0) {
2139 struct inpcb *inp = NULL;
2140 struct ifnet *ifp = NULL;
2141
2142 inp = sotoinpcb(fd_cb->so);
2143
2144 ifnet_head_lock_shared();
2145 if (out_if_index <= if_index) {
2146 ifp = ifindex2ifnet[out_if_index];
2147 }
2148
2149 if (ifp != NULL) {
2150 inp->inp_last_outifp = ifp;
2151 }
2152 ifnet_head_done();
2153 }
2154
2155 if (app_data_length > 0) {
2156 uint8_t *app_data = NULL;
2157 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2158 if (app_data != NULL) {
2159 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2160 if (error == 0) {
2161 if (fd_cb->app_data != NULL) {
2162 FREE(fd_cb->app_data, M_TEMP);
2163 }
2164 fd_cb->app_data = app_data;
2165 fd_cb->app_data_length = app_data_length;
2166 } else {
2167 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2168 FREE(app_data, M_TEMP);
2169 }
2170 } else {
2171 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2172 }
2173 }
2174
2175 socket_unlock(fd_cb->so, 0);
2176 }
2177 FDUNLOCK(fd_cb);
2178 }
2179
2180 static void
2181 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2182 {
2183 size_t bytes_mem_size;
2184 size_t child_maps_mem_size;
2185 int cursor;
2186 int error = 0;
2187 struct flow_divert_trie new_trie;
2188 int insert_error = 0;
2189 size_t nodes_mem_size;
2190 int prefix_count = 0;
2191 int signing_id_count = 0;
2192 size_t trie_memory_size = 0;
2193
2194 lck_rw_lock_exclusive(&group->lck);
2195
2196 /* Re-set the current trie */
2197 if (group->signing_id_trie.memory != NULL) {
2198 FREE(group->signing_id_trie.memory, M_TEMP);
2199 }
2200 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2201 group->signing_id_trie.root = NULL_TRIE_IDX;
2202
2203 memset(&new_trie, 0, sizeof(new_trie));
2204
2205 /* Get the number of shared prefixes in the new set of signing ID strings */
2206 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2207
2208 if (prefix_count < 0) {
2209 lck_rw_done(&group->lck);
2210 return;
2211 }
2212
2213 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2214 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2215 cursor >= 0;
2216 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2217 {
2218 uint32_t sid_size = 0;
2219 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2220 new_trie.bytes_count += sid_size;
2221 signing_id_count++;
2222 }
2223
2224 if (signing_id_count == 0) {
2225 lck_rw_done(&group->lck);
2226 return;
2227 }
2228
2229 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2230 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2231
2232 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2233 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2234
2235 nodes_mem_size = (sizeof(*new_trie.nodes) * new_trie.nodes_count);
2236 child_maps_mem_size = (sizeof(*new_trie.child_maps) * CHILD_MAP_SIZE * new_trie.child_maps_count);
2237 bytes_mem_size = (sizeof(*new_trie.bytes) * new_trie.bytes_count);
2238
2239 trie_memory_size = nodes_mem_size + child_maps_mem_size + bytes_mem_size;
2240 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2241 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2242 lck_rw_done(&group->lck);
2243 return;
2244 }
2245
2246 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2247 if (new_trie.memory == NULL) {
2248 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2249 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2250 lck_rw_done(&group->lck);
2251 return;
2252 }
2253
2254 /* Initialize the free lists */
2255 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2256 new_trie.nodes_free_next = 0;
2257 memset(new_trie.nodes, 0, nodes_mem_size);
2258
2259 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2260 new_trie.child_maps_free_next = 0;
2261 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2262
2263 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2264 new_trie.bytes_free_next = 0;
2265
2266 /* The root is an empty node */
2267 new_trie.root = trie_node_alloc(&new_trie);
2268
2269 /* Add each signing ID to the trie */
2270 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2271 cursor >= 0;
2272 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1))
2273 {
2274 uint32_t sid_size = 0;
2275 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2276 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2277 uint16_t new_node_idx;
2278 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2279 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2280 if (new_node_idx == NULL_TRIE_IDX) {
2281 insert_error = EINVAL;
2282 break;
2283 }
2284 } else {
2285 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2286 insert_error = ENOBUFS;
2287 break;
2288 }
2289 }
2290
2291 if (!insert_error) {
2292 group->signing_id_trie = new_trie;
2293 } else {
2294 FREE(new_trie.memory, M_TEMP);
2295 }
2296
2297 lck_rw_done(&group->lck);
2298 }
2299
2300 static int
2301 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2302 {
2303 struct flow_divert_packet_header hdr;
2304 int error = 0;
2305 struct flow_divert_pcb *fd_cb;
2306
2307 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2308 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2309 error = EINVAL;
2310 goto done;
2311 }
2312
2313 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2314 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2315 error = EINVAL;
2316 goto done;
2317 }
2318
2319 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2320 if (error) {
2321 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2322 error = ENOBUFS;
2323 goto done;
2324 }
2325
2326 hdr.conn_id = ntohl(hdr.conn_id);
2327
2328 if (hdr.conn_id == 0) {
2329 switch (hdr.packet_type) {
2330 case FLOW_DIVERT_PKT_GROUP_INIT:
2331 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2332 break;
2333 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2334 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2335 break;
2336 default:
2337 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2338 break;
2339 }
2340 goto done;
2341 }
2342
2343 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2344 if (fd_cb == NULL) {
2345 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2346 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2347 }
2348 goto done;
2349 }
2350
2351 switch (hdr.packet_type) {
2352 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2353 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2354 break;
2355 case FLOW_DIVERT_PKT_CLOSE:
2356 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2357 break;
2358 case FLOW_DIVERT_PKT_DATA:
2359 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2360 break;
2361 case FLOW_DIVERT_PKT_READ_NOTIFY:
2362 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2363 break;
2364 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2365 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2366 break;
2367 default:
2368 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2369 break;
2370 }
2371
2372 FDRELEASE(fd_cb);
2373
2374 done:
2375 mbuf_freem(packet);
2376 return error;
2377 }
2378
2379 static void
2380 flow_divert_close_all(struct flow_divert_group *group)
2381 {
2382 struct flow_divert_pcb *fd_cb;
2383 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2384
2385 SLIST_INIT(&tmp_list);
2386
2387 lck_rw_lock_exclusive(&group->lck);
2388
2389 MBUFQ_DRAIN(&group->send_queue);
2390
2391 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2392 FDRETAIN(fd_cb);
2393 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2394 }
2395
2396 lck_rw_done(&group->lck);
2397
2398 while (!SLIST_EMPTY(&tmp_list)) {
2399 fd_cb = SLIST_FIRST(&tmp_list);
2400 FDLOCK(fd_cb);
2401 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2402 if (fd_cb->so != NULL) {
2403 socket_lock(fd_cb->so, 0);
2404 flow_divert_pcb_remove(fd_cb);
2405 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2406 fd_cb->so->so_error = ECONNABORTED;
2407 flow_divert_disconnect_socket(fd_cb->so);
2408 socket_unlock(fd_cb->so, 0);
2409 }
2410 FDUNLOCK(fd_cb);
2411 FDRELEASE(fd_cb);
2412 }
2413 }
2414
2415 void
2416 flow_divert_detach(struct socket *so)
2417 {
2418 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2419
2420 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2421
2422 so->so_flags &= ~SOF_FLOW_DIVERT;
2423 so->so_fd_pcb = NULL;
2424
2425 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2426
2427 if (fd_cb->group != NULL) {
2428 /* Last-ditch effort to send any buffered data */
2429 flow_divert_send_buffered_data(fd_cb, TRUE);
2430
2431 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2432 flow_divert_send_close_if_needed(fd_cb);
2433 /* Remove from the group */
2434 flow_divert_pcb_remove(fd_cb);
2435 }
2436
2437 socket_unlock(so, 0);
2438 FDLOCK(fd_cb);
2439 fd_cb->so = NULL;
2440 FDUNLOCK(fd_cb);
2441 socket_lock(so, 0);
2442
2443 FDRELEASE(fd_cb); /* Release the socket's reference */
2444 }
2445
2446 static int
2447 flow_divert_close(struct socket *so)
2448 {
2449 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2450
2451 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2452
2453 FDLOG0(LOG_INFO, fd_cb, "Closing");
2454
2455 if (SOCK_TYPE(so) == SOCK_STREAM) {
2456 soisdisconnecting(so);
2457 sbflush(&so->so_rcv);
2458 }
2459
2460 flow_divert_send_buffered_data(fd_cb, TRUE);
2461 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2462 flow_divert_send_close_if_needed(fd_cb);
2463
2464 /* Remove from the group */
2465 flow_divert_pcb_remove(fd_cb);
2466
2467 return 0;
2468 }
2469
2470 static int
2471 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2472 sae_connid_t cid __unused)
2473 {
2474 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2475 return (EINVAL);
2476 }
2477
2478 return (flow_divert_close(so));
2479 }
2480
2481 static int
2482 flow_divert_shutdown(struct socket *so)
2483 {
2484 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2485
2486 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2487
2488 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2489
2490 socantsendmore(so);
2491
2492 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2493 flow_divert_send_close_if_needed(fd_cb);
2494
2495 return 0;
2496 }
2497
2498 static int
2499 flow_divert_rcvd(struct socket *so, int flags __unused)
2500 {
2501 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2502 uint32_t latest_sb_size;
2503 uint32_t read_count;
2504
2505 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2506
2507 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2508
2509 if (fd_cb->sb_size < latest_sb_size) {
2510 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2511 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2512 }
2513
2514 read_count = fd_cb->sb_size - latest_sb_size;
2515
2516 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2517
2518 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2519 fd_cb->bytes_read_by_app += read_count;
2520 fd_cb->sb_size = latest_sb_size;
2521 }
2522
2523 return 0;
2524 }
2525
2526 static int
2527 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2528 {
2529 int error = 0;
2530 int port = 0;
2531
2532 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2533 if (error) {
2534 goto done;
2535 }
2536
2537 if (toaddr->sa_family == AF_INET) {
2538 port = ntohs((satosin(toaddr))->sin_port);
2539 }
2540 #if INET6
2541 else {
2542 port = ntohs((satosin6(toaddr))->sin6_port);
2543 }
2544 #endif
2545
2546 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2547 if (error) {
2548 goto done;
2549 }
2550
2551 done:
2552 return error;
2553 }
2554
2555 struct sockaddr *
2556 flow_divert_get_buffered_target_address(mbuf_t buffer)
2557 {
2558 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2559 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2560 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2561 return toaddr;
2562 }
2563 }
2564 return NULL;
2565 }
2566
2567 static boolean_t
2568 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2569 {
2570 switch(addr->sa_family)
2571 {
2572 case AF_INET:
2573 if (addr->sa_len != sizeof(struct sockaddr_in)) {
2574 return FALSE;
2575 }
2576 break;
2577 #if INET6
2578 case AF_INET6:
2579 if (addr->sa_len != sizeof(struct sockaddr_in6)) {
2580 return FALSE;
2581 }
2582 break;
2583 #endif /* INET6 */
2584 default:
2585 return FALSE;
2586 }
2587 return TRUE;
2588 }
2589
2590 static errno_t
2591 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2592 {
2593 int error = 0;
2594 union sockaddr_in_4_6 sin46;
2595
2596 bzero(&sin46, sizeof(sin46));
2597 if (inp->inp_vflag & INP_IPV4) {
2598 struct sockaddr_in *sin = &sin46.sin;
2599
2600 sin->sin_family = AF_INET;
2601 sin->sin_len = sizeof(*sin);
2602 sin->sin_port = inp->inp_lport;
2603 sin->sin_addr = inp->inp_laddr;
2604 } else if (inp->inp_vflag & INP_IPV6) {
2605 struct sockaddr_in6 *sin6 = &sin46.sin6;
2606
2607 sin6->sin6_len = sizeof(*sin6);
2608 sin6->sin6_family = AF_INET6;
2609 sin6->sin6_port = inp->inp_lport;
2610 sin6->sin6_addr = inp->in6p_laddr;
2611 }
2612 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2613 if (*local_socket == NULL) {
2614 error = ENOBUFS;
2615 }
2616 return (error);
2617 }
2618
2619 static boolean_t
2620 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2621 {
2622 return (inp->inp_lport != 0
2623 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)));
2624 }
2625
2626 static errno_t
2627 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2628 struct sockaddr **dup)
2629 {
2630 int error = 0;
2631 struct sockaddr *result;
2632 struct sockaddr_storage ss;
2633
2634 if (addr != NULL) {
2635 result = addr;
2636 } else {
2637 memset(&ss, 0, sizeof(ss));
2638 ss.ss_family = family;
2639 if (ss.ss_family == AF_INET) {
2640 ss.ss_len = sizeof(struct sockaddr_in);
2641 }
2642 #if INET6
2643 else if (ss.ss_family == AF_INET6) {
2644 ss.ss_len = sizeof(struct sockaddr_in6);
2645 }
2646 #endif /* INET6 */
2647 else {
2648 error = EINVAL;
2649 }
2650 result = (struct sockaddr *)&ss;
2651 }
2652
2653 if (!error) {
2654 *dup = dup_sockaddr(result, 1);
2655 if (*dup == NULL) {
2656 error = ENOBUFS;
2657 }
2658 }
2659
2660 return error;
2661 }
2662
2663 static void
2664 flow_divert_disconnect_socket(struct socket *so)
2665 {
2666 soisdisconnected(so);
2667 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2668 struct inpcb *inp = NULL;
2669
2670 inp = sotoinpcb(so);
2671 if (inp != NULL) {
2672 #if INET6
2673 if (SOCK_CHECK_DOM(so, PF_INET6))
2674 in6_pcbdetach(inp);
2675 else
2676 #endif /* INET6 */
2677 in_pcbdetach(inp);
2678 }
2679 }
2680 }
2681
2682 static errno_t
2683 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2684 {
2685 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2686
2687 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2688
2689 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2690 fd_cb->remote_address,
2691 sa);
2692 }
2693
2694 static errno_t
2695 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2696 {
2697 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2698
2699 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2700
2701 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2702 fd_cb->local_address,
2703 sa);
2704 }
2705
2706 static errno_t
2707 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2708 {
2709 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2710
2711 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2712
2713 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2714 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2715 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2716 }
2717 }
2718
2719 if (SOCK_DOM(so) == PF_INET) {
2720 return g_tcp_protosw->pr_ctloutput(so, sopt);
2721 }
2722 #if INET6
2723 else if (SOCK_DOM(so) == PF_INET6) {
2724 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2725 }
2726 #endif
2727 return 0;
2728 }
2729
2730 errno_t
2731 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2732 {
2733 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2734 int error = 0;
2735 struct inpcb *inp = sotoinpcb(so);
2736 struct sockaddr_in *sinp;
2737 mbuf_t connect_packet = NULL;
2738 int do_send = 1;
2739
2740 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2741
2742 if (fd_cb->group == NULL) {
2743 error = ENETUNREACH;
2744 goto done;
2745 }
2746
2747 if (inp == NULL) {
2748 error = EINVAL;
2749 goto done;
2750 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2751 if (so->so_error) {
2752 error = so->so_error;
2753 so->so_error = 0;
2754 } else {
2755 error = EINVAL;
2756 }
2757 goto done;
2758 }
2759
2760 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2761 error = EALREADY;
2762 goto done;
2763 }
2764
2765 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2766 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2767 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2768 if (fd_cb->remote_address != NULL) {
2769 soisconnected(fd_cb->so);
2770 goto done;
2771 }
2772 }
2773
2774 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2775
2776 if (fd_cb->connect_packet == NULL) {
2777 if (to == NULL) {
2778 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2779 error = EINVAL;
2780 goto done;
2781 }
2782
2783 sinp = (struct sockaddr_in *)(void *)to;
2784 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2785 error = EAFNOSUPPORT;
2786 goto done;
2787 }
2788
2789 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2790 if (error) {
2791 goto done;
2792 }
2793
2794 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2795 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2796 do_send = 0;
2797 }
2798 } else {
2799 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2800 connect_packet = fd_cb->connect_packet;
2801 fd_cb->connect_packet = NULL;
2802 }
2803
2804 if (do_send) {
2805 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2806 if (error) {
2807 goto done;
2808 }
2809
2810 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2811 } else {
2812 fd_cb->connect_packet = connect_packet;
2813 connect_packet = NULL;
2814 }
2815
2816 soisconnecting(so);
2817
2818 done:
2819 if (error && connect_packet != NULL) {
2820 mbuf_freem(connect_packet);
2821 }
2822 return error;
2823 }
2824
2825 static int
2826 flow_divert_connectx_out_common(struct socket *so, int af,
2827 struct sockaddr_list **src_sl, struct sockaddr_list **dst_sl,
2828 struct proc *p, uint32_t ifscope __unused, sae_associd_t aid __unused,
2829 sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
2830 uint32_t arglen __unused, struct uio *auio, user_ssize_t *bytes_written)
2831 {
2832 struct sockaddr_entry *src_se = NULL, *dst_se = NULL;
2833 struct inpcb *inp = sotoinpcb(so);
2834 int error;
2835
2836 if (inp == NULL) {
2837 return (EINVAL);
2838 }
2839
2840 VERIFY(dst_sl != NULL);
2841
2842 /* select source (if specified) and destination addresses */
2843 error = in_selectaddrs(af, src_sl, &src_se, dst_sl, &dst_se);
2844 if (error != 0) {
2845 return (error);
2846 }
2847
2848 VERIFY(*dst_sl != NULL && dst_se != NULL);
2849 VERIFY(src_se == NULL || *src_sl != NULL);
2850 VERIFY(dst_se->se_addr->sa_family == af);
2851 VERIFY(src_se == NULL || src_se->se_addr->sa_family == af);
2852
2853 error = flow_divert_connect_out(so, dst_se->se_addr, p);
2854
2855 if (error != 0) {
2856 return error;
2857 }
2858
2859 /* if there is data, send it */
2860 if (auio != NULL) {
2861 user_ssize_t datalen = 0;
2862
2863 socket_unlock(so, 0);
2864
2865 VERIFY(bytes_written != NULL);
2866
2867 datalen = uio_resid(auio);
2868 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2869 socket_lock(so, 0);
2870
2871 if (error == 0 || error == EWOULDBLOCK) {
2872 *bytes_written = datalen - uio_resid(auio);
2873 }
2874
2875 /*
2876 * sosend returns EWOULDBLOCK if it's a non-blocking
2877 * socket or a timeout occured (this allows to return
2878 * the amount of queued data through sendit()).
2879 *
2880 * However, connectx() returns EINPROGRESS in case of a
2881 * blocking socket. So we change the return value here.
2882 */
2883 if (error == EWOULDBLOCK) {
2884 error = EINPROGRESS;
2885 }
2886 }
2887
2888 if (error == 0 && pcid != NULL) {
2889 *pcid = 1; /* there is only 1 connection for a TCP */
2890 }
2891
2892 return (error);
2893 }
2894
2895 static int
2896 flow_divert_connectx_out(struct socket *so, struct sockaddr_list **src_sl,
2897 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2898 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2899 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
2900 {
2901 #pragma unused(uio, bytes_written)
2902 return (flow_divert_connectx_out_common(so, AF_INET, src_sl, dst_sl,
2903 p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written));
2904 }
2905
2906 #if INET6
2907 static int
2908 flow_divert_connectx6_out(struct socket *so, struct sockaddr_list **src_sl,
2909 struct sockaddr_list **dst_sl, struct proc *p, uint32_t ifscope,
2910 sae_associd_t aid, sae_connid_t *pcid, uint32_t flags, void *arg,
2911 uint32_t arglen, struct uio *uio, user_ssize_t *bytes_written)
2912 {
2913 #pragma unused(uio, bytes_written)
2914 return (flow_divert_connectx_out_common(so, AF_INET6, src_sl, dst_sl,
2915 p, ifscope, aid, pcid, flags, arg, arglen, uio, bytes_written));
2916 }
2917 #endif /* INET6 */
2918
2919 static int
2920 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
2921 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
2922 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
2923 user_addr_t aux_data __unused, uint32_t *aux_len)
2924 {
2925 int error = 0;
2926 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2927 struct ifnet *ifp = NULL;
2928 struct inpcb *inp = sotoinpcb(so);
2929
2930 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
2931
2932 if (so->so_fd_pcb == NULL || inp == NULL) {
2933 error = EINVAL;
2934 goto out;
2935 }
2936
2937 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
2938 error = EINVAL;
2939 goto out;
2940 }
2941
2942 ifp = inp->inp_last_outifp;
2943 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
2944 *soerror = so->so_error;
2945 *flags = 0;
2946
2947 if (so->so_state & SS_ISCONNECTED) {
2948 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
2949 }
2950
2951 if (fd_cb->local_address == NULL) {
2952 struct sockaddr_in sin;
2953 bzero(&sin, sizeof(sin));
2954 sin.sin_len = sizeof(sin);
2955 sin.sin_family = AF_INET;
2956 *src_len = sin.sin_len;
2957 if (src != USER_ADDR_NULL) {
2958 error = copyout(&sin, src, sin.sin_len);
2959 if (error != 0) {
2960 goto out;
2961 }
2962 }
2963 } else {
2964 *src_len = fd_cb->local_address->sa_len;
2965 if (src != USER_ADDR_NULL) {
2966 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
2967 if (error != 0) {
2968 goto out;
2969 }
2970 }
2971 }
2972
2973 if (fd_cb->remote_address == NULL) {
2974 struct sockaddr_in sin;
2975 bzero(&sin, sizeof(sin));
2976 sin.sin_len = sizeof(sin);
2977 sin.sin_family = AF_INET;
2978 *dst_len = sin.sin_len;
2979 if (dst != USER_ADDR_NULL) {
2980 error = copyout(&sin, dst, sin.sin_len);
2981 if (error != 0) {
2982 goto out;
2983 }
2984 }
2985 } else {
2986 *dst_len = fd_cb->remote_address->sa_len;
2987 if (dst != USER_ADDR_NULL) {
2988 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
2989 if (error != 0) {
2990 goto out;
2991 }
2992 }
2993 }
2994
2995 *aux_type = 0;
2996 *aux_len = 0;
2997
2998 out:
2999 return error;
3000 }
3001
3002 static int
3003 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3004 {
3005 int error = 0;
3006
3007 switch (cmd) {
3008 case SIOCGCONNINFO32: {
3009 struct so_cinforeq32 cifr;
3010 bcopy(data, &cifr, sizeof (cifr));
3011 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3012 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3013 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3014 &cifr.scir_aux_type, cifr.scir_aux_data,
3015 &cifr.scir_aux_len);
3016 if (error == 0) {
3017 bcopy(&cifr, data, sizeof (cifr));
3018 }
3019 break;
3020 }
3021
3022 case SIOCGCONNINFO64: {
3023 struct so_cinforeq64 cifr;
3024 bcopy(data, &cifr, sizeof (cifr));
3025 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3026 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3027 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3028 &cifr.scir_aux_type, cifr.scir_aux_data,
3029 &cifr.scir_aux_len);
3030 if (error == 0) {
3031 bcopy(&cifr, data, sizeof (cifr));
3032 }
3033 break;
3034 }
3035
3036 default:
3037 error = EOPNOTSUPP;
3038 }
3039
3040 return error;
3041 }
3042
3043 static int
3044 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3045 {
3046 int error = flow_divert_control(so, cmd, data, ifp, p);
3047
3048 if (error == EOPNOTSUPP) {
3049 error = in_control(so, cmd, data, ifp, p);
3050 }
3051
3052 return error;
3053 }
3054
3055 static int
3056 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3057 {
3058 int error = flow_divert_control(so, cmd, data, ifp, p);
3059
3060 if (error == EOPNOTSUPP) {
3061 error = in6_control(so, cmd, data, ifp, p);
3062 }
3063
3064 return error;
3065 }
3066
3067 static errno_t
3068 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3069 {
3070 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3071 int error = 0;
3072 struct inpcb *inp;
3073
3074 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3075
3076 inp = sotoinpcb(so);
3077 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3078 error = ECONNRESET;
3079 goto done;
3080 }
3081
3082 if (control && mbuf_len(control) > 0) {
3083 error = EINVAL;
3084 goto done;
3085 }
3086
3087 if (flags & MSG_OOB) {
3088 error = EINVAL;
3089 goto done; /* We don't support OOB data */
3090 }
3091
3092 error = flow_divert_check_no_cellular(fd_cb) ||
3093 flow_divert_check_no_expensive(fd_cb);
3094 if (error) {
3095 goto done;
3096 }
3097
3098 /* Implicit connect */
3099 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3100 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3101 error = flow_divert_connect_out(so, to, p);
3102 if (error) {
3103 goto done;
3104 }
3105
3106 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3107 /* Open up the send window so that the data will get sent right away */
3108 fd_cb->send_window = mbuf_pkthdr_len(data);
3109 }
3110 }
3111
3112 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3113
3114 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3115 error = flow_divert_send_app_data(fd_cb, data, to);
3116 if (error) {
3117 goto done;
3118 }
3119
3120 data = NULL;
3121
3122 if (flags & PRUS_EOF) {
3123 flow_divert_shutdown(so);
3124 }
3125
3126 done:
3127 if (data) {
3128 mbuf_freem(data);
3129 }
3130 if (control) {
3131 mbuf_free(control);
3132 }
3133 return error;
3134 }
3135
3136 static int
3137 flow_divert_preconnect(struct socket *so)
3138 {
3139 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3140 int error = 0;
3141
3142 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3143 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3144 mbuf_t connect_packet = fd_cb->connect_packet;
3145 fd_cb->connect_packet = NULL;
3146
3147 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3148 if (error) {
3149 mbuf_freem(connect_packet);
3150 }
3151
3152 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3153 }
3154
3155 so->so_flags1 &= ~SOF1_PRECONNECT_DATA;
3156
3157 return error;
3158 }
3159
3160 static void
3161 flow_divert_set_protosw(struct socket *so)
3162 {
3163 so->so_flags |= SOF_FLOW_DIVERT;
3164 if (SOCK_DOM(so) == PF_INET) {
3165 so->so_proto = &g_flow_divert_in_protosw;
3166 }
3167 #if INET6
3168 else {
3169 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3170 }
3171 #endif /* INET6 */
3172 }
3173
3174 static void
3175 flow_divert_set_udp_protosw(struct socket *so)
3176 {
3177 so->so_flags |= SOF_FLOW_DIVERT;
3178 if (SOCK_DOM(so) == PF_INET) {
3179 so->so_proto = &g_flow_divert_in_udp_protosw;
3180 }
3181 #if INET6
3182 else {
3183 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3184 }
3185 #endif /* INET6 */
3186 }
3187
3188 static errno_t
3189 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3190 {
3191 int error = 0;
3192 struct flow_divert_pcb *fd_cb = NULL;
3193 struct ifnet *ifp = NULL;
3194 struct inpcb *inp = NULL;
3195 struct socket *old_so;
3196 mbuf_t recv_data = NULL;
3197
3198 socket_unlock(so, 0);
3199
3200 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3201
3202 /* Find the flow divert control block */
3203 lck_rw_lock_shared(&g_flow_divert_group_lck);
3204 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3205 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3206 if (group != NULL) {
3207 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3208 }
3209 }
3210 lck_rw_done(&g_flow_divert_group_lck);
3211
3212 if (fd_cb == NULL) {
3213 error = ENOENT;
3214 goto done;
3215 }
3216
3217 FDLOCK(fd_cb);
3218
3219 /* Dis-associate the flow divert control block from its current socket */
3220 old_so = fd_cb->so;
3221
3222 inp = sotoinpcb(old_so);
3223
3224 VERIFY(inp != NULL);
3225
3226 socket_lock(old_so, 0);
3227 flow_divert_disconnect_socket(old_so);
3228 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3229 old_so->so_fd_pcb = NULL;
3230 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3231 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3232 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3233 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3234 }
3235 fd_cb->so = NULL;
3236 /* Save the output interface */
3237 ifp = inp->inp_last_outifp;
3238 if (old_so->so_rcv.sb_cc > 0) {
3239 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3240 sbflush(&old_so->so_rcv);
3241 }
3242 socket_unlock(old_so, 0);
3243
3244 /* Associate the new socket with the flow divert control block */
3245 socket_lock(so, 0);
3246 so->so_fd_pcb = fd_cb;
3247 inp = sotoinpcb(so);
3248 inp->inp_last_outifp = ifp;
3249 if (recv_data != NULL) {
3250 if (sbappendstream(&so->so_rcv, recv_data)) {
3251 sorwakeup(so);
3252 }
3253 }
3254 flow_divert_set_protosw(so);
3255 socket_unlock(so, 0);
3256
3257 fd_cb->so = so;
3258 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3259
3260 FDUNLOCK(fd_cb);
3261
3262 done:
3263 socket_lock(so, 0);
3264
3265 if (fd_cb != NULL) {
3266 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3267 }
3268
3269 return error;
3270 }
3271
3272 errno_t
3273 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3274 {
3275 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3276 struct inpcb *inp;
3277 int error = 0;
3278
3279 inp = sotoinpcb(so);
3280 if (inp == NULL) {
3281 return (EINVAL);
3282 }
3283
3284 if (fd_cb == NULL) {
3285 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3286 if (fd_ctl_unit > 0) {
3287 error = flow_divert_pcb_init(so, fd_ctl_unit);
3288 fd_cb = so->so_fd_pcb;
3289 if (error != 0 || fd_cb == NULL) {
3290 goto done;
3291 }
3292 } else {
3293 error = ENETDOWN;
3294 goto done;
3295 }
3296 }
3297 return flow_divert_data_out(so, flags, data, to, control, p);
3298
3299 done:
3300 if (data) {
3301 mbuf_freem(data);
3302 }
3303 if (control) {
3304 mbuf_free(control);
3305 }
3306
3307 return error;
3308 }
3309
3310 errno_t
3311 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3312 {
3313 errno_t error = 0;
3314 struct flow_divert_pcb *fd_cb;
3315
3316 if (so->so_flags & SOF_FLOW_DIVERT) {
3317 return EALREADY;
3318 }
3319
3320 fd_cb = flow_divert_pcb_create(so);
3321 if (fd_cb != NULL) {
3322 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3323 if (error) {
3324 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3325 FDRELEASE(fd_cb);
3326 } else {
3327 fd_cb->control_group_unit = ctl_unit;
3328 so->so_fd_pcb = fd_cb;
3329
3330 if (SOCK_TYPE(so) == SOCK_STREAM) {
3331 flow_divert_set_protosw(so);
3332 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3333 flow_divert_set_udp_protosw(so);
3334 }
3335
3336 FDLOG0(LOG_INFO, fd_cb, "Created");
3337 }
3338 } else {
3339 error = ENOMEM;
3340 }
3341
3342 return error;
3343 }
3344
3345 errno_t
3346 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3347 {
3348 uint32_t ctl_unit = 0;
3349 uint32_t key_unit = 0;
3350 uint32_t flow_id = 0;
3351 int error = 0;
3352 int hmac_error = 0;
3353 mbuf_t token = NULL;
3354
3355 if (so->so_flags & SOF_FLOW_DIVERT) {
3356 error = EALREADY;
3357 goto done;
3358 }
3359
3360 if (g_init_result) {
3361 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3362 error = ENOPROTOOPT;
3363 goto done;
3364 }
3365
3366 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3367 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3368 (SOCK_DOM(so) != PF_INET
3369 #if INET6
3370 && SOCK_DOM(so) != PF_INET6
3371 #endif
3372 ))
3373 {
3374 error = EINVAL;
3375 goto done;
3376 } else {
3377 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3378 struct tcpcb *tp = sototcpcb(so);
3379 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3380 error = EINVAL;
3381 goto done;
3382 }
3383 }
3384 }
3385
3386 error = soopt_getm(sopt, &token);
3387 if (error) {
3388 goto done;
3389 }
3390
3391 error = soopt_mcopyin(sopt, token);
3392 if (error) {
3393 goto done;
3394 }
3395
3396 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3397 if (!error) {
3398 key_unit = ntohl(key_unit);
3399 } else if (error != ENOENT) {
3400 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3401 goto done;
3402 } else {
3403 key_unit = 0;
3404 }
3405
3406 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3407 if (error) {
3408 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3409 goto done;
3410 }
3411
3412 /* A valid kernel control unit is required */
3413 ctl_unit = ntohl(ctl_unit);
3414 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3415 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3416 error = EINVAL;
3417 goto done;
3418 }
3419
3420 socket_unlock(so, 0);
3421 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3422 socket_lock(so, 0);
3423
3424 if (hmac_error && hmac_error != ENOENT) {
3425 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3426 error = hmac_error;
3427 goto done;
3428 }
3429
3430 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3431 if (error && error != ENOENT) {
3432 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3433 goto done;
3434 }
3435
3436 if (flow_id == 0) {
3437 error = flow_divert_pcb_init(so, ctl_unit);
3438 if (error == 0) {
3439 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3440 int log_level = LOG_NOTICE;
3441
3442 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3443 sizeof(log_level), &log_level, NULL);
3444 if (error == 0) {
3445 fd_cb->log_level = log_level;
3446 }
3447 error = 0;
3448
3449 fd_cb->connect_token = token;
3450 token = NULL;
3451 }
3452 } else {
3453 error = flow_divert_attach(so, flow_id, ctl_unit);
3454 }
3455
3456 if (hmac_error == 0) {
3457 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3458 if (fd_cb != NULL) {
3459 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3460 }
3461 }
3462
3463 done:
3464 if (token != NULL) {
3465 mbuf_freem(token);
3466 }
3467
3468 return error;
3469 }
3470
3471 errno_t
3472 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3473 {
3474 uint32_t ctl_unit;
3475 int error = 0;
3476 uint8_t hmac[SHA_DIGEST_LENGTH];
3477 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3478 mbuf_t token = NULL;
3479 struct flow_divert_group *control_group = NULL;
3480
3481 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3482 error = EINVAL;
3483 goto done;
3484 }
3485
3486 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3487
3488 if (fd_cb->group == NULL) {
3489 error = EINVAL;
3490 goto done;
3491 }
3492
3493 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3494 if (error) {
3495 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3496 goto done;
3497 }
3498
3499 ctl_unit = htonl(fd_cb->group->ctl_unit);
3500
3501 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3502 if (error) {
3503 goto done;
3504 }
3505
3506 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3507 if (error) {
3508 goto done;
3509 }
3510
3511 if (fd_cb->app_data != NULL) {
3512 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3513 if (error) {
3514 goto done;
3515 }
3516 }
3517
3518 socket_unlock(so, 0);
3519 lck_rw_lock_shared(&g_flow_divert_group_lck);
3520
3521 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3522 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX)
3523 {
3524 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3525 }
3526
3527 if (control_group != NULL) {
3528 lck_rw_lock_shared(&control_group->lck);
3529 ctl_unit = htonl(control_group->ctl_unit);
3530 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3531 if (!error) {
3532 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3533 }
3534 lck_rw_done(&control_group->lck);
3535 } else {
3536 error = ENOPROTOOPT;
3537 }
3538
3539 lck_rw_done(&g_flow_divert_group_lck);
3540 socket_lock(so, 0);
3541
3542 if (error) {
3543 goto done;
3544 }
3545
3546 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3547 if (error) {
3548 goto done;
3549 }
3550
3551 if (sopt->sopt_val == USER_ADDR_NULL) {
3552 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3553 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3554 goto done;
3555 }
3556
3557 error = soopt_mcopyout(sopt, token);
3558 if (error) {
3559 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3560 goto done;
3561 }
3562
3563 done:
3564 if (token != NULL) {
3565 mbuf_freem(token);
3566 }
3567
3568 return error;
3569 }
3570
3571 static errno_t
3572 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3573 {
3574 struct flow_divert_group *new_group = NULL;
3575 int error = 0;
3576
3577 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3578 error = EINVAL;
3579 goto done;
3580 }
3581
3582 *unitinfo = NULL;
3583
3584 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3585 if (new_group == NULL) {
3586 error = ENOBUFS;
3587 goto done;
3588 }
3589
3590 memset(new_group, 0, sizeof(*new_group));
3591
3592 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3593 RB_INIT(&new_group->pcb_tree);
3594 new_group->ctl_unit = sac->sc_unit;
3595 MBUFQ_INIT(&new_group->send_queue);
3596 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3597
3598 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3599
3600 if (g_flow_divert_groups == NULL) {
3601 MALLOC(g_flow_divert_groups,
3602 struct flow_divert_group **,
3603 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3604 M_TEMP,
3605 M_WAITOK | M_ZERO);
3606 }
3607
3608 if (g_flow_divert_groups == NULL) {
3609 error = ENOBUFS;
3610 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3611 error = EALREADY;
3612 } else {
3613 g_flow_divert_groups[sac->sc_unit] = new_group;
3614 g_active_group_count++;
3615 }
3616
3617 lck_rw_done(&g_flow_divert_group_lck);
3618
3619 *unitinfo = new_group;
3620
3621 done:
3622 if (error != 0 && new_group != NULL) {
3623 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3624 }
3625 return error;
3626 }
3627
3628 static errno_t
3629 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3630 {
3631 struct flow_divert_group *group = NULL;
3632 errno_t error = 0;
3633
3634 if (unit >= GROUP_COUNT_MAX) {
3635 return EINVAL;
3636 }
3637
3638 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3639
3640 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3641
3642 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3643 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3644 g_flow_divert_groups, g_active_group_count);
3645 }
3646
3647 group = g_flow_divert_groups[unit];
3648
3649 if (group != (struct flow_divert_group *)unitinfo) {
3650 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3651 }
3652
3653 if (group != NULL) {
3654 flow_divert_close_all(group);
3655 if (group->token_key != NULL) {
3656 memset(group->token_key, 0, group->token_key_size);
3657 FREE(group->token_key, M_TEMP);
3658 group->token_key = NULL;
3659 group->token_key_size = 0;
3660 }
3661
3662 /* Re-set the current trie */
3663 if (group->signing_id_trie.memory != NULL) {
3664 FREE(group->signing_id_trie.memory, M_TEMP);
3665 }
3666 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3667 group->signing_id_trie.root = NULL_TRIE_IDX;
3668
3669 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3670 g_flow_divert_groups[unit] = NULL;
3671 g_active_group_count--;
3672 } else {
3673 error = EINVAL;
3674 }
3675
3676 if (g_active_group_count == 0) {
3677 FREE(g_flow_divert_groups, M_TEMP);
3678 g_flow_divert_groups = NULL;
3679 }
3680
3681 lck_rw_done(&g_flow_divert_group_lck);
3682
3683 return error;
3684 }
3685
3686 static errno_t
3687 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3688 {
3689 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3690 }
3691
3692 static void
3693 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3694 {
3695 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3696
3697 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3698 struct flow_divert_pcb *fd_cb;
3699 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3700
3701 lck_rw_lock_shared(&g_flow_divert_group_lck);
3702 lck_rw_lock_exclusive(&group->lck);
3703
3704 while (!MBUFQ_EMPTY(&group->send_queue)) {
3705 mbuf_t next_packet;
3706 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3707 next_packet = MBUFQ_FIRST(&group->send_queue);
3708 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3709 if (error) {
3710 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3711 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3712 lck_rw_done(&group->lck);
3713 lck_rw_done(&g_flow_divert_group_lck);
3714 return;
3715 }
3716 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3717 }
3718
3719 SLIST_INIT(&tmp_list);
3720
3721 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3722 FDRETAIN(fd_cb);
3723 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3724 }
3725
3726 lck_rw_done(&group->lck);
3727
3728 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3729 FDLOCK(fd_cb);
3730 if (fd_cb->so != NULL) {
3731 socket_lock(fd_cb->so, 0);
3732 if (fd_cb->group != NULL) {
3733 flow_divert_send_buffered_data(fd_cb, FALSE);
3734 }
3735 socket_unlock(fd_cb->so, 0);
3736 }
3737 FDUNLOCK(fd_cb);
3738 FDRELEASE(fd_cb);
3739 }
3740
3741 lck_rw_done(&g_flow_divert_group_lck);
3742 }
3743 }
3744
3745 static int
3746 flow_divert_kctl_init(void)
3747 {
3748 struct kern_ctl_reg ctl_reg;
3749 int result;
3750
3751 memset(&ctl_reg, 0, sizeof(ctl_reg));
3752
3753 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3754 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name)-1] = '\0';
3755 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3756 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3757 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3758
3759 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3760 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3761 ctl_reg.ctl_send = flow_divert_kctl_send;
3762 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3763
3764 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3765
3766 if (result) {
3767 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3768 return result;
3769 }
3770
3771 return 0;
3772 }
3773
3774 void
3775 flow_divert_init(void)
3776 {
3777 memset(&nil_pcb, 0, sizeof(nil_pcb));
3778 nil_pcb.log_level = LOG_NOTICE;
3779
3780 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3781
3782 VERIFY(g_tcp_protosw != NULL);
3783
3784 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3785 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3786
3787 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3788 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3789 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3790 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3791 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3792 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3793 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3794 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3795 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3796 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3797 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
3798
3799 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3800 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3801
3802 /*
3803 * Socket filters shouldn't attach/detach to/from this protosw
3804 * since pr_protosw is to be used instead, which points to the
3805 * real protocol; if they do, it is a bug and we should panic.
3806 */
3807 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3808 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3809 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3810 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3811
3812 /* UDP */
3813 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3814 VERIFY(g_udp_protosw != NULL);
3815
3816 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3817 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3818
3819 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3820 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3821 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3822 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3823 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3824 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3825 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3826 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3827 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3828 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3829 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3830 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3831 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3832
3833 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3834 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3835
3836 /*
3837 * Socket filters shouldn't attach/detach to/from this protosw
3838 * since pr_protosw is to be used instead, which points to the
3839 * real protocol; if they do, it is a bug and we should panic.
3840 */
3841 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3842 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3843 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3844 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3845
3846 #if INET6
3847 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3848
3849 VERIFY(g_tcp6_protosw != NULL);
3850
3851 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3852 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3853
3854 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3855 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3856 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3857 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3858 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3859 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3860 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3861 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3862 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3863 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3864 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
3865
3866 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3867 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3868 /*
3869 * Socket filters shouldn't attach/detach to/from this protosw
3870 * since pr_protosw is to be used instead, which points to the
3871 * real protocol; if they do, it is a bug and we should panic.
3872 */
3873 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
3874 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3875 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
3876 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3877
3878 /* UDP6 */
3879 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
3880
3881 VERIFY(g_udp6_protosw != NULL);
3882
3883 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
3884 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
3885
3886 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
3887 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
3888 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
3889 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
3890 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3891 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3892 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3893 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
3894 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3895 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3896 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3897 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
3898 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3899
3900 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
3901 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3902 /*
3903 * Socket filters shouldn't attach/detach to/from this protosw
3904 * since pr_protosw is to be used instead, which points to the
3905 * real protocol; if they do, it is a bug and we should panic.
3906 */
3907 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
3908 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3909 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
3910 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3911 #endif /* INET6 */
3912
3913 flow_divert_grp_attr = lck_grp_attr_alloc_init();
3914 if (flow_divert_grp_attr == NULL) {
3915 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
3916 g_init_result = ENOMEM;
3917 goto done;
3918 }
3919
3920 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
3921 if (flow_divert_mtx_grp == NULL) {
3922 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
3923 g_init_result = ENOMEM;
3924 goto done;
3925 }
3926
3927 flow_divert_mtx_attr = lck_attr_alloc_init();
3928 if (flow_divert_mtx_attr == NULL) {
3929 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
3930 g_init_result = ENOMEM;
3931 goto done;
3932 }
3933
3934 g_init_result = flow_divert_kctl_init();
3935 if (g_init_result) {
3936 goto done;
3937 }
3938
3939 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3940
3941 done:
3942 if (g_init_result != 0) {
3943 if (flow_divert_mtx_attr != NULL) {
3944 lck_attr_free(flow_divert_mtx_attr);
3945 flow_divert_mtx_attr = NULL;
3946 }
3947 if (flow_divert_mtx_grp != NULL) {
3948 lck_grp_free(flow_divert_mtx_grp);
3949 flow_divert_mtx_grp = NULL;
3950 }
3951 if (flow_divert_grp_attr != NULL) {
3952 lck_grp_attr_free(flow_divert_grp_attr);
3953 flow_divert_grp_attr = NULL;
3954 }
3955
3956 if (g_flow_divert_kctl_ref != NULL) {
3957 ctl_deregister(g_flow_divert_kctl_ref);
3958 g_flow_divert_kctl_ref = NULL;
3959 }
3960 }
3961 }
mirror of XNU