2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <net/if_var.h>
49 #include <net/route.h>
50 #include <net/flowhash.h>
51 #include <net/ntstat.h>
52 #include <netinet/in.h>
53 #include <netinet/in_var.h>
54 #include <netinet/tcp.h>
55 #include <netinet/tcp_var.h>
56 #include <netinet/tcp_fsm.h>
57 #include <netinet/flow_divert.h>
58 #include <netinet/flow_divert_proto.h>
60 #include <netinet6/in6_pcb.h>
61 #include <netinet6/ip6protosw.h>
63 #include <dev/random/randomdev.h>
64 #include <libkern/crypto/sha1.h>
65 #include <libkern/crypto/crypto_internal.h>
68 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
69 #define FLOW_DIVERT_READ_CLOSED 0x00000002
70 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
71 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
72 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
73 #define FLOW_DIVERT_TRANSFERRED 0x00000020
74 #define FLOW_DIVERT_HAS_HMAC 0x00000040
76 #define FDLOG(level, pcb, format, ...) \
77 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
79 #define FDLOG0(level, pcb, msg) \
80 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
82 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
83 #define FDRELEASE(pcb) \
85 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
86 flow_divert_pcb_destroy(pcb); \
90 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
91 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
93 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
94 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
96 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
98 #define GROUP_COUNT_MAX 32
99 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
100 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
101 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
103 struct flow_divert_trie_node
110 #define CHILD_MAP_SIZE 256
111 #define NULL_TRIE_IDX 0xffff
112 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
113 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
114 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
116 static struct flow_divert_pcb nil_pcb
;
118 decl_lck_rw_data(static, g_flow_divert_group_lck
);
119 static struct flow_divert_group
**g_flow_divert_groups
= NULL
;
120 static uint32_t g_active_group_count
= 0;
122 static lck_grp_attr_t
*flow_divert_grp_attr
= NULL
;
123 static lck_attr_t
*flow_divert_mtx_attr
= NULL
;
124 static lck_grp_t
*flow_divert_mtx_grp
= NULL
;
125 static errno_t g_init_result
= 0;
127 static kern_ctl_ref g_flow_divert_kctl_ref
= NULL
;
129 static struct protosw g_flow_divert_in_protosw
;
130 static struct pr_usrreqs g_flow_divert_in_usrreqs
;
131 static struct protosw g_flow_divert_in_udp_protosw
;
132 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs
;
134 static struct ip6protosw g_flow_divert_in6_protosw
;
135 static struct pr_usrreqs g_flow_divert_in6_usrreqs
;
136 static struct ip6protosw g_flow_divert_in6_udp_protosw
;
137 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs
;
140 static struct protosw
*g_tcp_protosw
= NULL
;
141 static struct ip6protosw
*g_tcp6_protosw
= NULL
;
142 static struct protosw
*g_udp_protosw
= NULL
;
143 static struct ip6protosw
*g_udp6_protosw
= NULL
;
146 flow_divert_dup_addr(sa_family_t family
, struct sockaddr
*addr
, struct sockaddr
**dup
);
149 flow_divert_inp_to_sockaddr(const struct inpcb
*inp
, struct sockaddr
**local_socket
);
152 flow_divert_is_sockaddr_valid(struct sockaddr
*addr
);
155 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet
, struct sockaddr
*toaddr
);
158 flow_divert_get_buffered_target_address(mbuf_t buffer
);
161 flow_divert_has_pcb_local_address(const struct inpcb
*inp
);
164 flow_divert_disconnect_socket(struct socket
*so
);
166 static inline uint8_t
167 flow_divert_syslog_type_to_oslog_type(int syslog_type
)
169 switch (syslog_type
) {
170 case LOG_ERR
: return OS_LOG_TYPE_ERROR
;
171 case LOG_INFO
: return OS_LOG_TYPE_INFO
;
172 case LOG_DEBUG
: return OS_LOG_TYPE_DEBUG
;
173 default: return OS_LOG_TYPE_DEFAULT
;
178 flow_divert_pcb_cmp(const struct flow_divert_pcb
*pcb_a
, const struct flow_divert_pcb
*pcb_b
)
180 return memcmp(&pcb_a
->hash
, &pcb_b
->hash
, sizeof(pcb_a
->hash
));
183 RB_PROTOTYPE(fd_pcb_tree
, flow_divert_pcb
, rb_link
, flow_divert_pcb_cmp
);
184 RB_GENERATE(fd_pcb_tree
, flow_divert_pcb
, rb_link
, flow_divert_pcb_cmp
);
187 flow_divert_packet_type2str(uint8_t packet_type
)
189 switch (packet_type
) {
190 case FLOW_DIVERT_PKT_CONNECT
:
192 case FLOW_DIVERT_PKT_CONNECT_RESULT
:
193 return "connect result";
194 case FLOW_DIVERT_PKT_DATA
:
196 case FLOW_DIVERT_PKT_CLOSE
:
198 case FLOW_DIVERT_PKT_READ_NOTIFY
:
199 return "read notification";
200 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE
:
201 return "properties update";
202 case FLOW_DIVERT_PKT_APP_MAP_CREATE
:
203 return "app map create";
209 static struct flow_divert_pcb
*
210 flow_divert_pcb_lookup(uint32_t hash
, struct flow_divert_group
*group
)
212 struct flow_divert_pcb key_item
;
213 struct flow_divert_pcb
*fd_cb
= NULL
;
215 key_item
.hash
= hash
;
217 lck_rw_lock_shared(&group
->lck
);
218 fd_cb
= RB_FIND(fd_pcb_tree
, &group
->pcb_tree
, &key_item
);
220 lck_rw_done(&group
->lck
);
226 flow_divert_pcb_insert(struct flow_divert_pcb
*fd_cb
, uint32_t ctl_unit
)
229 struct flow_divert_pcb
*exist
= NULL
;
230 struct flow_divert_group
*group
;
231 static uint32_t g_nextkey
= 1;
232 static uint32_t g_hash_seed
= 0;
235 if (ctl_unit
== 0 || ctl_unit
>= GROUP_COUNT_MAX
) {
239 socket_unlock(fd_cb
->so
, 0);
240 lck_rw_lock_shared(&g_flow_divert_group_lck
);
242 if (g_flow_divert_groups
== NULL
|| g_active_group_count
== 0) {
243 FDLOG0(LOG_ERR
, &nil_pcb
, "No active groups, flow divert cannot be used for this socket");
248 group
= g_flow_divert_groups
[ctl_unit
];
250 FDLOG(LOG_ERR
, &nil_pcb
, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit
);
255 socket_lock(fd_cb
->so
, 0);
261 key
[0] = g_nextkey
++;
262 key
[1] = RandomULong();
264 if (g_hash_seed
== 0) {
265 g_hash_seed
= RandomULong();
268 fd_cb
->hash
= net_flowhash(key
, sizeof(key
), g_hash_seed
);
270 for (idx
= 1; idx
< GROUP_COUNT_MAX
; idx
++) {
271 struct flow_divert_group
*curr_group
= g_flow_divert_groups
[idx
];
272 if (curr_group
!= NULL
&& curr_group
!= group
) {
273 lck_rw_lock_shared(&curr_group
->lck
);
274 exist
= RB_FIND(fd_pcb_tree
, &curr_group
->pcb_tree
, fd_cb
);
275 lck_rw_done(&curr_group
->lck
);
283 lck_rw_lock_exclusive(&group
->lck
);
284 exist
= RB_INSERT(fd_pcb_tree
, &group
->pcb_tree
, fd_cb
);
285 lck_rw_done(&group
->lck
);
287 } while (exist
!= NULL
&& try_count
++ < 3);
290 fd_cb
->group
= group
;
291 FDRETAIN(fd_cb
); /* The group now has a reference */
297 socket_unlock(fd_cb
->so
, 0);
300 lck_rw_done(&g_flow_divert_group_lck
);
301 socket_lock(fd_cb
->so
, 0);
306 static struct flow_divert_pcb
*
307 flow_divert_pcb_create(socket_t so
)
309 struct flow_divert_pcb
*new_pcb
= NULL
;
311 MALLOC_ZONE(new_pcb
, struct flow_divert_pcb
*, sizeof(*new_pcb
), M_FLOW_DIVERT_PCB
, M_WAITOK
);
312 if (new_pcb
== NULL
) {
313 FDLOG0(LOG_ERR
, &nil_pcb
, "failed to allocate a pcb");
317 memset(new_pcb
, 0, sizeof(*new_pcb
));
319 lck_mtx_init(&new_pcb
->mtx
, flow_divert_mtx_grp
, flow_divert_mtx_attr
);
321 new_pcb
->log_level
= nil_pcb
.log_level
;
323 FDRETAIN(new_pcb
); /* Represents the socket's reference */
329 flow_divert_pcb_destroy(struct flow_divert_pcb
*fd_cb
)
331 FDLOG(LOG_INFO
, fd_cb
, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
332 fd_cb
->bytes_written_by_app
, fd_cb
->bytes_read_by_app
, fd_cb
->bytes_sent
, fd_cb
->bytes_received
);
334 if (fd_cb
->local_address
!= NULL
) {
335 FREE(fd_cb
->local_address
, M_SONAME
);
337 if (fd_cb
->remote_address
!= NULL
) {
338 FREE(fd_cb
->remote_address
, M_SONAME
);
340 if (fd_cb
->connect_token
!= NULL
) {
341 mbuf_freem(fd_cb
->connect_token
);
343 if (fd_cb
->connect_packet
!= NULL
) {
344 mbuf_freem(fd_cb
->connect_packet
);
346 if (fd_cb
->app_data
!= NULL
) {
347 FREE(fd_cb
->app_data
, M_TEMP
);
349 FREE_ZONE(fd_cb
, sizeof(*fd_cb
), M_FLOW_DIVERT_PCB
);
353 flow_divert_pcb_remove(struct flow_divert_pcb
*fd_cb
)
355 if (fd_cb
->group
!= NULL
) {
356 struct flow_divert_group
*group
= fd_cb
->group
;
357 lck_rw_lock_exclusive(&group
->lck
);
358 FDLOG(LOG_INFO
, fd_cb
, "Removing from group %d, ref count = %d", group
->ctl_unit
, fd_cb
->ref_count
);
359 RB_REMOVE(fd_pcb_tree
, &group
->pcb_tree
, fd_cb
);
361 FDRELEASE(fd_cb
); /* Release the group's reference */
362 lck_rw_done(&group
->lck
);
367 flow_divert_packet_init(struct flow_divert_pcb
*fd_cb
, uint8_t packet_type
, mbuf_t
*packet
)
369 struct flow_divert_packet_header hdr
;
372 error
= mbuf_gethdr(MBUF_DONTWAIT
, MBUF_TYPE_HEADER
, packet
);
374 FDLOG(LOG_ERR
, fd_cb
, "failed to allocate the header mbuf: %d", error
);
378 hdr
.packet_type
= packet_type
;
379 hdr
.conn_id
= htonl(fd_cb
->hash
);
381 /* Lay down the header */
382 error
= mbuf_copyback(*packet
, 0, sizeof(hdr
), &hdr
, MBUF_DONTWAIT
);
384 FDLOG(LOG_ERR
, fd_cb
, "mbuf_copyback(hdr) failed: %d", error
);
394 flow_divert_packet_append_tlv(mbuf_t packet
, uint8_t type
, uint32_t length
, const void *value
)
396 uint32_t net_length
= htonl(length
);
399 error
= mbuf_copyback(packet
, mbuf_pkthdr_len(packet
), sizeof(type
), &type
, MBUF_DONTWAIT
);
401 FDLOG(LOG_ERR
, &nil_pcb
, "failed to append the type (%d)", type
);
405 error
= mbuf_copyback(packet
, mbuf_pkthdr_len(packet
), sizeof(net_length
), &net_length
, MBUF_DONTWAIT
);
407 FDLOG(LOG_ERR
, &nil_pcb
, "failed to append the length (%u)", length
);
411 error
= mbuf_copyback(packet
, mbuf_pkthdr_len(packet
), length
, value
, MBUF_DONTWAIT
);
413 FDLOG0(LOG_ERR
, &nil_pcb
, "failed to append the value");
421 flow_divert_packet_find_tlv(mbuf_t packet
, int offset
, uint8_t type
, int *err
, int next
)
423 size_t cursor
= offset
;
425 uint32_t curr_length
;
432 error
= mbuf_copydata(packet
, cursor
, sizeof(curr_type
), &curr_type
);
439 curr_type
= FLOW_DIVERT_TLV_NIL
;
442 if (curr_type
!= type
) {
443 cursor
+= sizeof(curr_type
);
444 error
= mbuf_copydata(packet
, cursor
, sizeof(curr_length
), &curr_length
);
450 cursor
+= (sizeof(curr_length
) + ntohl(curr_length
));
452 } while (curr_type
!= type
);
458 flow_divert_packet_get_tlv(mbuf_t packet
, int offset
, uint8_t type
, size_t buff_len
, void *buff
, uint32_t *val_size
)
464 tlv_offset
= flow_divert_packet_find_tlv(packet
, offset
, type
, &error
, 0);
465 if (tlv_offset
< 0) {
469 error
= mbuf_copydata(packet
, tlv_offset
+ sizeof(type
), sizeof(length
), &length
);
474 length
= ntohl(length
);
476 if (val_size
!= NULL
) {
480 if (buff
!= NULL
&& buff_len
> 0) {
481 size_t to_copy
= (length
< buff_len
) ? length
: buff_len
;
482 error
= mbuf_copydata(packet
, tlv_offset
+ sizeof(type
) + sizeof(length
), to_copy
, buff
);
492 flow_divert_packet_compute_hmac(mbuf_t packet
, struct flow_divert_group
*group
, uint8_t *hmac
)
494 mbuf_t curr_mbuf
= packet
;
496 if (g_crypto_funcs
== NULL
|| group
->token_key
== NULL
) {
500 cchmac_di_decl(g_crypto_funcs
->ccsha1_di
, hmac_ctx
);
501 g_crypto_funcs
->cchmac_init_fn(g_crypto_funcs
->ccsha1_di
, hmac_ctx
, group
->token_key_size
, group
->token_key
);
503 while (curr_mbuf
!= NULL
) {
504 g_crypto_funcs
->cchmac_update_fn(g_crypto_funcs
->ccsha1_di
, hmac_ctx
, mbuf_len(curr_mbuf
), mbuf_data(curr_mbuf
));
505 curr_mbuf
= mbuf_next(curr_mbuf
);
508 g_crypto_funcs
->cchmac_final_fn(g_crypto_funcs
->ccsha1_di
, hmac_ctx
, hmac
);
514 flow_divert_packet_verify_hmac(mbuf_t packet
, uint32_t ctl_unit
)
517 struct flow_divert_group
*group
= NULL
;
519 uint8_t packet_hmac
[SHA_DIGEST_LENGTH
];
520 uint8_t computed_hmac
[SHA_DIGEST_LENGTH
];
523 lck_rw_lock_shared(&g_flow_divert_group_lck
);
525 if (g_flow_divert_groups
!= NULL
&& g_active_group_count
> 0) {
526 group
= g_flow_divert_groups
[ctl_unit
];
530 lck_rw_done(&g_flow_divert_group_lck
);
534 lck_rw_lock_shared(&group
->lck
);
536 if (group
->token_key
== NULL
) {
541 hmac_offset
= flow_divert_packet_find_tlv(packet
, 0, FLOW_DIVERT_TLV_HMAC
, &error
, 0);
542 if (hmac_offset
< 0) {
546 error
= flow_divert_packet_get_tlv(packet
, hmac_offset
, FLOW_DIVERT_TLV_HMAC
, sizeof(packet_hmac
), packet_hmac
, NULL
);
551 /* Chop off the HMAC TLV */
552 error
= mbuf_split(packet
, hmac_offset
, MBUF_WAITOK
, &tail
);
559 error
= flow_divert_packet_compute_hmac(packet
, group
, computed_hmac
);
564 if (memcmp(packet_hmac
, computed_hmac
, sizeof(packet_hmac
))) {
565 FDLOG0(LOG_WARNING
, &nil_pcb
, "HMAC in token does not match computed HMAC");
571 lck_rw_done(&group
->lck
);
572 lck_rw_done(&g_flow_divert_group_lck
);
577 flow_divert_add_data_statistics(struct flow_divert_pcb
*fd_cb
, int data_len
, Boolean send
)
579 struct inpcb
*inp
= NULL
;
580 struct ifnet
*ifp
= NULL
;
581 Boolean cell
= FALSE
;
582 Boolean wifi
= FALSE
;
583 Boolean wired
= FALSE
;
585 inp
= sotoinpcb(fd_cb
->so
);
590 ifp
= inp
->inp_last_outifp
;
592 cell
= IFNET_IS_CELLULAR(ifp
);
593 wifi
= (!cell
&& IFNET_IS_WIFI(ifp
));
594 wired
= (!wifi
&& IFNET_IS_WIRED(ifp
));
598 INP_ADD_STAT(inp
, cell
, wifi
, wired
, txpackets
, 1);
599 INP_ADD_STAT(inp
, cell
, wifi
, wired
, txbytes
, data_len
);
601 INP_ADD_STAT(inp
, cell
, wifi
, wired
, rxpackets
, 1);
602 INP_ADD_STAT(inp
, cell
, wifi
, wired
, rxbytes
, data_len
);
604 inp_set_activity_bitmap(inp
);
608 flow_divert_check_no_cellular(struct flow_divert_pcb
*fd_cb
)
610 struct inpcb
*inp
= NULL
;
612 inp
= sotoinpcb(fd_cb
->so
);
613 if (inp
&& INP_NO_CELLULAR(inp
) && inp
->inp_last_outifp
&&
614 IFNET_IS_CELLULAR(inp
->inp_last_outifp
))
621 flow_divert_check_no_expensive(struct flow_divert_pcb
*fd_cb
)
623 struct inpcb
*inp
= NULL
;
625 inp
= sotoinpcb(fd_cb
->so
);
626 if (inp
&& INP_NO_EXPENSIVE(inp
) && inp
->inp_last_outifp
&&
627 IFNET_IS_EXPENSIVE(inp
->inp_last_outifp
))
634 flow_divert_update_closed_state(struct flow_divert_pcb
*fd_cb
, int how
, Boolean tunnel
)
636 if (how
!= SHUT_RD
) {
637 fd_cb
->flags
|= FLOW_DIVERT_WRITE_CLOSED
;
638 if (tunnel
|| !(fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
)) {
639 fd_cb
->flags
|= FLOW_DIVERT_TUNNEL_WR_CLOSED
;
640 /* If the tunnel is not accepting writes any more, then flush the send buffer */
641 sbflush(&fd_cb
->so
->so_snd
);
644 if (how
!= SHUT_WR
) {
645 fd_cb
->flags
|= FLOW_DIVERT_READ_CLOSED
;
646 if (tunnel
|| !(fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
)) {
647 fd_cb
->flags
|= FLOW_DIVERT_TUNNEL_RD_CLOSED
;
653 trie_node_alloc(struct flow_divert_trie
*trie
)
655 if (trie
->nodes_free_next
< trie
->nodes_count
) {
656 uint16_t node_idx
= trie
->nodes_free_next
++;
657 TRIE_NODE(trie
, node_idx
).child_map
= NULL_TRIE_IDX
;
660 return NULL_TRIE_IDX
;
665 trie_child_map_alloc(struct flow_divert_trie
*trie
)
667 if (trie
->child_maps_free_next
< trie
->child_maps_count
) {
668 return trie
->child_maps_free_next
++;
670 return NULL_TRIE_IDX
;
675 trie_bytes_move(struct flow_divert_trie
*trie
, uint16_t bytes_idx
, size_t bytes_size
)
677 uint16_t start
= trie
->bytes_free_next
;
678 if (start
+ bytes_size
<= trie
->bytes_count
) {
679 if (start
!= bytes_idx
) {
680 memmove(&TRIE_BYTE(trie
, start
), &TRIE_BYTE(trie
, bytes_idx
), bytes_size
);
682 trie
->bytes_free_next
+= bytes_size
;
685 return NULL_TRIE_IDX
;
690 flow_divert_trie_insert(struct flow_divert_trie
*trie
, uint16_t string_start
, size_t string_len
)
692 uint16_t current
= trie
->root
;
693 uint16_t child
= trie
->root
;
694 uint16_t string_end
= string_start
+ string_len
;
695 uint16_t string_idx
= string_start
;
696 uint16_t string_remainder
= string_len
;
698 while (child
!= NULL_TRIE_IDX
) {
699 uint16_t parent
= current
;
701 uint16_t current_end
;
704 child
= NULL_TRIE_IDX
;
706 current_end
= TRIE_NODE(trie
, current
).start
+ TRIE_NODE(trie
, current
).length
;
708 for (node_idx
= TRIE_NODE(trie
, current
).start
;
709 node_idx
< current_end
&&
710 string_idx
< string_end
&&
711 TRIE_BYTE(trie
, node_idx
) == TRIE_BYTE(trie
, string_idx
);
712 node_idx
++, string_idx
++);
714 string_remainder
= string_end
- string_idx
;
716 if (node_idx
< (TRIE_NODE(trie
, current
).start
+ TRIE_NODE(trie
, current
).length
)) {
718 * We did not reach the end of the current node's string.
719 * We need to split the current node into two:
720 * 1. A new node that contains the prefix of the node that matches
721 * the prefix of the string being inserted.
722 * 2. The current node modified to point to the remainder
723 * of the current node's string.
725 uint16_t prefix
= trie_node_alloc(trie
);
726 if (prefix
== NULL_TRIE_IDX
) {
727 FDLOG0(LOG_ERR
, &nil_pcb
, "Ran out of trie nodes while splitting an existing node");
728 return NULL_TRIE_IDX
;
732 * Prefix points to the portion of the current nodes's string that has matched
733 * the input string thus far.
735 TRIE_NODE(trie
, prefix
).start
= TRIE_NODE(trie
, current
).start
;
736 TRIE_NODE(trie
, prefix
).length
= (node_idx
- TRIE_NODE(trie
, current
).start
);
739 * Prefix has the current node as the child corresponding to the first byte
742 TRIE_NODE(trie
, prefix
).child_map
= trie_child_map_alloc(trie
);
743 if (TRIE_NODE(trie
, prefix
).child_map
== NULL_TRIE_IDX
) {
744 FDLOG0(LOG_ERR
, &nil_pcb
, "Ran out of child maps while splitting an existing node");
745 return NULL_TRIE_IDX
;
747 TRIE_CHILD(trie
, prefix
, TRIE_BYTE(trie
, node_idx
)) = current
;
749 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
750 TRIE_CHILD(trie
, parent
, TRIE_BYTE(trie
, TRIE_NODE(trie
, prefix
).start
)) = prefix
;
752 /* Current node is adjusted to point to the remainder */
753 TRIE_NODE(trie
, current
).start
= node_idx
;
754 TRIE_NODE(trie
, current
).length
-= TRIE_NODE(trie
, prefix
).length
;
756 /* We want to insert the new leaf (if any) as a child of the prefix */
760 if (string_remainder
> 0) {
762 * We still have bytes in the string that have not been matched yet.
763 * If the current node has children, iterate to the child corresponding
764 * to the next byte in the string.
766 if (TRIE_NODE(trie
, current
).child_map
!= NULL_TRIE_IDX
) {
767 child
= TRIE_CHILD(trie
, current
, TRIE_BYTE(trie
, string_idx
));
770 } /* while (child != NULL_TRIE_IDX) */
772 if (string_remainder
> 0) {
773 /* Add a new leaf containing the remainder of the string */
774 uint16_t leaf
= trie_node_alloc(trie
);
775 if (leaf
== NULL_TRIE_IDX
) {
776 FDLOG0(LOG_ERR
, &nil_pcb
, "Ran out of trie nodes while inserting a new leaf");
777 return NULL_TRIE_IDX
;
780 TRIE_NODE(trie
, leaf
).start
= trie_bytes_move(trie
, string_idx
, string_remainder
);
781 if (TRIE_NODE(trie
, leaf
).start
== NULL_TRIE_IDX
) {
782 FDLOG0(LOG_ERR
, &nil_pcb
, "Ran out of bytes while inserting a new leaf");
783 return NULL_TRIE_IDX
;
785 TRIE_NODE(trie
, leaf
).length
= string_remainder
;
787 /* Set the new leaf as the child of the current node */
788 if (TRIE_NODE(trie
, current
).child_map
== NULL_TRIE_IDX
) {
789 TRIE_NODE(trie
, current
).child_map
= trie_child_map_alloc(trie
);
790 if (TRIE_NODE(trie
, current
).child_map
== NULL_TRIE_IDX
) {
791 FDLOG0(LOG_ERR
, &nil_pcb
, "Ran out of child maps while inserting a new leaf");
792 return NULL_TRIE_IDX
;
795 TRIE_CHILD(trie
, current
, TRIE_BYTE(trie
, TRIE_NODE(trie
, leaf
).start
)) = leaf
;
797 } /* else duplicate or this string is a prefix of one of the existing strings */
802 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
804 flow_divert_trie_search(struct flow_divert_trie
*trie
, uint8_t *string_bytes
)
806 uint16_t current
= trie
->root
;
807 uint16_t string_idx
= 0;
809 while (current
!= NULL_TRIE_IDX
) {
810 uint16_t next
= NULL_TRIE_IDX
;
811 uint16_t node_end
= TRIE_NODE(trie
, current
).start
+ TRIE_NODE(trie
, current
).length
;
814 for (node_idx
= TRIE_NODE(trie
, current
).start
;
815 node_idx
< node_end
&& string_bytes
[string_idx
] != '\0' && string_bytes
[string_idx
] == TRIE_BYTE(trie
, node_idx
);
816 node_idx
++, string_idx
++);
818 if (node_idx
== node_end
) {
819 if (string_bytes
[string_idx
] == '\0') {
820 return current
; /* Got an exact match */
821 } else if (string_idx
== strlen(APPLE_WEBCLIP_ID_PREFIX
) &&
822 0 == strncmp((const char *)string_bytes
, APPLE_WEBCLIP_ID_PREFIX
, string_idx
)) {
823 string_bytes
[string_idx
] = '\0';
824 return current
; /* Got an apple webclip id prefix match */
825 } else if (TRIE_NODE(trie
, current
).child_map
!= NULL_TRIE_IDX
) {
826 next
= TRIE_CHILD(trie
, current
, string_bytes
[string_idx
]);
832 return NULL_TRIE_IDX
;
835 struct uuid_search_info
{
837 char *found_signing_id
;
838 boolean_t found_multiple_signing_ids
;
843 flow_divert_find_proc_by_uuid_callout(proc_t p
, void *arg
)
845 struct uuid_search_info
*info
= (struct uuid_search_info
*)arg
;
846 int result
= PROC_RETURNED_DONE
; /* By default, we didn't find the process */
848 if (info
->found_signing_id
!= NULL
) {
849 if (!info
->found_multiple_signing_ids
) {
850 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
851 info
->found_proc
= p
;
852 result
= PROC_CLAIMED_DONE
;
854 uuid_string_t uuid_str
;
855 uuid_unparse(info
->target_uuid
, uuid_str
);
856 FDLOG(LOG_WARNING
, &nil_pcb
, "Found multiple processes with UUID %s with different signing identifiers", uuid_str
);
858 FREE(info
->found_signing_id
, M_TEMP
);
859 info
->found_signing_id
= NULL
;
862 if (result
== PROC_RETURNED_DONE
) {
863 uuid_string_t uuid_str
;
864 uuid_unparse(info
->target_uuid
, uuid_str
);
865 FDLOG(LOG_WARNING
, &nil_pcb
, "Failed to find a process with UUID %s", uuid_str
);
872 flow_divert_find_proc_by_uuid_filter(proc_t p
, void *arg
)
874 struct uuid_search_info
*info
= (struct uuid_search_info
*)arg
;
877 if (info
->found_multiple_signing_ids
) {
881 include
= (uuid_compare(p
->p_uuid
, info
->target_uuid
) == 0);
883 const char *signing_id
= cs_identity_get(p
);
884 if (signing_id
!= NULL
) {
885 FDLOG(LOG_INFO
, &nil_pcb
, "Found process %d with signing identifier %s", p
->p_pid
, signing_id
);
886 size_t signing_id_size
= strlen(signing_id
) + 1;
887 if (info
->found_signing_id
== NULL
) {
888 MALLOC(info
->found_signing_id
, char *, signing_id_size
, M_TEMP
, M_WAITOK
);
889 memcpy(info
->found_signing_id
, signing_id
, signing_id_size
);
890 } else if (memcmp(signing_id
, info
->found_signing_id
, signing_id_size
)) {
891 info
->found_multiple_signing_ids
= TRUE
;
894 info
->found_multiple_signing_ids
= TRUE
;
896 include
= !info
->found_multiple_signing_ids
;
903 flow_divert_find_proc_by_uuid(uuid_t uuid
)
905 struct uuid_search_info info
;
907 if (LOG_INFO
<= nil_pcb
.log_level
) {
908 uuid_string_t uuid_str
;
909 uuid_unparse(uuid
, uuid_str
);
910 FDLOG(LOG_INFO
, &nil_pcb
, "Looking for process with UUID %s", uuid_str
);
913 memset(&info
, 0, sizeof(info
));
914 info
.found_proc
= PROC_NULL
;
915 uuid_copy(info
.target_uuid
, uuid
);
917 proc_iterate(PROC_ALLPROCLIST
, flow_divert_find_proc_by_uuid_callout
, &info
, flow_divert_find_proc_by_uuid_filter
, &info
);
919 return info
.found_proc
;
923 flow_divert_get_src_proc(struct socket
*so
, proc_t
*proc
)
927 if (so
->so_flags
& SOF_DELEGATED
) {
928 if ((*proc
)->p_pid
!= so
->e_pid
) {
929 *proc
= proc_find(so
->e_pid
);
931 } else if (uuid_compare((*proc
)->p_uuid
, so
->e_uuid
)) {
932 *proc
= flow_divert_find_proc_by_uuid(so
->e_uuid
);
935 } else if (*proc
== PROC_NULL
) {
936 *proc
= current_proc();
939 if (*proc
!= PROC_NULL
) {
940 if ((*proc
)->p_pid
== 0) {
953 flow_divert_send_packet(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, Boolean enqueue
)
957 if (fd_cb
->group
== NULL
) {
958 fd_cb
->so
->so_error
= ECONNABORTED
;
959 flow_divert_disconnect_socket(fd_cb
->so
);
963 lck_rw_lock_shared(&fd_cb
->group
->lck
);
965 if (MBUFQ_EMPTY(&fd_cb
->group
->send_queue
)) {
966 error
= ctl_enqueuembuf(g_flow_divert_kctl_ref
, fd_cb
->group
->ctl_unit
, packet
, CTL_DATA_EOR
);
971 if (error
== ENOBUFS
) {
973 if (!lck_rw_lock_shared_to_exclusive(&fd_cb
->group
->lck
)) {
974 lck_rw_lock_exclusive(&fd_cb
->group
->lck
);
976 MBUFQ_ENQUEUE(&fd_cb
->group
->send_queue
, packet
);
979 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED
, &fd_cb
->group
->atomic_bits
);
982 lck_rw_done(&fd_cb
->group
->lck
);
988 flow_divert_create_connect_packet(struct flow_divert_pcb
*fd_cb
, struct sockaddr
*to
, struct socket
*so
, proc_t p
, mbuf_t
*out_connect_packet
)
992 char *signing_id
= NULL
;
993 int free_signing_id
= 0;
994 mbuf_t connect_packet
= NULL
;
996 int release_proc
= 0;
998 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_CONNECT
, &connect_packet
);
1005 if (fd_cb
->connect_token
!= NULL
&& (fd_cb
->flags
& FLOW_DIVERT_HAS_HMAC
)) {
1006 uint32_t sid_size
= 0;
1007 int find_error
= flow_divert_packet_get_tlv(fd_cb
->connect_token
, 0, FLOW_DIVERT_TLV_SIGNING_ID
, 0, NULL
, &sid_size
);
1008 if (find_error
== 0 && sid_size
> 0) {
1009 MALLOC(signing_id
, char *, sid_size
+ 1, M_TEMP
, M_WAITOK
| M_ZERO
);
1010 if (signing_id
!= NULL
) {
1011 flow_divert_packet_get_tlv(fd_cb
->connect_token
, 0, FLOW_DIVERT_TLV_SIGNING_ID
, sid_size
, signing_id
, NULL
);
1012 FDLOG(LOG_INFO
, fd_cb
, "Got %s from token", signing_id
);
1013 free_signing_id
= 1;
1018 socket_unlock(so
, 0);
1020 if (signing_id
== NULL
) {
1021 release_proc
= flow_divert_get_src_proc(so
, &src_proc
);
1022 if (src_proc
!= PROC_NULL
) {
1023 proc_lock(src_proc
);
1024 if (src_proc
->p_csflags
& (CS_VALID
|CS_DEBUGGED
)) {
1026 cs_id
= cs_identity_get(src_proc
);
1027 signing_id
= __DECONST(char *, cs_id
);
1029 FDLOG0(LOG_WARNING
, fd_cb
, "Signature is invalid");
1032 FDLOG0(LOG_WARNING
, fd_cb
, "Failed to determine the current proc");
1035 src_proc
= PROC_NULL
;
1038 if (signing_id
!= NULL
) {
1039 uint16_t result
= NULL_TRIE_IDX
;
1040 lck_rw_lock_shared(&fd_cb
->group
->lck
);
1041 if (fd_cb
->group
->flags
& FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP
) {
1044 result
= flow_divert_trie_search(&fd_cb
->group
->signing_id_trie
, (uint8_t *)signing_id
);
1046 lck_rw_done(&fd_cb
->group
->lck
);
1047 if (result
!= NULL_TRIE_IDX
) {
1049 FDLOG(LOG_INFO
, fd_cb
, "%s matched", signing_id
);
1051 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_SIGNING_ID
, strlen(signing_id
), signing_id
);
1053 if (src_proc
!= PROC_NULL
) {
1054 unsigned char cdhash
[SHA1_RESULTLEN
];
1055 error
= proc_getcdhash(src_proc
, cdhash
);
1057 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_CDHASH
, sizeof(cdhash
), cdhash
);
1059 FDLOG(LOG_ERR
, fd_cb
, "failed to append the cdhash: %d", error
);
1062 FDLOG(LOG_ERR
, fd_cb
, "failed to get the cdhash: %d", error
);
1066 FDLOG(LOG_ERR
, fd_cb
, "failed to append the signing ID: %d", error
);
1069 FDLOG(LOG_WARNING
, fd_cb
, "%s did not match", signing_id
);
1072 FDLOG0(LOG_WARNING
, fd_cb
, "Failed to get the code signing identity");
1073 if (fd_cb
->group
->flags
& FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP
) {
1078 if (src_proc
!= PROC_NULL
) {
1079 proc_unlock(src_proc
);
1081 proc_rele(src_proc
);
1086 if (free_signing_id
) {
1087 FREE(signing_id
, M_TEMP
);
1094 error
= flow_divert_packet_append_tlv(connect_packet
,
1095 FLOW_DIVERT_TLV_TRAFFIC_CLASS
,
1096 sizeof(fd_cb
->so
->so_traffic_class
),
1097 &fd_cb
->so
->so_traffic_class
);
1102 if (SOCK_TYPE(fd_cb
->so
) == SOCK_STREAM
) {
1103 flow_type
= FLOW_DIVERT_FLOW_TYPE_TCP
;
1104 } else if (SOCK_TYPE(fd_cb
->so
) == SOCK_DGRAM
) {
1105 flow_type
= FLOW_DIVERT_FLOW_TYPE_UDP
;
1110 error
= flow_divert_packet_append_tlv(connect_packet
,
1111 FLOW_DIVERT_TLV_FLOW_TYPE
,
1119 if (fd_cb
->so
->so_flags
& SOF_DELEGATED
) {
1120 error
= flow_divert_packet_append_tlv(connect_packet
,
1121 FLOW_DIVERT_TLV_PID
,
1122 sizeof(fd_cb
->so
->e_pid
),
1128 error
= flow_divert_packet_append_tlv(connect_packet
,
1129 FLOW_DIVERT_TLV_UUID
,
1130 sizeof(fd_cb
->so
->e_uuid
),
1131 &fd_cb
->so
->e_uuid
);
1136 error
= flow_divert_packet_append_tlv(connect_packet
,
1137 FLOW_DIVERT_TLV_PID
,
1138 sizeof(fd_cb
->so
->e_pid
),
1139 &fd_cb
->so
->last_pid
);
1144 error
= flow_divert_packet_append_tlv(connect_packet
,
1145 FLOW_DIVERT_TLV_UUID
,
1146 sizeof(fd_cb
->so
->e_uuid
),
1147 &fd_cb
->so
->last_uuid
);
1153 if (fd_cb
->connect_token
!= NULL
) {
1154 unsigned int token_len
= m_length(fd_cb
->connect_token
);
1155 mbuf_concatenate(connect_packet
, fd_cb
->connect_token
);
1156 mbuf_pkthdr_adjustlen(connect_packet
, token_len
);
1157 fd_cb
->connect_token
= NULL
;
1159 uint32_t ctl_unit
= htonl(fd_cb
->control_group_unit
);
1161 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_CTL_UNIT
, sizeof(ctl_unit
), &ctl_unit
);
1166 error
= flow_divert_append_target_endpoint_tlv(connect_packet
, to
);
1172 if (fd_cb
->local_address
!= NULL
) {
1176 struct inpcb
*inp
= sotoinpcb(so
);
1177 if (flow_divert_has_pcb_local_address(inp
)) {
1178 error
= flow_divert_inp_to_sockaddr(inp
, &fd_cb
->local_address
);
1180 FDLOG0(LOG_ERR
, fd_cb
, "failed to get the local socket address.");
1186 if (fd_cb
->local_address
!= NULL
) {
1187 /* socket is bound. */
1188 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_LOCAL_ADDR
,
1189 sizeof(struct sockaddr_storage
), fd_cb
->local_address
);
1195 if (so
->so_flags1
& SOF1_DATA_IDEMPOTENT
) {
1196 uint32_t flags
= FLOW_DIVERT_TOKEN_FLAG_TFO
;
1197 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_FLAGS
, sizeof(flags
), &flags
);
1205 *out_connect_packet
= connect_packet
;
1206 } else if (connect_packet
!= NULL
) {
1207 mbuf_freem(connect_packet
);
1214 flow_divert_send_connect_result(struct flow_divert_pcb
*fd_cb
)
1217 mbuf_t packet
= NULL
;
1218 int rbuff_space
= 0;
1220 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_CONNECT_RESULT
, &packet
);
1222 FDLOG(LOG_ERR
, fd_cb
, "failed to create a connect result packet: %d", error
);
1226 rbuff_space
= fd_cb
->so
->so_rcv
.sb_hiwat
;
1227 if (rbuff_space
< 0) {
1230 rbuff_space
= htonl(rbuff_space
);
1231 error
= flow_divert_packet_append_tlv(packet
,
1232 FLOW_DIVERT_TLV_SPACE_AVAILABLE
,
1233 sizeof(rbuff_space
),
1239 error
= flow_divert_send_packet(fd_cb
, packet
, TRUE
);
1245 if (error
&& packet
!= NULL
) {
1253 flow_divert_send_close(struct flow_divert_pcb
*fd_cb
, int how
)
1256 mbuf_t packet
= NULL
;
1259 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_CLOSE
, &packet
);
1261 FDLOG(LOG_ERR
, fd_cb
, "failed to create a close packet: %d", error
);
1265 error
= flow_divert_packet_append_tlv(packet
, FLOW_DIVERT_TLV_ERROR_CODE
, sizeof(zero
), &zero
);
1267 FDLOG(LOG_ERR
, fd_cb
, "failed to add the error code TLV: %d", error
);
1272 error
= flow_divert_packet_append_tlv(packet
, FLOW_DIVERT_TLV_HOW
, sizeof(how
), &how
);
1274 FDLOG(LOG_ERR
, fd_cb
, "failed to add the how flag: %d", error
);
1278 error
= flow_divert_send_packet(fd_cb
, packet
, TRUE
);
1284 if (error
&& packet
!= NULL
) {
1292 flow_divert_tunnel_how_closed(struct flow_divert_pcb
*fd_cb
)
1294 if ((fd_cb
->flags
& (FLOW_DIVERT_TUNNEL_RD_CLOSED
|FLOW_DIVERT_TUNNEL_WR_CLOSED
)) ==
1295 (FLOW_DIVERT_TUNNEL_RD_CLOSED
|FLOW_DIVERT_TUNNEL_WR_CLOSED
))
1298 } else if (fd_cb
->flags
& FLOW_DIVERT_TUNNEL_RD_CLOSED
) {
1300 } else if (fd_cb
->flags
& FLOW_DIVERT_TUNNEL_WR_CLOSED
) {
1308 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1309 * writes. Returns FALSE otherwise.
1312 flow_divert_send_close_if_needed(struct flow_divert_pcb
*fd_cb
)
1316 /* Do not send any close messages if there is still data in the send buffer */
1317 if (fd_cb
->so
->so_snd
.sb_cc
== 0) {
1318 if ((fd_cb
->flags
& (FLOW_DIVERT_READ_CLOSED
|FLOW_DIVERT_TUNNEL_RD_CLOSED
)) == FLOW_DIVERT_READ_CLOSED
) {
1319 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1322 if ((fd_cb
->flags
& (FLOW_DIVERT_WRITE_CLOSED
|FLOW_DIVERT_TUNNEL_WR_CLOSED
)) == FLOW_DIVERT_WRITE_CLOSED
) {
1323 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1324 if (how
== SHUT_RD
) {
1333 FDLOG(LOG_INFO
, fd_cb
, "sending close, how = %d", how
);
1334 if (flow_divert_send_close(fd_cb
, how
) != ENOBUFS
) {
1335 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1336 if (how
!= SHUT_RD
) {
1337 fd_cb
->flags
|= FLOW_DIVERT_TUNNEL_WR_CLOSED
;
1339 if (how
!= SHUT_WR
) {
1340 fd_cb
->flags
|= FLOW_DIVERT_TUNNEL_RD_CLOSED
;
1345 if (flow_divert_tunnel_how_closed(fd_cb
) == SHUT_RDWR
) {
1346 flow_divert_disconnect_socket(fd_cb
->so
);
1351 flow_divert_send_data_packet(struct flow_divert_pcb
*fd_cb
, mbuf_t data
, size_t data_len
, struct sockaddr
*toaddr
, Boolean force
)
1357 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_DATA
, &packet
);
1359 FDLOG(LOG_ERR
, fd_cb
, "flow_divert_packet_init failed: %d", error
);
1363 if (toaddr
!= NULL
) {
1364 error
= flow_divert_append_target_endpoint_tlv(packet
, toaddr
);
1366 FDLOG(LOG_ERR
, fd_cb
, "flow_divert_append_target_endpoint_tlv() failed: %d", error
);
1371 if (data_len
> 0 && data
!= NULL
) {
1372 last
= m_last(packet
);
1373 mbuf_setnext(last
, data
);
1374 mbuf_pkthdr_adjustlen(packet
, data_len
);
1376 error
= flow_divert_send_packet(fd_cb
, packet
, force
);
1379 mbuf_setnext(last
, NULL
);
1382 fd_cb
->bytes_sent
+= data_len
;
1383 flow_divert_add_data_statistics(fd_cb
, data_len
, TRUE
);
1390 flow_divert_send_buffered_data(struct flow_divert_pcb
*fd_cb
, Boolean force
)
1397 to_send
= fd_cb
->so
->so_snd
.sb_cc
;
1398 buffer
= fd_cb
->so
->so_snd
.sb_mb
;
1400 if (buffer
== NULL
&& to_send
> 0) {
1401 FDLOG(LOG_ERR
, fd_cb
, "Send buffer is NULL, but size is supposed to be %lu", to_send
);
1405 /* Ignore the send window if force is enabled */
1406 if (!force
&& (to_send
> fd_cb
->send_window
)) {
1407 to_send
= fd_cb
->send_window
;
1410 if (SOCK_TYPE(fd_cb
->so
) == SOCK_STREAM
) {
1411 while (sent
< to_send
) {
1415 data_len
= to_send
- sent
;
1416 if (data_len
> FLOW_DIVERT_CHUNK_SIZE
) {
1417 data_len
= FLOW_DIVERT_CHUNK_SIZE
;
1420 error
= mbuf_copym(buffer
, sent
, data_len
, MBUF_DONTWAIT
, &data
);
1422 FDLOG(LOG_ERR
, fd_cb
, "mbuf_copym failed: %d", error
);
1426 error
= flow_divert_send_data_packet(fd_cb
, data
, data_len
, NULL
, force
);
1434 sbdrop(&fd_cb
->so
->so_snd
, sent
);
1435 sowwakeup(fd_cb
->so
);
1436 } else if (SOCK_TYPE(fd_cb
->so
) == SOCK_DGRAM
) {
1442 struct sockaddr
*toaddr
= flow_divert_get_buffered_target_address(buffer
);
1445 if (toaddr
!= NULL
) {
1446 /* look for data in the chain */
1449 if (m
!= NULL
&& m
->m_type
== MT_DATA
) {
1455 FDLOG0(LOG_ERR
, fd_cb
, "failed to find type MT_DATA in the mbuf chain.");
1459 data_len
= mbuf_pkthdr_len(m
);
1461 FDLOG(LOG_DEBUG
, fd_cb
, "mbuf_copym() data_len = %lu", data_len
);
1462 error
= mbuf_copym(m
, 0, data_len
, MBUF_DONTWAIT
, &data
);
1464 FDLOG(LOG_ERR
, fd_cb
, "mbuf_copym failed: %d", error
);
1470 error
= flow_divert_send_data_packet(fd_cb
, data
, data_len
, toaddr
, force
);
1477 buffer
= buffer
->m_nextpkt
;
1478 (void) sbdroprecord(&(fd_cb
->so
->so_snd
));
1483 FDLOG(LOG_DEBUG
, fd_cb
, "sent %lu bytes of buffered data", sent
);
1484 if (fd_cb
->send_window
>= sent
) {
1485 fd_cb
->send_window
-= sent
;
1487 fd_cb
->send_window
= 0;
1493 flow_divert_send_app_data(struct flow_divert_pcb
*fd_cb
, mbuf_t data
, struct sockaddr
*toaddr
)
1495 size_t to_send
= mbuf_pkthdr_len(data
);
1498 if (to_send
> fd_cb
->send_window
) {
1499 to_send
= fd_cb
->send_window
;
1502 if (fd_cb
->so
->so_snd
.sb_cc
> 0) {
1503 to_send
= 0; /* If the send buffer is non-empty, then we can't send anything */
1506 if (SOCK_TYPE(fd_cb
->so
) == SOCK_STREAM
) {
1508 mbuf_t remaining_data
= data
;
1509 mbuf_t pkt_data
= NULL
;
1510 while (sent
< to_send
&& remaining_data
!= NULL
) {
1511 size_t pkt_data_len
;
1513 pkt_data
= remaining_data
;
1515 if ((to_send
- sent
) > FLOW_DIVERT_CHUNK_SIZE
) {
1516 pkt_data_len
= FLOW_DIVERT_CHUNK_SIZE
;
1518 pkt_data_len
= to_send
- sent
;
1521 if (pkt_data_len
< mbuf_pkthdr_len(pkt_data
)) {
1522 error
= mbuf_split(pkt_data
, pkt_data_len
, MBUF_DONTWAIT
, &remaining_data
);
1524 FDLOG(LOG_ERR
, fd_cb
, "mbuf_split failed: %d", error
);
1529 remaining_data
= NULL
;
1532 error
= flow_divert_send_data_packet(fd_cb
, pkt_data
, pkt_data_len
, NULL
, FALSE
);
1539 sent
+= pkt_data_len
;
1542 fd_cb
->send_window
-= sent
;
1546 if (pkt_data
!= NULL
) {
1547 if (sbspace(&fd_cb
->so
->so_snd
) > 0) {
1548 if (!sbappendstream(&fd_cb
->so
->so_snd
, pkt_data
)) {
1549 FDLOG(LOG_ERR
, fd_cb
, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1550 fd_cb
->so
->so_snd
.sb_cc
, fd_cb
->send_window
);
1557 if (remaining_data
!= NULL
) {
1558 if (sbspace(&fd_cb
->so
->so_snd
) > 0) {
1559 if (!sbappendstream(&fd_cb
->so
->so_snd
, remaining_data
)) {
1560 FDLOG(LOG_ERR
, fd_cb
, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1561 fd_cb
->so
->so_snd
.sb_cc
, fd_cb
->send_window
);
1567 } else if (SOCK_TYPE(fd_cb
->so
) == SOCK_DGRAM
) {
1568 if (to_send
|| mbuf_pkthdr_len(data
) == 0) {
1569 error
= flow_divert_send_data_packet(fd_cb
, data
, to_send
, toaddr
, FALSE
);
1571 FDLOG(LOG_ERR
, fd_cb
, "flow_divert_send_data_packet failed. send data size = %lu", to_send
);
1573 fd_cb
->send_window
-= to_send
;
1577 if (sbspace(&fd_cb
->so
->so_snd
) >= (int)mbuf_pkthdr_len(data
)) {
1578 if (toaddr
!= NULL
) {
1579 if (!sbappendaddr(&fd_cb
->so
->so_snd
, toaddr
, data
, NULL
, &error
)) {
1580 FDLOG(LOG_ERR
, fd_cb
,
1581 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1582 fd_cb
->so
->so_snd
.sb_cc
, fd_cb
->send_window
, error
);
1585 if (!sbappendrecord(&fd_cb
->so
->so_snd
, data
)) {
1586 FDLOG(LOG_ERR
, fd_cb
,
1587 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1588 fd_cb
->so
->so_snd
.sb_cc
, fd_cb
->send_window
, error
);
1601 flow_divert_send_read_notification(struct flow_divert_pcb
*fd_cb
, uint32_t read_count
)
1604 mbuf_t packet
= NULL
;
1605 uint32_t net_read_count
= htonl(read_count
);
1607 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_READ_NOTIFY
, &packet
);
1609 FDLOG(LOG_ERR
, fd_cb
, "failed to create a read notification packet: %d", error
);
1613 error
= flow_divert_packet_append_tlv(packet
, FLOW_DIVERT_TLV_READ_COUNT
, sizeof(net_read_count
), &net_read_count
);
1615 FDLOG(LOG_ERR
, fd_cb
, "failed to add the read count: %d", error
);
1619 error
= flow_divert_send_packet(fd_cb
, packet
, TRUE
);
1625 if (error
&& packet
!= NULL
) {
1633 flow_divert_send_traffic_class_update(struct flow_divert_pcb
*fd_cb
, int traffic_class
)
1636 mbuf_t packet
= NULL
;
1638 error
= flow_divert_packet_init(fd_cb
, FLOW_DIVERT_PKT_PROPERTIES_UPDATE
, &packet
);
1640 FDLOG(LOG_ERR
, fd_cb
, "failed to create a properties update packet: %d", error
);
1644 error
= flow_divert_packet_append_tlv(packet
, FLOW_DIVERT_TLV_TRAFFIC_CLASS
, sizeof(traffic_class
), &traffic_class
);
1646 FDLOG(LOG_ERR
, fd_cb
, "failed to add the traffic class: %d", error
);
1650 error
= flow_divert_send_packet(fd_cb
, packet
, TRUE
);
1656 if (error
&& packet
!= NULL
) {
1664 flow_divert_handle_connect_result(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, int offset
)
1666 uint32_t connect_error
;
1667 uint32_t ctl_unit
= 0;
1669 struct flow_divert_group
*grp
= NULL
;
1670 struct sockaddr_storage local_address
;
1671 int out_if_index
= 0;
1672 struct sockaddr_storage remote_address
;
1673 uint32_t send_window
;
1674 uint32_t app_data_length
= 0;
1676 memset(&local_address
, 0, sizeof(local_address
));
1677 memset(&remote_address
, 0, sizeof(remote_address
));
1679 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_ERROR_CODE
, sizeof(connect_error
), &connect_error
, NULL
);
1681 FDLOG(LOG_ERR
, fd_cb
, "failed to get the connect result: %d", error
);
1685 FDLOG(LOG_INFO
, fd_cb
, "received connect result %u", connect_error
);
1687 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_SPACE_AVAILABLE
, sizeof(send_window
), &send_window
, NULL
);
1689 FDLOG(LOG_ERR
, fd_cb
, "failed to get the send window: %d", error
);
1693 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_CTL_UNIT
, sizeof(ctl_unit
), &ctl_unit
, NULL
);
1695 FDLOG0(LOG_INFO
, fd_cb
, "No control unit provided in the connect result");
1698 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_LOCAL_ADDR
, sizeof(local_address
), &local_address
, NULL
);
1700 FDLOG0(LOG_INFO
, fd_cb
, "No local address provided");
1703 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_REMOTE_ADDR
, sizeof(remote_address
), &remote_address
, NULL
);
1705 FDLOG0(LOG_INFO
, fd_cb
, "No remote address provided");
1708 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_OUT_IF_INDEX
, sizeof(out_if_index
), &out_if_index
, NULL
);
1710 FDLOG0(LOG_INFO
, fd_cb
, "No output if index provided");
1713 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_APP_DATA
, 0, NULL
, &app_data_length
);
1715 FDLOG0(LOG_INFO
, fd_cb
, "No application data provided in connect result");
1719 connect_error
= ntohl(connect_error
);
1720 ctl_unit
= ntohl(ctl_unit
);
1722 lck_rw_lock_shared(&g_flow_divert_group_lck
);
1724 if (connect_error
== 0 && ctl_unit
> 0) {
1725 if (ctl_unit
>= GROUP_COUNT_MAX
) {
1726 FDLOG(LOG_ERR
, fd_cb
, "Connect result contains an invalid control unit: %u", ctl_unit
);
1728 } else if (g_flow_divert_groups
== NULL
|| g_active_group_count
== 0) {
1729 FDLOG0(LOG_ERR
, fd_cb
, "No active groups, dropping connection");
1732 grp
= g_flow_divert_groups
[ctl_unit
];
1740 if (fd_cb
->so
!= NULL
) {
1741 struct inpcb
*inp
= NULL
;
1742 struct ifnet
*ifp
= NULL
;
1743 struct flow_divert_group
*old_group
;
1745 socket_lock(fd_cb
->so
, 0);
1747 if (!(fd_cb
->so
->so_state
& SS_ISCONNECTING
)) {
1751 inp
= sotoinpcb(fd_cb
->so
);
1753 if (connect_error
|| error
) {
1754 goto set_socket_state
;
1757 if (local_address
.ss_family
== 0 && fd_cb
->local_address
== NULL
) {
1759 goto set_socket_state
;
1761 if (local_address
.ss_family
!= 0 && fd_cb
->local_address
== NULL
) {
1762 if (local_address
.ss_len
> sizeof(local_address
)) {
1763 local_address
.ss_len
= sizeof(local_address
);
1765 fd_cb
->local_address
= dup_sockaddr((struct sockaddr
*)&local_address
, 1);
1768 if (remote_address
.ss_family
!= 0) {
1769 if (remote_address
.ss_len
> sizeof(remote_address
)) {
1770 remote_address
.ss_len
= sizeof(remote_address
);
1772 fd_cb
->remote_address
= dup_sockaddr((struct sockaddr
*)&remote_address
, 1);
1775 goto set_socket_state
;
1778 if (app_data_length
> 0) {
1779 uint8_t *app_data
= NULL
;
1780 MALLOC(app_data
, uint8_t *, app_data_length
, M_TEMP
, M_WAITOK
);
1781 if (app_data
!= NULL
) {
1782 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_APP_DATA
, app_data_length
, app_data
, NULL
);
1784 FDLOG(LOG_INFO
, fd_cb
, "Got %u bytes of app data from the connect result", app_data_length
);
1785 if (fd_cb
->app_data
!= NULL
) {
1786 FREE(fd_cb
->app_data
, M_TEMP
);
1788 fd_cb
->app_data
= app_data
;
1789 fd_cb
->app_data_length
= app_data_length
;
1791 FDLOG(LOG_ERR
, fd_cb
, "Failed to copy %u bytes of application data from the connect result packet", app_data_length
);
1792 FREE(app_data
, M_TEMP
);
1795 FDLOG(LOG_ERR
, fd_cb
, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length
);
1799 ifnet_head_lock_shared();
1800 if (out_if_index
> 0 && out_if_index
<= if_index
) {
1801 ifp
= ifindex2ifnet
[out_if_index
];
1805 inp
->inp_last_outifp
= ifp
;
1812 goto set_socket_state
;
1815 if (fd_cb
->group
== NULL
) {
1817 goto set_socket_state
;
1821 old_group
= fd_cb
->group
;
1823 lck_rw_lock_exclusive(&old_group
->lck
);
1824 lck_rw_lock_exclusive(&grp
->lck
);
1826 RB_REMOVE(fd_pcb_tree
, &old_group
->pcb_tree
, fd_cb
);
1827 if (RB_INSERT(fd_pcb_tree
, &grp
->pcb_tree
, fd_cb
) != NULL
) {
1828 panic("group with unit %u already contains a connection with hash %u", grp
->ctl_unit
, fd_cb
->hash
);
1833 lck_rw_done(&grp
->lck
);
1834 lck_rw_done(&old_group
->lck
);
1837 fd_cb
->send_window
= ntohl(send_window
);
1840 if (!connect_error
&& !error
) {
1841 FDLOG0(LOG_INFO
, fd_cb
, "sending connect result");
1842 error
= flow_divert_send_connect_result(fd_cb
);
1845 if (connect_error
|| error
) {
1846 if (!connect_error
) {
1847 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, FALSE
);
1848 fd_cb
->so
->so_error
= error
;
1849 flow_divert_send_close_if_needed(fd_cb
);
1851 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, TRUE
);
1852 fd_cb
->so
->so_error
= connect_error
;
1854 flow_divert_disconnect_socket(fd_cb
->so
);
1856 flow_divert_send_buffered_data(fd_cb
, FALSE
);
1857 soisconnected(fd_cb
->so
);
1861 socket_unlock(fd_cb
->so
, 0);
1865 lck_rw_done(&g_flow_divert_group_lck
);
1869 flow_divert_handle_close(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, int offset
)
1871 uint32_t close_error
;
1875 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_ERROR_CODE
, sizeof(close_error
), &close_error
, NULL
);
1877 FDLOG(LOG_ERR
, fd_cb
, "failed to get the close error: %d", error
);
1881 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_HOW
, sizeof(how
), &how
, NULL
);
1883 FDLOG(LOG_ERR
, fd_cb
, "failed to get the close how flag: %d", error
);
1889 FDLOG(LOG_INFO
, fd_cb
, "close received, how = %d", how
);
1892 if (fd_cb
->so
!= NULL
) {
1893 socket_lock(fd_cb
->so
, 0);
1895 fd_cb
->so
->so_error
= ntohl(close_error
);
1897 flow_divert_update_closed_state(fd_cb
, how
, TRUE
);
1899 how
= flow_divert_tunnel_how_closed(fd_cb
);
1900 if (how
== SHUT_RDWR
) {
1901 flow_divert_disconnect_socket(fd_cb
->so
);
1902 } else if (how
== SHUT_RD
) {
1903 socantrcvmore(fd_cb
->so
);
1904 } else if (how
== SHUT_WR
) {
1905 socantsendmore(fd_cb
->so
);
1908 socket_unlock(fd_cb
->so
, 0);
1914 flow_divert_get_control_mbuf(struct flow_divert_pcb
*fd_cb
)
1916 struct inpcb
*inp
= sotoinpcb(fd_cb
->so
);
1917 if (inp
->inp_vflag
& INP_IPV4
&& inp
->inp_flags
& INP_RECVDSTADDR
) {
1918 struct sockaddr_in
*sin
= (struct sockaddr_in
*)(void *)fd_cb
->local_address
;
1920 return sbcreatecontrol((caddr_t
) &sin
->sin_addr
, sizeof(struct in_addr
), IP_RECVDSTADDR
, IPPROTO_IP
);
1921 } else if (inp
->inp_vflag
& INP_IPV6
&& (inp
->inp_flags
& IN6P_PKTINFO
) != 0) {
1922 struct sockaddr_in6
*sin6
= (struct sockaddr_in6
*)(void *)fd_cb
->local_address
;
1923 struct in6_pktinfo pi6
;
1925 bcopy(&sin6
->sin6_addr
, &pi6
.ipi6_addr
, sizeof (struct in6_addr
));
1926 pi6
.ipi6_ifindex
= 0;
1927 return sbcreatecontrol((caddr_t
)&pi6
, sizeof (struct in6_pktinfo
), IPV6_PKTINFO
, IPPROTO_IPV6
);
1933 flow_divert_handle_data(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, size_t offset
)
1936 if (fd_cb
->so
!= NULL
) {
1940 struct sockaddr_storage remote_address
;
1941 boolean_t got_remote_sa
= FALSE
;
1943 socket_lock(fd_cb
->so
, 0);
1945 if (SOCK_TYPE(fd_cb
->so
) == SOCK_DGRAM
) {
1946 uint32_t val_size
= 0;
1948 /* check if we got remote address with data */
1949 memset(&remote_address
, 0, sizeof(remote_address
));
1950 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_REMOTE_ADDR
, sizeof(remote_address
), &remote_address
, &val_size
);
1951 if (error
|| val_size
> sizeof(remote_address
)) {
1952 FDLOG0(LOG_INFO
, fd_cb
, "No remote address provided");
1955 /* validate the address */
1956 if (flow_divert_is_sockaddr_valid((struct sockaddr
*)&remote_address
)) {
1957 got_remote_sa
= TRUE
;
1959 offset
+= (sizeof(uint8_t) + sizeof(uint32_t) + val_size
);
1963 data_size
= (mbuf_pkthdr_len(packet
) - offset
);
1965 FDLOG(LOG_DEBUG
, fd_cb
, "received %lu bytes of data", data_size
);
1967 error
= mbuf_split(packet
, offset
, MBUF_DONTWAIT
, &data
);
1968 if (error
|| data
== NULL
) {
1969 FDLOG(LOG_ERR
, fd_cb
, "mbuf_split failed: %d", error
);
1971 if (flow_divert_check_no_cellular(fd_cb
) ||
1972 flow_divert_check_no_expensive(fd_cb
))
1974 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, TRUE
);
1975 flow_divert_send_close(fd_cb
, SHUT_RDWR
);
1976 flow_divert_disconnect_socket(fd_cb
->so
);
1977 } else if (!(fd_cb
->so
->so_state
& SS_CANTRCVMORE
)) {
1978 if (SOCK_TYPE(fd_cb
->so
) == SOCK_STREAM
) {
1979 if (sbappendstream(&fd_cb
->so
->so_rcv
, data
)) {
1980 fd_cb
->bytes_received
+= data_size
;
1981 flow_divert_add_data_statistics(fd_cb
, data_size
, FALSE
);
1982 fd_cb
->sb_size
= fd_cb
->so
->so_rcv
.sb_cc
;
1983 sorwakeup(fd_cb
->so
);
1986 FDLOG0(LOG_ERR
, fd_cb
, "received data, but appendstream failed");
1988 } else if (SOCK_TYPE(fd_cb
->so
) == SOCK_DGRAM
) {
1989 struct sockaddr
*append_sa
;
1992 if (got_remote_sa
== TRUE
) {
1993 error
= flow_divert_dup_addr(fd_cb
->so
->so_proto
->pr_domain
->dom_family
,
1994 (struct sockaddr
*)&remote_address
, &append_sa
);
1996 error
= flow_divert_dup_addr(fd_cb
->so
->so_proto
->pr_domain
->dom_family
,
1997 fd_cb
->remote_address
, &append_sa
);
2000 FDLOG0(LOG_ERR
, fd_cb
, "failed to dup the socket address.");
2003 mctl
= flow_divert_get_control_mbuf(fd_cb
);
2004 if (sbappendaddr(&fd_cb
->so
->so_rcv
, append_sa
, data
, mctl
, NULL
)) {
2005 fd_cb
->bytes_received
+= data_size
;
2006 flow_divert_add_data_statistics(fd_cb
, data_size
, FALSE
);
2007 fd_cb
->sb_size
= fd_cb
->so
->so_rcv
.sb_cc
;
2008 sorwakeup(fd_cb
->so
);
2011 FDLOG0(LOG_ERR
, fd_cb
, "received data, but sbappendaddr failed");
2014 FREE(append_sa
, M_TEMP
);
2019 socket_unlock(fd_cb
->so
, 0);
2029 flow_divert_handle_read_notification(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, int offset
)
2031 uint32_t read_count
;
2034 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_READ_COUNT
, sizeof(read_count
), &read_count
, NULL
);
2036 FDLOG(LOG_ERR
, fd_cb
, "failed to get the read count: %d", error
);
2040 FDLOG(LOG_DEBUG
, fd_cb
, "received a read notification for %u bytes", ntohl(read_count
));
2043 if (fd_cb
->so
!= NULL
) {
2044 socket_lock(fd_cb
->so
, 0);
2045 fd_cb
->send_window
+= ntohl(read_count
);
2046 flow_divert_send_buffered_data(fd_cb
, FALSE
);
2047 socket_unlock(fd_cb
->so
, 0);
2053 flow_divert_handle_group_init(struct flow_divert_group
*group
, mbuf_t packet
, int offset
)
2056 uint32_t key_size
= 0;
2060 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_TOKEN_KEY
, 0, NULL
, &key_size
);
2062 FDLOG(LOG_ERR
, &nil_pcb
, "failed to get the key size: %d", error
);
2066 if (key_size
== 0 || key_size
> FLOW_DIVERT_MAX_KEY_SIZE
) {
2067 FDLOG(LOG_ERR
, &nil_pcb
, "Invalid key size: %u", key_size
);
2071 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_LOG_LEVEL
, sizeof(log_level
), &log_level
, NULL
);
2073 nil_pcb
.log_level
= log_level
;
2076 lck_rw_lock_exclusive(&group
->lck
);
2078 MALLOC(group
->token_key
, uint8_t *, key_size
, M_TEMP
, M_WAITOK
);
2079 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_TOKEN_KEY
, key_size
, group
->token_key
, NULL
);
2081 FDLOG(LOG_ERR
, &nil_pcb
, "failed to get the token key: %d", error
);
2082 FREE(group
->token_key
, M_TEMP
);
2083 group
->token_key
= NULL
;
2084 lck_rw_done(&group
->lck
);
2088 group
->token_key_size
= key_size
;
2090 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_FLAGS
, sizeof(flags
), &flags
, NULL
);
2092 group
->flags
= flags
;
2095 lck_rw_done(&group
->lck
);
2099 flow_divert_handle_properties_update(struct flow_divert_pcb
*fd_cb
, mbuf_t packet
, int offset
)
2102 struct sockaddr_storage local_address
;
2103 int out_if_index
= 0;
2104 struct sockaddr_storage remote_address
;
2105 uint32_t app_data_length
= 0;
2107 FDLOG0(LOG_INFO
, fd_cb
, "received a properties update");
2109 memset(&local_address
, 0, sizeof(local_address
));
2110 memset(&remote_address
, 0, sizeof(remote_address
));
2112 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_LOCAL_ADDR
, sizeof(local_address
), &local_address
, NULL
);
2114 FDLOG0(LOG_INFO
, fd_cb
, "No local address provided in properties update");
2117 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_REMOTE_ADDR
, sizeof(remote_address
), &remote_address
, NULL
);
2119 FDLOG0(LOG_INFO
, fd_cb
, "No remote address provided in properties update");
2122 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_OUT_IF_INDEX
, sizeof(out_if_index
), &out_if_index
, NULL
);
2124 FDLOG0(LOG_INFO
, fd_cb
, "No output if index provided in properties update");
2127 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_APP_DATA
, 0, NULL
, &app_data_length
);
2129 FDLOG0(LOG_INFO
, fd_cb
, "No application data provided in properties update");
2133 if (fd_cb
->so
!= NULL
) {
2134 socket_lock(fd_cb
->so
, 0);
2136 if (local_address
.ss_family
!= 0) {
2137 if (local_address
.ss_len
> sizeof(local_address
)) {
2138 local_address
.ss_len
= sizeof(local_address
);
2140 if (fd_cb
->local_address
!= NULL
) {
2141 FREE(fd_cb
->local_address
, M_SONAME
);
2142 fd_cb
->local_address
= NULL
;
2144 fd_cb
->local_address
= dup_sockaddr((struct sockaddr
*)&local_address
, 1);
2147 if (remote_address
.ss_family
!= 0) {
2148 if (remote_address
.ss_len
> sizeof(remote_address
)) {
2149 remote_address
.ss_len
= sizeof(remote_address
);
2151 if (fd_cb
->remote_address
!= NULL
) {
2152 FREE(fd_cb
->remote_address
, M_SONAME
);
2153 fd_cb
->remote_address
= NULL
;
2155 fd_cb
->remote_address
= dup_sockaddr((struct sockaddr
*)&remote_address
, 1);
2158 if (out_if_index
> 0) {
2159 struct inpcb
*inp
= NULL
;
2160 struct ifnet
*ifp
= NULL
;
2162 inp
= sotoinpcb(fd_cb
->so
);
2164 ifnet_head_lock_shared();
2165 if (out_if_index
<= if_index
) {
2166 ifp
= ifindex2ifnet
[out_if_index
];
2170 inp
->inp_last_outifp
= ifp
;
2175 if (app_data_length
> 0) {
2176 uint8_t *app_data
= NULL
;
2177 MALLOC(app_data
, uint8_t *, app_data_length
, M_TEMP
, M_WAITOK
);
2178 if (app_data
!= NULL
) {
2179 error
= flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_APP_DATA
, app_data_length
, app_data
, NULL
);
2181 if (fd_cb
->app_data
!= NULL
) {
2182 FREE(fd_cb
->app_data
, M_TEMP
);
2184 fd_cb
->app_data
= app_data
;
2185 fd_cb
->app_data_length
= app_data_length
;
2187 FDLOG(LOG_ERR
, fd_cb
, "Failed to copy %u bytes of application data from the properties update packet", app_data_length
);
2188 FREE(app_data
, M_TEMP
);
2191 FDLOG(LOG_ERR
, fd_cb
, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length
);
2195 socket_unlock(fd_cb
->so
, 0);
2201 flow_divert_handle_app_map_create(struct flow_divert_group
*group
, mbuf_t packet
, int offset
)
2203 size_t bytes_mem_size
;
2204 size_t child_maps_mem_size
;
2207 struct flow_divert_trie new_trie
;
2208 int insert_error
= 0;
2209 size_t nodes_mem_size
;
2210 int prefix_count
= 0;
2211 int signing_id_count
= 0;
2212 size_t trie_memory_size
= 0;
2214 lck_rw_lock_exclusive(&group
->lck
);
2216 /* Re-set the current trie */
2217 if (group
->signing_id_trie
.memory
!= NULL
) {
2218 FREE(group
->signing_id_trie
.memory
, M_TEMP
);
2220 memset(&group
->signing_id_trie
, 0, sizeof(group
->signing_id_trie
));
2221 group
->signing_id_trie
.root
= NULL_TRIE_IDX
;
2223 memset(&new_trie
, 0, sizeof(new_trie
));
2225 /* Get the number of shared prefixes in the new set of signing ID strings */
2226 flow_divert_packet_get_tlv(packet
, offset
, FLOW_DIVERT_TLV_PREFIX_COUNT
, sizeof(prefix_count
), &prefix_count
, NULL
);
2228 if (prefix_count
< 0) {
2229 lck_rw_done(&group
->lck
);
2233 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2234 for (cursor
= flow_divert_packet_find_tlv(packet
, offset
, FLOW_DIVERT_TLV_SIGNING_ID
, &error
, 0);
2236 cursor
= flow_divert_packet_find_tlv(packet
, cursor
, FLOW_DIVERT_TLV_SIGNING_ID
, &error
, 1))
2238 uint32_t sid_size
= 0;
2239 flow_divert_packet_get_tlv(packet
, cursor
, FLOW_DIVERT_TLV_SIGNING_ID
, 0, NULL
, &sid_size
);
2240 new_trie
.bytes_count
+= sid_size
;
2244 if (signing_id_count
== 0) {
2245 lck_rw_done(&group
->lck
);
2249 new_trie
.nodes_count
= (prefix_count
+ signing_id_count
+ 1); /* + 1 for the root node */
2250 new_trie
.child_maps_count
= (prefix_count
+ 1); /* + 1 for the root node */
2252 FDLOG(LOG_INFO
, &nil_pcb
, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2253 new_trie
.nodes_count
, new_trie
.child_maps_count
, new_trie
.bytes_count
);
2255 nodes_mem_size
= (sizeof(*new_trie
.nodes
) * new_trie
.nodes_count
);
2256 child_maps_mem_size
= (sizeof(*new_trie
.child_maps
) * CHILD_MAP_SIZE
* new_trie
.child_maps_count
);
2257 bytes_mem_size
= (sizeof(*new_trie
.bytes
) * new_trie
.bytes_count
);
2259 trie_memory_size
= nodes_mem_size
+ child_maps_mem_size
+ bytes_mem_size
;
2260 if (trie_memory_size
> FLOW_DIVERT_MAX_TRIE_MEMORY
) {
2261 FDLOG(LOG_ERR
, &nil_pcb
, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size
, FLOW_DIVERT_MAX_TRIE_MEMORY
);
2262 lck_rw_done(&group
->lck
);
2266 MALLOC(new_trie
.memory
, void *, trie_memory_size
, M_TEMP
, M_WAITOK
);
2267 if (new_trie
.memory
== NULL
) {
2268 FDLOG(LOG_ERR
, &nil_pcb
, "Failed to allocate %lu bytes of memory for the signing ID trie",
2269 nodes_mem_size
+ child_maps_mem_size
+ bytes_mem_size
);
2270 lck_rw_done(&group
->lck
);
2274 /* Initialize the free lists */
2275 new_trie
.nodes
= (struct flow_divert_trie_node
*)new_trie
.memory
;
2276 new_trie
.nodes_free_next
= 0;
2277 memset(new_trie
.nodes
, 0, nodes_mem_size
);
2279 new_trie
.child_maps
= (uint16_t *)(void *)((uint8_t *)new_trie
.memory
+ nodes_mem_size
);
2280 new_trie
.child_maps_free_next
= 0;
2281 memset(new_trie
.child_maps
, 0xff, child_maps_mem_size
);
2283 new_trie
.bytes
= (uint8_t *)(void *)((uint8_t *)new_trie
.memory
+ nodes_mem_size
+ child_maps_mem_size
);
2284 new_trie
.bytes_free_next
= 0;
2286 /* The root is an empty node */
2287 new_trie
.root
= trie_node_alloc(&new_trie
);
2289 /* Add each signing ID to the trie */
2290 for (cursor
= flow_divert_packet_find_tlv(packet
, offset
, FLOW_DIVERT_TLV_SIGNING_ID
, &error
, 0);
2292 cursor
= flow_divert_packet_find_tlv(packet
, cursor
, FLOW_DIVERT_TLV_SIGNING_ID
, &error
, 1))
2294 uint32_t sid_size
= 0;
2295 flow_divert_packet_get_tlv(packet
, cursor
, FLOW_DIVERT_TLV_SIGNING_ID
, 0, NULL
, &sid_size
);
2296 if (new_trie
.bytes_free_next
+ sid_size
<= new_trie
.bytes_count
) {
2297 uint16_t new_node_idx
;
2298 flow_divert_packet_get_tlv(packet
, cursor
, FLOW_DIVERT_TLV_SIGNING_ID
, sid_size
, &TRIE_BYTE(&new_trie
, new_trie
.bytes_free_next
), NULL
);
2299 new_node_idx
= flow_divert_trie_insert(&new_trie
, new_trie
.bytes_free_next
, sid_size
);
2300 if (new_node_idx
== NULL_TRIE_IDX
) {
2301 insert_error
= EINVAL
;
2305 FDLOG0(LOG_ERR
, &nil_pcb
, "No place to put signing ID for insertion");
2306 insert_error
= ENOBUFS
;
2311 if (!insert_error
) {
2312 group
->signing_id_trie
= new_trie
;
2314 FREE(new_trie
.memory
, M_TEMP
);
2317 lck_rw_done(&group
->lck
);
2321 flow_divert_input(mbuf_t packet
, struct flow_divert_group
*group
)
2323 struct flow_divert_packet_header hdr
;
2325 struct flow_divert_pcb
*fd_cb
;
2327 if (mbuf_pkthdr_len(packet
) < sizeof(hdr
)) {
2328 FDLOG(LOG_ERR
, &nil_pcb
, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet
), sizeof(hdr
));
2333 if (mbuf_pkthdr_len(packet
) > FD_CTL_RCVBUFF_SIZE
) {
2334 FDLOG(LOG_ERR
, &nil_pcb
, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet
), FD_CTL_RCVBUFF_SIZE
);
2339 error
= mbuf_copydata(packet
, 0, sizeof(hdr
), &hdr
);
2341 FDLOG(LOG_ERR
, &nil_pcb
, "mbuf_copydata failed for the header: %d", error
);
2346 hdr
.conn_id
= ntohl(hdr
.conn_id
);
2348 if (hdr
.conn_id
== 0) {
2349 switch (hdr
.packet_type
) {
2350 case FLOW_DIVERT_PKT_GROUP_INIT
:
2351 flow_divert_handle_group_init(group
, packet
, sizeof(hdr
));
2353 case FLOW_DIVERT_PKT_APP_MAP_CREATE
:
2354 flow_divert_handle_app_map_create(group
, packet
, sizeof(hdr
));
2357 FDLOG(LOG_WARNING
, &nil_pcb
, "got an unknown message type: %d", hdr
.packet_type
);
2363 fd_cb
= flow_divert_pcb_lookup(hdr
.conn_id
, group
); /* This retains the PCB */
2364 if (fd_cb
== NULL
) {
2365 if (hdr
.packet_type
!= FLOW_DIVERT_PKT_CLOSE
&& hdr
.packet_type
!= FLOW_DIVERT_PKT_READ_NOTIFY
) {
2366 FDLOG(LOG_NOTICE
, &nil_pcb
, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr
.packet_type
), group
->ctl_unit
, hdr
.conn_id
);
2371 switch (hdr
.packet_type
) {
2372 case FLOW_DIVERT_PKT_CONNECT_RESULT
:
2373 flow_divert_handle_connect_result(fd_cb
, packet
, sizeof(hdr
));
2375 case FLOW_DIVERT_PKT_CLOSE
:
2376 flow_divert_handle_close(fd_cb
, packet
, sizeof(hdr
));
2378 case FLOW_DIVERT_PKT_DATA
:
2379 flow_divert_handle_data(fd_cb
, packet
, sizeof(hdr
));
2381 case FLOW_DIVERT_PKT_READ_NOTIFY
:
2382 flow_divert_handle_read_notification(fd_cb
, packet
, sizeof(hdr
));
2384 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE
:
2385 flow_divert_handle_properties_update(fd_cb
, packet
, sizeof(hdr
));
2388 FDLOG(LOG_WARNING
, fd_cb
, "got an unknown message type: %d", hdr
.packet_type
);
2400 flow_divert_close_all(struct flow_divert_group
*group
)
2402 struct flow_divert_pcb
*fd_cb
;
2403 SLIST_HEAD(, flow_divert_pcb
) tmp_list
;
2405 SLIST_INIT(&tmp_list
);
2407 lck_rw_lock_exclusive(&group
->lck
);
2409 MBUFQ_DRAIN(&group
->send_queue
);
2411 RB_FOREACH(fd_cb
, fd_pcb_tree
, &group
->pcb_tree
) {
2413 SLIST_INSERT_HEAD(&tmp_list
, fd_cb
, tmp_list_entry
);
2416 lck_rw_done(&group
->lck
);
2418 while (!SLIST_EMPTY(&tmp_list
)) {
2419 fd_cb
= SLIST_FIRST(&tmp_list
);
2421 SLIST_REMOVE_HEAD(&tmp_list
, tmp_list_entry
);
2422 if (fd_cb
->so
!= NULL
) {
2423 socket_lock(fd_cb
->so
, 0);
2424 flow_divert_pcb_remove(fd_cb
);
2425 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, TRUE
);
2426 fd_cb
->so
->so_error
= ECONNABORTED
;
2427 flow_divert_disconnect_socket(fd_cb
->so
);
2428 socket_unlock(fd_cb
->so
, 0);
2436 flow_divert_detach(struct socket
*so
)
2438 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2440 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2442 so
->so_flags
&= ~SOF_FLOW_DIVERT
;
2443 so
->so_fd_pcb
= NULL
;
2445 FDLOG(LOG_INFO
, fd_cb
, "Detaching, ref count = %d", fd_cb
->ref_count
);
2447 if (fd_cb
->group
!= NULL
) {
2448 /* Last-ditch effort to send any buffered data */
2449 flow_divert_send_buffered_data(fd_cb
, TRUE
);
2451 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, FALSE
);
2452 flow_divert_send_close_if_needed(fd_cb
);
2453 /* Remove from the group */
2454 flow_divert_pcb_remove(fd_cb
);
2457 socket_unlock(so
, 0);
2463 FDRELEASE(fd_cb
); /* Release the socket's reference */
2467 flow_divert_close(struct socket
*so
)
2469 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2471 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2473 FDLOG0(LOG_INFO
, fd_cb
, "Closing");
2475 if (SOCK_TYPE(so
) == SOCK_STREAM
) {
2476 soisdisconnecting(so
);
2477 sbflush(&so
->so_rcv
);
2480 flow_divert_send_buffered_data(fd_cb
, TRUE
);
2481 flow_divert_update_closed_state(fd_cb
, SHUT_RDWR
, FALSE
);
2482 flow_divert_send_close_if_needed(fd_cb
);
2484 /* Remove from the group */
2485 flow_divert_pcb_remove(fd_cb
);
2491 flow_divert_disconnectx(struct socket
*so
, sae_associd_t aid
,
2492 sae_connid_t cid __unused
)
2494 if (aid
!= SAE_ASSOCID_ANY
&& aid
!= SAE_ASSOCID_ALL
) {
2498 return (flow_divert_close(so
));
2502 flow_divert_shutdown(struct socket
*so
)
2504 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2506 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2508 FDLOG0(LOG_INFO
, fd_cb
, "Can't send more");
2512 flow_divert_update_closed_state(fd_cb
, SHUT_WR
, FALSE
);
2513 flow_divert_send_close_if_needed(fd_cb
);
2519 flow_divert_rcvd(struct socket
*so
, int flags __unused
)
2521 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2522 uint32_t latest_sb_size
;
2523 uint32_t read_count
;
2525 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2527 latest_sb_size
= fd_cb
->so
->so_rcv
.sb_cc
;
2529 if (fd_cb
->sb_size
< latest_sb_size
) {
2530 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2531 fd_cb
->hash
, fd_cb
->sb_size
, latest_sb_size
);
2534 read_count
= fd_cb
->sb_size
- latest_sb_size
;
2536 FDLOG(LOG_DEBUG
, fd_cb
, "app read %u bytes", read_count
);
2538 if (read_count
> 0 && flow_divert_send_read_notification(fd_cb
, read_count
) == 0) {
2539 fd_cb
->bytes_read_by_app
+= read_count
;
2540 fd_cb
->sb_size
= latest_sb_size
;
2547 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet
, struct sockaddr
*toaddr
)
2552 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_TARGET_ADDRESS
, toaddr
->sa_len
, toaddr
);
2557 if (toaddr
->sa_family
== AF_INET
) {
2558 port
= ntohs((satosin(toaddr
))->sin_port
);
2562 port
= ntohs((satosin6(toaddr
))->sin6_port
);
2566 error
= flow_divert_packet_append_tlv(connect_packet
, FLOW_DIVERT_TLV_TARGET_PORT
, sizeof(port
), &port
);
2576 flow_divert_get_buffered_target_address(mbuf_t buffer
)
2578 if (buffer
!= NULL
&& buffer
->m_type
== MT_SONAME
) {
2579 struct sockaddr
*toaddr
= mtod(buffer
, struct sockaddr
*);
2580 if (toaddr
!= NULL
&& flow_divert_is_sockaddr_valid(toaddr
)) {
2588 flow_divert_is_sockaddr_valid(struct sockaddr
*addr
)
2590 switch(addr
->sa_family
)
2593 if (addr
->sa_len
!= sizeof(struct sockaddr_in
)) {
2599 if (addr
->sa_len
!= sizeof(struct sockaddr_in6
)) {
2611 flow_divert_inp_to_sockaddr(const struct inpcb
*inp
, struct sockaddr
**local_socket
)
2614 union sockaddr_in_4_6 sin46
;
2616 bzero(&sin46
, sizeof(sin46
));
2617 if (inp
->inp_vflag
& INP_IPV4
) {
2618 struct sockaddr_in
*sin
= &sin46
.sin
;
2620 sin
->sin_family
= AF_INET
;
2621 sin
->sin_len
= sizeof(*sin
);
2622 sin
->sin_port
= inp
->inp_lport
;
2623 sin
->sin_addr
= inp
->inp_laddr
;
2624 } else if (inp
->inp_vflag
& INP_IPV6
) {
2625 struct sockaddr_in6
*sin6
= &sin46
.sin6
;
2627 sin6
->sin6_len
= sizeof(*sin6
);
2628 sin6
->sin6_family
= AF_INET6
;
2629 sin6
->sin6_port
= inp
->inp_lport
;
2630 sin6
->sin6_addr
= inp
->in6p_laddr
;
2632 *local_socket
= dup_sockaddr((struct sockaddr
*)&sin46
, 1);
2633 if (*local_socket
== NULL
) {
2640 flow_divert_has_pcb_local_address(const struct inpcb
*inp
)
2642 return (inp
->inp_lport
!= 0
2643 && (inp
->inp_laddr
.s_addr
!= INADDR_ANY
|| !IN6_IS_ADDR_UNSPECIFIED(&inp
->in6p_laddr
)));
2647 flow_divert_dup_addr(sa_family_t family
, struct sockaddr
*addr
,
2648 struct sockaddr
**dup
)
2651 struct sockaddr
*result
;
2652 struct sockaddr_storage ss
;
2657 memset(&ss
, 0, sizeof(ss
));
2658 ss
.ss_family
= family
;
2659 if (ss
.ss_family
== AF_INET
) {
2660 ss
.ss_len
= sizeof(struct sockaddr_in
);
2663 else if (ss
.ss_family
== AF_INET6
) {
2664 ss
.ss_len
= sizeof(struct sockaddr_in6
);
2670 result
= (struct sockaddr
*)&ss
;
2674 *dup
= dup_sockaddr(result
, 1);
2684 flow_divert_disconnect_socket(struct socket
*so
)
2686 soisdisconnected(so
);
2687 if (SOCK_TYPE(so
) == SOCK_DGRAM
) {
2688 struct inpcb
*inp
= NULL
;
2690 inp
= sotoinpcb(so
);
2693 if (SOCK_CHECK_DOM(so
, PF_INET6
))
2703 flow_divert_getpeername(struct socket
*so
, struct sockaddr
**sa
)
2705 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2707 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2709 return flow_divert_dup_addr(so
->so_proto
->pr_domain
->dom_family
,
2710 fd_cb
->remote_address
,
2715 flow_divert_getsockaddr(struct socket
*so
, struct sockaddr
**sa
)
2717 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2719 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2721 return flow_divert_dup_addr(so
->so_proto
->pr_domain
->dom_family
,
2722 fd_cb
->local_address
,
2727 flow_divert_ctloutput(struct socket
*so
, struct sockopt
*sopt
)
2729 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2731 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2733 if (sopt
->sopt_name
== SO_TRAFFIC_CLASS
) {
2734 if (sopt
->sopt_dir
== SOPT_SET
&& fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
) {
2735 flow_divert_send_traffic_class_update(fd_cb
, so
->so_traffic_class
);
2739 if (SOCK_DOM(so
) == PF_INET
) {
2740 return g_tcp_protosw
->pr_ctloutput(so
, sopt
);
2743 else if (SOCK_DOM(so
) == PF_INET6
) {
2744 return g_tcp6_protosw
->pr_ctloutput(so
, sopt
);
2751 flow_divert_connect_out(struct socket
*so
, struct sockaddr
*to
, proc_t p
)
2753 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2755 struct inpcb
*inp
= sotoinpcb(so
);
2756 struct sockaddr_in
*sinp
;
2757 mbuf_t connect_packet
= NULL
;
2760 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
2762 if (fd_cb
->group
== NULL
) {
2763 error
= ENETUNREACH
;
2770 } else if (inp
->inp_state
== INPCB_STATE_DEAD
) {
2772 error
= so
->so_error
;
2780 if ((fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
) && !(fd_cb
->flags
& FLOW_DIVERT_TRANSFERRED
)) {
2785 if (fd_cb
->flags
& FLOW_DIVERT_TRANSFERRED
) {
2786 FDLOG0(LOG_INFO
, fd_cb
, "fully transferred");
2787 fd_cb
->flags
&= ~FLOW_DIVERT_TRANSFERRED
;
2788 if (fd_cb
->remote_address
!= NULL
) {
2789 soisconnected(fd_cb
->so
);
2794 FDLOG0(LOG_INFO
, fd_cb
, "Connecting");
2796 if (fd_cb
->connect_packet
== NULL
) {
2798 FDLOG0(LOG_ERR
, fd_cb
, "No destination address available when creating connect packet");
2803 sinp
= (struct sockaddr_in
*)(void *)to
;
2804 if (sinp
->sin_family
== AF_INET
&& IN_MULTICAST(ntohl(sinp
->sin_addr
.s_addr
))) {
2805 error
= EAFNOSUPPORT
;
2809 error
= flow_divert_create_connect_packet(fd_cb
, to
, so
, p
, &connect_packet
);
2814 if (so
->so_flags1
& SOF1_PRECONNECT_DATA
) {
2815 FDLOG0(LOG_INFO
, fd_cb
, "Delaying sending the connect packet until send or receive");
2819 FDLOG0(LOG_INFO
, fd_cb
, "Sending saved connect packet");
2820 connect_packet
= fd_cb
->connect_packet
;
2821 fd_cb
->connect_packet
= NULL
;
2825 error
= flow_divert_send_packet(fd_cb
, connect_packet
, TRUE
);
2830 fd_cb
->flags
|= FLOW_DIVERT_CONNECT_STARTED
;
2832 fd_cb
->connect_packet
= connect_packet
;
2833 connect_packet
= NULL
;
2839 if (error
&& connect_packet
!= NULL
) {
2840 mbuf_freem(connect_packet
);
2846 flow_divert_connectx_out_common(struct socket
*so
, struct sockaddr
*dst
,
2847 struct proc
*p
, sae_connid_t
*pcid
, struct uio
*auio
, user_ssize_t
*bytes_written
)
2849 struct inpcb
*inp
= sotoinpcb(so
);
2856 VERIFY(dst
!= NULL
);
2858 error
= flow_divert_connect_out(so
, dst
, p
);
2864 /* if there is data, send it */
2866 user_ssize_t datalen
= 0;
2868 socket_unlock(so
, 0);
2870 VERIFY(bytes_written
!= NULL
);
2872 datalen
= uio_resid(auio
);
2873 error
= so
->so_proto
->pr_usrreqs
->pru_sosend(so
, NULL
, (uio_t
)auio
, NULL
, NULL
, 0);
2876 if (error
== 0 || error
== EWOULDBLOCK
) {
2877 *bytes_written
= datalen
- uio_resid(auio
);
2881 * sosend returns EWOULDBLOCK if it's a non-blocking
2882 * socket or a timeout occured (this allows to return
2883 * the amount of queued data through sendit()).
2885 * However, connectx() returns EINPROGRESS in case of a
2886 * blocking socket. So we change the return value here.
2888 if (error
== EWOULDBLOCK
) {
2889 error
= EINPROGRESS
;
2893 if (error
== 0 && pcid
!= NULL
) {
2894 *pcid
= 1; /* there is only 1 connection for a TCP */
2901 flow_divert_connectx_out(struct socket
*so
, struct sockaddr
*src __unused
,
2902 struct sockaddr
*dst
, struct proc
*p
, uint32_t ifscope __unused
,
2903 sae_associd_t aid __unused
, sae_connid_t
*pcid
, uint32_t flags __unused
, void *arg __unused
,
2904 uint32_t arglen __unused
, struct uio
*uio
, user_ssize_t
*bytes_written
)
2906 return (flow_divert_connectx_out_common(so
, dst
, p
, pcid
, uio
, bytes_written
));
2911 flow_divert_connectx6_out(struct socket
*so
, struct sockaddr
*src __unused
,
2912 struct sockaddr
*dst
, struct proc
*p
, uint32_t ifscope __unused
,
2913 sae_associd_t aid __unused
, sae_connid_t
*pcid
, uint32_t flags __unused
, void *arg __unused
,
2914 uint32_t arglen __unused
, struct uio
*uio
, user_ssize_t
*bytes_written
)
2916 return (flow_divert_connectx_out_common(so
, dst
, p
, pcid
, uio
, bytes_written
));
2921 flow_divert_getconninfo(struct socket
*so
, sae_connid_t cid
, uint32_t *flags
,
2922 uint32_t *ifindex
, int32_t *soerror
, user_addr_t src
, socklen_t
*src_len
,
2923 user_addr_t dst
, socklen_t
*dst_len
, uint32_t *aux_type
,
2924 user_addr_t aux_data __unused
, uint32_t *aux_len
)
2927 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
2928 struct ifnet
*ifp
= NULL
;
2929 struct inpcb
*inp
= sotoinpcb(so
);
2931 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
));
2933 if (so
->so_fd_pcb
== NULL
|| inp
== NULL
) {
2938 if (cid
!= SAE_CONNID_ANY
&& cid
!= SAE_CONNID_ALL
&& cid
!= 1) {
2943 ifp
= inp
->inp_last_outifp
;
2944 *ifindex
= ((ifp
!= NULL
) ? ifp
->if_index
: 0);
2945 *soerror
= so
->so_error
;
2948 if (so
->so_state
& SS_ISCONNECTED
) {
2949 *flags
|= (CIF_CONNECTED
| CIF_PREFERRED
);
2952 if (fd_cb
->local_address
== NULL
) {
2953 struct sockaddr_in sin
;
2954 bzero(&sin
, sizeof(sin
));
2955 sin
.sin_len
= sizeof(sin
);
2956 sin
.sin_family
= AF_INET
;
2957 *src_len
= sin
.sin_len
;
2958 if (src
!= USER_ADDR_NULL
) {
2959 error
= copyout(&sin
, src
, sin
.sin_len
);
2965 *src_len
= fd_cb
->local_address
->sa_len
;
2966 if (src
!= USER_ADDR_NULL
) {
2967 error
= copyout(fd_cb
->local_address
, src
, fd_cb
->local_address
->sa_len
);
2974 if (fd_cb
->remote_address
== NULL
) {
2975 struct sockaddr_in sin
;
2976 bzero(&sin
, sizeof(sin
));
2977 sin
.sin_len
= sizeof(sin
);
2978 sin
.sin_family
= AF_INET
;
2979 *dst_len
= sin
.sin_len
;
2980 if (dst
!= USER_ADDR_NULL
) {
2981 error
= copyout(&sin
, dst
, sin
.sin_len
);
2987 *dst_len
= fd_cb
->remote_address
->sa_len
;
2988 if (dst
!= USER_ADDR_NULL
) {
2989 error
= copyout(fd_cb
->remote_address
, dst
, fd_cb
->remote_address
->sa_len
);
3004 flow_divert_control(struct socket
*so
, u_long cmd
, caddr_t data
, struct ifnet
*ifp __unused
, struct proc
*p __unused
)
3009 case SIOCGCONNINFO32
: {
3010 struct so_cinforeq32 cifr
;
3011 bcopy(data
, &cifr
, sizeof (cifr
));
3012 error
= flow_divert_getconninfo(so
, cifr
.scir_cid
, &cifr
.scir_flags
,
3013 &cifr
.scir_ifindex
, &cifr
.scir_error
, cifr
.scir_src
,
3014 &cifr
.scir_src_len
, cifr
.scir_dst
, &cifr
.scir_dst_len
,
3015 &cifr
.scir_aux_type
, cifr
.scir_aux_data
,
3016 &cifr
.scir_aux_len
);
3018 bcopy(&cifr
, data
, sizeof (cifr
));
3023 case SIOCGCONNINFO64
: {
3024 struct so_cinforeq64 cifr
;
3025 bcopy(data
, &cifr
, sizeof (cifr
));
3026 error
= flow_divert_getconninfo(so
, cifr
.scir_cid
, &cifr
.scir_flags
,
3027 &cifr
.scir_ifindex
, &cifr
.scir_error
, cifr
.scir_src
,
3028 &cifr
.scir_src_len
, cifr
.scir_dst
, &cifr
.scir_dst_len
,
3029 &cifr
.scir_aux_type
, cifr
.scir_aux_data
,
3030 &cifr
.scir_aux_len
);
3032 bcopy(&cifr
, data
, sizeof (cifr
));
3045 flow_divert_in_control(struct socket
*so
, u_long cmd
, caddr_t data
, struct ifnet
*ifp
, struct proc
*p
)
3047 int error
= flow_divert_control(so
, cmd
, data
, ifp
, p
);
3049 if (error
== EOPNOTSUPP
) {
3050 error
= in_control(so
, cmd
, data
, ifp
, p
);
3057 flow_divert_in6_control(struct socket
*so
, u_long cmd
, caddr_t data
, struct ifnet
*ifp
, struct proc
*p
)
3059 int error
= flow_divert_control(so
, cmd
, data
, ifp
, p
);
3061 if (error
== EOPNOTSUPP
) {
3062 error
= in6_control(so
, cmd
, data
, ifp
, p
);
3069 flow_divert_data_out(struct socket
*so
, int flags
, mbuf_t data
, struct sockaddr
*to
, mbuf_t control
, struct proc
*p
)
3071 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3075 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
3077 inp
= sotoinpcb(so
);
3078 if (inp
== NULL
|| inp
->inp_state
== INPCB_STATE_DEAD
) {
3083 if (control
&& mbuf_len(control
) > 0) {
3088 if (flags
& MSG_OOB
) {
3090 goto done
; /* We don't support OOB data */
3093 error
= flow_divert_check_no_cellular(fd_cb
) ||
3094 flow_divert_check_no_expensive(fd_cb
);
3099 /* Implicit connect */
3100 if (!(fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
)) {
3101 FDLOG0(LOG_INFO
, fd_cb
, "implicit connect");
3102 error
= flow_divert_connect_out(so
, to
, p
);
3107 if (so
->so_flags1
& SOF1_DATA_IDEMPOTENT
) {
3108 /* Open up the send window so that the data will get sent right away */
3109 fd_cb
->send_window
= mbuf_pkthdr_len(data
);
3113 FDLOG(LOG_DEBUG
, fd_cb
, "app wrote %lu bytes", mbuf_pkthdr_len(data
));
3115 fd_cb
->bytes_written_by_app
+= mbuf_pkthdr_len(data
);
3116 error
= flow_divert_send_app_data(fd_cb
, data
, to
);
3123 if (flags
& PRUS_EOF
) {
3124 flow_divert_shutdown(so
);
3138 flow_divert_preconnect(struct socket
*so
)
3140 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3143 if (!(fd_cb
->flags
& FLOW_DIVERT_CONNECT_STARTED
) && fd_cb
->connect_packet
!= NULL
) {
3144 FDLOG0(LOG_INFO
, fd_cb
, "Pre-connect read: sending saved connect packet");
3145 mbuf_t connect_packet
= fd_cb
->connect_packet
;
3146 fd_cb
->connect_packet
= NULL
;
3148 error
= flow_divert_send_packet(fd_cb
, connect_packet
, TRUE
);
3150 mbuf_freem(connect_packet
);
3153 fd_cb
->flags
|= FLOW_DIVERT_CONNECT_STARTED
;
3156 soclearfastopen(so
);
3162 flow_divert_set_protosw(struct socket
*so
)
3164 so
->so_flags
|= SOF_FLOW_DIVERT
;
3165 if (SOCK_DOM(so
) == PF_INET
) {
3166 so
->so_proto
= &g_flow_divert_in_protosw
;
3170 so
->so_proto
= (struct protosw
*)&g_flow_divert_in6_protosw
;
3176 flow_divert_set_udp_protosw(struct socket
*so
)
3178 so
->so_flags
|= SOF_FLOW_DIVERT
;
3179 if (SOCK_DOM(so
) == PF_INET
) {
3180 so
->so_proto
= &g_flow_divert_in_udp_protosw
;
3184 so
->so_proto
= (struct protosw
*)&g_flow_divert_in6_udp_protosw
;
3190 flow_divert_attach(struct socket
*so
, uint32_t flow_id
, uint32_t ctl_unit
)
3193 struct flow_divert_pcb
*fd_cb
= NULL
;
3194 struct ifnet
*ifp
= NULL
;
3195 struct inpcb
*inp
= NULL
;
3196 struct socket
*old_so
;
3197 mbuf_t recv_data
= NULL
;
3199 socket_unlock(so
, 0);
3201 FDLOG(LOG_INFO
, &nil_pcb
, "Attaching socket to flow %u", flow_id
);
3203 /* Find the flow divert control block */
3204 lck_rw_lock_shared(&g_flow_divert_group_lck
);
3205 if (g_flow_divert_groups
!= NULL
&& g_active_group_count
> 0) {
3206 struct flow_divert_group
*group
= g_flow_divert_groups
[ctl_unit
];
3207 if (group
!= NULL
) {
3208 fd_cb
= flow_divert_pcb_lookup(flow_id
, group
);
3211 lck_rw_done(&g_flow_divert_group_lck
);
3213 if (fd_cb
== NULL
) {
3220 /* Dis-associate the flow divert control block from its current socket */
3223 inp
= sotoinpcb(old_so
);
3225 VERIFY(inp
!= NULL
);
3227 socket_lock(old_so
, 0);
3228 flow_divert_disconnect_socket(old_so
);
3229 old_so
->so_flags
&= ~SOF_FLOW_DIVERT
;
3230 old_so
->so_fd_pcb
= NULL
;
3231 if (SOCK_TYPE(old_so
) == SOCK_STREAM
) {
3232 old_so
->so_proto
= pffindproto(SOCK_DOM(old_so
), IPPROTO_TCP
, SOCK_STREAM
);
3233 } else if (SOCK_TYPE(old_so
) == SOCK_DGRAM
) {
3234 old_so
->so_proto
= pffindproto(SOCK_DOM(old_so
), IPPROTO_UDP
, SOCK_DGRAM
);
3237 /* Save the output interface */
3238 ifp
= inp
->inp_last_outifp
;
3239 if (old_so
->so_rcv
.sb_cc
> 0) {
3240 error
= mbuf_dup(old_so
->so_rcv
.sb_mb
, MBUF_DONTWAIT
, &recv_data
);
3241 sbflush(&old_so
->so_rcv
);
3243 socket_unlock(old_so
, 0);
3245 /* Associate the new socket with the flow divert control block */
3247 so
->so_fd_pcb
= fd_cb
;
3248 inp
= sotoinpcb(so
);
3249 inp
->inp_last_outifp
= ifp
;
3250 if (recv_data
!= NULL
) {
3251 if (sbappendstream(&so
->so_rcv
, recv_data
)) {
3255 flow_divert_set_protosw(so
);
3256 socket_unlock(so
, 0);
3259 fd_cb
->flags
|= FLOW_DIVERT_TRANSFERRED
;
3266 if (fd_cb
!= NULL
) {
3267 FDRELEASE(fd_cb
); /* Release the reference obtained via flow_divert_pcb_lookup */
3274 flow_divert_implicit_data_out(struct socket
*so
, int flags
, mbuf_t data
, struct sockaddr
*to
, mbuf_t control
, struct proc
*p
)
3276 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3280 inp
= sotoinpcb(so
);
3285 if (fd_cb
== NULL
) {
3286 uint32_t fd_ctl_unit
= necp_socket_get_flow_divert_control_unit(inp
);
3287 if (fd_ctl_unit
> 0) {
3288 error
= flow_divert_pcb_init(so
, fd_ctl_unit
);
3289 fd_cb
= so
->so_fd_pcb
;
3290 if (error
!= 0 || fd_cb
== NULL
) {
3298 return flow_divert_data_out(so
, flags
, data
, to
, control
, p
);
3312 flow_divert_pcb_init(struct socket
*so
, uint32_t ctl_unit
)
3315 struct flow_divert_pcb
*fd_cb
;
3317 if (so
->so_flags
& SOF_FLOW_DIVERT
) {
3321 fd_cb
= flow_divert_pcb_create(so
);
3322 if (fd_cb
!= NULL
) {
3323 error
= flow_divert_pcb_insert(fd_cb
, ctl_unit
);
3325 FDLOG(LOG_ERR
, fd_cb
, "pcb insert failed: %d", error
);
3328 fd_cb
->control_group_unit
= ctl_unit
;
3329 so
->so_fd_pcb
= fd_cb
;
3331 if (SOCK_TYPE(so
) == SOCK_STREAM
) {
3332 flow_divert_set_protosw(so
);
3333 } else if (SOCK_TYPE(so
) == SOCK_DGRAM
) {
3334 flow_divert_set_udp_protosw(so
);
3337 FDLOG0(LOG_INFO
, fd_cb
, "Created");
3347 flow_divert_token_set(struct socket
*so
, struct sockopt
*sopt
)
3349 uint32_t ctl_unit
= 0;
3350 uint32_t key_unit
= 0;
3351 uint32_t flow_id
= 0;
3354 mbuf_t token
= NULL
;
3356 if (so
->so_flags
& SOF_FLOW_DIVERT
) {
3361 if (g_init_result
) {
3362 FDLOG(LOG_ERR
, &nil_pcb
, "flow_divert_init failed (%d), cannot use flow divert", g_init_result
);
3363 error
= ENOPROTOOPT
;
3367 if ((SOCK_TYPE(so
) != SOCK_STREAM
&& SOCK_TYPE(so
) != SOCK_DGRAM
) ||
3368 (SOCK_PROTO(so
) != IPPROTO_TCP
&& SOCK_PROTO(so
) != IPPROTO_UDP
) ||
3369 (SOCK_DOM(so
) != PF_INET
3371 && SOCK_DOM(so
) != PF_INET6
3378 if (SOCK_TYPE(so
) == SOCK_STREAM
&& SOCK_PROTO(so
) == IPPROTO_TCP
) {
3379 struct tcpcb
*tp
= sototcpcb(so
);
3380 if (tp
== NULL
|| tp
->t_state
!= TCPS_CLOSED
) {
3387 error
= soopt_getm(sopt
, &token
);
3393 error
= soopt_mcopyin(sopt
, token
);
3399 error
= flow_divert_packet_get_tlv(token
, 0, FLOW_DIVERT_TLV_KEY_UNIT
, sizeof(key_unit
), (void *)&key_unit
, NULL
);
3401 key_unit
= ntohl(key_unit
);
3402 if (key_unit
>= GROUP_COUNT_MAX
) {
3405 } else if (error
!= ENOENT
) {
3406 FDLOG(LOG_ERR
, &nil_pcb
, "Failed to get the key unit from the token: %d", error
);
3412 error
= flow_divert_packet_get_tlv(token
, 0, FLOW_DIVERT_TLV_CTL_UNIT
, sizeof(ctl_unit
), (void *)&ctl_unit
, NULL
);
3414 FDLOG(LOG_ERR
, &nil_pcb
, "Failed to get the control socket unit from the token: %d", error
);
3418 /* A valid kernel control unit is required */
3419 ctl_unit
= ntohl(ctl_unit
);
3420 if (ctl_unit
== 0 || ctl_unit
>= GROUP_COUNT_MAX
) {
3421 FDLOG(LOG_ERR
, &nil_pcb
, "Got an invalid control socket unit: %u", ctl_unit
);
3426 socket_unlock(so
, 0);
3427 hmac_error
= flow_divert_packet_verify_hmac(token
, (key_unit
!= 0 ? key_unit
: ctl_unit
));
3430 if (hmac_error
&& hmac_error
!= ENOENT
) {
3431 FDLOG(LOG_ERR
, &nil_pcb
, "HMAC verfication failed: %d", hmac_error
);
3436 error
= flow_divert_packet_get_tlv(token
, 0, FLOW_DIVERT_TLV_FLOW_ID
, sizeof(flow_id
), (void *)&flow_id
, NULL
);
3437 if (error
&& error
!= ENOENT
) {
3438 FDLOG(LOG_ERR
, &nil_pcb
, "Failed to get the flow ID from the token: %d", error
);
3443 error
= flow_divert_pcb_init(so
, ctl_unit
);
3445 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3446 int log_level
= LOG_NOTICE
;
3448 error
= flow_divert_packet_get_tlv(token
, 0, FLOW_DIVERT_TLV_LOG_LEVEL
,
3449 sizeof(log_level
), &log_level
, NULL
);
3451 fd_cb
->log_level
= log_level
;
3455 fd_cb
->connect_token
= token
;
3459 error
= flow_divert_attach(so
, flow_id
, ctl_unit
);
3462 if (hmac_error
== 0) {
3463 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3464 if (fd_cb
!= NULL
) {
3465 fd_cb
->flags
|= FLOW_DIVERT_HAS_HMAC
;
3470 if (token
!= NULL
) {
3478 flow_divert_token_get(struct socket
*so
, struct sockopt
*sopt
)
3482 uint8_t hmac
[SHA_DIGEST_LENGTH
];
3483 struct flow_divert_pcb
*fd_cb
= so
->so_fd_pcb
;
3484 mbuf_t token
= NULL
;
3485 struct flow_divert_group
*control_group
= NULL
;
3487 if (!(so
->so_flags
& SOF_FLOW_DIVERT
)) {
3492 VERIFY((so
->so_flags
& SOF_FLOW_DIVERT
) && so
->so_fd_pcb
!= NULL
);
3494 if (fd_cb
->group
== NULL
) {
3499 error
= mbuf_gethdr(MBUF_DONTWAIT
, MBUF_TYPE_HEADER
, &token
);
3501 FDLOG(LOG_ERR
, fd_cb
, "failed to allocate the header mbuf: %d", error
);
3505 ctl_unit
= htonl(fd_cb
->group
->ctl_unit
);
3507 error
= flow_divert_packet_append_tlv(token
, FLOW_DIVERT_TLV_CTL_UNIT
, sizeof(ctl_unit
), &ctl_unit
);
3512 error
= flow_divert_packet_append_tlv(token
, FLOW_DIVERT_TLV_FLOW_ID
, sizeof(fd_cb
->hash
), &fd_cb
->hash
);
3517 if (fd_cb
->app_data
!= NULL
) {
3518 error
= flow_divert_packet_append_tlv(token
, FLOW_DIVERT_TLV_APP_DATA
, fd_cb
->app_data_length
, fd_cb
->app_data
);
3524 socket_unlock(so
, 0);
3525 lck_rw_lock_shared(&g_flow_divert_group_lck
);
3527 if (g_flow_divert_groups
!= NULL
&& g_active_group_count
> 0 &&
3528 fd_cb
->control_group_unit
> 0 && fd_cb
->control_group_unit
< GROUP_COUNT_MAX
)
3530 control_group
= g_flow_divert_groups
[fd_cb
->control_group_unit
];
3533 if (control_group
!= NULL
) {
3534 lck_rw_lock_shared(&control_group
->lck
);
3535 ctl_unit
= htonl(control_group
->ctl_unit
);
3536 error
= flow_divert_packet_append_tlv(token
, FLOW_DIVERT_TLV_KEY_UNIT
, sizeof(ctl_unit
), &ctl_unit
);
3538 error
= flow_divert_packet_compute_hmac(token
, control_group
, hmac
);
3540 lck_rw_done(&control_group
->lck
);
3542 error
= ENOPROTOOPT
;
3545 lck_rw_done(&g_flow_divert_group_lck
);
3552 error
= flow_divert_packet_append_tlv(token
, FLOW_DIVERT_TLV_HMAC
, sizeof(hmac
), hmac
);
3557 if (sopt
->sopt_val
== USER_ADDR_NULL
) {
3558 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3559 sopt
->sopt_valsize
= mbuf_pkthdr_len(token
);
3563 error
= soopt_mcopyout(sopt
, token
);
3565 token
= NULL
; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3570 if (token
!= NULL
) {
3578 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused
, struct sockaddr_ctl
*sac
, void **unitinfo
)
3580 struct flow_divert_group
*new_group
= NULL
;
3583 if (sac
->sc_unit
>= GROUP_COUNT_MAX
) {
3590 MALLOC_ZONE(new_group
, struct flow_divert_group
*, sizeof(*new_group
), M_FLOW_DIVERT_GROUP
, M_WAITOK
);
3591 if (new_group
== NULL
) {
3596 memset(new_group
, 0, sizeof(*new_group
));
3598 lck_rw_init(&new_group
->lck
, flow_divert_mtx_grp
, flow_divert_mtx_attr
);
3599 RB_INIT(&new_group
->pcb_tree
);
3600 new_group
->ctl_unit
= sac
->sc_unit
;
3601 MBUFQ_INIT(&new_group
->send_queue
);
3602 new_group
->signing_id_trie
.root
= NULL_TRIE_IDX
;
3604 lck_rw_lock_exclusive(&g_flow_divert_group_lck
);
3606 if (g_flow_divert_groups
== NULL
) {
3607 MALLOC(g_flow_divert_groups
,
3608 struct flow_divert_group
**,
3609 GROUP_COUNT_MAX
* sizeof(struct flow_divert_group
*),
3614 if (g_flow_divert_groups
== NULL
) {
3616 } else if (g_flow_divert_groups
[sac
->sc_unit
] != NULL
) {
3619 g_flow_divert_groups
[sac
->sc_unit
] = new_group
;
3620 g_active_group_count
++;
3623 lck_rw_done(&g_flow_divert_group_lck
);
3625 *unitinfo
= new_group
;
3628 if (error
!= 0 && new_group
!= NULL
) {
3629 FREE_ZONE(new_group
, sizeof(*new_group
), M_FLOW_DIVERT_GROUP
);
3635 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused
, uint32_t unit
, void *unitinfo
)
3637 struct flow_divert_group
*group
= NULL
;
3640 if (unit
>= GROUP_COUNT_MAX
) {
3644 FDLOG(LOG_INFO
, &nil_pcb
, "disconnecting group %d", unit
);
3646 lck_rw_lock_exclusive(&g_flow_divert_group_lck
);
3648 if (g_flow_divert_groups
== NULL
|| g_active_group_count
== 0) {
3649 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit
,
3650 g_flow_divert_groups
, g_active_group_count
);
3653 group
= g_flow_divert_groups
[unit
];
3655 if (group
!= (struct flow_divert_group
*)unitinfo
) {
3656 panic("group with unit %d (%p) != unit info (%p)", unit
, group
, unitinfo
);
3659 if (group
!= NULL
) {
3660 flow_divert_close_all(group
);
3661 if (group
->token_key
!= NULL
) {
3662 memset(group
->token_key
, 0, group
->token_key_size
);
3663 FREE(group
->token_key
, M_TEMP
);
3664 group
->token_key
= NULL
;
3665 group
->token_key_size
= 0;
3668 /* Re-set the current trie */
3669 if (group
->signing_id_trie
.memory
!= NULL
) {
3670 FREE(group
->signing_id_trie
.memory
, M_TEMP
);
3672 memset(&group
->signing_id_trie
, 0, sizeof(group
->signing_id_trie
));
3673 group
->signing_id_trie
.root
= NULL_TRIE_IDX
;
3675 FREE_ZONE(group
, sizeof(*group
), M_FLOW_DIVERT_GROUP
);
3676 g_flow_divert_groups
[unit
] = NULL
;
3677 g_active_group_count
--;
3682 if (g_active_group_count
== 0) {
3683 FREE(g_flow_divert_groups
, M_TEMP
);
3684 g_flow_divert_groups
= NULL
;
3687 lck_rw_done(&g_flow_divert_group_lck
);
3693 flow_divert_kctl_send(kern_ctl_ref kctlref __unused
, uint32_t unit __unused
, void *unitinfo
, mbuf_t m
, int flags __unused
)
3695 return flow_divert_input(m
, (struct flow_divert_group
*)unitinfo
);
3699 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused
, uint32_t unit __unused
, void *unitinfo
, int flags __unused
)
3701 struct flow_divert_group
*group
= (struct flow_divert_group
*)unitinfo
;
3703 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED
, &group
->atomic_bits
)) {
3704 struct flow_divert_pcb
*fd_cb
;
3705 SLIST_HEAD(, flow_divert_pcb
) tmp_list
;
3707 lck_rw_lock_shared(&g_flow_divert_group_lck
);
3708 lck_rw_lock_exclusive(&group
->lck
);
3710 while (!MBUFQ_EMPTY(&group
->send_queue
)) {
3712 FDLOG0(LOG_DEBUG
, &nil_pcb
, "trying ctl_enqueuembuf again");
3713 next_packet
= MBUFQ_FIRST(&group
->send_queue
);
3714 int error
= ctl_enqueuembuf(g_flow_divert_kctl_ref
, group
->ctl_unit
, next_packet
, CTL_DATA_EOR
);
3716 FDLOG(LOG_DEBUG
, &nil_pcb
, "ctl_enqueuembuf returned an error: %d", error
);
3717 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED
, &group
->atomic_bits
);
3718 lck_rw_done(&group
->lck
);
3719 lck_rw_done(&g_flow_divert_group_lck
);
3722 MBUFQ_DEQUEUE(&group
->send_queue
, next_packet
);
3725 SLIST_INIT(&tmp_list
);
3727 RB_FOREACH(fd_cb
, fd_pcb_tree
, &group
->pcb_tree
) {
3729 SLIST_INSERT_HEAD(&tmp_list
, fd_cb
, tmp_list_entry
);
3732 lck_rw_done(&group
->lck
);
3734 SLIST_FOREACH(fd_cb
, &tmp_list
, tmp_list_entry
) {
3736 if (fd_cb
->so
!= NULL
) {
3737 socket_lock(fd_cb
->so
, 0);
3738 if (fd_cb
->group
!= NULL
) {
3739 flow_divert_send_buffered_data(fd_cb
, FALSE
);
3741 socket_unlock(fd_cb
->so
, 0);
3747 lck_rw_done(&g_flow_divert_group_lck
);
3752 flow_divert_kctl_init(void)
3754 struct kern_ctl_reg ctl_reg
;
3757 memset(&ctl_reg
, 0, sizeof(ctl_reg
));
3759 strlcpy(ctl_reg
.ctl_name
, FLOW_DIVERT_CONTROL_NAME
, sizeof(ctl_reg
.ctl_name
));
3760 ctl_reg
.ctl_name
[sizeof(ctl_reg
.ctl_name
)-1] = '\0';
3761 ctl_reg
.ctl_flags
= CTL_FLAG_PRIVILEGED
| CTL_FLAG_REG_EXTENDED
;
3762 ctl_reg
.ctl_sendsize
= FD_CTL_SENDBUFF_SIZE
;
3763 ctl_reg
.ctl_recvsize
= FD_CTL_RCVBUFF_SIZE
;
3765 ctl_reg
.ctl_connect
= flow_divert_kctl_connect
;
3766 ctl_reg
.ctl_disconnect
= flow_divert_kctl_disconnect
;
3767 ctl_reg
.ctl_send
= flow_divert_kctl_send
;
3768 ctl_reg
.ctl_rcvd
= flow_divert_kctl_rcvd
;
3770 result
= ctl_register(&ctl_reg
, &g_flow_divert_kctl_ref
);
3773 FDLOG(LOG_ERR
, &nil_pcb
, "flow_divert_kctl_init - ctl_register failed: %d\n", result
);
3781 flow_divert_init(void)
3783 memset(&nil_pcb
, 0, sizeof(nil_pcb
));
3784 nil_pcb
.log_level
= LOG_NOTICE
;
3786 g_tcp_protosw
= pffindproto(AF_INET
, IPPROTO_TCP
, SOCK_STREAM
);
3788 VERIFY(g_tcp_protosw
!= NULL
);
3790 memcpy(&g_flow_divert_in_protosw
, g_tcp_protosw
, sizeof(g_flow_divert_in_protosw
));
3791 memcpy(&g_flow_divert_in_usrreqs
, g_tcp_protosw
->pr_usrreqs
, sizeof(g_flow_divert_in_usrreqs
));
3793 g_flow_divert_in_usrreqs
.pru_connect
= flow_divert_connect_out
;
3794 g_flow_divert_in_usrreqs
.pru_connectx
= flow_divert_connectx_out
;
3795 g_flow_divert_in_usrreqs
.pru_control
= flow_divert_in_control
;
3796 g_flow_divert_in_usrreqs
.pru_disconnect
= flow_divert_close
;
3797 g_flow_divert_in_usrreqs
.pru_disconnectx
= flow_divert_disconnectx
;
3798 g_flow_divert_in_usrreqs
.pru_peeraddr
= flow_divert_getpeername
;
3799 g_flow_divert_in_usrreqs
.pru_rcvd
= flow_divert_rcvd
;
3800 g_flow_divert_in_usrreqs
.pru_send
= flow_divert_data_out
;
3801 g_flow_divert_in_usrreqs
.pru_shutdown
= flow_divert_shutdown
;
3802 g_flow_divert_in_usrreqs
.pru_sockaddr
= flow_divert_getsockaddr
;
3803 g_flow_divert_in_usrreqs
.pru_preconnect
= flow_divert_preconnect
;
3805 g_flow_divert_in_protosw
.pr_usrreqs
= &g_flow_divert_in_usrreqs
;
3806 g_flow_divert_in_protosw
.pr_ctloutput
= flow_divert_ctloutput
;
3809 * Socket filters shouldn't attach/detach to/from this protosw
3810 * since pr_protosw is to be used instead, which points to the
3811 * real protocol; if they do, it is a bug and we should panic.
3813 g_flow_divert_in_protosw
.pr_filter_head
.tqh_first
=
3814 (struct socket_filter
*)(uintptr_t)0xdeadbeefdeadbeef;
3815 g_flow_divert_in_protosw
.pr_filter_head
.tqh_last
=
3816 (struct socket_filter
**)(uintptr_t)0xdeadbeefdeadbeef;
3819 g_udp_protosw
= pffindproto(AF_INET
, IPPROTO_UDP
, SOCK_DGRAM
);
3820 VERIFY(g_udp_protosw
!= NULL
);
3822 memcpy(&g_flow_divert_in_udp_protosw
, g_udp_protosw
, sizeof(g_flow_divert_in_udp_protosw
));
3823 memcpy(&g_flow_divert_in_udp_usrreqs
, g_udp_protosw
->pr_usrreqs
, sizeof(g_flow_divert_in_udp_usrreqs
));
3825 g_flow_divert_in_udp_usrreqs
.pru_connect
= flow_divert_connect_out
;
3826 g_flow_divert_in_udp_usrreqs
.pru_connectx
= flow_divert_connectx_out
;
3827 g_flow_divert_in_udp_usrreqs
.pru_control
= flow_divert_in_control
;
3828 g_flow_divert_in_udp_usrreqs
.pru_disconnect
= flow_divert_close
;
3829 g_flow_divert_in_udp_usrreqs
.pru_disconnectx
= flow_divert_disconnectx
;
3830 g_flow_divert_in_udp_usrreqs
.pru_peeraddr
= flow_divert_getpeername
;
3831 g_flow_divert_in_udp_usrreqs
.pru_rcvd
= flow_divert_rcvd
;
3832 g_flow_divert_in_udp_usrreqs
.pru_send
= flow_divert_data_out
;
3833 g_flow_divert_in_udp_usrreqs
.pru_shutdown
= flow_divert_shutdown
;
3834 g_flow_divert_in_udp_usrreqs
.pru_sockaddr
= flow_divert_getsockaddr
;
3835 g_flow_divert_in_udp_usrreqs
.pru_sosend_list
= pru_sosend_list_notsupp
;
3836 g_flow_divert_in_udp_usrreqs
.pru_soreceive_list
= pru_soreceive_list_notsupp
;
3837 g_flow_divert_in_udp_usrreqs
.pru_preconnect
= flow_divert_preconnect
;
3839 g_flow_divert_in_udp_protosw
.pr_usrreqs
= &g_flow_divert_in_usrreqs
;
3840 g_flow_divert_in_udp_protosw
.pr_ctloutput
= flow_divert_ctloutput
;
3843 * Socket filters shouldn't attach/detach to/from this protosw
3844 * since pr_protosw is to be used instead, which points to the
3845 * real protocol; if they do, it is a bug and we should panic.
3847 g_flow_divert_in_udp_protosw
.pr_filter_head
.tqh_first
=
3848 (struct socket_filter
*)(uintptr_t)0xdeadbeefdeadbeef;
3849 g_flow_divert_in_udp_protosw
.pr_filter_head
.tqh_last
=
3850 (struct socket_filter
**)(uintptr_t)0xdeadbeefdeadbeef;
3853 g_tcp6_protosw
= (struct ip6protosw
*)pffindproto(AF_INET6
, IPPROTO_TCP
, SOCK_STREAM
);
3855 VERIFY(g_tcp6_protosw
!= NULL
);
3857 memcpy(&g_flow_divert_in6_protosw
, g_tcp6_protosw
, sizeof(g_flow_divert_in6_protosw
));
3858 memcpy(&g_flow_divert_in6_usrreqs
, g_tcp6_protosw
->pr_usrreqs
, sizeof(g_flow_divert_in6_usrreqs
));
3860 g_flow_divert_in6_usrreqs
.pru_connect
= flow_divert_connect_out
;
3861 g_flow_divert_in6_usrreqs
.pru_connectx
= flow_divert_connectx6_out
;
3862 g_flow_divert_in6_usrreqs
.pru_control
= flow_divert_in6_control
;
3863 g_flow_divert_in6_usrreqs
.pru_disconnect
= flow_divert_close
;
3864 g_flow_divert_in6_usrreqs
.pru_disconnectx
= flow_divert_disconnectx
;
3865 g_flow_divert_in6_usrreqs
.pru_peeraddr
= flow_divert_getpeername
;
3866 g_flow_divert_in6_usrreqs
.pru_rcvd
= flow_divert_rcvd
;
3867 g_flow_divert_in6_usrreqs
.pru_send
= flow_divert_data_out
;
3868 g_flow_divert_in6_usrreqs
.pru_shutdown
= flow_divert_shutdown
;
3869 g_flow_divert_in6_usrreqs
.pru_sockaddr
= flow_divert_getsockaddr
;
3870 g_flow_divert_in6_usrreqs
.pru_preconnect
= flow_divert_preconnect
;
3872 g_flow_divert_in6_protosw
.pr_usrreqs
= &g_flow_divert_in6_usrreqs
;
3873 g_flow_divert_in6_protosw
.pr_ctloutput
= flow_divert_ctloutput
;
3875 * Socket filters shouldn't attach/detach to/from this protosw
3876 * since pr_protosw is to be used instead, which points to the
3877 * real protocol; if they do, it is a bug and we should panic.
3879 g_flow_divert_in6_protosw
.pr_filter_head
.tqh_first
=
3880 (struct socket_filter
*)(uintptr_t)0xdeadbeefdeadbeef;
3881 g_flow_divert_in6_protosw
.pr_filter_head
.tqh_last
=
3882 (struct socket_filter
**)(uintptr_t)0xdeadbeefdeadbeef;
3885 g_udp6_protosw
= (struct ip6protosw
*)pffindproto(AF_INET6
, IPPROTO_UDP
, SOCK_DGRAM
);
3887 VERIFY(g_udp6_protosw
!= NULL
);
3889 memcpy(&g_flow_divert_in6_udp_protosw
, g_udp6_protosw
, sizeof(g_flow_divert_in6_udp_protosw
));
3890 memcpy(&g_flow_divert_in6_udp_usrreqs
, g_udp6_protosw
->pr_usrreqs
, sizeof(g_flow_divert_in6_udp_usrreqs
));
3892 g_flow_divert_in6_udp_usrreqs
.pru_connect
= flow_divert_connect_out
;
3893 g_flow_divert_in6_udp_usrreqs
.pru_connectx
= flow_divert_connectx6_out
;
3894 g_flow_divert_in6_udp_usrreqs
.pru_control
= flow_divert_in6_control
;
3895 g_flow_divert_in6_udp_usrreqs
.pru_disconnect
= flow_divert_close
;
3896 g_flow_divert_in6_udp_usrreqs
.pru_disconnectx
= flow_divert_disconnectx
;
3897 g_flow_divert_in6_udp_usrreqs
.pru_peeraddr
= flow_divert_getpeername
;
3898 g_flow_divert_in6_udp_usrreqs
.pru_rcvd
= flow_divert_rcvd
;
3899 g_flow_divert_in6_udp_usrreqs
.pru_send
= flow_divert_data_out
;
3900 g_flow_divert_in6_udp_usrreqs
.pru_shutdown
= flow_divert_shutdown
;
3901 g_flow_divert_in6_udp_usrreqs
.pru_sockaddr
= flow_divert_getsockaddr
;
3902 g_flow_divert_in6_udp_usrreqs
.pru_sosend_list
= pru_sosend_list_notsupp
;
3903 g_flow_divert_in6_udp_usrreqs
.pru_soreceive_list
= pru_soreceive_list_notsupp
;
3904 g_flow_divert_in6_udp_usrreqs
.pru_preconnect
= flow_divert_preconnect
;
3906 g_flow_divert_in6_udp_protosw
.pr_usrreqs
= &g_flow_divert_in6_udp_usrreqs
;
3907 g_flow_divert_in6_udp_protosw
.pr_ctloutput
= flow_divert_ctloutput
;
3909 * Socket filters shouldn't attach/detach to/from this protosw
3910 * since pr_protosw is to be used instead, which points to the
3911 * real protocol; if they do, it is a bug and we should panic.
3913 g_flow_divert_in6_udp_protosw
.pr_filter_head
.tqh_first
=
3914 (struct socket_filter
*)(uintptr_t)0xdeadbeefdeadbeef;
3915 g_flow_divert_in6_udp_protosw
.pr_filter_head
.tqh_last
=
3916 (struct socket_filter
**)(uintptr_t)0xdeadbeefdeadbeef;
3919 flow_divert_grp_attr
= lck_grp_attr_alloc_init();
3920 if (flow_divert_grp_attr
== NULL
) {
3921 FDLOG0(LOG_ERR
, &nil_pcb
, "lck_grp_attr_alloc_init failed");
3922 g_init_result
= ENOMEM
;
3926 flow_divert_mtx_grp
= lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME
, flow_divert_grp_attr
);
3927 if (flow_divert_mtx_grp
== NULL
) {
3928 FDLOG0(LOG_ERR
, &nil_pcb
, "lck_grp_alloc_init failed");
3929 g_init_result
= ENOMEM
;
3933 flow_divert_mtx_attr
= lck_attr_alloc_init();
3934 if (flow_divert_mtx_attr
== NULL
) {
3935 FDLOG0(LOG_ERR
, &nil_pcb
, "lck_attr_alloc_init failed");
3936 g_init_result
= ENOMEM
;
3940 g_init_result
= flow_divert_kctl_init();
3941 if (g_init_result
) {
3945 lck_rw_init(&g_flow_divert_group_lck
, flow_divert_mtx_grp
, flow_divert_mtx_attr
);
3948 if (g_init_result
!= 0) {
3949 if (flow_divert_mtx_attr
!= NULL
) {
3950 lck_attr_free(flow_divert_mtx_attr
);
3951 flow_divert_mtx_attr
= NULL
;
3953 if (flow_divert_mtx_grp
!= NULL
) {
3954 lck_grp_free(flow_divert_mtx_grp
);
3955 flow_divert_mtx_grp
= NULL
;
3957 if (flow_divert_grp_attr
!= NULL
) {
3958 lck_grp_attr_free(flow_divert_grp_attr
);
3959 flow_divert_grp_attr
= NULL
;
3962 if (g_flow_divert_kctl_ref
!= NULL
) {
3963 ctl_deregister(g_flow_divert_kctl_ref
);
3964 g_flow_divert_kctl_ref
= NULL
;