]> git.saurik.com Git - apple/xnu.git/blame - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-6153.61.1.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
CommitLineData
39236c6e 1/*
5ba3f43e 2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
39236c6e
A		 3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745 5 *
39236c6e
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0a7de745 14 *
39236c6e
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745 17 *
39236c6e
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0a7de745 25 *
39236c6e
A		 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <string.h>
30#include <sys/types.h>
31#include <sys/syslog.h>
32#include <sys/queue.h>
33#include <sys/malloc.h>
34#include <sys/socket.h>
35#include <sys/kpi_mbuf.h>
36#include <sys/mbuf.h>
37#include <sys/domain.h>
38#include <sys/protosw.h>
39#include <sys/socketvar.h>
40#include <sys/kernel.h>
41#include <sys/systm.h>
42#include <sys/kern_control.h>
43#include <sys/ubc.h>
44#include <sys/codesign.h>
45#include <libkern/tree.h>
46#include <kern/locks.h>
47#include <kern/debug.h>
cb323159
A		 48#include <kern/task.h>
49#include <mach/task_info.h>
39236c6e
A		 50#include <net/if_var.h>
51#include <net/route.h>
52#include <net/flowhash.h>
53#include <net/ntstat.h>
54#include <netinet/in.h>
55#include <netinet/in_var.h>
56#include <netinet/tcp.h>
57#include <netinet/tcp_var.h>
58#include <netinet/tcp_fsm.h>
59#include <netinet/flow_divert.h>
60#include <netinet/flow_divert_proto.h>
61#if INET6
3e170ce0 62#include <netinet6/in6_pcb.h>
39236c6e 63#include <netinet6/ip6protosw.h>
0a7de745 64#endif /* INET6 */
39236c6e
A		 65#include <dev/random/randomdev.h>
66#include <libkern/crypto/sha1.h>
67#include <libkern/crypto/crypto_internal.h>
39037602 68#include <os/log.h>
cb323159
A		 69#include <corecrypto/cc.h>
70#if CONTENT_FILTER
71#include <net/content_filter.h>
72#endif /* CONTENT_FILTER */
39236c6e 73
0a7de745
A		 74#define FLOW_DIVERT_CONNECT_STARTED 0x00000001
75#define FLOW_DIVERT_READ_CLOSED 0x00000002
76#define FLOW_DIVERT_WRITE_CLOSED 0x00000004
77#define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
78#define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
79#define FLOW_DIVERT_TRANSFERRED 0x00000020
490019cf 80#define FLOW_DIVERT_HAS_HMAC 0x00000040
39236c6e 81
39037602
A		 82#define FDLOG(level, pcb, format, ...) \
83 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
39236c6e 84
39037602
A		 85#define FDLOG0(level, pcb, msg) \
86 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
39236c6e 87
0a7de745
A		 88#define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
89#define FDRELEASE(pcb) \
90 do { \
91 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
92 flow_divert_pcb_destroy(pcb); \
93 } \
39236c6e
A		 94 } while (0)
95
0a7de745
A		 96#define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
97#define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
39236c6e 98
0a7de745
A		 99#define FD_CTL_SENDBUFF_SIZE (128 * 1024)
100#define FD_CTL_RCVBUFF_SIZE (128 * 1024)
39236c6e 101
0a7de745 102#define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
39236c6e 103
0a7de745
A		 104#define GROUP_COUNT_MAX 32
105#define FLOW_DIVERT_MAX_NAME_SIZE 4096
106#define FLOW_DIVERT_MAX_KEY_SIZE 1024
107#define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
39236c6e 108
0a7de745 109struct flow_divert_trie_node {
39236c6e
A		 110 uint16_t start;
111 uint16_t length;
112 uint16_t child_map;
39236c6e
A		 113};
114
0a7de745
A		 115#define CHILD_MAP_SIZE 256
116#define NULL_TRIE_IDX 0xffff
117#define TRIE_NODE(t, i) ((t)->nodes[(i)])
118#define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
119#define TRIE_BYTE(t, i) ((t)->bytes[(i)])
39236c6e 120
0a7de745 121static struct flow_divert_pcb nil_pcb;
39236c6e
A		 122
123decl_lck_rw_data(static, g_flow_divert_group_lck);
0a7de745
A		 124static struct flow_divert_group **g_flow_divert_groups = NULL;
125static uint32_t g_active_group_count = 0;
39236c6e 126
0a7de745
A		 127static lck_grp_attr_t *flow_divert_grp_attr = NULL;
128static lck_attr_t *flow_divert_mtx_attr = NULL;
129static lck_grp_t *flow_divert_mtx_grp = NULL;
130static errno_t g_init_result = 0;
39236c6e 131
0a7de745 132static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
39236c6e 133
0a7de745
A		 134static struct protosw g_flow_divert_in_protosw;
135static struct pr_usrreqs g_flow_divert_in_usrreqs;
136static struct protosw g_flow_divert_in_udp_protosw;
137static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
39236c6e 138#if INET6
0a7de745
A		 139static struct ip6protosw g_flow_divert_in6_protosw;
140static struct pr_usrreqs g_flow_divert_in6_usrreqs;
141static struct ip6protosw g_flow_divert_in6_udp_protosw;
142static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
143#endif /* INET6 */
39236c6e 144
0a7de745
A		 145static struct protosw *g_tcp_protosw = NULL;
146static struct ip6protosw *g_tcp6_protosw = NULL;
147static struct protosw *g_udp_protosw = NULL;
148static struct ip6protosw *g_udp6_protosw = NULL;
3e170ce0
A		 149
150static errno_t
151flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
152
153static errno_t
154flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
155
156static boolean_t
157flow_divert_is_sockaddr_valid(struct sockaddr *addr);
158
159static int
160flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
161
162struct sockaddr *
163flow_divert_get_buffered_target_address(mbuf_t buffer);
164
165static boolean_t
166flow_divert_has_pcb_local_address(const struct inpcb *inp);
167
168static void
169flow_divert_disconnect_socket(struct socket *so);
39236c6e 170
39037602
A		 171static inline uint8_t
172flow_divert_syslog_type_to_oslog_type(int syslog_type)
173{
174 switch (syslog_type) {
0a7de745
A		 175 case LOG_ERR: return OS_LOG_TYPE_ERROR;
176 case LOG_INFO: return OS_LOG_TYPE_INFO;
177 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
178 default: return OS_LOG_TYPE_DEFAULT;
39037602
A		 179 }
180}
181
39236c6e
A		 182static inline int
183flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
184{
185 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
186}
187
188RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
189RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
190
191static const char *
192flow_divert_packet_type2str(uint8_t packet_type)
193{
194 switch (packet_type) {
0a7de745
A		 195 case FLOW_DIVERT_PKT_CONNECT:
196 return "connect";
197 case FLOW_DIVERT_PKT_CONNECT_RESULT:
198 return "connect result";
199 case FLOW_DIVERT_PKT_DATA:
200 return "data";
201 case FLOW_DIVERT_PKT_CLOSE:
202 return "close";
203 case FLOW_DIVERT_PKT_READ_NOTIFY:
204 return "read notification";
205 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
206 return "properties update";
207 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
208 return "app map create";
209 default:
210 return "unknown";
39236c6e
A		 211 }
212}
213
214static struct flow_divert_pcb *
215flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
216{
0a7de745
A		 217 struct flow_divert_pcb key_item;
218 struct flow_divert_pcb *fd_cb = NULL;
39236c6e
A		 219
220 key_item.hash = hash;
221
222 lck_rw_lock_shared(&group->lck);
223 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
224 FDRETAIN(fd_cb);
225 lck_rw_done(&group->lck);
226
227 return fd_cb;
228}
229
230static errno_t
231flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
232{
0a7de745
A		 233 errno_t error = 0;
234 struct flow_divert_pcb *exist = NULL;
235 struct flow_divert_group *group;
236 static uint32_t g_nextkey = 1;
237 static uint32_t g_hash_seed = 0;
238 int try_count = 0;
39236c6e
A		 239
240 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
241 return EINVAL;
242 }
243
244 socket_unlock(fd_cb->so, 0);
245 lck_rw_lock_shared(&g_flow_divert_group_lck);
246
247 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
248 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
249 error = ENETUNREACH;
250 goto done;
251 }
252
253 group = g_flow_divert_groups[ctl_unit];
254 if (group == NULL) {
255 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
256 error = ENETUNREACH;
257 goto done;
258 }
259
260 socket_lock(fd_cb->so, 0);
261
262 do {
0a7de745
A		 263 uint32_t key[2];
264 uint32_t idx;
39236c6e
A		 265
266 key[0] = g_nextkey++;
267 key[1] = RandomULong();
268
269 if (g_hash_seed == 0) {
270 g_hash_seed = RandomULong();
271 }
272
273 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
274
275 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
276 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
277 if (curr_group != NULL && curr_group != group) {
278 lck_rw_lock_shared(&curr_group->lck);
279 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
280 lck_rw_done(&curr_group->lck);
281 if (exist != NULL) {
282 break;
283 }
284 }
285 }
286
287 if (exist == NULL) {
288 lck_rw_lock_exclusive(&group->lck);
289 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
290 lck_rw_done(&group->lck);
291 }
292 } while (exist != NULL && try_count++ < 3);
293
294 if (exist == NULL) {
295 fd_cb->group = group;
0a7de745 296 FDRETAIN(fd_cb); /* The group now has a reference */
39236c6e
A		 297 } else {
298 fd_cb->hash = 0;
3e170ce0 299 error = EEXIST;
39236c6e
A		 300 }
301
302 socket_unlock(fd_cb->so, 0);
303
304done:
305 lck_rw_done(&g_flow_divert_group_lck);
306 socket_lock(fd_cb->so, 0);
307
3e170ce0 308 return error;
39236c6e
A		 309}
310
311static struct flow_divert_pcb *
312flow_divert_pcb_create(socket_t so)
313{
0a7de745 314 struct flow_divert_pcb *new_pcb = NULL;
39236c6e
A		 315
316 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
317 if (new_pcb == NULL) {
318 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
319 return NULL;
320 }
321
322 memset(new_pcb, 0, sizeof(*new_pcb));
323
324 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
325 new_pcb->so = so;
326 new_pcb->log_level = nil_pcb.log_level;
327
0a7de745 328 FDRETAIN(new_pcb); /* Represents the socket's reference */
39236c6e
A		 329
330 return new_pcb;
331}
332
333static void
334flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
335{
336 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
0a7de745 337 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
39236c6e
A		 338
339 if (fd_cb->local_address != NULL) {
340 FREE(fd_cb->local_address, M_SONAME);
341 }
342 if (fd_cb->remote_address != NULL) {
343 FREE(fd_cb->remote_address, M_SONAME);
344 }
345 if (fd_cb->connect_token != NULL) {
346 mbuf_freem(fd_cb->connect_token);
347 }
490019cf
A		 348 if (fd_cb->connect_packet != NULL) {
349 mbuf_freem(fd_cb->connect_packet);
350 }
351 if (fd_cb->app_data != NULL) {
352 FREE(fd_cb->app_data, M_TEMP);
353 }
39236c6e
A		 354 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
355}
356
357static void
358flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
359{
360 if (fd_cb->group != NULL) {
361 struct flow_divert_group *group = fd_cb->group;
362 lck_rw_lock_exclusive(&group->lck);
363 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
364 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
365 fd_cb->group = NULL;
0a7de745 366 FDRELEASE(fd_cb); /* Release the group's reference */
39236c6e
A		 367 lck_rw_done(&group->lck);
368 }
369}
370
371static int
372flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
373{
0a7de745
A		 374 struct flow_divert_packet_header hdr;
375 int error = 0;
39236c6e
A		 376
377 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
378 if (error) {
379 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
380 return error;
381 }
382
383 hdr.packet_type = packet_type;
384 hdr.conn_id = htonl(fd_cb->hash);
385
386 /* Lay down the header */
387 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
388 if (error) {
389 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
390 mbuf_freem(*packet);
391 *packet = NULL;
392 return error;
393 }
394
395 return 0;
396}
397
398static int
3e170ce0 399flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
39236c6e 400{
0a7de745
A		 401 uint32_t net_length = htonl(length);
402 int error = 0;
39236c6e
A		 403
404 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
405 if (error) {
406 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
407 return error;
408 }
409
410 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
411 if (error) {
39037602 412 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
39236c6e
A		 413 return error;
414 }
415
416 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
417 if (error) {
418 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
419 return error;
420 }
421
422 return error;
423}
424
425static int
426flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
427{
0a7de745
A		 428 size_t cursor = offset;
429 int error = 0;
430 uint32_t curr_length;
431 uint8_t curr_type;
39236c6e
A		 432
433 *err = 0;
434
435 do {
436 if (!next) {
437 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
438 if (error) {
439 *err = ENOENT;
440 return -1;
441 }
442 } else {
443 next = 0;
444 curr_type = FLOW_DIVERT_TLV_NIL;
445 }
446
447 if (curr_type != type) {
448 cursor += sizeof(curr_type);
449 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
450 if (error) {
451 *err = error;
452 return -1;
453 }
454
455 cursor += (sizeof(curr_length) + ntohl(curr_length));
456 }
457 } while (curr_type != type);
458
459 return cursor;
460}
461
462static int
3e170ce0 463flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
39236c6e 464{
0a7de745
A		 465 int error = 0;
466 uint32_t length;
467 int tlv_offset;
39236c6e
A		 468
469 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
470 if (tlv_offset < 0) {
471 return error;
472 }
473
474 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
475 if (error) {
476 return error;
477 }
478
479 length = ntohl(length);
480
cb323159
A		 481 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
482
483 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
484 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
485 return EINVAL;
486 }
487
39236c6e
A		 488 if (val_size != NULL) {
489 *val_size = length;
490 }
491
492 if (buff != NULL && buff_len > 0) {
cb323159 493 memset(buff, 0, buff_len);
39236c6e 494 size_t to_copy = (length < buff_len) ? length : buff_len;
cb323159 495 error = mbuf_copydata(packet, data_offset, to_copy, buff);
39236c6e
A		 496 if (error) {
497 return error;
498 }
499 }
500
501 return 0;
502}
503
504static int
505flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
506{
0a7de745 507 mbuf_t curr_mbuf = packet;
39236c6e
A		 508
509 if (g_crypto_funcs == NULL || group->token_key == NULL) {
510 return ENOPROTOOPT;
511 }
512
513 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
514 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
515
516 while (curr_mbuf != NULL) {
517 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
518 curr_mbuf = mbuf_next(curr_mbuf);
519 }
520
521 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
522
523 return 0;
524}
525
526static int
527flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
528{
0a7de745
A		 529 int error = 0;
530 struct flow_divert_group *group = NULL;
531 int hmac_offset;
532 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
533 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
534 mbuf_t tail;
39236c6e
A		 535
536 lck_rw_lock_shared(&g_flow_divert_group_lck);
537
538 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
539 group = g_flow_divert_groups[ctl_unit];
540 }
541
542 if (group == NULL) {
543 lck_rw_done(&g_flow_divert_group_lck);
544 return ENOPROTOOPT;
545 }
546
547 lck_rw_lock_shared(&group->lck);
548
549 if (group->token_key == NULL) {
550 error = ENOPROTOOPT;
551 goto done;
552 }
553
554 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
555 if (hmac_offset < 0) {
556 goto done;
557 }
558
559 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
560 if (error) {
561 goto done;
562 }
563
564 /* Chop off the HMAC TLV */
565 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
566 if (error) {
567 goto done;
568 }
569
570 mbuf_free(tail);
571
572 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
573 if (error) {
574 goto done;
575 }
576
cb323159 577 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
39236c6e
A		 578 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
579 error = EINVAL;
580 goto done;
581 }
582
583done:
584 lck_rw_done(&group->lck);
585 lck_rw_done(&g_flow_divert_group_lck);
586 return error;
587}
588
589static void
590flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
591{
592 struct inpcb *inp = NULL;
593 struct ifnet *ifp = NULL;
594 Boolean cell = FALSE;
595 Boolean wifi = FALSE;
fe8ab488 596 Boolean wired = FALSE;
0a7de745 597
39236c6e
A		 598 inp = sotoinpcb(fd_cb->so);
599 if (inp == NULL) {
600 return;
601 }
602
603 ifp = inp->inp_last_outifp;
604 if (ifp != NULL) {
605 cell = IFNET_IS_CELLULAR(ifp);
606 wifi = (!cell && IFNET_IS_WIFI(ifp));
fe8ab488 607 wired = (!wifi && IFNET_IS_WIRED(ifp));
39236c6e 608 }
0a7de745 609
39236c6e 610 if (send) {
fe8ab488
A		 611 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
612 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
39236c6e 613 } else {
fe8ab488
A		 614 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
615 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
39236c6e 616 }
5ba3f43e 617 inp_set_activity_bitmap(inp);
39236c6e
A		 618}
619
620static errno_t
621flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
622{
623 struct inpcb *inp = NULL;
39236c6e
A		 624
625 inp = sotoinpcb(fd_cb->so);
fe8ab488 626 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
0a7de745 627 IFNET_IS_CELLULAR(inp->inp_last_outifp)) {
fe8ab488 628 return EHOSTUNREACH;
0a7de745
A		 629 }
630
fe8ab488
A		 631 return 0;
632}
633
634static errno_t
635flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
636{
637 struct inpcb *inp = NULL;
638
639 inp = sotoinpcb(fd_cb->so);
640 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
0a7de745 641 IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) {
fe8ab488 642 return EHOSTUNREACH;
0a7de745
A		 643 }
644
39236c6e
A		 645 return 0;
646}
647
cb323159
A		 648static errno_t
649flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
650{
651 struct inpcb *inp = NULL;
652
653 inp = sotoinpcb(fd_cb->so);
654 if (inp && INP_NO_CONSTRAINED(inp) && inp->inp_last_outifp &&
655 IFNET_IS_CONSTRAINED(inp->inp_last_outifp)) {
656 return EHOSTUNREACH;
657 }
658
659 return 0;
660}
661
39236c6e
A		 662static void
663flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
664{
665 if (how != SHUT_RD) {
666 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
667 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
668 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
669 /* If the tunnel is not accepting writes any more, then flush the send buffer */
670 sbflush(&fd_cb->so->so_snd);
671 }
672 }
673 if (how != SHUT_WR) {
674 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
675 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
676 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
677 }
678 }
679}
680
681static uint16_t
682trie_node_alloc(struct flow_divert_trie *trie)
683{
684 if (trie->nodes_free_next < trie->nodes_count) {
685 uint16_t node_idx = trie->nodes_free_next++;
686 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
687 return node_idx;
688 } else {
689 return NULL_TRIE_IDX;
690 }
691}
692
693static uint16_t
694trie_child_map_alloc(struct flow_divert_trie *trie)
695{
696 if (trie->child_maps_free_next < trie->child_maps_count) {
697 return trie->child_maps_free_next++;
698 } else {
699 return NULL_TRIE_IDX;
700 }
701}
702
703static uint16_t
704trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
705{
706 uint16_t start = trie->bytes_free_next;
707 if (start + bytes_size <= trie->bytes_count) {
708 if (start != bytes_idx) {
709 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
710 }
711 trie->bytes_free_next += bytes_size;
712 return start;
713 } else {
714 return NULL_TRIE_IDX;
715 }
716}
717
718static uint16_t
719flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
720{
721 uint16_t current = trie->root;
722 uint16_t child = trie->root;
723 uint16_t string_end = string_start + string_len;
724 uint16_t string_idx = string_start;
725 uint16_t string_remainder = string_len;
726
727 while (child != NULL_TRIE_IDX) {
728 uint16_t parent = current;
729 uint16_t node_idx;
730 uint16_t current_end;
731
732 current = child;
733 child = NULL_TRIE_IDX;
734
735 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
736
737 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 738 node_idx < current_end &&
739 string_idx < string_end &&
740 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
741 node_idx++, string_idx++) {
742 ;
743 }
39236c6e
A		 744
745 string_remainder = string_end - string_idx;
746
747 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
748 /*
749 * We did not reach the end of the current node's string.
750 * We need to split the current node into two:
751 * 1. A new node that contains the prefix of the node that matches
752 * the prefix of the string being inserted.
753 * 2. The current node modified to point to the remainder
754 * of the current node's string.
755 */
756 uint16_t prefix = trie_node_alloc(trie);
757 if (prefix == NULL_TRIE_IDX) {
758 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
759 return NULL_TRIE_IDX;
760 }
761
762 /*
763 * Prefix points to the portion of the current nodes's string that has matched
764 * the input string thus far.
765 */
766 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
767 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
768
769 /*
770 * Prefix has the current node as the child corresponding to the first byte
771 * after the split.
772 */
773 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
774 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
775 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
776 return NULL_TRIE_IDX;
777 }
778 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
779
780 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
781 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
782
783 /* Current node is adjusted to point to the remainder */
784 TRIE_NODE(trie, current).start = node_idx;
785 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
786
787 /* We want to insert the new leaf (if any) as a child of the prefix */
788 current = prefix;
789 }
790
791 if (string_remainder > 0) {
792 /*
793 * We still have bytes in the string that have not been matched yet.
794 * If the current node has children, iterate to the child corresponding
795 * to the next byte in the string.
796 */
797 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
798 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
799 }
800 }
801 } /* while (child != NULL_TRIE_IDX) */
802
803 if (string_remainder > 0) {
804 /* Add a new leaf containing the remainder of the string */
805 uint16_t leaf = trie_node_alloc(trie);
806 if (leaf == NULL_TRIE_IDX) {
807 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
808 return NULL_TRIE_IDX;
809 }
810
811 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
812 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
813 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
814 return NULL_TRIE_IDX;
815 }
816 TRIE_NODE(trie, leaf).length = string_remainder;
817
818 /* Set the new leaf as the child of the current node */
819 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
820 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
821 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
822 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
823 return NULL_TRIE_IDX;
824 }
825 }
826 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
827 current = leaf;
828 } /* else duplicate or this string is a prefix of one of the existing strings */
829
830 return current;
831}
832
0a7de745 833#define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
39236c6e 834static uint16_t
3e170ce0 835flow_divert_trie_search(struct flow_divert_trie *trie, uint8_t *string_bytes)
39236c6e
A		 836{
837 uint16_t current = trie->root;
838 uint16_t string_idx = 0;
839
840 while (current != NULL_TRIE_IDX) {
841 uint16_t next = NULL_TRIE_IDX;
842 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
843 uint16_t node_idx;
844
845 for (node_idx = TRIE_NODE(trie, current).start;
0a7de745
A		 846 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
847 node_idx++, string_idx++) {
848 ;
849 }
39236c6e
A		 850
851 if (node_idx == node_end) {
852 if (string_bytes[string_idx] == '\0') {
853 return current; /* Got an exact match */
3e170ce0 854 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
0a7de745
A		 855 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
856 string_bytes[string_idx] = '\0';
3e170ce0 857 return current; /* Got an apple webclip id prefix match */
39236c6e
A		 858 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
859 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
860 }
861 }
862 current = next;
863 }
864
865 return NULL_TRIE_IDX;
866}
867
490019cf
A		 868struct uuid_search_info {
869 uuid_t target_uuid;
870 char *found_signing_id;
871 boolean_t found_multiple_signing_ids;
872 proc_t found_proc;
873};
874
875static int
876flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
877{
878 struct uuid_search_info *info = (struct uuid_search_info *)arg;
879 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
880
881 if (info->found_signing_id != NULL) {
882 if (!info->found_multiple_signing_ids) {
883 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
884 info->found_proc = p;
885 result = PROC_CLAIMED_DONE;
886 } else {
887 uuid_string_t uuid_str;
888 uuid_unparse(info->target_uuid, uuid_str);
889 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
890 }
891 FREE(info->found_signing_id, M_TEMP);
892 info->found_signing_id = NULL;
893 }
894
895 if (result == PROC_RETURNED_DONE) {
896 uuid_string_t uuid_str;
897 uuid_unparse(info->target_uuid, uuid_str);
898 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
899 }
900
901 return result;
902}
903
904static int
905flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
906{
907 struct uuid_search_info *info = (struct uuid_search_info *)arg;
908 int include = 0;
909
910 if (info->found_multiple_signing_ids) {
911 return include;
912 }
913
914 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
915 if (include) {
916 const char *signing_id = cs_identity_get(p);
917 if (signing_id != NULL) {
918 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
919 size_t signing_id_size = strlen(signing_id) + 1;
920 if (info->found_signing_id == NULL) {
921 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
922 memcpy(info->found_signing_id, signing_id, signing_id_size);
923 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
924 info->found_multiple_signing_ids = TRUE;
925 }
926 } else {
927 info->found_multiple_signing_ids = TRUE;
928 }
929 include = !info->found_multiple_signing_ids;
930 }
931
932 return include;
933}
934
935static proc_t
936flow_divert_find_proc_by_uuid(uuid_t uuid)
937{
938 struct uuid_search_info info;
939
940 if (LOG_INFO <= nil_pcb.log_level) {
941 uuid_string_t uuid_str;
942 uuid_unparse(uuid, uuid_str);
943 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
944 }
945
946 memset(&info, 0, sizeof(info));
947 info.found_proc = PROC_NULL;
948 uuid_copy(info.target_uuid, uuid);
949
950 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
951
952 return info.found_proc;
953}
954
39236c6e 955static int
490019cf 956flow_divert_get_src_proc(struct socket *so, proc_t *proc)
39236c6e
A		 957{
958 int release = 0;
959
490019cf
A		 960 if (so->so_flags & SOF_DELEGATED) {
961 if ((*proc)->p_pid != so->e_pid) {
962 *proc = proc_find(so->e_pid);
963 release = 1;
964 } else if (uuid_compare((*proc)->p_uuid, so->e_uuid)) {
965 *proc = flow_divert_find_proc_by_uuid(so->e_uuid);
966 release = 1;
967 }
39236c6e
A		 968 } else if (*proc == PROC_NULL) {
969 *proc = current_proc();
970 }
971
972 if (*proc != PROC_NULL) {
973 if ((*proc)->p_pid == 0) {
974 if (release) {
975 proc_rele(*proc);
976 }
977 release = 0;
978 *proc = PROC_NULL;
979 }
980 }
981
982 return release;
983}
984
985static int
986flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
987{
0a7de745 988 int error;
39236c6e
A		 989
990 if (fd_cb->group == NULL) {
991 fd_cb->so->so_error = ECONNABORTED;
3e170ce0 992 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 993 return ECONNABORTED;
994 }
995
996 lck_rw_lock_shared(&fd_cb->group->lck);
997
998 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
999 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1000 } else {
1001 error = ENOBUFS;
1002 }
1003
1004 if (error == ENOBUFS) {
1005 if (enqueue) {
1006 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1007 lck_rw_lock_exclusive(&fd_cb->group->lck);
1008 }
1009 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1010 error = 0;
1011 }
1012 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1013 }
1014
1015 lck_rw_done(&fd_cb->group->lck);
1016
1017 return error;
1018}
1019
1020static int
490019cf 1021flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
39236c6e 1022{
0a7de745
A		 1023 int error = 0;
1024 int flow_type = 0;
1025 char *signing_id = NULL;
1026 int free_signing_id = 0;
1027 mbuf_t connect_packet = NULL;
1028 proc_t src_proc = p;
1029 int release_proc = 0;
490019cf
A		 1030
1031 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1032 if (error) {
1033 goto done;
1034 }
1035
1036 error = EPERM;
1037
1038 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1039 uint32_t sid_size = 0;
1040 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1041 if (find_error == 0 && sid_size > 0) {
1042 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1043 if (signing_id != NULL) {
1044 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1045 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1046 free_signing_id = 1;
1047 }
1048 }
1049 }
1050
1051 socket_unlock(so, 0);
39037602 1052
cb323159
A		 1053 release_proc = flow_divert_get_src_proc(so, &src_proc);
1054 if (src_proc != PROC_NULL) {
1055 proc_lock(src_proc);
1056 if (signing_id == NULL) {
0a7de745 1057 if (src_proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
39037602
A		 1058 const char * cs_id;
1059 cs_id = cs_identity_get(src_proc);
1060 signing_id = __DECONST(char *, cs_id);
490019cf 1061 } else {
39037602 1062 FDLOG0(LOG_WARNING, fd_cb, "Signature is invalid");
490019cf 1063 }
490019cf 1064 }
39037602 1065 } else {
cb323159 1066 FDLOG0(LOG_WARNING, fd_cb, "Failed to determine the current proc");
39037602 1067 }
490019cf 1068
39037602
A		 1069 if (signing_id != NULL) {
1070 uint16_t result = NULL_TRIE_IDX;
1071 lck_rw_lock_shared(&fd_cb->group->lck);
5ba3f43e
A		 1072 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1073 result = 1;
1074 } else {
1075 result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (uint8_t *)signing_id);
1076 }
39037602
A		 1077 lck_rw_done(&fd_cb->group->lck);
1078 if (result != NULL_TRIE_IDX) {
1079 error = 0;
1080 FDLOG(LOG_INFO, fd_cb, "%s matched", signing_id);
490019cf 1081
39037602
A		 1082 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_SIGNING_ID, strlen(signing_id), signing_id);
1083 if (error == 0) {
1084 if (src_proc != PROC_NULL) {
1085 unsigned char cdhash[SHA1_RESULTLEN];
1086 error = proc_getcdhash(src_proc, cdhash);
1087 if (error == 0) {
1088 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CDHASH, sizeof(cdhash), cdhash);
1089 if (error) {
1090 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
490019cf 1091 }
39037602
A		 1092 } else {
1093 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", error);
490019cf 1094 }
490019cf
A		 1095 }
1096 } else {
39037602 1097 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
490019cf
A		 1098 }
1099 } else {
39037602 1100 FDLOG(LOG_WARNING, fd_cb, "%s did not match", signing_id);
490019cf 1101 }
39037602
A		 1102 } else {
1103 FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
5c9f4661
A		 1104 if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
1105 error = 0;
1106 }
39037602 1107 }
490019cf 1108
cb323159
A		 1109 if (error == 0 && src_proc != PROC_NULL) {
1110 task_t task = proc_task(src_proc);
1111 if (task != TASK_NULL) {
1112 audit_token_t audit_token;
1113 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1114 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1115 if (rc == KERN_SUCCESS) {
1116 error = flow_divert_packet_append_tlv(connect_packet,
1117 FLOW_DIVERT_TLV_APP_AUDIT_TOKEN,
1118 sizeof(audit_token_t),
1119 &audit_token);
1120 if (error) {
1121 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", error);
1122 error = 0; /* do not treat this as fatal error, proceed */
1123 }
1124 } else {
1125 FDLOG(LOG_ERR, fd_cb, "failed to retrieve app audit token: %d", rc);
1126 }
1127 }
1128 }
1129
39037602
A		 1130 if (src_proc != PROC_NULL) {
1131 proc_unlock(src_proc);
1132 if (release_proc) {
1133 proc_rele(src_proc);
490019cf 1134 }
490019cf
A		 1135 }
1136 socket_lock(so, 0);
1137
1138 if (free_signing_id) {
1139 FREE(signing_id, M_TEMP);
1140 }
1141
1142 if (error) {
1143 goto done;
1144 }
39236c6e 1145
39236c6e 1146 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1147 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1148 sizeof(fd_cb->so->so_traffic_class),
1149 &fd_cb->so->so_traffic_class);
39236c6e
A		 1150 if (error) {
1151 goto done;
1152 }
1153
3e170ce0
A		 1154 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1155 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1156 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1157 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1158 } else {
1159 error = EINVAL;
1160 goto done;
1161 }
1162 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1163 FLOW_DIVERT_TLV_FLOW_TYPE,
1164 sizeof(flow_type),
1165 &flow_type);
3e170ce0
A		 1166
1167 if (error) {
1168 goto done;
1169 }
1170
39236c6e
A		 1171 if (fd_cb->so->so_flags & SOF_DELEGATED) {
1172 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1173 FLOW_DIVERT_TLV_PID,
1174 sizeof(fd_cb->so->e_pid),
1175 &fd_cb->so->e_pid);
39236c6e
A		 1176 if (error) {
1177 goto done;
1178 }
1179
1180 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1181 FLOW_DIVERT_TLV_UUID,
1182 sizeof(fd_cb->so->e_uuid),
1183 &fd_cb->so->e_uuid);
39236c6e
A		 1184 if (error) {
1185 goto done;
1186 }
1187 } else {
1188 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1189 FLOW_DIVERT_TLV_PID,
1190 sizeof(fd_cb->so->e_pid),
1191 &fd_cb->so->last_pid);
39236c6e
A		 1192 if (error) {
1193 goto done;
1194 }
1195
1196 error = flow_divert_packet_append_tlv(connect_packet,
0a7de745
A		 1197 FLOW_DIVERT_TLV_UUID,
1198 sizeof(fd_cb->so->e_uuid),
1199 &fd_cb->so->last_uuid);
39236c6e
A		 1200 if (error) {
1201 goto done;
1202 }
1203 }
1204
1205 if (fd_cb->connect_token != NULL) {
1206 unsigned int token_len = m_length(fd_cb->connect_token);
1207 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1208 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1209 fd_cb->connect_token = NULL;
1210 } else {
1211 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
39236c6e
A		 1212
1213 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1214 if (error) {
1215 goto done;
1216 }
1217
3e170ce0 1218 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
39236c6e
A		 1219 if (error) {
1220 goto done;
1221 }
39236c6e
A		 1222 }
1223
490019cf
A		 1224 if (fd_cb->local_address != NULL) {
1225 error = EALREADY;
1226 goto done;
1227 } else {
1228 struct inpcb *inp = sotoinpcb(so);
1229 if (flow_divert_has_pcb_local_address(inp)) {
1230 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1231 if (error) {
1232 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1233 goto done;
1234 }
1235 }
1236 }
1237
3e170ce0
A		 1238 if (fd_cb->local_address != NULL) {
1239 /* socket is bound. */
490019cf 1240 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
0a7de745 1241 fd_cb->local_address->sa_len, fd_cb->local_address);
490019cf
A		 1242 if (error) {
1243 goto done;
1244 }
1245 }
3e170ce0 1246
490019cf
A		 1247 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1248 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1249 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1250 if (error) {
1251 goto done;
1252 }
39236c6e
A		 1253 }
1254
1255done:
490019cf
A		 1256 if (!error) {
1257 *out_connect_packet = connect_packet;
1258 } else if (connect_packet != NULL) {
1259 mbuf_freem(connect_packet);
1260 }
1261
39236c6e
A		 1262 return error;
1263}
1264
1265static int
1266flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1267{
0a7de745
A		 1268 int error = 0;
1269 mbuf_t packet = NULL;
1270 int rbuff_space = 0;
39236c6e
A		 1271
1272 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1273 if (error) {
1274 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1275 goto done;
1276 }
1277
3e170ce0 1278 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
39236c6e
A		 1279 if (rbuff_space < 0) {
1280 rbuff_space = 0;
1281 }
1282 rbuff_space = htonl(rbuff_space);
1283 error = flow_divert_packet_append_tlv(packet,
0a7de745
A		 1284 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1285 sizeof(rbuff_space),
1286 &rbuff_space);
39236c6e
A		 1287 if (error) {
1288 goto done;
1289 }
1290
1291 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1292 if (error) {
1293 goto done;
1294 }
1295
1296done:
1297 if (error && packet != NULL) {
3e170ce0 1298 mbuf_freem(packet);
39236c6e
A		 1299 }
1300
1301 return error;
1302}
1303
1304static int
1305flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1306{
0a7de745
A		 1307 int error = 0;
1308 mbuf_t packet = NULL;
1309 uint32_t zero = 0;
39236c6e
A		 1310
1311 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1312 if (error) {
1313 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1314 goto done;
1315 }
1316
1317 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1318 if (error) {
1319 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1320 goto done;
1321 }
1322
1323 how = htonl(how);
1324 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1325 if (error) {
1326 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1327 goto done;
1328 }
1329
1330 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1331 if (error) {
1332 goto done;
1333 }
1334
1335done:
1336 if (error && packet != NULL) {
1337 mbuf_free(packet);
1338 }
1339
1340 return error;
1341}
1342
1343static int
1344flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1345{
0a7de745
A		 1346 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1347 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
39236c6e
A		 1348 return SHUT_RDWR;
1349 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1350 return SHUT_RD;
1351 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1352 return SHUT_WR;
1353 }
1354
1355 return -1;
1356}
1357
1358/*
1359 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1360 * writes. Returns FALSE otherwise.
1361 */
1362static void
1363flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1364{
0a7de745 1365 int how = -1;
39236c6e
A		 1366
1367 /* Do not send any close messages if there is still data in the send buffer */
1368 if (fd_cb->so->so_snd.sb_cc == 0) {
0a7de745 1369 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
39236c6e
A		 1370 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1371 how = SHUT_RD;
1372 }
0a7de745 1373 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
39236c6e
A		 1374 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1375 if (how == SHUT_RD) {
1376 how = SHUT_RDWR;
1377 } else {
1378 how = SHUT_WR;
1379 }
1380 }
1381 }
1382
1383 if (how != -1) {
1384 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1385 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1386 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1387 if (how != SHUT_RD) {
1388 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1389 }
1390 if (how != SHUT_WR) {
1391 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1392 }
1393 }
1394 }
1395
1396 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
3e170ce0 1397 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1398 }
1399}
1400
1401static errno_t
3e170ce0 1402flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
39236c6e 1403{
0a7de745
A		 1404 mbuf_t packet;
1405 mbuf_t last;
1406 int error = 0;
39236c6e
A		 1407
1408 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1409 if (error) {
1410 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1411 return error;
1412 }
1413
3e170ce0
A		 1414 if (toaddr != NULL) {
1415 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1416 if (error) {
1417 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1418 return error;
1419 }
1420 }
1421
5ba3f43e
A		 1422 if (data_len > 0 && data != NULL) {
1423 last = m_last(packet);
1424 mbuf_setnext(last, data);
1425 mbuf_pkthdr_adjustlen(packet, data_len);
1426 }
39236c6e
A		 1427 error = flow_divert_send_packet(fd_cb, packet, force);
1428
1429 if (error) {
1430 mbuf_setnext(last, NULL);
3e170ce0 1431 mbuf_freem(packet);
39236c6e
A		 1432 } else {
1433 fd_cb->bytes_sent += data_len;
1434 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1435 }
1436
1437 return error;
1438}
1439
1440static void
1441flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1442{
0a7de745
A		 1443 size_t to_send;
1444 size_t sent = 0;
1445 int error = 0;
1446 mbuf_t buffer;
39236c6e
A		 1447
1448 to_send = fd_cb->so->so_snd.sb_cc;
1449 buffer = fd_cb->so->so_snd.sb_mb;
1450
1451 if (buffer == NULL && to_send > 0) {
1452 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1453 return;
1454 }
1455
1456 /* Ignore the send window if force is enabled */
1457 if (!force && (to_send > fd_cb->send_window)) {
1458 to_send = fd_cb->send_window;
1459 }
1460
3e170ce0
A		 1461 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1462 while (sent < to_send) {
0a7de745
A		 1463 mbuf_t data;
1464 size_t data_len;
39236c6e 1465
3e170ce0
A		 1466 data_len = to_send - sent;
1467 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1468 data_len = FLOW_DIVERT_CHUNK_SIZE;
1469 }
39236c6e 1470
3e170ce0
A		 1471 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1472 if (error) {
1473 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1474 break;
1475 }
39236c6e 1476
3e170ce0
A		 1477 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1478 if (error) {
1479 mbuf_freem(data);
1480 break;
1481 }
39236c6e 1482
3e170ce0
A		 1483 sent += data_len;
1484 }
1485 sbdrop(&fd_cb->so->so_snd, sent);
1486 sowwakeup(fd_cb->so);
1487 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1488 mbuf_t data;
1489 mbuf_t m;
1490 size_t data_len;
1491
0a7de745 1492 while (buffer) {
3e170ce0
A		 1493 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1494
1495 m = buffer;
1496 if (toaddr != NULL) {
1497 /* look for data in the chain */
1498 do {
1499 m = m->m_next;
1500 if (m != NULL && m->m_type == MT_DATA) {
1501 break;
1502 }
0a7de745 1503 } while (m);
3e170ce0
A		 1504 if (m == NULL) {
1505 /* unexpected */
1506 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1507 goto move_on;
1508 }
1509 }
1510 data_len = mbuf_pkthdr_len(m);
5ba3f43e
A		 1511 if (data_len > 0) {
1512 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1513 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1514 if (error) {
1515 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1516 break;
1517 }
1518 } else {
1519 data = NULL;
3e170ce0
A		 1520 }
1521 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1522 if (error) {
1523 mbuf_freem(data);
1524 break;
1525 }
1526 sent += data_len;
1527move_on:
1528 buffer = buffer->m_nextpkt;
1529 (void) sbdroprecord(&(fd_cb->so->so_snd));
1530 }
39236c6e
A		 1531 }
1532
1533 if (sent > 0) {
1534 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1535 if (fd_cb->send_window >= sent) {
1536 fd_cb->send_window -= sent;
1537 } else {
1538 fd_cb->send_window = 0;
1539 }
39236c6e
A		 1540 }
1541}
1542
1543static int
3e170ce0 1544flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
39236c6e 1545{
0a7de745
A		 1546 size_t to_send = mbuf_pkthdr_len(data);
1547 int error = 0;
39236c6e
A		 1548
1549 if (to_send > fd_cb->send_window) {
1550 to_send = fd_cb->send_window;
1551 }
1552
1553 if (fd_cb->so->so_snd.sb_cc > 0) {
0a7de745 1554 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
39236c6e
A		 1555 }
1556
3e170ce0 1557 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
0a7de745
A		 1558 size_t sent = 0;
1559 mbuf_t remaining_data = data;
1560 mbuf_t pkt_data = NULL;
490019cf 1561 while (sent < to_send && remaining_data != NULL) {
0a7de745 1562 size_t pkt_data_len;
3e170ce0
A		 1563
1564 pkt_data = remaining_data;
39236c6e 1565
3e170ce0
A		 1566 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1567 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1568 } else {
1569 pkt_data_len = to_send - sent;
1570 }
1571
1572 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1573 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1574 if (error) {
1575 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1576 pkt_data = NULL;
1577 break;
1578 }
1579 } else {
1580 remaining_data = NULL;
1581 }
1582
1583 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
39236c6e 1584
39236c6e 1585 if (error) {
39236c6e
A		 1586 break;
1587 }
39236c6e 1588
3e170ce0
A		 1589 pkt_data = NULL;
1590 sent += pkt_data_len;
39236c6e
A		 1591 }
1592
3e170ce0 1593 fd_cb->send_window -= sent;
39236c6e 1594
3e170ce0 1595 error = 0;
39236c6e 1596
3e170ce0
A		 1597 if (pkt_data != NULL) {
1598 if (sbspace(&fd_cb->so->so_snd) > 0) {
1599 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1600 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
0a7de745 1601 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1602 }
1603 } else {
1604 error = ENOBUFS;
39236c6e 1605 }
39236c6e 1606 }
39236c6e 1607
3e170ce0
A		 1608 if (remaining_data != NULL) {
1609 if (sbspace(&fd_cb->so->so_snd) > 0) {
1610 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1611 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
0a7de745 1612 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
3e170ce0
A		 1613 }
1614 } else {
1615 error = ENOBUFS;
1616 }
1617 }
1618 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
5ba3f43e 1619 if (to_send || mbuf_pkthdr_len(data) == 0) {
3e170ce0
A		 1620 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1621 if (error) {
39037602 1622 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
3e170ce0
A		 1623 } else {
1624 fd_cb->send_window -= to_send;
39236c6e
A		 1625 }
1626 } else {
3e170ce0
A		 1627 /* buffer it */
1628 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1629 if (toaddr != NULL) {
1630 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1631 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1632 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1633 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0
A		 1634 }
1635 } else {
1636 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1637 FDLOG(LOG_ERR, fd_cb,
0a7de745
A		 1638 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1639 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
3e170ce0
A		 1640 }
1641 }
1642 } else {
1643 error = ENOBUFS;
1644 }
39236c6e
A		 1645 }
1646 }
1647
1648 return error;
1649}
1650
1651static int
1652flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1653{
0a7de745
A		 1654 int error = 0;
1655 mbuf_t packet = NULL;
1656 uint32_t net_read_count = htonl(read_count);
39236c6e
A		 1657
1658 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1659 if (error) {
1660 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1661 goto done;
1662 }
1663
1664 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1665 if (error) {
1666 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1667 goto done;
1668 }
1669
1670 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1671 if (error) {
1672 goto done;
1673 }
1674
1675done:
1676 if (error && packet != NULL) {
1677 mbuf_free(packet);
1678 }
1679
1680 return error;
1681}
1682
1683static int
1684flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1685{
0a7de745
A		 1686 int error = 0;
1687 mbuf_t packet = NULL;
39236c6e
A		 1688
1689 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1690 if (error) {
1691 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1692 goto done;
1693 }
1694
1695 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1696 if (error) {
1697 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1698 goto done;
1699 }
1700
1701 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1702 if (error) {
1703 goto done;
1704 }
1705
1706done:
1707 if (error && packet != NULL) {
1708 mbuf_free(packet);
1709 }
1710
1711 return error;
1712}
1713
1714static void
1715flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1716{
0a7de745
A		 1717 uint32_t connect_error;
1718 uint32_t ctl_unit = 0;
1719 int error = 0;
1720 struct flow_divert_group *grp = NULL;
1721 struct sockaddr_storage local_address;
1722 int out_if_index = 0;
1723 struct sockaddr_storage remote_address;
1724 uint32_t send_window;
1725 uint32_t app_data_length = 0;
39236c6e
A		 1726
1727 memset(&local_address, 0, sizeof(local_address));
1728 memset(&remote_address, 0, sizeof(remote_address));
1729
1730 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1731 if (error) {
1732 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1733 return;
1734 }
1735
1736 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1737
1738 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1739 if (error) {
1740 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1741 return;
1742 }
1743
1744 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1745 if (error) {
39037602 1746 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
39236c6e
A		 1747 }
1748
1749 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1750 if (error) {
39037602 1751 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
39236c6e
A		 1752 }
1753
1754 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1755 if (error) {
39037602 1756 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
39236c6e
A		 1757 }
1758
1759 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1760 if (error) {
39037602
A		 1761 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1762 }
1763
1764 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1765 if (error) {
1766 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
39236c6e
A		 1767 }
1768
39037602 1769 error = 0;
0a7de745
A		 1770 connect_error = ntohl(connect_error);
1771 ctl_unit = ntohl(ctl_unit);
39236c6e
A		 1772
1773 lck_rw_lock_shared(&g_flow_divert_group_lck);
1774
39037602
A		 1775 if (connect_error == 0 && ctl_unit > 0) {
1776 if (ctl_unit >= GROUP_COUNT_MAX) {
39236c6e
A		 1777 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1778 error = EINVAL;
1779 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1780 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1781 error = EINVAL;
1782 } else {
1783 grp = g_flow_divert_groups[ctl_unit];
1784 if (grp == NULL) {
1785 error = ECONNRESET;
1786 }
1787 }
1788 }
1789
1790 FDLOCK(fd_cb);
1791 if (fd_cb->so != NULL) {
0a7de745
A		 1792 struct inpcb *inp = NULL;
1793 struct ifnet *ifp = NULL;
1794 struct flow_divert_group *old_group;
39236c6e
A		 1795
1796 socket_lock(fd_cb->so, 0);
1797
1798 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1799 goto done;
1800 }
1801
1802 inp = sotoinpcb(fd_cb->so);
1803
1804 if (connect_error || error) {
1805 goto set_socket_state;
1806 }
1807
3e170ce0
A		 1808 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1809 error = EINVAL;
1810 goto set_socket_state;
1811 }
1812 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
39236c6e
A		 1813 if (local_address.ss_len > sizeof(local_address)) {
1814 local_address.ss_len = sizeof(local_address);
1815 }
1816 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
39236c6e 1817 }
cb323159
A		 1818 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&local_address)) {
1819 if (inp->inp_vflag & INP_IPV4 && local_address.ss_family == AF_INET) {
1820 struct sockaddr_in *local_in_address = (struct sockaddr_in *)&local_address;
1821 inp->inp_lport = local_in_address->sin_port;
1822 memcpy(&inp->inp_laddr, &local_in_address->sin_addr, sizeof(struct in_addr));
1823 } else if (inp->inp_vflag & INP_IPV6 && local_address.ss_family == AF_INET6) {
1824 struct sockaddr_in6 *local_in6_address = (struct sockaddr_in6 *)&local_address;
1825 inp->inp_lport = local_in6_address->sin6_port;
1826 memcpy(&inp->in6p_laddr, &local_in6_address->sin6_addr, sizeof(struct in6_addr));
1827 }
1828 }
39236c6e
A		 1829
1830 if (remote_address.ss_family != 0) {
cb323159
A		 1831 if (fd_cb->remote_address != NULL) {
1832 FREE(fd_cb->remote_address, M_SONAME);
1833 fd_cb->remote_address = NULL;
1834 }
39236c6e
A		 1835 if (remote_address.ss_len > sizeof(remote_address)) {
1836 remote_address.ss_len = sizeof(remote_address);
1837 }
1838 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
cb323159
A		 1839 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1840 if (inp->inp_vflag & INP_IPV4 && remote_address.ss_family == AF_INET) {
1841 struct sockaddr_in *remote_in_address = (struct sockaddr_in *)&remote_address;
1842 inp->inp_fport = remote_in_address->sin_port;
1843 memcpy(&inp->inp_faddr, &remote_in_address->sin_addr, sizeof(struct in_addr));
1844 } else if (inp->inp_vflag & INP_IPV6 && remote_address.ss_family == AF_INET6) {
1845 struct sockaddr_in6 *remote_in6_address = (struct sockaddr_in6 *)&remote_address;
1846 inp->inp_fport = remote_in6_address->sin6_port;
1847 memcpy(&inp->in6p_faddr, &remote_in6_address->sin6_addr, sizeof(struct in6_addr));
1848 }
1849 }
39236c6e
A		 1850 } else {
1851 error = EINVAL;
1852 goto set_socket_state;
1853 }
1854
39037602 1855 if (app_data_length > 0) {
0a7de745 1856 uint8_t *app_data = NULL;
39037602
A		 1857 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1858 if (app_data != NULL) {
1859 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1860 if (error == 0) {
1861 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1862 if (fd_cb->app_data != NULL) {
1863 FREE(fd_cb->app_data, M_TEMP);
1864 }
1865 fd_cb->app_data = app_data;
1866 fd_cb->app_data_length = app_data_length;
1867 } else {
1868 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1869 FREE(app_data, M_TEMP);
1870 }
1871 } else {
1872 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1873 }
1874 }
1875
39236c6e
A		 1876 ifnet_head_lock_shared();
1877 if (out_if_index > 0 && out_if_index <= if_index) {
1878 ifp = ifindex2ifnet[out_if_index];
1879 }
1880
1881 if (ifp != NULL) {
1882 inp->inp_last_outifp = ifp;
1883 } else {
1884 error = EINVAL;
1885 }
1886 ifnet_head_done();
1887
1888 if (error) {
1889 goto set_socket_state;
1890 }
1891
1892 if (fd_cb->group == NULL) {
1893 error = EINVAL;
1894 goto set_socket_state;
1895 }
1896
39037602
A		 1897 if (grp != NULL) {
1898 old_group = fd_cb->group;
39236c6e 1899
39037602
A		 1900 lck_rw_lock_exclusive(&old_group->lck);
1901 lck_rw_lock_exclusive(&grp->lck);
39236c6e 1902
39037602
A		 1903 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1904 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1905 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1906 }
39236c6e 1907
39037602 1908 fd_cb->group = grp;
39236c6e 1909
39037602
A		 1910 lck_rw_done(&grp->lck);
1911 lck_rw_done(&old_group->lck);
1912 }
39236c6e
A		 1913
1914 fd_cb->send_window = ntohl(send_window);
39236c6e
A		 1915
1916set_socket_state:
1917 if (!connect_error && !error) {
1918 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1919 error = flow_divert_send_connect_result(fd_cb);
1920 }
1921
1922 if (connect_error || error) {
1923 if (!connect_error) {
1924 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1925 fd_cb->so->so_error = error;
1926 flow_divert_send_close_if_needed(fd_cb);
1927 } else {
1928 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1929 fd_cb->so->so_error = connect_error;
1930 }
3e170ce0 1931 flow_divert_disconnect_socket(fd_cb->so);
39236c6e 1932 } else {
cb323159
A		 1933#if NECP
1934 /* Update NECP client with connected five-tuple */
1935 if (!uuid_is_null(inp->necp_client_uuid)) {
1936 socket_unlock(fd_cb->so, 0);
1937 necp_client_assign_from_socket(fd_cb->so->last_pid, inp->necp_client_uuid, inp);
1938 socket_lock(fd_cb->so, 0);
1939 }
1940#endif /* NECP */
1941
490019cf 1942 flow_divert_send_buffered_data(fd_cb, FALSE);
39236c6e
A		 1943 soisconnected(fd_cb->so);
1944 }
1945
1946done:
1947 socket_unlock(fd_cb->so, 0);
1948 }
1949 FDUNLOCK(fd_cb);
1950
1951 lck_rw_done(&g_flow_divert_group_lck);
1952}
1953
1954static void
1955flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1956{
0a7de745
A		 1957 uint32_t close_error;
1958 int error = 0;
1959 int how;
39236c6e
A		 1960
1961 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
1962 if (error) {
1963 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
1964 return;
1965 }
1966
1967 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
1968 if (error) {
1969 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
1970 return;
1971 }
1972
1973 how = ntohl(how);
1974
1975 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
1976
1977 FDLOCK(fd_cb);
1978 if (fd_cb->so != NULL) {
1979 socket_lock(fd_cb->so, 0);
1980
1981 fd_cb->so->so_error = ntohl(close_error);
1982
1983 flow_divert_update_closed_state(fd_cb, how, TRUE);
0a7de745 1984
39236c6e
A		 1985 how = flow_divert_tunnel_how_closed(fd_cb);
1986 if (how == SHUT_RDWR) {
3e170ce0 1987 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 1988 } else if (how == SHUT_RD) {
1989 socantrcvmore(fd_cb->so);
1990 } else if (how == SHUT_WR) {
1991 socantsendmore(fd_cb->so);
1992 }
1993
1994 socket_unlock(fd_cb->so, 0);
1995 }
1996 FDUNLOCK(fd_cb);
1997}
1998
3e170ce0
A		 1999static mbuf_t
2000flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
39236c6e 2001{
cb323159
A		 2002 struct inpcb *inp = sotoinpcb(fd_cb->so);
2003 if ((inp->inp_vflag & INP_IPV4) && (inp->inp_flags & INP_RECVDSTADDR)) {
2004 struct in_addr ia = { };
2005
2006 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in)) {
b226f5e5 2007 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
cb323159
A		 2008 bcopy(&sin->sin_addr, &ia, sizeof(struct in_addr));
2009 }
b226f5e5 2010
cb323159
A		 2011 return sbcreatecontrol((caddr_t)&ia, sizeof(ia), IP_RECVDSTADDR, IPPROTO_IP);
2012 } else if ((inp->inp_vflag & INP_IPV6) && (inp->inp_flags & IN6P_PKTINFO)) {
2013 struct in6_pktinfo pi6;
2014 memset(&pi6, 0, sizeof(pi6));
b226f5e5 2015
cb323159
A		 2016 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET6 && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in6)) {
2017 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
0a7de745 2018 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof(struct in6_addr));
b226f5e5 2019 pi6.ipi6_ifindex = 0;
b226f5e5 2020 }
cb323159
A		 2021
2022 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
39236c6e 2023 }
0a7de745 2024 return NULL;
3e170ce0 2025}
39236c6e 2026
3e170ce0
A		 2027static void
2028flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2029{
39236c6e
A		 2030 FDLOCK(fd_cb);
2031 if (fd_cb->so != NULL) {
0a7de745
A		 2032 int error = 0;
2033 mbuf_t data = NULL;
2034 size_t data_size;
3e170ce0
A		 2035 struct sockaddr_storage remote_address;
2036 boolean_t got_remote_sa = FALSE;
2037
39236c6e 2038 socket_lock(fd_cb->so, 0);
3e170ce0
A		 2039
2040 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2041 uint32_t val_size = 0;
2042
2043 /* check if we got remote address with data */
2044 memset(&remote_address, 0, sizeof(remote_address));
2045 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2046 if (error || val_size > sizeof(remote_address)) {
2047 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2048 error = 0;
39236c6e 2049 } else {
3e170ce0
A		 2050 /* validate the address */
2051 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2052 got_remote_sa = TRUE;
2053 }
2054 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2055 }
2056 }
2057
2058 data_size = (mbuf_pkthdr_len(packet) - offset);
2059
2060 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2061
2062 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2063 if (error || data == NULL) {
2064 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2065 } else {
0a7de745 2066 if (flow_divert_check_no_cellular(fd_cb) ||
cb323159
A		 2067 flow_divert_check_no_expensive(fd_cb) ||
2068 flow_divert_check_no_constrained(fd_cb)) {
3e170ce0
A		 2069 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2070 flow_divert_send_close(fd_cb, SHUT_RDWR);
2071 flow_divert_disconnect_socket(fd_cb->so);
2072 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
2073 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2074 if (sbappendstream(&fd_cb->so->so_rcv, data)) {
2075 fd_cb->bytes_received += data_size;
2076 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2077 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2078 sorwakeup(fd_cb->so);
2079 data = NULL;
2080 } else {
2081 FDLOG0(LOG_ERR, fd_cb, "received data, but appendstream failed");
2082 }
2083 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2084 struct sockaddr *append_sa;
2085 mbuf_t mctl;
2086
2087 if (got_remote_sa == TRUE) {
2088 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
0a7de745 2089 (struct sockaddr *)&remote_address, &append_sa);
3e170ce0
A		 2090 } else {
2091 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
0a7de745 2092 fd_cb->remote_address, &append_sa);
3e170ce0
A		 2093 }
2094 if (error) {
2095 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2096 }
2097
2098 mctl = flow_divert_get_control_mbuf(fd_cb);
cb323159
A		 2099 int append_error = 0;
2100 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error)) {
3e170ce0
A		 2101 fd_cb->bytes_received += data_size;
2102 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2103 fd_cb->sb_size = fd_cb->so->so_rcv.sb_cc;
2104 sorwakeup(fd_cb->so);
2105 data = NULL;
cb323159 2106 } else if (append_error != EJUSTRETURN) {
3e170ce0
A		 2107 FDLOG0(LOG_ERR, fd_cb, "received data, but sbappendaddr failed");
2108 }
2109 if (!error) {
2110 FREE(append_sa, M_TEMP);
2111 }
2112 }
39236c6e
A		 2113 }
2114 }
2115 socket_unlock(fd_cb->so, 0);
39236c6e 2116 }
3e170ce0 2117 FDUNLOCK(fd_cb);
39236c6e
A		 2118}
2119
2120static void
2121flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2122{
0a7de745
A		 2123 uint32_t read_count;
2124 int error = 0;
39236c6e
A		 2125
2126 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2127 if (error) {
2128 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2129 return;
2130 }
2131
3e170ce0 2132 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
39236c6e
A		 2133
2134 FDLOCK(fd_cb);
2135 if (fd_cb->so != NULL) {
2136 socket_lock(fd_cb->so, 0);
2137 fd_cb->send_window += ntohl(read_count);
2138 flow_divert_send_buffered_data(fd_cb, FALSE);
2139 socket_unlock(fd_cb->so, 0);
2140 }
2141 FDUNLOCK(fd_cb);
2142}
2143
2144static void
2145flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2146{
2147 int error = 0;
3e170ce0 2148 uint32_t key_size = 0;
39236c6e 2149 int log_level;
5ba3f43e 2150 uint32_t flags = 0;
39236c6e
A		 2151
2152 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2153 if (error) {
2154 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2155 return;
2156 }
2157
2158 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
39037602 2159 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
39236c6e
A		 2160 return;
2161 }
2162
2163 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2164 if (!error) {
2165 nil_pcb.log_level = log_level;
2166 }
2167
2168 lck_rw_lock_exclusive(&group->lck);
2169
cb323159
A		 2170 if (group->token_key != NULL) {
2171 FREE(group->token_key, M_TEMP);
2172 group->token_key = NULL;
2173 }
2174
39236c6e
A		 2175 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2176 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2177 if (error) {
2178 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2179 FREE(group->token_key, M_TEMP);
2180 group->token_key = NULL;
2181 lck_rw_done(&group->lck);
2182 return;
2183 }
2184
2185 group->token_key_size = key_size;
2186
5ba3f43e
A		 2187 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2188 if (!error) {
2189 group->flags = flags;
2190 }
2191
39236c6e
A		 2192 lck_rw_done(&group->lck);
2193}
2194
2195static void
2196flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2197{
0a7de745
A		 2198 int error = 0;
2199 struct sockaddr_storage local_address;
2200 int out_if_index = 0;
2201 struct sockaddr_storage remote_address;
2202 uint32_t app_data_length = 0;
39236c6e
A		 2203
2204 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2205
2206 memset(&local_address, 0, sizeof(local_address));
2207 memset(&remote_address, 0, sizeof(remote_address));
2208
2209 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2210 if (error) {
490019cf 2211 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
39236c6e
A		 2212 }
2213
2214 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2215 if (error) {
490019cf 2216 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
39236c6e
A		 2217 }
2218
2219 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2220 if (error) {
490019cf
A		 2221 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2222 }
2223
2224 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2225 if (error) {
2226 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
39236c6e
A		 2227 }
2228
2229 FDLOCK(fd_cb);
2230 if (fd_cb->so != NULL) {
39236c6e
A		 2231 socket_lock(fd_cb->so, 0);
2232
39236c6e
A		 2233 if (local_address.ss_family != 0) {
2234 if (local_address.ss_len > sizeof(local_address)) {
2235 local_address.ss_len = sizeof(local_address);
2236 }
490019cf
A		 2237 if (fd_cb->local_address != NULL) {
2238 FREE(fd_cb->local_address, M_SONAME);
2239 fd_cb->local_address = NULL;
2240 }
39236c6e
A		 2241 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2242 }
2243
2244 if (remote_address.ss_family != 0) {
2245 if (remote_address.ss_len > sizeof(remote_address)) {
2246 remote_address.ss_len = sizeof(remote_address);
2247 }
490019cf
A		 2248 if (fd_cb->remote_address != NULL) {
2249 FREE(fd_cb->remote_address, M_SONAME);
2250 fd_cb->remote_address = NULL;
2251 }
39236c6e
A		 2252 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2253 }
2254
490019cf
A		 2255 if (out_if_index > 0) {
2256 struct inpcb *inp = NULL;
2257 struct ifnet *ifp = NULL;
2258
2259 inp = sotoinpcb(fd_cb->so);
2260
2261 ifnet_head_lock_shared();
2262 if (out_if_index <= if_index) {
2263 ifp = ifindex2ifnet[out_if_index];
2264 }
2265
2266 if (ifp != NULL) {
2267 inp->inp_last_outifp = ifp;
2268 }
2269 ifnet_head_done();
39236c6e
A		 2270 }
2271
490019cf 2272 if (app_data_length > 0) {
0a7de745 2273 uint8_t *app_data = NULL;
490019cf
A		 2274 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2275 if (app_data != NULL) {
2276 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2277 if (error == 0) {
2278 if (fd_cb->app_data != NULL) {
2279 FREE(fd_cb->app_data, M_TEMP);
2280 }
2281 fd_cb->app_data = app_data;
2282 fd_cb->app_data_length = app_data_length;
2283 } else {
2284 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2285 FREE(app_data, M_TEMP);
2286 }
2287 } else {
2288 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2289 }
39236c6e 2290 }
39236c6e
A		 2291
2292 socket_unlock(fd_cb->so, 0);
2293 }
2294 FDUNLOCK(fd_cb);
2295}
2296
2297static void
39037602 2298flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
39236c6e
A		 2299{
2300 size_t bytes_mem_size;
2301 size_t child_maps_mem_size;
2302 int cursor;
2303 int error = 0;
2304 struct flow_divert_trie new_trie;
2305 int insert_error = 0;
2306 size_t nodes_mem_size;
2307 int prefix_count = 0;
2308 int signing_id_count = 0;
ecc0ceb4 2309 size_t trie_memory_size = 0;
39236c6e 2310
39037602 2311 lck_rw_lock_exclusive(&group->lck);
39236c6e
A		 2312
2313 /* Re-set the current trie */
39037602
A		 2314 if (group->signing_id_trie.memory != NULL) {
2315 FREE(group->signing_id_trie.memory, M_TEMP);
39236c6e 2316 }
39037602
A		 2317 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2318 group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 2319
2320 memset(&new_trie, 0, sizeof(new_trie));
2321
2322 /* Get the number of shared prefixes in the new set of signing ID strings */
2323 flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2324
ecc0ceb4 2325 if (prefix_count < 0) {
39037602 2326 lck_rw_done(&group->lck);
ecc0ceb4
A		 2327 return;
2328 }
2329
39236c6e
A		 2330 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2331 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2332 cursor >= 0;
2333 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2334 uint32_t sid_size = 0;
39236c6e
A		 2335 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2336 new_trie.bytes_count += sid_size;
2337 signing_id_count++;
2338 }
2339
2340 if (signing_id_count == 0) {
39037602 2341 lck_rw_done(&group->lck);
39236c6e
A		 2342 return;
2343 }
2344
2345 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2346 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2347
2348 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
0a7de745 2349 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
39236c6e 2350
4ba76501
A		 2351 if (os_mul_overflow(sizeof(*new_trie.nodes), new_trie.nodes_count, &nodes_mem_size) ||
2352 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, new_trie.child_maps_count, &child_maps_mem_size) ||
2353 os_mul_overflow(sizeof(*new_trie.bytes), new_trie.bytes_count, &bytes_mem_size) ||
2354 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2355 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2356 lck_rw_done(&group->lck);
2357 return;
2358 }
39236c6e 2359
ecc0ceb4 2360 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
39037602
A		 2361 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2362 lck_rw_done(&group->lck);
ecc0ceb4
A		 2363 return;
2364 }
2365
2366 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
39236c6e
A		 2367 if (new_trie.memory == NULL) {
2368 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
0a7de745 2369 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
39037602 2370 lck_rw_done(&group->lck);
39236c6e
A		 2371 return;
2372 }
2373
2374 /* Initialize the free lists */
2375 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2376 new_trie.nodes_free_next = 0;
2377 memset(new_trie.nodes, 0, nodes_mem_size);
2378
2379 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2380 new_trie.child_maps_free_next = 0;
2381 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2382
2383 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2384 new_trie.bytes_free_next = 0;
2385
2386 /* The root is an empty node */
2387 new_trie.root = trie_node_alloc(&new_trie);
2388
2389 /* Add each signing ID to the trie */
2390 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
0a7de745
A		 2391 cursor >= 0;
2392 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
3e170ce0 2393 uint32_t sid_size = 0;
39236c6e
A		 2394 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2395 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
39236c6e
A		 2396 uint16_t new_node_idx;
2397 flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
39236c6e 2398 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
39037602 2399 if (new_node_idx == NULL_TRIE_IDX) {
39236c6e
A		 2400 insert_error = EINVAL;
2401 break;
2402 }
2403 } else {
2404 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2405 insert_error = ENOBUFS;
2406 break;
2407 }
2408 }
2409
2410 if (!insert_error) {
39037602 2411 group->signing_id_trie = new_trie;
39236c6e
A		 2412 } else {
2413 FREE(new_trie.memory, M_TEMP);
2414 }
2415
39236c6e 2416 lck_rw_done(&group->lck);
39236c6e
A		 2417}
2418
2419static int
2420flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2421{
0a7de745
A		 2422 struct flow_divert_packet_header hdr;
2423 int error = 0;
2424 struct flow_divert_pcb *fd_cb;
39236c6e
A		 2425
2426 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2427 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2428 error = EINVAL;
2429 goto done;
2430 }
2431
3e170ce0 2432 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
39037602 2433 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
3e170ce0
A		 2434 error = EINVAL;
2435 goto done;
2436 }
2437
39236c6e
A		 2438 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2439 if (error) {
2440 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2441 error = ENOBUFS;
2442 goto done;
2443 }
2444
2445 hdr.conn_id = ntohl(hdr.conn_id);
2446
2447 if (hdr.conn_id == 0) {
2448 switch (hdr.packet_type) {
0a7de745
A		 2449 case FLOW_DIVERT_PKT_GROUP_INIT:
2450 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2451 break;
2452 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2453 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2454 break;
2455 default:
2456 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2457 break;
39236c6e
A		 2458 }
2459 goto done;
2460 }
2461
0a7de745 2462 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
39236c6e
A		 2463 if (fd_cb == NULL) {
2464 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2465 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2466 }
2467 goto done;
2468 }
2469
2470 switch (hdr.packet_type) {
0a7de745
A		 2471 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2472 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2473 break;
2474 case FLOW_DIVERT_PKT_CLOSE:
2475 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2476 break;
2477 case FLOW_DIVERT_PKT_DATA:
2478 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2479 break;
2480 case FLOW_DIVERT_PKT_READ_NOTIFY:
2481 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2482 break;
2483 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2484 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2485 break;
2486 default:
2487 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2488 break;
39236c6e
A		 2489 }
2490
2491 FDRELEASE(fd_cb);
2492
2493done:
3e170ce0 2494 mbuf_freem(packet);
39236c6e
A		 2495 return error;
2496}
2497
2498static void
2499flow_divert_close_all(struct flow_divert_group *group)
2500{
0a7de745
A		 2501 struct flow_divert_pcb *fd_cb;
2502 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 2503
2504 SLIST_INIT(&tmp_list);
2505
2506 lck_rw_lock_exclusive(&group->lck);
2507
2508 MBUFQ_DRAIN(&group->send_queue);
2509
2510 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2511 FDRETAIN(fd_cb);
2512 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2513 }
2514
2515 lck_rw_done(&group->lck);
2516
2517 while (!SLIST_EMPTY(&tmp_list)) {
2518 fd_cb = SLIST_FIRST(&tmp_list);
2519 FDLOCK(fd_cb);
2520 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2521 if (fd_cb->so != NULL) {
2522 socket_lock(fd_cb->so, 0);
2523 flow_divert_pcb_remove(fd_cb);
2524 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2525 fd_cb->so->so_error = ECONNABORTED;
39037602 2526 flow_divert_disconnect_socket(fd_cb->so);
39236c6e
A		 2527 socket_unlock(fd_cb->so, 0);
2528 }
2529 FDUNLOCK(fd_cb);
2530 FDRELEASE(fd_cb);
2531 }
2532}
2533
2534void
2535flow_divert_detach(struct socket *so)
2536{
0a7de745 2537 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2538
2539 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2540
2541 so->so_flags &= ~SOF_FLOW_DIVERT;
2542 so->so_fd_pcb = NULL;
2543
2544 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2545
2546 if (fd_cb->group != NULL) {
2547 /* Last-ditch effort to send any buffered data */
2548 flow_divert_send_buffered_data(fd_cb, TRUE);
2549
3e170ce0
A		 2550 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2551 flow_divert_send_close_if_needed(fd_cb);
39236c6e
A		 2552 /* Remove from the group */
2553 flow_divert_pcb_remove(fd_cb);
2554 }
2555
2556 socket_unlock(so, 0);
2557 FDLOCK(fd_cb);
2558 fd_cb->so = NULL;
2559 FDUNLOCK(fd_cb);
2560 socket_lock(so, 0);
2561
0a7de745 2562 FDRELEASE(fd_cb); /* Release the socket's reference */
39236c6e
A		 2563}
2564
2565static int
2566flow_divert_close(struct socket *so)
2567{
0a7de745 2568 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2569
2570 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2571
2572 FDLOG0(LOG_INFO, fd_cb, "Closing");
2573
3e170ce0
A		 2574 if (SOCK_TYPE(so) == SOCK_STREAM) {
2575 soisdisconnecting(so);
2576 sbflush(&so->so_rcv);
2577 }
39236c6e
A		 2578
2579 flow_divert_send_buffered_data(fd_cb, TRUE);
2580 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2581 flow_divert_send_close_if_needed(fd_cb);
2582
2583 /* Remove from the group */
2584 flow_divert_pcb_remove(fd_cb);
2585
2586 return 0;
2587}
2588
2589static int
3e170ce0
A		 2590flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2591 sae_connid_t cid __unused)
39236c6e 2592{
3e170ce0 2593 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
0a7de745 2594 return EINVAL;
39236c6e
A		 2595 }
2596
0a7de745 2597 return flow_divert_close(so);
39236c6e
A		 2598}
2599
2600static int
2601flow_divert_shutdown(struct socket *so)
2602{
0a7de745 2603 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2604
2605 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2606
2607 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2608
2609 socantsendmore(so);
2610
2611 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2612 flow_divert_send_close_if_needed(fd_cb);
2613
2614 return 0;
2615}
2616
2617static int
2618flow_divert_rcvd(struct socket *so, int flags __unused)
2619{
0a7de745
A		 2620 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2621 uint32_t latest_sb_size;
2622 uint32_t read_count;
39236c6e
A		 2623
2624 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2625
2626 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2627
2628 if (fd_cb->sb_size < latest_sb_size) {
2629 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
0a7de745 2630 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
39236c6e
A		 2631 }
2632
2633 read_count = fd_cb->sb_size - latest_sb_size;
2634
2635 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2636
2637 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2638 fd_cb->bytes_read_by_app += read_count;
2639 fd_cb->sb_size = latest_sb_size;
2640 }
2641
2642 return 0;
2643}
2644
3e170ce0
A		 2645static int
2646flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2647{
2648 int error = 0;
2649 int port = 0;
2650
cb323159
A		 2651 if (!flow_divert_is_sockaddr_valid(toaddr)) {
2652 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
2653 error = EINVAL;
2654 goto done;
2655 }
2656
3e170ce0
A		 2657 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2658 if (error) {
2659 goto done;
2660 }
2661
2662 if (toaddr->sa_family == AF_INET) {
2663 port = ntohs((satosin(toaddr))->sin_port);
2664 }
2665#if INET6
2666 else {
2667 port = ntohs((satosin6(toaddr))->sin6_port);
2668 }
2669#endif
2670
2671 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2672 if (error) {
2673 goto done;
2674 }
2675
2676done:
2677 return error;
2678}
2679
2680struct sockaddr *
2681flow_divert_get_buffered_target_address(mbuf_t buffer)
2682{
2683 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2684 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2685 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2686 return toaddr;
2687 }
2688 }
2689 return NULL;
2690}
2691
2692static boolean_t
2693flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2694{
0a7de745
A		 2695 switch (addr->sa_family) {
2696 case AF_INET:
cb323159 2697 if (addr->sa_len < sizeof(struct sockaddr_in)) {
0a7de745
A		 2698 return FALSE;
2699 }
2700 break;
3e170ce0 2701#if INET6
0a7de745 2702 case AF_INET6:
cb323159 2703 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
3e170ce0 2704 return FALSE;
0a7de745
A		 2705 }
2706 break;
2707#endif /* INET6 */
2708 default:
2709 return FALSE;
3e170ce0
A		 2710 }
2711 return TRUE;
2712}
2713
2714static errno_t
2715flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2716{
2717 int error = 0;
2718 union sockaddr_in_4_6 sin46;
2719
2720 bzero(&sin46, sizeof(sin46));
2721 if (inp->inp_vflag & INP_IPV4) {
2722 struct sockaddr_in *sin = &sin46.sin;
2723
2724 sin->sin_family = AF_INET;
2725 sin->sin_len = sizeof(*sin);
2726 sin->sin_port = inp->inp_lport;
2727 sin->sin_addr = inp->inp_laddr;
2728 } else if (inp->inp_vflag & INP_IPV6) {
2729 struct sockaddr_in6 *sin6 = &sin46.sin6;
2730
2731 sin6->sin6_len = sizeof(*sin6);
2732 sin6->sin6_family = AF_INET6;
2733 sin6->sin6_port = inp->inp_lport;
2734 sin6->sin6_addr = inp->in6p_laddr;
2735 }
2736 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2737 if (*local_socket == NULL) {
2738 error = ENOBUFS;
2739 }
0a7de745 2740 return error;
3e170ce0
A		 2741}
2742
2743static boolean_t
2744flow_divert_has_pcb_local_address(const struct inpcb *inp)
2745{
0a7de745
A		 2746 return inp->inp_lport != 0
2747 && (inp->inp_laddr.s_addr != INADDR_ANY || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr));
3e170ce0
A		 2748}
2749
39236c6e
A		 2750static errno_t
2751flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
0a7de745 2752 struct sockaddr **dup)
39236c6e 2753{
0a7de745
A		 2754 int error = 0;
2755 struct sockaddr *result;
2756 struct sockaddr_storage ss;
39236c6e
A		 2757
2758 if (addr != NULL) {
2759 result = addr;
2760 } else {
2761 memset(&ss, 0, sizeof(ss));
2762 ss.ss_family = family;
2763 if (ss.ss_family == AF_INET) {
2764 ss.ss_len = sizeof(struct sockaddr_in);
2765 }
2766#if INET6
2767 else if (ss.ss_family == AF_INET6) {
2768 ss.ss_len = sizeof(struct sockaddr_in6);
2769 }
0a7de745 2770#endif /* INET6 */
39236c6e
A		 2771 else {
2772 error = EINVAL;
2773 }
2774 result = (struct sockaddr *)&ss;
2775 }
2776
2777 if (!error) {
2778 *dup = dup_sockaddr(result, 1);
2779 if (*dup == NULL) {
2780 error = ENOBUFS;
2781 }
2782 }
2783
2784 return error;
2785}
2786
3e170ce0
A		 2787static void
2788flow_divert_disconnect_socket(struct socket *so)
2789{
2790 soisdisconnected(so);
2791 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2792 struct inpcb *inp = NULL;
2793
2794 inp = sotoinpcb(so);
2795 if (inp != NULL) {
2796#if INET6
0a7de745 2797 if (SOCK_CHECK_DOM(so, PF_INET6)) {
3e170ce0 2798 in6_pcbdetach(inp);
0a7de745 2799 } else
3e170ce0 2800#endif /* INET6 */
0a7de745 2801 in_pcbdetach(inp);
3e170ce0
A		 2802 }
2803 }
2804}
2805
39236c6e
A		 2806static errno_t
2807flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2808{
0a7de745 2809 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2810
2811 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2812
0a7de745
A		 2813 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2814 fd_cb->remote_address,
2815 sa);
39236c6e
A		 2816}
2817
2818static errno_t
2819flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2820{
0a7de745 2821 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2822
2823 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2824
0a7de745
A		 2825 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2826 fd_cb->local_address,
2827 sa);
39236c6e
A		 2828}
2829
2830static errno_t
2831flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2832{
0a7de745 2833 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
39236c6e
A		 2834
2835 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2836
2837 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2838 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2839 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2840 }
2841 }
2842
2843 if (SOCK_DOM(so) == PF_INET) {
2844 return g_tcp_protosw->pr_ctloutput(so, sopt);
2845 }
2846#if INET6
2847 else if (SOCK_DOM(so) == PF_INET6) {
2848 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2849 }
2850#endif
2851 return 0;
2852}
2853
2854errno_t
2855flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2856{
0a7de745
A		 2857 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2858 int error = 0;
2859 struct inpcb *inp = sotoinpcb(so);
2860 struct sockaddr_in *sinp;
2861 mbuf_t connect_packet = NULL;
2862 int do_send = 1;
39236c6e
A		 2863
2864 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2865
2866 if (fd_cb->group == NULL) {
2867 error = ENETUNREACH;
2868 goto done;
2869 }
2870
2871 if (inp == NULL) {
2872 error = EINVAL;
2873 goto done;
2874 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2875 if (so->so_error) {
2876 error = so->so_error;
2877 so->so_error = 0;
2878 } else {
2879 error = EINVAL;
2880 }
2881 goto done;
2882 }
2883
39236c6e
A		 2884 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2885 error = EALREADY;
2886 goto done;
2887 }
2888
2889 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2890 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2891 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2892 if (fd_cb->remote_address != NULL) {
2893 soisconnected(fd_cb->so);
2894 goto done;
2895 }
2896 }
2897
490019cf 2898 FDLOG0(LOG_INFO, fd_cb, "Connecting");
fe8ab488 2899
490019cf
A		 2900 if (fd_cb->connect_packet == NULL) {
2901 if (to == NULL) {
2902 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2903 error = EINVAL;
2904 goto done;
fe8ab488 2905 }
fe8ab488 2906
490019cf
A		 2907 sinp = (struct sockaddr_in *)(void *)to;
2908 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2909 error = EAFNOSUPPORT;
2910 goto done;
fe8ab488
A		 2911 }
2912
490019cf
A		 2913 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2914 if (error) {
2915 goto done;
fe8ab488
A		 2916 }
2917
490019cf
A		 2918 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2919 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2920 do_send = 0;
fe8ab488
A		 2921 }
2922 } else {
490019cf
A		 2923 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
2924 connect_packet = fd_cb->connect_packet;
2925 fd_cb->connect_packet = NULL;
fe8ab488
A		 2926 }
2927
490019cf
A		 2928 if (do_send) {
2929 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
2930 if (error) {
2931 goto done;
2932 }
39236c6e 2933
490019cf
A		 2934 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
2935 } else {
2936 fd_cb->connect_packet = connect_packet;
2937 connect_packet = NULL;
39236c6e
A		 2938 }
2939
39236c6e
A		 2940 soisconnecting(so);
2941
2942done:
fe8ab488 2943 if (error && connect_packet != NULL) {
3e170ce0 2944 mbuf_freem(connect_packet);
fe8ab488 2945 }
39236c6e
A		 2946 return error;
2947}
2948
2949static int
813fb2f6
A		 2950flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
2951 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
39236c6e 2952{
39236c6e
A		 2953 struct inpcb *inp = sotoinpcb(so);
2954 int error;
2955
2956 if (inp == NULL) {
0a7de745 2957 return EINVAL;
39236c6e
A		 2958 }
2959
813fb2f6 2960 VERIFY(dst != NULL);
39236c6e 2961
813fb2f6 2962 error = flow_divert_connect_out(so, dst, p);
39236c6e 2963
490019cf
A		 2964 if (error != 0) {
2965 return error;
2966 }
2967
2968 /* if there is data, send it */
2969 if (auio != NULL) {
2970 user_ssize_t datalen = 0;
2971
2972 socket_unlock(so, 0);
2973
2974 VERIFY(bytes_written != NULL);
2975
2976 datalen = uio_resid(auio);
2977 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
2978 socket_lock(so, 0);
2979
2980 if (error == 0 || error == EWOULDBLOCK) {
2981 *bytes_written = datalen - uio_resid(auio);
2982 }
2983
2984 /*
2985 * sosend returns EWOULDBLOCK if it's a non-blocking
2986 * socket or a timeout occured (this allows to return
2987 * the amount of queued data through sendit()).
2988 *
2989 * However, connectx() returns EINPROGRESS in case of a
2990 * blocking socket. So we change the return value here.
2991 */
2992 if (error == EWOULDBLOCK) {
2993 error = EINPROGRESS;
2994 }
2995 }
2996
39236c6e 2997 if (error == 0 && pcid != NULL) {
0a7de745 2998 *pcid = 1; /* there is only 1 connection for a TCP */
39236c6e
A		 2999 }
3000
0a7de745 3001 return error;
39236c6e
A		 3002}
3003
3004static int
813fb2f6
A		 3005flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3006 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3007 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3008 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
39236c6e 3009{
0a7de745 3010 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e
A		 3011}
3012
3013#if INET6
3014static int
813fb2f6
A		 3015flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3016 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3017 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3018 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3019{
0a7de745 3020 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
39236c6e
A		 3021}
3022#endif /* INET6 */
3023
3024static int
3e170ce0 3025flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
0a7de745
A		 3026 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
3027 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
3028 user_addr_t aux_data __unused, uint32_t *aux_len)
39236c6e 3029{
0a7de745
A		 3030 int error = 0;
3031 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3032 struct ifnet *ifp = NULL;
3033 struct inpcb *inp = sotoinpcb(so);
39236c6e
A		 3034
3035 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
3036
3037 if (so->so_fd_pcb == NULL || inp == NULL) {
3038 error = EINVAL;
3039 goto out;
3040 }
3041
3e170ce0 3042 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
39236c6e
A		 3043 error = EINVAL;
3044 goto out;
3045 }
3046
3047 ifp = inp->inp_last_outifp;
3048 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
3049 *soerror = so->so_error;
3050 *flags = 0;
3051
3052 if (so->so_state & SS_ISCONNECTED) {
3053 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
3054 }
3055
3056 if (fd_cb->local_address == NULL) {
3057 struct sockaddr_in sin;
3058 bzero(&sin, sizeof(sin));
3059 sin.sin_len = sizeof(sin);
3060 sin.sin_family = AF_INET;
3061 *src_len = sin.sin_len;
3062 if (src != USER_ADDR_NULL) {
3063 error = copyout(&sin, src, sin.sin_len);
3064 if (error != 0) {
3065 goto out;
3066 }
3067 }
3068 } else {
3069 *src_len = fd_cb->local_address->sa_len;
3070 if (src != USER_ADDR_NULL) {
3071 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
3072 if (error != 0) {
3073 goto out;
3074 }
3075 }
3076 }
3077
3078 if (fd_cb->remote_address == NULL) {
3079 struct sockaddr_in sin;
3080 bzero(&sin, sizeof(sin));
3081 sin.sin_len = sizeof(sin);
3082 sin.sin_family = AF_INET;
3083 *dst_len = sin.sin_len;
3084 if (dst != USER_ADDR_NULL) {
3085 error = copyout(&sin, dst, sin.sin_len);
3086 if (error != 0) {
3087 goto out;
3088 }
3089 }
3090 } else {
3091 *dst_len = fd_cb->remote_address->sa_len;
3092 if (dst != USER_ADDR_NULL) {
3093 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
3094 if (error != 0) {
3095 goto out;
3096 }
3097 }
3098 }
3099
3100 *aux_type = 0;
3101 *aux_len = 0;
3102
3103out:
3104 return error;
3105}
3106
3107static int
3108flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3109{
3110 int error = 0;
3111
3112 switch (cmd) {
0a7de745
A		 3113 case SIOCGCONNINFO32: {
3114 struct so_cinforeq32 cifr;
3115 bcopy(data, &cifr, sizeof(cifr));
3116 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3117 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3118 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3119 &cifr.scir_aux_type, cifr.scir_aux_data,
3120 &cifr.scir_aux_len);
3121 if (error == 0) {
3122 bcopy(&cifr, data, sizeof(cifr));
39236c6e 3123 }
0a7de745
A		 3124 break;
3125 }
39236c6e 3126
0a7de745
A		 3127 case SIOCGCONNINFO64: {
3128 struct so_cinforeq64 cifr;
3129 bcopy(data, &cifr, sizeof(cifr));
3130 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3131 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3132 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3133 &cifr.scir_aux_type, cifr.scir_aux_data,
3134 &cifr.scir_aux_len);
3135 if (error == 0) {
3136 bcopy(&cifr, data, sizeof(cifr));
39236c6e 3137 }
0a7de745
A		 3138 break;
3139 }
39236c6e 3140
0a7de745
A		 3141 default:
3142 error = EOPNOTSUPP;
39236c6e
A		 3143 }
3144
3145 return error;
3146}
3147
3148static int
3149flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3150{
3151 int error = flow_divert_control(so, cmd, data, ifp, p);
3152
3153 if (error == EOPNOTSUPP) {
3154 error = in_control(so, cmd, data, ifp, p);
3155 }
3156
3157 return error;
3158}
3159
3160static int
3161flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3162{
3163 int error = flow_divert_control(so, cmd, data, ifp, p);
3164
3165 if (error == EOPNOTSUPP) {
3166 error = in6_control(so, cmd, data, ifp, p);
3167 }
3168
3169 return error;
3170}
3171
3172static errno_t
490019cf 3173flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
39236c6e 3174{
0a7de745
A		 3175 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3176 int error = 0;
39236c6e
A		 3177 struct inpcb *inp;
3178
3179 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3180
3181 inp = sotoinpcb(so);
3182 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3183 error = ECONNRESET;
3184 goto done;
3185 }
3186
3187 if (control && mbuf_len(control) > 0) {
3188 error = EINVAL;
3189 goto done;
3190 }
3191
3192 if (flags & MSG_OOB) {
3193 error = EINVAL;
3194 goto done; /* We don't support OOB data */
3195 }
0a7de745
A		 3196
3197 error = flow_divert_check_no_cellular(fd_cb) ||
cb323159
A		 3198 flow_divert_check_no_expensive(fd_cb) ||
3199 flow_divert_check_no_constrained(fd_cb);
39236c6e
A		 3200 if (error) {
3201 goto done;
3202 }
3203
3204 /* Implicit connect */
3205 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3206 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
cb323159
A		 3207
3208#if CONTENT_FILTER
3209 /*
3210 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3211 * retrieve the CFIL saved remote address from the mbuf and use it.
3212 */
3213 if (to == NULL && so->so_cfil_db) {
3214 struct sockaddr *cfil_faddr = NULL;
3215 struct m_tag *cfil_tag = cfil_udp_get_socket_state(data, NULL, NULL, &cfil_faddr);
3216 if (cfil_tag) {
3217 to = (struct sockaddr *)(void *)cfil_faddr;
3218 }
3219 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3220 }
3221#endif
490019cf 3222 error = flow_divert_connect_out(so, to, p);
39236c6e
A		 3223 if (error) {
3224 goto done;
3225 }
490019cf
A		 3226
3227 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3228 /* Open up the send window so that the data will get sent right away */
3229 fd_cb->send_window = mbuf_pkthdr_len(data);
3230 }
39236c6e
A		 3231 }
3232
3233 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3234
3235 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3e170ce0 3236 error = flow_divert_send_app_data(fd_cb, data, to);
39236c6e
A		 3237 if (error) {
3238 goto done;
3239 }
3240
3241 data = NULL;
3242
3243 if (flags & PRUS_EOF) {
3244 flow_divert_shutdown(so);
3245 }
3246
3247done:
3248 if (data) {
3e170ce0 3249 mbuf_freem(data);
39236c6e
A		 3250 }
3251 if (control) {
3252 mbuf_free(control);
3253 }
3254 return error;
3255}
3256
490019cf
A		 3257static int
3258flow_divert_preconnect(struct socket *so)
3259{
0a7de745 3260 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
490019cf
A		 3261 int error = 0;
3262
3263 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3264 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3265 mbuf_t connect_packet = fd_cb->connect_packet;
3266 fd_cb->connect_packet = NULL;
3267
3268 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3269 if (error) {
3270 mbuf_freem(connect_packet);
3271 }
3272
3273 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3274 }
3275
5ba3f43e 3276 soclearfastopen(so);
490019cf
A		 3277
3278 return error;
3279}
3280
39236c6e
A		 3281static void
3282flow_divert_set_protosw(struct socket *so)
3283{
3284 so->so_flags |= SOF_FLOW_DIVERT;
3285 if (SOCK_DOM(so) == PF_INET) {
3286 so->so_proto = &g_flow_divert_in_protosw;
3287 }
3288#if INET6
3289 else {
3290 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3291 }
0a7de745 3292#endif /* INET6 */
39236c6e
A		 3293}
3294
3e170ce0
A		 3295static void
3296flow_divert_set_udp_protosw(struct socket *so)
3297{
0a7de745
A		 3298 so->so_flags |= SOF_FLOW_DIVERT;
3299 if (SOCK_DOM(so) == PF_INET) {
3300 so->so_proto = &g_flow_divert_in_udp_protosw;
3301 }
3e170ce0 3302#if INET6
0a7de745
A		 3303 else {
3304 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3305 }
3e170ce0
A		 3306#endif /* INET6 */
3307}
3308
39236c6e
A		 3309static errno_t
3310flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3311{
0a7de745
A		 3312 int error = 0;
3313 struct flow_divert_pcb *fd_cb = NULL;
3314 struct ifnet *ifp = NULL;
3315 struct inpcb *inp = NULL;
3316 struct socket *old_so;
3317 mbuf_t recv_data = NULL;
39236c6e
A		 3318
3319 socket_unlock(so, 0);
3320
3321 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3322
3323 /* Find the flow divert control block */
3324 lck_rw_lock_shared(&g_flow_divert_group_lck);
3325 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3326 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3327 if (group != NULL) {
3328 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3329 }
3330 }
3331 lck_rw_done(&g_flow_divert_group_lck);
3332
3333 if (fd_cb == NULL) {
3334 error = ENOENT;
3335 goto done;
3336 }
3337
3338 FDLOCK(fd_cb);
3339
3340 /* Dis-associate the flow divert control block from its current socket */
3341 old_so = fd_cb->so;
3342
0a7de745 3343 inp = sotoinpcb(old_so);
39236c6e
A		 3344
3345 VERIFY(inp != NULL);
3346
3347 socket_lock(old_so, 0);
3e170ce0 3348 flow_divert_disconnect_socket(old_so);
39236c6e
A		 3349 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3350 old_so->so_fd_pcb = NULL;
3e170ce0
A		 3351 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3352 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3353 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3354 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3355 }
39236c6e
A		 3356 fd_cb->so = NULL;
3357 /* Save the output interface */
3358 ifp = inp->inp_last_outifp;
3359 if (old_so->so_rcv.sb_cc > 0) {
3360 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3361 sbflush(&old_so->so_rcv);
3362 }
3363 socket_unlock(old_so, 0);
3364
3365 /* Associate the new socket with the flow divert control block */
3366 socket_lock(so, 0);
3367 so->so_fd_pcb = fd_cb;
3368 inp = sotoinpcb(so);
3369 inp->inp_last_outifp = ifp;
3370 if (recv_data != NULL) {
3371 if (sbappendstream(&so->so_rcv, recv_data)) {
3372 sorwakeup(so);
3373 }
3374 }
3375 flow_divert_set_protosw(so);
3376 socket_unlock(so, 0);
3377
3378 fd_cb->so = so;
3379 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3380
3381 FDUNLOCK(fd_cb);
3382
3383done:
3384 socket_lock(so, 0);
3385
3386 if (fd_cb != NULL) {
0a7de745 3387 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
39236c6e
A		 3388 }
3389
3390 return error;
3391}
3392
3e170ce0
A		 3393errno_t
3394flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3395{
0a7de745 3396 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3e170ce0 3397 struct inpcb *inp;
0a7de745 3398 int error = 0;
3e170ce0
A		 3399
3400 inp = sotoinpcb(so);
3401 if (inp == NULL) {
0a7de745
A		 3402 return EINVAL;
3403 }
3404
3405 if (fd_cb == NULL) {
3406 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3407 if (fd_ctl_unit > 0) {
3408 error = flow_divert_pcb_init(so, fd_ctl_unit);
3409 fd_cb = so->so_fd_pcb;
3410 if (error != 0 || fd_cb == NULL) {
3411 goto done;
3412 }
3413 } else {
3414 error = ENETDOWN;
3415 goto done;
3416 }
3417 }
3418 return flow_divert_data_out(so, flags, data, to, control, p);
3e170ce0
A		 3419
3420done:
0a7de745
A		 3421 if (data) {
3422 mbuf_freem(data);
3423 }
3424 if (control) {
3425 mbuf_free(control);
3426 }
3e170ce0 3427
0a7de745 3428 return error;
3e170ce0
A		 3429}
3430
39236c6e
A		 3431errno_t
3432flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3433{
3434 errno_t error = 0;
3435 struct flow_divert_pcb *fd_cb;
3436
3437 if (so->so_flags & SOF_FLOW_DIVERT) {
3438 return EALREADY;
3439 }
0a7de745 3440
39236c6e
A		 3441 fd_cb = flow_divert_pcb_create(so);
3442 if (fd_cb != NULL) {
3443 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3444 if (error) {
3445 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3446 FDRELEASE(fd_cb);
3447 } else {
39236c6e
A		 3448 fd_cb->control_group_unit = ctl_unit;
3449 so->so_fd_pcb = fd_cb;
3450
3e170ce0
A		 3451 if (SOCK_TYPE(so) == SOCK_STREAM) {
3452 flow_divert_set_protosw(so);
3453 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3454 flow_divert_set_udp_protosw(so);
3455 }
39236c6e
A		 3456
3457 FDLOG0(LOG_INFO, fd_cb, "Created");
3458 }
3459 } else {
3460 error = ENOMEM;
3461 }
3462
3463 return error;
3464}
3465
3466errno_t
3467flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3468{
0a7de745
A		 3469 uint32_t ctl_unit = 0;
3470 uint32_t key_unit = 0;
3471 uint32_t flow_id = 0;
3472 int error = 0;
3473 int hmac_error = 0;
3474 mbuf_t token = NULL;
39236c6e
A		 3475
3476 if (so->so_flags & SOF_FLOW_DIVERT) {
3477 error = EALREADY;
3478 goto done;
3479 }
3480
3481 if (g_init_result) {
3482 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3483 error = ENOPROTOOPT;
3484 goto done;
3485 }
3486
3e170ce0
A		 3487 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3488 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
39236c6e
A		 3489 (SOCK_DOM(so) != PF_INET
3490#if INET6
0a7de745 3491 && SOCK_DOM(so) != PF_INET6
39236c6e 3492#endif
0a7de745 3493 )) {
39236c6e
A		 3494 error = EINVAL;
3495 goto done;
3496 } else {
3e170ce0
A		 3497 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3498 struct tcpcb *tp = sototcpcb(so);
3499 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3500 error = EINVAL;
3501 goto done;
3502 }
39236c6e
A		 3503 }
3504 }
3505
3506 error = soopt_getm(sopt, &token);
3507 if (error) {
5c9f4661 3508 token = NULL;
39236c6e
A		 3509 goto done;
3510 }
3511
3512 error = soopt_mcopyin(sopt, token);
3513 if (error) {
5c9f4661 3514 token = NULL;
39236c6e
A		 3515 goto done;
3516 }
3517
3518 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3519 if (!error) {
3520 key_unit = ntohl(key_unit);
4d15aeb1
A		 3521 if (key_unit >= GROUP_COUNT_MAX) {
3522 key_unit = 0;
3523 }
39236c6e
A		 3524 } else if (error != ENOENT) {
3525 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3526 goto done;
3527 } else {
3528 key_unit = 0;
3529 }
3530
3531 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3532 if (error) {
3533 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3534 goto done;
3535 }
3536
3537 /* A valid kernel control unit is required */
3538 ctl_unit = ntohl(ctl_unit);
3539 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3540 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3541 error = EINVAL;
3542 goto done;
3543 }
3544
3545 socket_unlock(so, 0);
490019cf 3546 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
39236c6e
A		 3547 socket_lock(so, 0);
3548
490019cf
A		 3549 if (hmac_error && hmac_error != ENOENT) {
3550 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3551 error = hmac_error;
39236c6e
A		 3552 goto done;
3553 }
3554
3555 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3556 if (error && error != ENOENT) {
3557 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3558 goto done;
3559 }
3560
3561 if (flow_id == 0) {
3562 error = flow_divert_pcb_init(so, ctl_unit);
3563 if (error == 0) {
3564 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3565 int log_level = LOG_NOTICE;
3566
3567 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
0a7de745 3568 sizeof(log_level), &log_level, NULL);
39236c6e
A		 3569 if (error == 0) {
3570 fd_cb->log_level = log_level;
3571 }
3572 error = 0;
3573
3574 fd_cb->connect_token = token;
3575 token = NULL;
3576 }
3577 } else {
3578 error = flow_divert_attach(so, flow_id, ctl_unit);
3579 }
3580
490019cf
A		 3581 if (hmac_error == 0) {
3582 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3583 if (fd_cb != NULL) {
3584 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3585 }
3586 }
3587
39236c6e
A		 3588done:
3589 if (token != NULL) {
3590 mbuf_freem(token);
3591 }
3592
3593 return error;
3594}
3595
3596errno_t
3597flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3598{
0a7de745
A		 3599 uint32_t ctl_unit;
3600 int error = 0;
3601 uint8_t hmac[SHA_DIGEST_LENGTH];
3602 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3603 mbuf_t token = NULL;
3604 struct flow_divert_group *control_group = NULL;
39236c6e
A		 3605
3606 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3607 error = EINVAL;
3608 goto done;
3609 }
3610
3611 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3612
3613 if (fd_cb->group == NULL) {
3614 error = EINVAL;
3615 goto done;
3616 }
3617
3618 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3619 if (error) {
3620 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3621 goto done;
3622 }
3623
3624 ctl_unit = htonl(fd_cb->group->ctl_unit);
3625
3626 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3627 if (error) {
3628 goto done;
3629 }
3630
3631 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3632 if (error) {
3633 goto done;
3634 }
3635
490019cf
A		 3636 if (fd_cb->app_data != NULL) {
3637 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3638 if (error) {
3639 goto done;
3640 }
3641 }
3642
39236c6e
A		 3643 socket_unlock(so, 0);
3644 lck_rw_lock_shared(&g_flow_divert_group_lck);
3645
3646 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
0a7de745 3647 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
39236c6e
A		 3648 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3649 }
3650
3651 if (control_group != NULL) {
3652 lck_rw_lock_shared(&control_group->lck);
3653 ctl_unit = htonl(control_group->ctl_unit);
3654 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3655 if (!error) {
3656 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3657 }
3658 lck_rw_done(&control_group->lck);
3659 } else {
3660 error = ENOPROTOOPT;
3661 }
3662
3663 lck_rw_done(&g_flow_divert_group_lck);
3664 socket_lock(so, 0);
3665
3666 if (error) {
3667 goto done;
3668 }
3669
3670 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3671 if (error) {
3672 goto done;
3673 }
3674
39037602
A		 3675 if (sopt->sopt_val == USER_ADDR_NULL) {
3676 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3677 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3678 goto done;
3679 }
3680
39236c6e
A		 3681 error = soopt_mcopyout(sopt, token);
3682 if (error) {
0a7de745 3683 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
39236c6e
A		 3684 goto done;
3685 }
3686
3687done:
3688 if (token != NULL) {
3689 mbuf_freem(token);
3690 }
3691
3692 return error;
3693}
3694
3695static errno_t
3696flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3697{
0a7de745
A		 3698 struct flow_divert_group *new_group = NULL;
3699 int error = 0;
39236c6e
A		 3700
3701 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3702 error = EINVAL;
3703 goto done;
3704 }
3705
3706 *unitinfo = NULL;
3707
3708 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3709 if (new_group == NULL) {
3710 error = ENOBUFS;
3711 goto done;
3712 }
3713
3714 memset(new_group, 0, sizeof(*new_group));
3715
3716 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3717 RB_INIT(&new_group->pcb_tree);
3718 new_group->ctl_unit = sac->sc_unit;
3719 MBUFQ_INIT(&new_group->send_queue);
39037602 3720 new_group->signing_id_trie.root = NULL_TRIE_IDX;
39236c6e
A		 3721
3722 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3723
3724 if (g_flow_divert_groups == NULL) {
3725 MALLOC(g_flow_divert_groups,
0a7de745
A		 3726 struct flow_divert_group **,
3727 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3728 M_TEMP,
3729 M_WAITOK | M_ZERO);
39236c6e
A		 3730 }
3731
3732 if (g_flow_divert_groups == NULL) {
3733 error = ENOBUFS;
3734 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3735 error = EALREADY;
3736 } else {
3737 g_flow_divert_groups[sac->sc_unit] = new_group;
3738 g_active_group_count++;
3739 }
3740
3741 lck_rw_done(&g_flow_divert_group_lck);
3742
3743 *unitinfo = new_group;
3744
3745done:
3746 if (error != 0 && new_group != NULL) {
3747 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3748 }
3749 return error;
3750}
3751
3752static errno_t
3753flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3754{
0a7de745
A		 3755 struct flow_divert_group *group = NULL;
3756 errno_t error = 0;
39236c6e
A		 3757
3758 if (unit >= GROUP_COUNT_MAX) {
3759 return EINVAL;
3760 }
3761
3762 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3763
3764 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3765
3766 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3767 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
0a7de745 3768 g_flow_divert_groups, g_active_group_count);
39236c6e
A		 3769 }
3770
3771 group = g_flow_divert_groups[unit];
3772
3773 if (group != (struct flow_divert_group *)unitinfo) {
3774 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3775 }
3776
cb323159
A		 3777 g_flow_divert_groups[unit] = NULL;
3778 g_active_group_count--;
3779
3780 if (g_active_group_count == 0) {
3781 FREE(g_flow_divert_groups, M_TEMP);
3782 g_flow_divert_groups = NULL;
3783 }
3784
3785 lck_rw_done(&g_flow_divert_group_lck);
3786
39236c6e
A		 3787 if (group != NULL) {
3788 flow_divert_close_all(group);
cb323159
A		 3789
3790 lck_rw_lock_exclusive(&group->lck);
3791
39236c6e
A		 3792 if (group->token_key != NULL) {
3793 memset(group->token_key, 0, group->token_key_size);
3794 FREE(group->token_key, M_TEMP);
3795 group->token_key = NULL;
3796 group->token_key_size = 0;
3797 }
39037602
A		 3798
3799 /* Re-set the current trie */
3800 if (group->signing_id_trie.memory != NULL) {
3801 FREE(group->signing_id_trie.memory, M_TEMP);
3802 }
3803 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3804 group->signing_id_trie.root = NULL_TRIE_IDX;
3805
cb323159
A		 3806 lck_rw_done(&group->lck);
3807
39236c6e 3808 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
39236c6e
A		 3809 } else {
3810 error = EINVAL;
3811 }
3812
39236c6e
A		 3813 return error;
3814}
3815
3816static errno_t
3817flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3818{
3819 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3820}
3821
3822static void
3823flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3824{
0a7de745 3825 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
39236c6e
A		 3826
3827 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
0a7de745
A		 3828 struct flow_divert_pcb *fd_cb;
3829 SLIST_HEAD(, flow_divert_pcb) tmp_list;
39236c6e
A		 3830
3831 lck_rw_lock_shared(&g_flow_divert_group_lck);
3832 lck_rw_lock_exclusive(&group->lck);
3833
3834 while (!MBUFQ_EMPTY(&group->send_queue)) {
3835 mbuf_t next_packet;
3836 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3837 next_packet = MBUFQ_FIRST(&group->send_queue);
3838 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3839 if (error) {
3840 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3841 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3842 lck_rw_done(&group->lck);
3843 lck_rw_done(&g_flow_divert_group_lck);
3844 return;
3845 }
3846 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3847 }
3848
3849 SLIST_INIT(&tmp_list);
3850
3851 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3852 FDRETAIN(fd_cb);
3853 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3854 }
3855
3856 lck_rw_done(&group->lck);
3857
3858 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3859 FDLOCK(fd_cb);
3860 if (fd_cb->so != NULL) {
3861 socket_lock(fd_cb->so, 0);
3862 if (fd_cb->group != NULL) {
3863 flow_divert_send_buffered_data(fd_cb, FALSE);
3864 }
3865 socket_unlock(fd_cb->so, 0);
3866 }
3867 FDUNLOCK(fd_cb);
3868 FDRELEASE(fd_cb);
3869 }
3870
3871 lck_rw_done(&g_flow_divert_group_lck);
3872 }
3873}
3874
3875static int
3876flow_divert_kctl_init(void)
3877{
0a7de745
A		 3878 struct kern_ctl_reg ctl_reg;
3879 int result;
39236c6e
A		 3880
3881 memset(&ctl_reg, 0, sizeof(ctl_reg));
3882
fe8ab488 3883 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
0a7de745 3884 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
39236c6e
A		 3885 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3886 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3887 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3888
3889 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3890 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3891 ctl_reg.ctl_send = flow_divert_kctl_send;
3892 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3893
3894 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3895
3896 if (result) {
3897 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3898 return result;
3899 }
3900
3901 return 0;
3902}
3903
3904void
3905flow_divert_init(void)
3906{
3907 memset(&nil_pcb, 0, sizeof(nil_pcb));
fe8ab488 3908 nil_pcb.log_level = LOG_NOTICE;
39236c6e
A		 3909
3910 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
3911
3912 VERIFY(g_tcp_protosw != NULL);
3913
3914 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
3915 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
3916
3917 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
3918 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
3919 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
3920 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
3921 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3922 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
3923 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
3924 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
3925 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
3926 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3927 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3928
3929 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3930 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
3931
3932 /*
3933 * Socket filters shouldn't attach/detach to/from this protosw
3934 * since pr_protosw is to be used instead, which points to the
3935 * real protocol; if they do, it is a bug and we should panic.
3936 */
3937 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
3938 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3939 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
3940 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3941
3e170ce0
A		 3942 /* UDP */
3943 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
3944 VERIFY(g_udp_protosw != NULL);
3945
3946 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
3947 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
3948
3949 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
3950 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
3951 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
3952 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
3953 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3954 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
3955 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
3956 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
3957 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
3958 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
3959 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
3960 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 3961 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 3962
3963 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
3964 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
3965
3966 /*
0a7de745
A		 3967 * Socket filters shouldn't attach/detach to/from this protosw
3968 * since pr_protosw is to be used instead, which points to the
3969 * real protocol; if they do, it is a bug and we should panic.
3970 */
3e170ce0
A		 3971 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
3972 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
3973 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
3974 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3975
39236c6e
A		 3976#if INET6
3977 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
3978
3979 VERIFY(g_tcp6_protosw != NULL);
3980
3981 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
3982 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
3983
3984 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
3985 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
3986 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
3987 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
3988 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
3989 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
3990 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
3991 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
3992 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
3993 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
490019cf 3994 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
39236c6e
A		 3995
3996 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
3997 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
3998 /*
3999 * Socket filters shouldn't attach/detach to/from this protosw
4000 * since pr_protosw is to be used instead, which points to the
4001 * real protocol; if they do, it is a bug and we should panic.
4002 */
4003 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4004 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4005 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4006 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
3e170ce0
A		 4007
4008 /* UDP6 */
4009 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4010
4011 VERIFY(g_udp6_protosw != NULL);
4012
4013 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4014 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4015
4016 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4017 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4018 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
4019 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4020 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4021 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
4022 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4023 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4024 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4025 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4026 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4027 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
490019cf 4028 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
3e170ce0
A		 4029
4030 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4031 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4032 /*
0a7de745
A		 4033 * Socket filters shouldn't attach/detach to/from this protosw
4034 * since pr_protosw is to be used instead, which points to the
4035 * real protocol; if they do, it is a bug and we should panic.
4036 */
3e170ce0
A		 4037 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4038 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4039 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4040 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
0a7de745 4041#endif /* INET6 */
39236c6e
A		 4042
4043 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4044 if (flow_divert_grp_attr == NULL) {
4045 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4046 g_init_result = ENOMEM;
4047 goto done;
4048 }
4049
4050 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4051 if (flow_divert_mtx_grp == NULL) {
4052 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4053 g_init_result = ENOMEM;
4054 goto done;
4055 }
4056
4057 flow_divert_mtx_attr = lck_attr_alloc_init();
4058 if (flow_divert_mtx_attr == NULL) {
4059 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4060 g_init_result = ENOMEM;
4061 goto done;
4062 }
4063
4064 g_init_result = flow_divert_kctl_init();
4065 if (g_init_result) {
4066 goto done;
4067 }
4068
4069 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4070
39236c6e
A		 4071done:
4072 if (g_init_result != 0) {
4073 if (flow_divert_mtx_attr != NULL) {
4074 lck_attr_free(flow_divert_mtx_attr);
4075 flow_divert_mtx_attr = NULL;
4076 }
4077 if (flow_divert_mtx_grp != NULL) {
4078 lck_grp_free(flow_divert_mtx_grp);
4079 flow_divert_mtx_grp = NULL;
4080 }
4081 if (flow_divert_grp_attr != NULL) {
4082 lck_grp_attr_free(flow_divert_grp_attr);
4083 flow_divert_grp_attr = NULL;
4084 }
4085
4086 if (g_flow_divert_kctl_ref != NULL) {
4087 ctl_deregister(g_flow_divert_kctl_ref);
4088 g_flow_divert_kctl_ref = NULL;
4089 }
4090 }
4091}
mirror of XNU