]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/flow_divert.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-6153.121.1.tar.gz
[apple/xnu.git] / bsd / netinet / flow_divert.c
1 /*
2 * Copyright (c) 2012-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <string.h>
30 #include <sys/types.h>
31 #include <sys/syslog.h>
32 #include <sys/queue.h>
33 #include <sys/malloc.h>
34 #include <sys/socket.h>
35 #include <sys/kpi_mbuf.h>
36 #include <sys/mbuf.h>
37 #include <sys/domain.h>
38 #include <sys/protosw.h>
39 #include <sys/socketvar.h>
40 #include <sys/kernel.h>
41 #include <sys/systm.h>
42 #include <sys/kern_control.h>
43 #include <sys/ubc.h>
44 #include <sys/codesign.h>
45 #include <libkern/tree.h>
46 #include <kern/locks.h>
47 #include <kern/debug.h>
48 #include <kern/task.h>
49 #include <mach/task_info.h>
50 #include <net/if_var.h>
51 #include <net/route.h>
52 #include <net/flowhash.h>
53 #include <net/ntstat.h>
54 #include <net/content_filter.h>
55 #include <netinet/in.h>
56 #include <netinet/in_var.h>
57 #include <netinet/tcp.h>
58 #include <netinet/tcp_var.h>
59 #include <netinet/tcp_fsm.h>
60 #include <netinet/flow_divert.h>
61 #include <netinet/flow_divert_proto.h>
62 #if INET6
63 #include <netinet6/in6_pcb.h>
64 #include <netinet6/ip6protosw.h>
65 #endif /* INET6 */
66 #include <dev/random/randomdev.h>
67 #include <libkern/crypto/sha1.h>
68 #include <libkern/crypto/crypto_internal.h>
69 #include <os/log.h>
70 #include <corecrypto/cc.h>
71 #if CONTENT_FILTER
72 #include <net/content_filter.h>
73 #endif /* CONTENT_FILTER */
74
75 #define FLOW_DIVERT_CONNECT_STARTED 0x00000001
76 #define FLOW_DIVERT_READ_CLOSED 0x00000002
77 #define FLOW_DIVERT_WRITE_CLOSED 0x00000004
78 #define FLOW_DIVERT_TUNNEL_RD_CLOSED 0x00000008
79 #define FLOW_DIVERT_TUNNEL_WR_CLOSED 0x00000010
80 #define FLOW_DIVERT_TRANSFERRED 0x00000020
81 #define FLOW_DIVERT_HAS_HMAC 0x00000040
82
83 #define FDLOG(level, pcb, format, ...) \
84 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " format "\n", (pcb)->hash, __VA_ARGS__)
85
86 #define FDLOG0(level, pcb, msg) \
87 os_log_with_type(OS_LOG_DEFAULT, flow_divert_syslog_type_to_oslog_type(level), "(%u): " msg "\n", (pcb)->hash)
88
89 #define FDRETAIN(pcb) if ((pcb) != NULL) OSIncrementAtomic(&(pcb)->ref_count)
90 #define FDRELEASE(pcb) \
91 do { \
92 if ((pcb) != NULL && 1 == OSDecrementAtomic(&(pcb)->ref_count)) { \
93 flow_divert_pcb_destroy(pcb); \
94 } \
95 } while (0)
96
97 #define FDLOCK(pcb) lck_mtx_lock(&(pcb)->mtx)
98 #define FDUNLOCK(pcb) lck_mtx_unlock(&(pcb)->mtx)
99
100 #define FD_CTL_SENDBUFF_SIZE (128 * 1024)
101 #define FD_CTL_RCVBUFF_SIZE (128 * 1024)
102
103 #define GROUP_BIT_CTL_ENQUEUE_BLOCKED 0
104
105 #define GROUP_COUNT_MAX 32
106 #define FLOW_DIVERT_MAX_NAME_SIZE 4096
107 #define FLOW_DIVERT_MAX_KEY_SIZE 1024
108 #define FLOW_DIVERT_MAX_TRIE_MEMORY (1024 * 1024)
109
110 struct flow_divert_trie_node {
111 uint16_t start;
112 uint16_t length;
113 uint16_t child_map;
114 };
115
116 #define CHILD_MAP_SIZE 256
117 #define NULL_TRIE_IDX 0xffff
118 #define TRIE_NODE(t, i) ((t)->nodes[(i)])
119 #define TRIE_CHILD(t, i, b) (((t)->child_maps + (CHILD_MAP_SIZE * TRIE_NODE(t, i).child_map))[(b)])
120 #define TRIE_BYTE(t, i) ((t)->bytes[(i)])
121
122 static struct flow_divert_pcb nil_pcb;
123
124 decl_lck_rw_data(static, g_flow_divert_group_lck);
125 static struct flow_divert_group **g_flow_divert_groups = NULL;
126 static uint32_t g_active_group_count = 0;
127
128 static lck_grp_attr_t *flow_divert_grp_attr = NULL;
129 static lck_attr_t *flow_divert_mtx_attr = NULL;
130 static lck_grp_t *flow_divert_mtx_grp = NULL;
131 static errno_t g_init_result = 0;
132
133 static kern_ctl_ref g_flow_divert_kctl_ref = NULL;
134
135 static struct protosw g_flow_divert_in_protosw;
136 static struct pr_usrreqs g_flow_divert_in_usrreqs;
137 static struct protosw g_flow_divert_in_udp_protosw;
138 static struct pr_usrreqs g_flow_divert_in_udp_usrreqs;
139 #if INET6
140 static struct ip6protosw g_flow_divert_in6_protosw;
141 static struct pr_usrreqs g_flow_divert_in6_usrreqs;
142 static struct ip6protosw g_flow_divert_in6_udp_protosw;
143 static struct pr_usrreqs g_flow_divert_in6_udp_usrreqs;
144 #endif /* INET6 */
145
146 static struct protosw *g_tcp_protosw = NULL;
147 static struct ip6protosw *g_tcp6_protosw = NULL;
148 static struct protosw *g_udp_protosw = NULL;
149 static struct ip6protosw *g_udp6_protosw = NULL;
150
151 static errno_t
152 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr, struct sockaddr **dup);
153
154 static errno_t
155 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket);
156
157 static boolean_t
158 flow_divert_is_sockaddr_valid(struct sockaddr *addr);
159
160 static int
161 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr);
162
163 struct sockaddr *
164 flow_divert_get_buffered_target_address(mbuf_t buffer);
165
166 static boolean_t
167 flow_divert_has_pcb_local_address(const struct inpcb *inp);
168
169 static void
170 flow_divert_disconnect_socket(struct socket *so);
171
172 static inline uint8_t
173 flow_divert_syslog_type_to_oslog_type(int syslog_type)
174 {
175 switch (syslog_type) {
176 case LOG_ERR: return OS_LOG_TYPE_ERROR;
177 case LOG_INFO: return OS_LOG_TYPE_INFO;
178 case LOG_DEBUG: return OS_LOG_TYPE_DEBUG;
179 default: return OS_LOG_TYPE_DEFAULT;
180 }
181 }
182
183 static inline int
184 flow_divert_pcb_cmp(const struct flow_divert_pcb *pcb_a, const struct flow_divert_pcb *pcb_b)
185 {
186 return memcmp(&pcb_a->hash, &pcb_b->hash, sizeof(pcb_a->hash));
187 }
188
189 RB_PROTOTYPE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
190 RB_GENERATE(fd_pcb_tree, flow_divert_pcb, rb_link, flow_divert_pcb_cmp);
191
192 static const char *
193 flow_divert_packet_type2str(uint8_t packet_type)
194 {
195 switch (packet_type) {
196 case FLOW_DIVERT_PKT_CONNECT:
197 return "connect";
198 case FLOW_DIVERT_PKT_CONNECT_RESULT:
199 return "connect result";
200 case FLOW_DIVERT_PKT_DATA:
201 return "data";
202 case FLOW_DIVERT_PKT_CLOSE:
203 return "close";
204 case FLOW_DIVERT_PKT_READ_NOTIFY:
205 return "read notification";
206 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
207 return "properties update";
208 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
209 return "app map create";
210 default:
211 return "unknown";
212 }
213 }
214
215 static struct flow_divert_pcb *
216 flow_divert_pcb_lookup(uint32_t hash, struct flow_divert_group *group)
217 {
218 struct flow_divert_pcb key_item;
219 struct flow_divert_pcb *fd_cb = NULL;
220
221 key_item.hash = hash;
222
223 lck_rw_lock_shared(&group->lck);
224 fd_cb = RB_FIND(fd_pcb_tree, &group->pcb_tree, &key_item);
225 FDRETAIN(fd_cb);
226 lck_rw_done(&group->lck);
227
228 return fd_cb;
229 }
230
231 static errno_t
232 flow_divert_pcb_insert(struct flow_divert_pcb *fd_cb, uint32_t ctl_unit)
233 {
234 errno_t error = 0;
235 struct flow_divert_pcb *exist = NULL;
236 struct flow_divert_group *group;
237 static uint32_t g_nextkey = 1;
238 static uint32_t g_hash_seed = 0;
239 int try_count = 0;
240
241 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
242 return EINVAL;
243 }
244
245 socket_unlock(fd_cb->so, 0);
246 lck_rw_lock_shared(&g_flow_divert_group_lck);
247
248 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
249 FDLOG0(LOG_ERR, &nil_pcb, "No active groups, flow divert cannot be used for this socket");
250 error = ENETUNREACH;
251 goto done;
252 }
253
254 group = g_flow_divert_groups[ctl_unit];
255 if (group == NULL) {
256 FDLOG(LOG_ERR, &nil_pcb, "Group for control unit %u is NULL, flow divert cannot be used for this socket", ctl_unit);
257 error = ENETUNREACH;
258 goto done;
259 }
260
261 socket_lock(fd_cb->so, 0);
262
263 do {
264 uint32_t key[2];
265 uint32_t idx;
266
267 key[0] = g_nextkey++;
268 key[1] = RandomULong();
269
270 if (g_hash_seed == 0) {
271 g_hash_seed = RandomULong();
272 }
273
274 fd_cb->hash = net_flowhash(key, sizeof(key), g_hash_seed);
275
276 for (idx = 1; idx < GROUP_COUNT_MAX; idx++) {
277 struct flow_divert_group *curr_group = g_flow_divert_groups[idx];
278 if (curr_group != NULL && curr_group != group) {
279 lck_rw_lock_shared(&curr_group->lck);
280 exist = RB_FIND(fd_pcb_tree, &curr_group->pcb_tree, fd_cb);
281 lck_rw_done(&curr_group->lck);
282 if (exist != NULL) {
283 break;
284 }
285 }
286 }
287
288 if (exist == NULL) {
289 lck_rw_lock_exclusive(&group->lck);
290 exist = RB_INSERT(fd_pcb_tree, &group->pcb_tree, fd_cb);
291 lck_rw_done(&group->lck);
292 }
293 } while (exist != NULL && try_count++ < 3);
294
295 if (exist == NULL) {
296 fd_cb->group = group;
297 FDRETAIN(fd_cb); /* The group now has a reference */
298 } else {
299 fd_cb->hash = 0;
300 error = EEXIST;
301 }
302
303 socket_unlock(fd_cb->so, 0);
304
305 done:
306 lck_rw_done(&g_flow_divert_group_lck);
307 socket_lock(fd_cb->so, 0);
308
309 return error;
310 }
311
312 static struct flow_divert_pcb *
313 flow_divert_pcb_create(socket_t so)
314 {
315 struct flow_divert_pcb *new_pcb = NULL;
316
317 MALLOC_ZONE(new_pcb, struct flow_divert_pcb *, sizeof(*new_pcb), M_FLOW_DIVERT_PCB, M_WAITOK);
318 if (new_pcb == NULL) {
319 FDLOG0(LOG_ERR, &nil_pcb, "failed to allocate a pcb");
320 return NULL;
321 }
322
323 memset(new_pcb, 0, sizeof(*new_pcb));
324
325 lck_mtx_init(&new_pcb->mtx, flow_divert_mtx_grp, flow_divert_mtx_attr);
326 new_pcb->so = so;
327 new_pcb->log_level = nil_pcb.log_level;
328
329 FDRETAIN(new_pcb); /* Represents the socket's reference */
330
331 return new_pcb;
332 }
333
334 static void
335 flow_divert_pcb_destroy(struct flow_divert_pcb *fd_cb)
336 {
337 FDLOG(LOG_INFO, fd_cb, "Destroying, app tx %u, app rx %u, tunnel tx %u, tunnel rx %u",
338 fd_cb->bytes_written_by_app, fd_cb->bytes_read_by_app, fd_cb->bytes_sent, fd_cb->bytes_received);
339
340 if (fd_cb->local_address != NULL) {
341 FREE(fd_cb->local_address, M_SONAME);
342 }
343 if (fd_cb->remote_address != NULL) {
344 FREE(fd_cb->remote_address, M_SONAME);
345 }
346 if (fd_cb->connect_token != NULL) {
347 mbuf_freem(fd_cb->connect_token);
348 }
349 if (fd_cb->connect_packet != NULL) {
350 mbuf_freem(fd_cb->connect_packet);
351 }
352 if (fd_cb->app_data != NULL) {
353 FREE(fd_cb->app_data, M_TEMP);
354 }
355 FREE_ZONE(fd_cb, sizeof(*fd_cb), M_FLOW_DIVERT_PCB);
356 }
357
358 static void
359 flow_divert_pcb_remove(struct flow_divert_pcb *fd_cb)
360 {
361 if (fd_cb->group != NULL) {
362 struct flow_divert_group *group = fd_cb->group;
363 lck_rw_lock_exclusive(&group->lck);
364 FDLOG(LOG_INFO, fd_cb, "Removing from group %d, ref count = %d", group->ctl_unit, fd_cb->ref_count);
365 RB_REMOVE(fd_pcb_tree, &group->pcb_tree, fd_cb);
366 fd_cb->group = NULL;
367 FDRELEASE(fd_cb); /* Release the group's reference */
368 lck_rw_done(&group->lck);
369 }
370 }
371
372 static int
373 flow_divert_packet_init(struct flow_divert_pcb *fd_cb, uint8_t packet_type, mbuf_t *packet)
374 {
375 struct flow_divert_packet_header hdr;
376 int error = 0;
377
378 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, packet);
379 if (error) {
380 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
381 return error;
382 }
383
384 hdr.packet_type = packet_type;
385 hdr.conn_id = htonl(fd_cb->hash);
386
387 /* Lay down the header */
388 error = mbuf_copyback(*packet, 0, sizeof(hdr), &hdr, MBUF_DONTWAIT);
389 if (error) {
390 FDLOG(LOG_ERR, fd_cb, "mbuf_copyback(hdr) failed: %d", error);
391 mbuf_freem(*packet);
392 *packet = NULL;
393 return error;
394 }
395
396 return 0;
397 }
398
399 static int
400 flow_divert_packet_append_tlv(mbuf_t packet, uint8_t type, uint32_t length, const void *value)
401 {
402 uint32_t net_length = htonl(length);
403 int error = 0;
404
405 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(type), &type, MBUF_DONTWAIT);
406 if (error) {
407 FDLOG(LOG_ERR, &nil_pcb, "failed to append the type (%d)", type);
408 return error;
409 }
410
411 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), sizeof(net_length), &net_length, MBUF_DONTWAIT);
412 if (error) {
413 FDLOG(LOG_ERR, &nil_pcb, "failed to append the length (%u)", length);
414 return error;
415 }
416
417 error = mbuf_copyback(packet, mbuf_pkthdr_len(packet), length, value, MBUF_DONTWAIT);
418 if (error) {
419 FDLOG0(LOG_ERR, &nil_pcb, "failed to append the value");
420 return error;
421 }
422
423 return error;
424 }
425
426 static int
427 flow_divert_packet_find_tlv(mbuf_t packet, int offset, uint8_t type, int *err, int next)
428 {
429 size_t cursor = offset;
430 int error = 0;
431 uint32_t curr_length;
432 uint8_t curr_type;
433
434 *err = 0;
435
436 do {
437 if (!next) {
438 error = mbuf_copydata(packet, cursor, sizeof(curr_type), &curr_type);
439 if (error) {
440 *err = ENOENT;
441 return -1;
442 }
443 } else {
444 next = 0;
445 curr_type = FLOW_DIVERT_TLV_NIL;
446 }
447
448 if (curr_type != type) {
449 cursor += sizeof(curr_type);
450 error = mbuf_copydata(packet, cursor, sizeof(curr_length), &curr_length);
451 if (error) {
452 *err = error;
453 return -1;
454 }
455
456 cursor += (sizeof(curr_length) + ntohl(curr_length));
457 }
458 } while (curr_type != type);
459
460 return cursor;
461 }
462
463 static int
464 flow_divert_packet_get_tlv(mbuf_t packet, int offset, uint8_t type, size_t buff_len, void *buff, uint32_t *val_size)
465 {
466 int error = 0;
467 uint32_t length;
468 int tlv_offset;
469
470 tlv_offset = flow_divert_packet_find_tlv(packet, offset, type, &error, 0);
471 if (tlv_offset < 0) {
472 return error;
473 }
474
475 error = mbuf_copydata(packet, tlv_offset + sizeof(type), sizeof(length), &length);
476 if (error) {
477 return error;
478 }
479
480 length = ntohl(length);
481
482 uint32_t data_offset = tlv_offset + sizeof(type) + sizeof(length);
483
484 if (length > (mbuf_pkthdr_len(packet) - data_offset)) {
485 FDLOG(LOG_ERR, &nil_pcb, "Length of %u TLV (%u) is larger than remaining packet data (%lu)", type, length, (mbuf_pkthdr_len(packet) - data_offset));
486 return EINVAL;
487 }
488
489 if (val_size != NULL) {
490 *val_size = length;
491 }
492
493 if (buff != NULL && buff_len > 0) {
494 memset(buff, 0, buff_len);
495 size_t to_copy = (length < buff_len) ? length : buff_len;
496 error = mbuf_copydata(packet, data_offset, to_copy, buff);
497 if (error) {
498 return error;
499 }
500 }
501
502 return 0;
503 }
504
505 static int
506 flow_divert_packet_compute_hmac(mbuf_t packet, struct flow_divert_group *group, uint8_t *hmac)
507 {
508 mbuf_t curr_mbuf = packet;
509
510 if (g_crypto_funcs == NULL || group->token_key == NULL) {
511 return ENOPROTOOPT;
512 }
513
514 cchmac_di_decl(g_crypto_funcs->ccsha1_di, hmac_ctx);
515 g_crypto_funcs->cchmac_init_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, group->token_key_size, group->token_key);
516
517 while (curr_mbuf != NULL) {
518 g_crypto_funcs->cchmac_update_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, mbuf_len(curr_mbuf), mbuf_data(curr_mbuf));
519 curr_mbuf = mbuf_next(curr_mbuf);
520 }
521
522 g_crypto_funcs->cchmac_final_fn(g_crypto_funcs->ccsha1_di, hmac_ctx, hmac);
523
524 return 0;
525 }
526
527 static int
528 flow_divert_packet_verify_hmac(mbuf_t packet, uint32_t ctl_unit)
529 {
530 int error = 0;
531 struct flow_divert_group *group = NULL;
532 int hmac_offset;
533 uint8_t packet_hmac[SHA_DIGEST_LENGTH];
534 uint8_t computed_hmac[SHA_DIGEST_LENGTH];
535 mbuf_t tail;
536
537 lck_rw_lock_shared(&g_flow_divert_group_lck);
538
539 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
540 group = g_flow_divert_groups[ctl_unit];
541 }
542
543 if (group == NULL) {
544 lck_rw_done(&g_flow_divert_group_lck);
545 return ENOPROTOOPT;
546 }
547
548 lck_rw_lock_shared(&group->lck);
549
550 if (group->token_key == NULL) {
551 error = ENOPROTOOPT;
552 goto done;
553 }
554
555 hmac_offset = flow_divert_packet_find_tlv(packet, 0, FLOW_DIVERT_TLV_HMAC, &error, 0);
556 if (hmac_offset < 0) {
557 goto done;
558 }
559
560 error = flow_divert_packet_get_tlv(packet, hmac_offset, FLOW_DIVERT_TLV_HMAC, sizeof(packet_hmac), packet_hmac, NULL);
561 if (error) {
562 goto done;
563 }
564
565 /* Chop off the HMAC TLV */
566 error = mbuf_split(packet, hmac_offset, MBUF_WAITOK, &tail);
567 if (error) {
568 goto done;
569 }
570
571 mbuf_free(tail);
572
573 error = flow_divert_packet_compute_hmac(packet, group, computed_hmac);
574 if (error) {
575 goto done;
576 }
577
578 if (cc_cmp_safe(sizeof(packet_hmac), packet_hmac, computed_hmac)) {
579 FDLOG0(LOG_WARNING, &nil_pcb, "HMAC in token does not match computed HMAC");
580 error = EINVAL;
581 goto done;
582 }
583
584 done:
585 lck_rw_done(&group->lck);
586 lck_rw_done(&g_flow_divert_group_lck);
587 return error;
588 }
589
590 static void
591 flow_divert_add_data_statistics(struct flow_divert_pcb *fd_cb, int data_len, Boolean send)
592 {
593 struct inpcb *inp = NULL;
594 struct ifnet *ifp = NULL;
595 Boolean cell = FALSE;
596 Boolean wifi = FALSE;
597 Boolean wired = FALSE;
598
599 inp = sotoinpcb(fd_cb->so);
600 if (inp == NULL) {
601 return;
602 }
603
604 ifp = inp->inp_last_outifp;
605 if (ifp != NULL) {
606 cell = IFNET_IS_CELLULAR(ifp);
607 wifi = (!cell && IFNET_IS_WIFI(ifp));
608 wired = (!wifi && IFNET_IS_WIRED(ifp));
609 }
610
611 if (send) {
612 INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1);
613 INP_ADD_STAT(inp, cell, wifi, wired, txbytes, data_len);
614 } else {
615 INP_ADD_STAT(inp, cell, wifi, wired, rxpackets, 1);
616 INP_ADD_STAT(inp, cell, wifi, wired, rxbytes, data_len);
617 }
618 inp_set_activity_bitmap(inp);
619 }
620
621 static errno_t
622 flow_divert_check_no_cellular(struct flow_divert_pcb *fd_cb)
623 {
624 struct inpcb *inp = NULL;
625
626 inp = sotoinpcb(fd_cb->so);
627 if (inp && INP_NO_CELLULAR(inp) && inp->inp_last_outifp &&
628 IFNET_IS_CELLULAR(inp->inp_last_outifp)) {
629 return EHOSTUNREACH;
630 }
631
632 return 0;
633 }
634
635 static errno_t
636 flow_divert_check_no_expensive(struct flow_divert_pcb *fd_cb)
637 {
638 struct inpcb *inp = NULL;
639
640 inp = sotoinpcb(fd_cb->so);
641 if (inp && INP_NO_EXPENSIVE(inp) && inp->inp_last_outifp &&
642 IFNET_IS_EXPENSIVE(inp->inp_last_outifp)) {
643 return EHOSTUNREACH;
644 }
645
646 return 0;
647 }
648
649 static errno_t
650 flow_divert_check_no_constrained(struct flow_divert_pcb *fd_cb)
651 {
652 struct inpcb *inp = NULL;
653
654 inp = sotoinpcb(fd_cb->so);
655 if (inp && INP_NO_CONSTRAINED(inp) && inp->inp_last_outifp &&
656 IFNET_IS_CONSTRAINED(inp->inp_last_outifp)) {
657 return EHOSTUNREACH;
658 }
659
660 return 0;
661 }
662
663 static void
664 flow_divert_update_closed_state(struct flow_divert_pcb *fd_cb, int how, Boolean tunnel)
665 {
666 if (how != SHUT_RD) {
667 fd_cb->flags |= FLOW_DIVERT_WRITE_CLOSED;
668 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
669 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
670 /* If the tunnel is not accepting writes any more, then flush the send buffer */
671 sbflush(&fd_cb->so->so_snd);
672 }
673 }
674 if (how != SHUT_WR) {
675 fd_cb->flags |= FLOW_DIVERT_READ_CLOSED;
676 if (tunnel || !(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
677 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
678 }
679 }
680 }
681
682 static uint16_t
683 trie_node_alloc(struct flow_divert_trie *trie)
684 {
685 if (trie->nodes_free_next < trie->nodes_count) {
686 uint16_t node_idx = trie->nodes_free_next++;
687 TRIE_NODE(trie, node_idx).child_map = NULL_TRIE_IDX;
688 return node_idx;
689 } else {
690 return NULL_TRIE_IDX;
691 }
692 }
693
694 static uint16_t
695 trie_child_map_alloc(struct flow_divert_trie *trie)
696 {
697 if (trie->child_maps_free_next < trie->child_maps_count) {
698 return trie->child_maps_free_next++;
699 } else {
700 return NULL_TRIE_IDX;
701 }
702 }
703
704 static uint16_t
705 trie_bytes_move(struct flow_divert_trie *trie, uint16_t bytes_idx, size_t bytes_size)
706 {
707 uint16_t start = trie->bytes_free_next;
708 if (start + bytes_size <= trie->bytes_count) {
709 if (start != bytes_idx) {
710 memmove(&TRIE_BYTE(trie, start), &TRIE_BYTE(trie, bytes_idx), bytes_size);
711 }
712 trie->bytes_free_next += bytes_size;
713 return start;
714 } else {
715 return NULL_TRIE_IDX;
716 }
717 }
718
719 static uint16_t
720 flow_divert_trie_insert(struct flow_divert_trie *trie, uint16_t string_start, size_t string_len)
721 {
722 uint16_t current = trie->root;
723 uint16_t child = trie->root;
724 uint16_t string_end = string_start + string_len;
725 uint16_t string_idx = string_start;
726 uint16_t string_remainder = string_len;
727
728 while (child != NULL_TRIE_IDX) {
729 uint16_t parent = current;
730 uint16_t node_idx;
731 uint16_t current_end;
732
733 current = child;
734 child = NULL_TRIE_IDX;
735
736 current_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
737
738 for (node_idx = TRIE_NODE(trie, current).start;
739 node_idx < current_end &&
740 string_idx < string_end &&
741 TRIE_BYTE(trie, node_idx) == TRIE_BYTE(trie, string_idx);
742 node_idx++, string_idx++) {
743 ;
744 }
745
746 string_remainder = string_end - string_idx;
747
748 if (node_idx < (TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length)) {
749 /*
750 * We did not reach the end of the current node's string.
751 * We need to split the current node into two:
752 * 1. A new node that contains the prefix of the node that matches
753 * the prefix of the string being inserted.
754 * 2. The current node modified to point to the remainder
755 * of the current node's string.
756 */
757 uint16_t prefix = trie_node_alloc(trie);
758 if (prefix == NULL_TRIE_IDX) {
759 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while splitting an existing node");
760 return NULL_TRIE_IDX;
761 }
762
763 /*
764 * Prefix points to the portion of the current nodes's string that has matched
765 * the input string thus far.
766 */
767 TRIE_NODE(trie, prefix).start = TRIE_NODE(trie, current).start;
768 TRIE_NODE(trie, prefix).length = (node_idx - TRIE_NODE(trie, current).start);
769
770 /*
771 * Prefix has the current node as the child corresponding to the first byte
772 * after the split.
773 */
774 TRIE_NODE(trie, prefix).child_map = trie_child_map_alloc(trie);
775 if (TRIE_NODE(trie, prefix).child_map == NULL_TRIE_IDX) {
776 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while splitting an existing node");
777 return NULL_TRIE_IDX;
778 }
779 TRIE_CHILD(trie, prefix, TRIE_BYTE(trie, node_idx)) = current;
780
781 /* Parent has the prefix as the child correspoding to the first byte in the prefix */
782 TRIE_CHILD(trie, parent, TRIE_BYTE(trie, TRIE_NODE(trie, prefix).start)) = prefix;
783
784 /* Current node is adjusted to point to the remainder */
785 TRIE_NODE(trie, current).start = node_idx;
786 TRIE_NODE(trie, current).length -= TRIE_NODE(trie, prefix).length;
787
788 /* We want to insert the new leaf (if any) as a child of the prefix */
789 current = prefix;
790 }
791
792 if (string_remainder > 0) {
793 /*
794 * We still have bytes in the string that have not been matched yet.
795 * If the current node has children, iterate to the child corresponding
796 * to the next byte in the string.
797 */
798 if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
799 child = TRIE_CHILD(trie, current, TRIE_BYTE(trie, string_idx));
800 }
801 }
802 } /* while (child != NULL_TRIE_IDX) */
803
804 if (string_remainder > 0) {
805 /* Add a new leaf containing the remainder of the string */
806 uint16_t leaf = trie_node_alloc(trie);
807 if (leaf == NULL_TRIE_IDX) {
808 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of trie nodes while inserting a new leaf");
809 return NULL_TRIE_IDX;
810 }
811
812 TRIE_NODE(trie, leaf).start = trie_bytes_move(trie, string_idx, string_remainder);
813 if (TRIE_NODE(trie, leaf).start == NULL_TRIE_IDX) {
814 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of bytes while inserting a new leaf");
815 return NULL_TRIE_IDX;
816 }
817 TRIE_NODE(trie, leaf).length = string_remainder;
818
819 /* Set the new leaf as the child of the current node */
820 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
821 TRIE_NODE(trie, current).child_map = trie_child_map_alloc(trie);
822 if (TRIE_NODE(trie, current).child_map == NULL_TRIE_IDX) {
823 FDLOG0(LOG_ERR, &nil_pcb, "Ran out of child maps while inserting a new leaf");
824 return NULL_TRIE_IDX;
825 }
826 }
827 TRIE_CHILD(trie, current, TRIE_BYTE(trie, TRIE_NODE(trie, leaf).start)) = leaf;
828 current = leaf;
829 } /* else duplicate or this string is a prefix of one of the existing strings */
830
831 return current;
832 }
833
834 #define APPLE_WEBCLIP_ID_PREFIX "com.apple.webapp"
835 static uint16_t
836 flow_divert_trie_search(struct flow_divert_trie *trie, const uint8_t *string_bytes)
837 {
838 uint16_t current = trie->root;
839 uint16_t string_idx = 0;
840
841 while (current != NULL_TRIE_IDX) {
842 uint16_t next = NULL_TRIE_IDX;
843 uint16_t node_end = TRIE_NODE(trie, current).start + TRIE_NODE(trie, current).length;
844 uint16_t node_idx;
845
846 for (node_idx = TRIE_NODE(trie, current).start;
847 node_idx < node_end && string_bytes[string_idx] != '\0' && string_bytes[string_idx] == TRIE_BYTE(trie, node_idx);
848 node_idx++, string_idx++) {
849 ;
850 }
851
852 if (node_idx == node_end) {
853 if (string_bytes[string_idx] == '\0') {
854 return current; /* Got an exact match */
855 } else if (string_idx == strlen(APPLE_WEBCLIP_ID_PREFIX) &&
856 0 == strncmp((const char *)string_bytes, APPLE_WEBCLIP_ID_PREFIX, string_idx)) {
857 return current; /* Got an apple webclip id prefix match */
858 } else if (TRIE_NODE(trie, current).child_map != NULL_TRIE_IDX) {
859 next = TRIE_CHILD(trie, current, string_bytes[string_idx]);
860 }
861 }
862 current = next;
863 }
864
865 return NULL_TRIE_IDX;
866 }
867
868 struct uuid_search_info {
869 uuid_t target_uuid;
870 char *found_signing_id;
871 boolean_t found_multiple_signing_ids;
872 proc_t found_proc;
873 };
874
875 static int
876 flow_divert_find_proc_by_uuid_callout(proc_t p, void *arg)
877 {
878 struct uuid_search_info *info = (struct uuid_search_info *)arg;
879 int result = PROC_RETURNED_DONE; /* By default, we didn't find the process */
880
881 if (info->found_signing_id != NULL) {
882 if (!info->found_multiple_signing_ids) {
883 /* All processes that were found had the same signing identifier, so just claim this first one and be done. */
884 info->found_proc = p;
885 result = PROC_CLAIMED_DONE;
886 } else {
887 uuid_string_t uuid_str;
888 uuid_unparse(info->target_uuid, uuid_str);
889 FDLOG(LOG_WARNING, &nil_pcb, "Found multiple processes with UUID %s with different signing identifiers", uuid_str);
890 }
891 FREE(info->found_signing_id, M_TEMP);
892 info->found_signing_id = NULL;
893 }
894
895 if (result == PROC_RETURNED_DONE) {
896 uuid_string_t uuid_str;
897 uuid_unparse(info->target_uuid, uuid_str);
898 FDLOG(LOG_WARNING, &nil_pcb, "Failed to find a process with UUID %s", uuid_str);
899 }
900
901 return result;
902 }
903
904 static int
905 flow_divert_find_proc_by_uuid_filter(proc_t p, void *arg)
906 {
907 struct uuid_search_info *info = (struct uuid_search_info *)arg;
908 int include = 0;
909
910 if (info->found_multiple_signing_ids) {
911 return include;
912 }
913
914 include = (uuid_compare(p->p_uuid, info->target_uuid) == 0);
915 if (include) {
916 const char *signing_id = cs_identity_get(p);
917 if (signing_id != NULL) {
918 FDLOG(LOG_INFO, &nil_pcb, "Found process %d with signing identifier %s", p->p_pid, signing_id);
919 size_t signing_id_size = strlen(signing_id) + 1;
920 if (info->found_signing_id == NULL) {
921 MALLOC(info->found_signing_id, char *, signing_id_size, M_TEMP, M_WAITOK);
922 memcpy(info->found_signing_id, signing_id, signing_id_size);
923 } else if (memcmp(signing_id, info->found_signing_id, signing_id_size)) {
924 info->found_multiple_signing_ids = TRUE;
925 }
926 } else {
927 info->found_multiple_signing_ids = TRUE;
928 }
929 include = !info->found_multiple_signing_ids;
930 }
931
932 return include;
933 }
934
935 static proc_t
936 flow_divert_find_proc_by_uuid(uuid_t uuid)
937 {
938 struct uuid_search_info info;
939
940 if (LOG_INFO <= nil_pcb.log_level) {
941 uuid_string_t uuid_str;
942 uuid_unparse(uuid, uuid_str);
943 FDLOG(LOG_INFO, &nil_pcb, "Looking for process with UUID %s", uuid_str);
944 }
945
946 memset(&info, 0, sizeof(info));
947 info.found_proc = PROC_NULL;
948 uuid_copy(info.target_uuid, uuid);
949
950 proc_iterate(PROC_ALLPROCLIST, flow_divert_find_proc_by_uuid_callout, &info, flow_divert_find_proc_by_uuid_filter, &info);
951
952 return info.found_proc;
953 }
954
955 static int
956 flow_divert_add_proc_info(struct flow_divert_pcb *fd_cb, proc_t proc, const char *signing_id, mbuf_t connect_packet, bool is_effective)
957 {
958 int error = 0;
959 int cdhash_error = 0;
960 unsigned char cdhash[SHA1_RESULTLEN] = { 0 };
961 audit_token_t audit_token = {};
962 const char *proc_cs_id = signing_id;
963
964 proc_lock(proc);
965
966 if (proc_cs_id == NULL) {
967 if (proc->p_csflags & (CS_VALID | CS_DEBUGGED)) {
968 proc_cs_id = cs_identity_get(proc);
969 } else {
970 FDLOG0(LOG_ERR, fd_cb, "Signature of proc is invalid");
971 }
972 }
973
974 if (is_effective) {
975 lck_rw_lock_shared(&fd_cb->group->lck);
976 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
977 if (proc_cs_id != NULL) {
978 uint16_t result = flow_divert_trie_search(&fd_cb->group->signing_id_trie, (const uint8_t *)proc_cs_id);
979 if (result == NULL_TRIE_IDX) {
980 FDLOG(LOG_WARNING, fd_cb, "%s did not match", proc_cs_id);
981 error = EPERM;
982 } else {
983 FDLOG(LOG_INFO, fd_cb, "%s matched", proc_cs_id);
984 }
985 } else {
986 error = EPERM;
987 }
988 }
989 lck_rw_done(&fd_cb->group->lck);
990 }
991
992 if (error != 0) {
993 goto done;
994 }
995
996 /*
997 * If signing_id is not NULL then it came from the flow divert token and will be added
998 * as part of the token, so there is no need to add it here.
999 */
1000 if (signing_id == NULL && proc_cs_id != NULL) {
1001 error = flow_divert_packet_append_tlv(connect_packet,
1002 (is_effective ? FLOW_DIVERT_TLV_SIGNING_ID : FLOW_DIVERT_TLV_APP_REAL_SIGNING_ID),
1003 strlen(proc_cs_id),
1004 proc_cs_id);
1005 if (error != 0) {
1006 FDLOG(LOG_ERR, fd_cb, "failed to append the signing ID: %d", error);
1007 goto done;
1008 }
1009 }
1010
1011 cdhash_error = proc_getcdhash(proc, cdhash);
1012 if (cdhash_error == 0) {
1013 error = flow_divert_packet_append_tlv(connect_packet,
1014 (is_effective ? FLOW_DIVERT_TLV_CDHASH : FLOW_DIVERT_TLV_APP_REAL_CDHASH),
1015 sizeof(cdhash),
1016 cdhash);
1017 if (error) {
1018 FDLOG(LOG_ERR, fd_cb, "failed to append the cdhash: %d", error);
1019 goto done;
1020 }
1021 } else {
1022 FDLOG(LOG_ERR, fd_cb, "failed to get the cdhash: %d", cdhash_error);
1023 }
1024
1025 task_t task = proc_task(proc);
1026 if (task != TASK_NULL) {
1027 mach_msg_type_number_t count = TASK_AUDIT_TOKEN_COUNT;
1028 kern_return_t rc = task_info(task, TASK_AUDIT_TOKEN, (task_info_t)&audit_token, &count);
1029 if (rc == KERN_SUCCESS) {
1030 int append_error = flow_divert_packet_append_tlv(connect_packet,
1031 (is_effective ? FLOW_DIVERT_TLV_APP_AUDIT_TOKEN : FLOW_DIVERT_TLV_APP_REAL_AUDIT_TOKEN),
1032 sizeof(audit_token_t),
1033 &audit_token);
1034 if (append_error) {
1035 FDLOG(LOG_ERR, fd_cb, "failed to append app audit token: %d", append_error);
1036 }
1037 }
1038 }
1039
1040 done:
1041 proc_unlock(proc);
1042
1043 return error;
1044 }
1045
1046 static int
1047 flow_divert_add_all_proc_info(struct flow_divert_pcb *fd_cb, struct socket *so, proc_t proc, const char *signing_id, mbuf_t connect_packet)
1048 {
1049 int error = 0;
1050 proc_t effective_proc = PROC_NULL;
1051 proc_t responsible_proc = PROC_NULL;
1052 proc_t real_proc = proc_find(so->last_pid);
1053 bool release_real_proc = true;
1054
1055 proc_t src_proc = PROC_NULL;
1056 proc_t real_src_proc = PROC_NULL;
1057
1058 if (real_proc == PROC_NULL) {
1059 FDLOG(LOG_ERR, fd_cb, "failed to find the real proc record for %d", so->last_pid);
1060 release_real_proc = false;
1061 real_proc = proc;
1062 if (real_proc == PROC_NULL) {
1063 real_proc = current_proc();
1064 }
1065 }
1066
1067 if (so->so_flags & SOF_DELEGATED) {
1068 if (real_proc->p_pid != so->e_pid) {
1069 effective_proc = proc_find(so->e_pid);
1070 } else if (uuid_compare(real_proc->p_uuid, so->e_uuid)) {
1071 effective_proc = flow_divert_find_proc_by_uuid(so->e_uuid);
1072 }
1073 }
1074
1075 #if defined(XNU_TARGET_OS_OSX)
1076 lck_rw_lock_shared(&fd_cb->group->lck);
1077 if (!(fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP)) {
1078 if (so->so_rpid > 0) {
1079 responsible_proc = proc_find(so->so_rpid);
1080 }
1081 }
1082 lck_rw_done(&fd_cb->group->lck);
1083 #endif
1084
1085 real_src_proc = real_proc;
1086
1087 if (responsible_proc != PROC_NULL) {
1088 src_proc = responsible_proc;
1089 if (effective_proc != NULL) {
1090 real_src_proc = effective_proc;
1091 }
1092 } else if (effective_proc != PROC_NULL) {
1093 src_proc = effective_proc;
1094 } else {
1095 src_proc = real_proc;
1096 }
1097
1098 error = flow_divert_add_proc_info(fd_cb, src_proc, signing_id, connect_packet, true);
1099 if (error != 0) {
1100 goto done;
1101 }
1102
1103 if (real_src_proc != NULL && real_src_proc != src_proc) {
1104 error = flow_divert_add_proc_info(fd_cb, real_src_proc, NULL, connect_packet, false);
1105 if (error != 0) {
1106 goto done;
1107 }
1108 }
1109
1110 done:
1111 if (responsible_proc != PROC_NULL) {
1112 proc_rele(responsible_proc);
1113 }
1114
1115 if (effective_proc != PROC_NULL) {
1116 proc_rele(effective_proc);
1117 }
1118
1119 if (real_proc != PROC_NULL && release_real_proc) {
1120 proc_rele(real_proc);
1121 }
1122
1123 return error;
1124 }
1125
1126 static int
1127 flow_divert_send_packet(struct flow_divert_pcb *fd_cb, mbuf_t packet, Boolean enqueue)
1128 {
1129 int error;
1130
1131 if (fd_cb->group == NULL) {
1132 fd_cb->so->so_error = ECONNABORTED;
1133 flow_divert_disconnect_socket(fd_cb->so);
1134 return ECONNABORTED;
1135 }
1136
1137 lck_rw_lock_shared(&fd_cb->group->lck);
1138
1139 if (MBUFQ_EMPTY(&fd_cb->group->send_queue)) {
1140 error = ctl_enqueuembuf(g_flow_divert_kctl_ref, fd_cb->group->ctl_unit, packet, CTL_DATA_EOR);
1141 } else {
1142 error = ENOBUFS;
1143 }
1144
1145 if (error == ENOBUFS) {
1146 if (enqueue) {
1147 if (!lck_rw_lock_shared_to_exclusive(&fd_cb->group->lck)) {
1148 lck_rw_lock_exclusive(&fd_cb->group->lck);
1149 }
1150 MBUFQ_ENQUEUE(&fd_cb->group->send_queue, packet);
1151 error = 0;
1152 }
1153 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &fd_cb->group->atomic_bits);
1154 }
1155
1156 lck_rw_done(&fd_cb->group->lck);
1157
1158 return error;
1159 }
1160
1161 static int
1162 flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr *to, struct socket *so, proc_t p, mbuf_t *out_connect_packet)
1163 {
1164 int error = 0;
1165 int flow_type = 0;
1166 char *signing_id = NULL;
1167 mbuf_t connect_packet = NULL;
1168 cfil_sock_id_t cfil_sock_id = CFIL_SOCK_ID_NONE;
1169 const void *cfil_id = NULL;
1170 size_t cfil_id_size = 0;
1171 struct inpcb *inp = sotoinpcb(so);
1172 struct ifnet *ifp = NULL;
1173
1174 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT, &connect_packet);
1175 if (error) {
1176 goto done;
1177 }
1178
1179
1180 if (fd_cb->connect_token != NULL && (fd_cb->flags & FLOW_DIVERT_HAS_HMAC)) {
1181 uint32_t sid_size = 0;
1182 int find_error = flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
1183 if (find_error == 0 && sid_size > 0) {
1184 MALLOC(signing_id, char *, sid_size + 1, M_TEMP, M_WAITOK | M_ZERO);
1185 if (signing_id != NULL) {
1186 flow_divert_packet_get_tlv(fd_cb->connect_token, 0, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, signing_id, NULL);
1187 FDLOG(LOG_INFO, fd_cb, "Got %s from token", signing_id);
1188 }
1189 }
1190 }
1191
1192 socket_unlock(so, 0);
1193
1194 error = flow_divert_add_all_proc_info(fd_cb, so, p, signing_id, connect_packet);
1195
1196 socket_lock(so, 0);
1197
1198 if (signing_id != NULL) {
1199 FREE(signing_id, M_TEMP);
1200 }
1201
1202 if (error) {
1203 FDLOG(LOG_ERR, fd_cb, "Failed to add source proc info: %d", error);
1204 goto done;
1205 }
1206
1207 error = flow_divert_packet_append_tlv(connect_packet,
1208 FLOW_DIVERT_TLV_TRAFFIC_CLASS,
1209 sizeof(fd_cb->so->so_traffic_class),
1210 &fd_cb->so->so_traffic_class);
1211 if (error) {
1212 goto done;
1213 }
1214
1215 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1216 flow_type = FLOW_DIVERT_FLOW_TYPE_TCP;
1217 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1218 flow_type = FLOW_DIVERT_FLOW_TYPE_UDP;
1219 } else {
1220 error = EINVAL;
1221 goto done;
1222 }
1223 error = flow_divert_packet_append_tlv(connect_packet,
1224 FLOW_DIVERT_TLV_FLOW_TYPE,
1225 sizeof(flow_type),
1226 &flow_type);
1227
1228 if (error) {
1229 goto done;
1230 }
1231
1232 if (fd_cb->connect_token != NULL) {
1233 unsigned int token_len = m_length(fd_cb->connect_token);
1234 mbuf_concatenate(connect_packet, fd_cb->connect_token);
1235 mbuf_pkthdr_adjustlen(connect_packet, token_len);
1236 fd_cb->connect_token = NULL;
1237 } else {
1238 uint32_t ctl_unit = htonl(fd_cb->control_group_unit);
1239
1240 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
1241 if (error) {
1242 goto done;
1243 }
1244
1245 error = flow_divert_append_target_endpoint_tlv(connect_packet, to);
1246 if (error) {
1247 goto done;
1248 }
1249 }
1250
1251 if (fd_cb->local_address != NULL) {
1252 error = EALREADY;
1253 goto done;
1254 } else {
1255 if (flow_divert_has_pcb_local_address(inp)) {
1256 error = flow_divert_inp_to_sockaddr(inp, &fd_cb->local_address);
1257 if (error) {
1258 FDLOG0(LOG_ERR, fd_cb, "failed to get the local socket address.");
1259 goto done;
1260 }
1261 }
1262 }
1263
1264 if (fd_cb->local_address != NULL) {
1265 /* socket is bound. */
1266 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_LOCAL_ADDR,
1267 fd_cb->local_address->sa_len, fd_cb->local_address);
1268 if (error) {
1269 goto done;
1270 }
1271 }
1272
1273 if ((inp->inp_flags | INP_BOUND_IF) && inp->inp_boundifp != NULL) {
1274 ifp = inp->inp_boundifp;
1275 } else if (inp->inp_last_outifp != NULL) {
1276 ifp = inp->inp_last_outifp;
1277 }
1278
1279 if (ifp != NULL) {
1280 uint32_t flow_if_index = ifp->if_index;
1281 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_OUT_IF_INDEX,
1282 sizeof(flow_if_index), &flow_if_index);
1283 if (error) {
1284 goto done;
1285 }
1286 }
1287
1288 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
1289 uint32_t flags = FLOW_DIVERT_TOKEN_FLAG_TFO;
1290 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags);
1291 if (error) {
1292 goto done;
1293 }
1294 }
1295
1296 cfil_sock_id = cfil_sock_id_from_socket(so);
1297 if (cfil_sock_id != CFIL_SOCK_ID_NONE) {
1298 cfil_id = &cfil_sock_id;
1299 cfil_id_size = sizeof(cfil_sock_id);
1300 } else if (so->so_flags1 & SOF1_CONTENT_FILTER_SKIP) {
1301 cfil_id = &inp->necp_client_uuid;
1302 cfil_id_size = sizeof(inp->necp_client_uuid);
1303 }
1304
1305 if (cfil_id != NULL && cfil_id_size > 0 && cfil_id_size <= sizeof(uuid_t)) {
1306 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_CFIL_ID, cfil_id_size, cfil_id);
1307 if (error) {
1308 goto done;
1309 }
1310 }
1311
1312 done:
1313 if (!error) {
1314 *out_connect_packet = connect_packet;
1315 } else if (connect_packet != NULL) {
1316 mbuf_freem(connect_packet);
1317 }
1318
1319 return error;
1320 }
1321
1322 static int
1323 flow_divert_send_connect_result(struct flow_divert_pcb *fd_cb)
1324 {
1325 int error = 0;
1326 mbuf_t packet = NULL;
1327 int rbuff_space = 0;
1328
1329 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CONNECT_RESULT, &packet);
1330 if (error) {
1331 FDLOG(LOG_ERR, fd_cb, "failed to create a connect result packet: %d", error);
1332 goto done;
1333 }
1334
1335 rbuff_space = fd_cb->so->so_rcv.sb_hiwat;
1336 if (rbuff_space < 0) {
1337 rbuff_space = 0;
1338 }
1339 rbuff_space = htonl(rbuff_space);
1340 error = flow_divert_packet_append_tlv(packet,
1341 FLOW_DIVERT_TLV_SPACE_AVAILABLE,
1342 sizeof(rbuff_space),
1343 &rbuff_space);
1344 if (error) {
1345 goto done;
1346 }
1347
1348 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1349 if (error) {
1350 goto done;
1351 }
1352
1353 done:
1354 if (error && packet != NULL) {
1355 mbuf_freem(packet);
1356 }
1357
1358 return error;
1359 }
1360
1361 static int
1362 flow_divert_send_close(struct flow_divert_pcb *fd_cb, int how)
1363 {
1364 int error = 0;
1365 mbuf_t packet = NULL;
1366 uint32_t zero = 0;
1367
1368 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_CLOSE, &packet);
1369 if (error) {
1370 FDLOG(LOG_ERR, fd_cb, "failed to create a close packet: %d", error);
1371 goto done;
1372 }
1373
1374 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(zero), &zero);
1375 if (error) {
1376 FDLOG(LOG_ERR, fd_cb, "failed to add the error code TLV: %d", error);
1377 goto done;
1378 }
1379
1380 how = htonl(how);
1381 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_HOW, sizeof(how), &how);
1382 if (error) {
1383 FDLOG(LOG_ERR, fd_cb, "failed to add the how flag: %d", error);
1384 goto done;
1385 }
1386
1387 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1388 if (error) {
1389 goto done;
1390 }
1391
1392 done:
1393 if (error && packet != NULL) {
1394 mbuf_free(packet);
1395 }
1396
1397 return error;
1398 }
1399
1400 static int
1401 flow_divert_tunnel_how_closed(struct flow_divert_pcb *fd_cb)
1402 {
1403 if ((fd_cb->flags & (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) ==
1404 (FLOW_DIVERT_TUNNEL_RD_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) {
1405 return SHUT_RDWR;
1406 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_RD_CLOSED) {
1407 return SHUT_RD;
1408 } else if (fd_cb->flags & FLOW_DIVERT_TUNNEL_WR_CLOSED) {
1409 return SHUT_WR;
1410 }
1411
1412 return -1;
1413 }
1414
1415 /*
1416 * Determine what close messages if any need to be sent to the tunnel. Returns TRUE if the tunnel is closed for both reads and
1417 * writes. Returns FALSE otherwise.
1418 */
1419 static void
1420 flow_divert_send_close_if_needed(struct flow_divert_pcb *fd_cb)
1421 {
1422 int how = -1;
1423
1424 /* Do not send any close messages if there is still data in the send buffer */
1425 if (fd_cb->so->so_snd.sb_cc == 0) {
1426 if ((fd_cb->flags & (FLOW_DIVERT_READ_CLOSED | FLOW_DIVERT_TUNNEL_RD_CLOSED)) == FLOW_DIVERT_READ_CLOSED) {
1427 /* Socket closed reads, but tunnel did not. Tell tunnel to close reads */
1428 how = SHUT_RD;
1429 }
1430 if ((fd_cb->flags & (FLOW_DIVERT_WRITE_CLOSED | FLOW_DIVERT_TUNNEL_WR_CLOSED)) == FLOW_DIVERT_WRITE_CLOSED) {
1431 /* Socket closed writes, but tunnel did not. Tell tunnel to close writes */
1432 if (how == SHUT_RD) {
1433 how = SHUT_RDWR;
1434 } else {
1435 how = SHUT_WR;
1436 }
1437 }
1438 }
1439
1440 if (how != -1) {
1441 FDLOG(LOG_INFO, fd_cb, "sending close, how = %d", how);
1442 if (flow_divert_send_close(fd_cb, how) != ENOBUFS) {
1443 /* Successfully sent the close packet. Record the ways in which the tunnel has been closed */
1444 if (how != SHUT_RD) {
1445 fd_cb->flags |= FLOW_DIVERT_TUNNEL_WR_CLOSED;
1446 }
1447 if (how != SHUT_WR) {
1448 fd_cb->flags |= FLOW_DIVERT_TUNNEL_RD_CLOSED;
1449 }
1450 }
1451 }
1452
1453 if (flow_divert_tunnel_how_closed(fd_cb) == SHUT_RDWR) {
1454 flow_divert_disconnect_socket(fd_cb->so);
1455 }
1456 }
1457
1458 static errno_t
1459 flow_divert_send_data_packet(struct flow_divert_pcb *fd_cb, mbuf_t data, size_t data_len, struct sockaddr *toaddr, Boolean force)
1460 {
1461 mbuf_t packet;
1462 mbuf_t last;
1463 int error = 0;
1464
1465 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_DATA, &packet);
1466 if (error) {
1467 FDLOG(LOG_ERR, fd_cb, "flow_divert_packet_init failed: %d", error);
1468 return error;
1469 }
1470
1471 if (toaddr != NULL) {
1472 error = flow_divert_append_target_endpoint_tlv(packet, toaddr);
1473 if (error) {
1474 FDLOG(LOG_ERR, fd_cb, "flow_divert_append_target_endpoint_tlv() failed: %d", error);
1475 return error;
1476 }
1477 }
1478
1479 if (data_len > 0 && data != NULL) {
1480 last = m_last(packet);
1481 mbuf_setnext(last, data);
1482 mbuf_pkthdr_adjustlen(packet, data_len);
1483 }
1484 error = flow_divert_send_packet(fd_cb, packet, force);
1485
1486 if (error) {
1487 mbuf_setnext(last, NULL);
1488 mbuf_freem(packet);
1489 } else {
1490 fd_cb->bytes_sent += data_len;
1491 flow_divert_add_data_statistics(fd_cb, data_len, TRUE);
1492 }
1493
1494 return error;
1495 }
1496
1497 static void
1498 flow_divert_send_buffered_data(struct flow_divert_pcb *fd_cb, Boolean force)
1499 {
1500 size_t to_send;
1501 size_t sent = 0;
1502 int error = 0;
1503 mbuf_t buffer;
1504
1505 to_send = fd_cb->so->so_snd.sb_cc;
1506 buffer = fd_cb->so->so_snd.sb_mb;
1507
1508 if (buffer == NULL && to_send > 0) {
1509 FDLOG(LOG_ERR, fd_cb, "Send buffer is NULL, but size is supposed to be %lu", to_send);
1510 return;
1511 }
1512
1513 /* Ignore the send window if force is enabled */
1514 if (!force && (to_send > fd_cb->send_window)) {
1515 to_send = fd_cb->send_window;
1516 }
1517
1518 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1519 while (sent < to_send) {
1520 mbuf_t data;
1521 size_t data_len;
1522
1523 data_len = to_send - sent;
1524 if (data_len > FLOW_DIVERT_CHUNK_SIZE) {
1525 data_len = FLOW_DIVERT_CHUNK_SIZE;
1526 }
1527
1528 error = mbuf_copym(buffer, sent, data_len, MBUF_DONTWAIT, &data);
1529 if (error) {
1530 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1531 break;
1532 }
1533
1534 error = flow_divert_send_data_packet(fd_cb, data, data_len, NULL, force);
1535 if (error) {
1536 mbuf_freem(data);
1537 break;
1538 }
1539
1540 sent += data_len;
1541 }
1542 sbdrop(&fd_cb->so->so_snd, sent);
1543 sowwakeup(fd_cb->so);
1544 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1545 mbuf_t data;
1546 mbuf_t m;
1547 size_t data_len;
1548
1549 while (buffer) {
1550 struct sockaddr *toaddr = flow_divert_get_buffered_target_address(buffer);
1551
1552 m = buffer;
1553 if (toaddr != NULL) {
1554 /* look for data in the chain */
1555 do {
1556 m = m->m_next;
1557 if (m != NULL && m->m_type == MT_DATA) {
1558 break;
1559 }
1560 } while (m);
1561 if (m == NULL) {
1562 /* unexpected */
1563 FDLOG0(LOG_ERR, fd_cb, "failed to find type MT_DATA in the mbuf chain.");
1564 goto move_on;
1565 }
1566 }
1567 data_len = mbuf_pkthdr_len(m);
1568 if (data_len > 0) {
1569 FDLOG(LOG_DEBUG, fd_cb, "mbuf_copym() data_len = %lu", data_len);
1570 error = mbuf_copym(m, 0, data_len, MBUF_DONTWAIT, &data);
1571 if (error) {
1572 FDLOG(LOG_ERR, fd_cb, "mbuf_copym failed: %d", error);
1573 break;
1574 }
1575 } else {
1576 data = NULL;
1577 }
1578 error = flow_divert_send_data_packet(fd_cb, data, data_len, toaddr, force);
1579 if (error) {
1580 mbuf_freem(data);
1581 break;
1582 }
1583 sent += data_len;
1584 move_on:
1585 buffer = buffer->m_nextpkt;
1586 (void) sbdroprecord(&(fd_cb->so->so_snd));
1587 }
1588 }
1589
1590 if (sent > 0) {
1591 FDLOG(LOG_DEBUG, fd_cb, "sent %lu bytes of buffered data", sent);
1592 if (fd_cb->send_window >= sent) {
1593 fd_cb->send_window -= sent;
1594 } else {
1595 fd_cb->send_window = 0;
1596 }
1597 }
1598 }
1599
1600 static int
1601 flow_divert_send_app_data(struct flow_divert_pcb *fd_cb, mbuf_t data, struct sockaddr *toaddr)
1602 {
1603 size_t to_send = mbuf_pkthdr_len(data);
1604 int error = 0;
1605
1606 if (to_send > fd_cb->send_window) {
1607 to_send = fd_cb->send_window;
1608 }
1609
1610 if (fd_cb->so->so_snd.sb_cc > 0) {
1611 to_send = 0; /* If the send buffer is non-empty, then we can't send anything */
1612 }
1613
1614 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
1615 size_t sent = 0;
1616 mbuf_t remaining_data = data;
1617 mbuf_t pkt_data = NULL;
1618 while (sent < to_send && remaining_data != NULL) {
1619 size_t pkt_data_len;
1620
1621 pkt_data = remaining_data;
1622
1623 if ((to_send - sent) > FLOW_DIVERT_CHUNK_SIZE) {
1624 pkt_data_len = FLOW_DIVERT_CHUNK_SIZE;
1625 } else {
1626 pkt_data_len = to_send - sent;
1627 }
1628
1629 if (pkt_data_len < mbuf_pkthdr_len(pkt_data)) {
1630 error = mbuf_split(pkt_data, pkt_data_len, MBUF_DONTWAIT, &remaining_data);
1631 if (error) {
1632 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
1633 pkt_data = NULL;
1634 break;
1635 }
1636 } else {
1637 remaining_data = NULL;
1638 }
1639
1640 error = flow_divert_send_data_packet(fd_cb, pkt_data, pkt_data_len, NULL, FALSE);
1641
1642 if (error) {
1643 break;
1644 }
1645
1646 pkt_data = NULL;
1647 sent += pkt_data_len;
1648 }
1649
1650 fd_cb->send_window -= sent;
1651
1652 error = 0;
1653
1654 if (pkt_data != NULL) {
1655 if (sbspace(&fd_cb->so->so_snd) > 0) {
1656 if (!sbappendstream(&fd_cb->so->so_snd, pkt_data)) {
1657 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with pkt_data, send buffer size = %u, send_window = %u\n",
1658 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1659 }
1660 } else {
1661 error = ENOBUFS;
1662 }
1663 }
1664
1665 if (remaining_data != NULL) {
1666 if (sbspace(&fd_cb->so->so_snd) > 0) {
1667 if (!sbappendstream(&fd_cb->so->so_snd, remaining_data)) {
1668 FDLOG(LOG_ERR, fd_cb, "sbappendstream failed with remaining_data, send buffer size = %u, send_window = %u\n",
1669 fd_cb->so->so_snd.sb_cc, fd_cb->send_window);
1670 }
1671 } else {
1672 error = ENOBUFS;
1673 }
1674 }
1675 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
1676 if (to_send || mbuf_pkthdr_len(data) == 0) {
1677 error = flow_divert_send_data_packet(fd_cb, data, to_send, toaddr, FALSE);
1678 if (error) {
1679 FDLOG(LOG_ERR, fd_cb, "flow_divert_send_data_packet failed. send data size = %lu", to_send);
1680 } else {
1681 fd_cb->send_window -= to_send;
1682 }
1683 } else {
1684 /* buffer it */
1685 if (sbspace(&fd_cb->so->so_snd) >= (int)mbuf_pkthdr_len(data)) {
1686 if (toaddr != NULL) {
1687 if (!sbappendaddr(&fd_cb->so->so_snd, toaddr, data, NULL, &error)) {
1688 FDLOG(LOG_ERR, fd_cb,
1689 "sbappendaddr failed. send buffer size = %u, send_window = %u, error = %d\n",
1690 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1691 }
1692 error = 0;
1693 } else {
1694 if (!sbappendrecord(&fd_cb->so->so_snd, data)) {
1695 FDLOG(LOG_ERR, fd_cb,
1696 "sbappendrecord failed. send buffer size = %u, send_window = %u, error = %d\n",
1697 fd_cb->so->so_snd.sb_cc, fd_cb->send_window, error);
1698 }
1699 }
1700 } else {
1701 error = ENOBUFS;
1702 }
1703 }
1704 }
1705
1706 return error;
1707 }
1708
1709 static int
1710 flow_divert_send_read_notification(struct flow_divert_pcb *fd_cb, uint32_t read_count)
1711 {
1712 int error = 0;
1713 mbuf_t packet = NULL;
1714 uint32_t net_read_count = htonl(read_count);
1715
1716 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_READ_NOTIFY, &packet);
1717 if (error) {
1718 FDLOG(LOG_ERR, fd_cb, "failed to create a read notification packet: %d", error);
1719 goto done;
1720 }
1721
1722 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_READ_COUNT, sizeof(net_read_count), &net_read_count);
1723 if (error) {
1724 FDLOG(LOG_ERR, fd_cb, "failed to add the read count: %d", error);
1725 goto done;
1726 }
1727
1728 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1729 if (error) {
1730 goto done;
1731 }
1732
1733 done:
1734 if (error && packet != NULL) {
1735 mbuf_free(packet);
1736 }
1737
1738 return error;
1739 }
1740
1741 static int
1742 flow_divert_send_traffic_class_update(struct flow_divert_pcb *fd_cb, int traffic_class)
1743 {
1744 int error = 0;
1745 mbuf_t packet = NULL;
1746
1747 error = flow_divert_packet_init(fd_cb, FLOW_DIVERT_PKT_PROPERTIES_UPDATE, &packet);
1748 if (error) {
1749 FDLOG(LOG_ERR, fd_cb, "failed to create a properties update packet: %d", error);
1750 goto done;
1751 }
1752
1753 error = flow_divert_packet_append_tlv(packet, FLOW_DIVERT_TLV_TRAFFIC_CLASS, sizeof(traffic_class), &traffic_class);
1754 if (error) {
1755 FDLOG(LOG_ERR, fd_cb, "failed to add the traffic class: %d", error);
1756 goto done;
1757 }
1758
1759 error = flow_divert_send_packet(fd_cb, packet, TRUE);
1760 if (error) {
1761 goto done;
1762 }
1763
1764 done:
1765 if (error && packet != NULL) {
1766 mbuf_free(packet);
1767 }
1768
1769 return error;
1770 }
1771
1772 static void
1773 flow_divert_handle_connect_result(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
1774 {
1775 uint32_t connect_error;
1776 uint32_t ctl_unit = 0;
1777 int error = 0;
1778 struct flow_divert_group *grp = NULL;
1779 struct sockaddr_storage local_address;
1780 int out_if_index = 0;
1781 struct sockaddr_storage remote_address;
1782 uint32_t send_window;
1783 uint32_t app_data_length = 0;
1784
1785 memset(&local_address, 0, sizeof(local_address));
1786 memset(&remote_address, 0, sizeof(remote_address));
1787
1788 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(connect_error), &connect_error, NULL);
1789 if (error) {
1790 FDLOG(LOG_ERR, fd_cb, "failed to get the connect result: %d", error);
1791 return;
1792 }
1793
1794 FDLOG(LOG_INFO, fd_cb, "received connect result %u", connect_error);
1795
1796 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_SPACE_AVAILABLE, sizeof(send_window), &send_window, NULL);
1797 if (error) {
1798 FDLOG(LOG_ERR, fd_cb, "failed to get the send window: %d", error);
1799 return;
1800 }
1801
1802 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit, NULL);
1803 if (error) {
1804 FDLOG0(LOG_INFO, fd_cb, "No control unit provided in the connect result");
1805 }
1806
1807 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
1808 if (error) {
1809 FDLOG0(LOG_INFO, fd_cb, "No local address provided");
1810 }
1811
1812 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
1813 if (error) {
1814 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
1815 }
1816
1817 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
1818 if (error) {
1819 FDLOG0(LOG_INFO, fd_cb, "No output if index provided");
1820 }
1821
1822 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
1823 if (error) {
1824 FDLOG0(LOG_INFO, fd_cb, "No application data provided in connect result");
1825 }
1826
1827 error = 0;
1828 connect_error = ntohl(connect_error);
1829 ctl_unit = ntohl(ctl_unit);
1830
1831 lck_rw_lock_shared(&g_flow_divert_group_lck);
1832
1833 if (connect_error == 0 && ctl_unit > 0) {
1834 if (ctl_unit >= GROUP_COUNT_MAX) {
1835 FDLOG(LOG_ERR, fd_cb, "Connect result contains an invalid control unit: %u", ctl_unit);
1836 error = EINVAL;
1837 } else if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
1838 FDLOG0(LOG_ERR, fd_cb, "No active groups, dropping connection");
1839 error = EINVAL;
1840 } else {
1841 grp = g_flow_divert_groups[ctl_unit];
1842 if (grp == NULL) {
1843 error = ECONNRESET;
1844 }
1845 }
1846 }
1847
1848 FDLOCK(fd_cb);
1849 if (fd_cb->so != NULL) {
1850 struct inpcb *inp = NULL;
1851 struct ifnet *ifp = NULL;
1852 struct flow_divert_group *old_group;
1853
1854 socket_lock(fd_cb->so, 0);
1855
1856 if (!(fd_cb->so->so_state & SS_ISCONNECTING)) {
1857 goto done;
1858 }
1859
1860 inp = sotoinpcb(fd_cb->so);
1861
1862 if (connect_error || error) {
1863 goto set_socket_state;
1864 }
1865
1866 if (local_address.ss_family == 0 && fd_cb->local_address == NULL) {
1867 error = EINVAL;
1868 goto set_socket_state;
1869 }
1870 if (local_address.ss_family != 0 && fd_cb->local_address == NULL) {
1871 if (local_address.ss_len > sizeof(local_address)) {
1872 local_address.ss_len = sizeof(local_address);
1873 }
1874 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
1875 }
1876 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&local_address)) {
1877 if (inp->inp_vflag & INP_IPV4 && local_address.ss_family == AF_INET) {
1878 struct sockaddr_in *local_in_address = (struct sockaddr_in *)&local_address;
1879 inp->inp_lport = local_in_address->sin_port;
1880 memcpy(&inp->inp_laddr, &local_in_address->sin_addr, sizeof(struct in_addr));
1881 } else if (inp->inp_vflag & INP_IPV6 && local_address.ss_family == AF_INET6) {
1882 struct sockaddr_in6 *local_in6_address = (struct sockaddr_in6 *)&local_address;
1883 inp->inp_lport = local_in6_address->sin6_port;
1884 memcpy(&inp->in6p_laddr, &local_in6_address->sin6_addr, sizeof(struct in6_addr));
1885 }
1886 }
1887
1888 if (remote_address.ss_family != 0) {
1889 if (fd_cb->remote_address != NULL) {
1890 FREE(fd_cb->remote_address, M_SONAME);
1891 fd_cb->remote_address = NULL;
1892 }
1893 if (remote_address.ss_len > sizeof(remote_address)) {
1894 remote_address.ss_len = sizeof(remote_address);
1895 }
1896 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
1897 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
1898 if (inp->inp_vflag & INP_IPV4 && remote_address.ss_family == AF_INET) {
1899 struct sockaddr_in *remote_in_address = (struct sockaddr_in *)&remote_address;
1900 inp->inp_fport = remote_in_address->sin_port;
1901 memcpy(&inp->inp_faddr, &remote_in_address->sin_addr, sizeof(struct in_addr));
1902 } else if (inp->inp_vflag & INP_IPV6 && remote_address.ss_family == AF_INET6) {
1903 struct sockaddr_in6 *remote_in6_address = (struct sockaddr_in6 *)&remote_address;
1904 inp->inp_fport = remote_in6_address->sin6_port;
1905 memcpy(&inp->in6p_faddr, &remote_in6_address->sin6_addr, sizeof(struct in6_addr));
1906 }
1907 }
1908 } else {
1909 error = EINVAL;
1910 goto set_socket_state;
1911 }
1912
1913 if (app_data_length > 0) {
1914 uint8_t *app_data = NULL;
1915 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
1916 if (app_data != NULL) {
1917 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
1918 if (error == 0) {
1919 FDLOG(LOG_INFO, fd_cb, "Got %u bytes of app data from the connect result", app_data_length);
1920 if (fd_cb->app_data != NULL) {
1921 FREE(fd_cb->app_data, M_TEMP);
1922 }
1923 fd_cb->app_data = app_data;
1924 fd_cb->app_data_length = app_data_length;
1925 } else {
1926 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the connect result packet", app_data_length);
1927 FREE(app_data, M_TEMP);
1928 }
1929 } else {
1930 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the connect result", app_data_length);
1931 }
1932 }
1933
1934 ifnet_head_lock_shared();
1935 if (out_if_index > 0 && out_if_index <= if_index) {
1936 ifp = ifindex2ifnet[out_if_index];
1937 }
1938
1939 if (ifp != NULL) {
1940 inp->inp_last_outifp = ifp;
1941 } else {
1942 error = EINVAL;
1943 }
1944 ifnet_head_done();
1945
1946 if (error) {
1947 goto set_socket_state;
1948 }
1949
1950 if (fd_cb->group == NULL) {
1951 error = EINVAL;
1952 goto set_socket_state;
1953 }
1954
1955 if (grp != NULL) {
1956 old_group = fd_cb->group;
1957
1958 lck_rw_lock_exclusive(&old_group->lck);
1959 lck_rw_lock_exclusive(&grp->lck);
1960
1961 RB_REMOVE(fd_pcb_tree, &old_group->pcb_tree, fd_cb);
1962 if (RB_INSERT(fd_pcb_tree, &grp->pcb_tree, fd_cb) != NULL) {
1963 panic("group with unit %u already contains a connection with hash %u", grp->ctl_unit, fd_cb->hash);
1964 }
1965
1966 fd_cb->group = grp;
1967
1968 lck_rw_done(&grp->lck);
1969 lck_rw_done(&old_group->lck);
1970 }
1971
1972 fd_cb->send_window = ntohl(send_window);
1973
1974 set_socket_state:
1975 if (!connect_error && !error) {
1976 FDLOG0(LOG_INFO, fd_cb, "sending connect result");
1977 error = flow_divert_send_connect_result(fd_cb);
1978 }
1979
1980 if (connect_error || error) {
1981 if (!connect_error) {
1982 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
1983 fd_cb->so->so_error = error;
1984 flow_divert_send_close_if_needed(fd_cb);
1985 } else {
1986 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
1987 fd_cb->so->so_error = connect_error;
1988 }
1989 flow_divert_disconnect_socket(fd_cb->so);
1990 } else {
1991 #if NECP
1992 /* Update NECP client with connected five-tuple */
1993 if (!uuid_is_null(inp->necp_client_uuid)) {
1994 socket_unlock(fd_cb->so, 0);
1995 necp_client_assign_from_socket(fd_cb->so->last_pid, inp->necp_client_uuid, inp);
1996 socket_lock(fd_cb->so, 0);
1997 }
1998 #endif /* NECP */
1999
2000 flow_divert_send_buffered_data(fd_cb, FALSE);
2001 soisconnected(fd_cb->so);
2002 }
2003
2004 done:
2005 socket_unlock(fd_cb->so, 0);
2006 }
2007 FDUNLOCK(fd_cb);
2008
2009 lck_rw_done(&g_flow_divert_group_lck);
2010 }
2011
2012 static void
2013 flow_divert_handle_close(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2014 {
2015 uint32_t close_error;
2016 int error = 0;
2017 int how;
2018
2019 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_ERROR_CODE, sizeof(close_error), &close_error, NULL);
2020 if (error) {
2021 FDLOG(LOG_ERR, fd_cb, "failed to get the close error: %d", error);
2022 return;
2023 }
2024
2025 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_HOW, sizeof(how), &how, NULL);
2026 if (error) {
2027 FDLOG(LOG_ERR, fd_cb, "failed to get the close how flag: %d", error);
2028 return;
2029 }
2030
2031 how = ntohl(how);
2032
2033 FDLOG(LOG_INFO, fd_cb, "close received, how = %d", how);
2034
2035 FDLOCK(fd_cb);
2036 if (fd_cb->so != NULL) {
2037 socket_lock(fd_cb->so, 0);
2038
2039 fd_cb->so->so_error = ntohl(close_error);
2040
2041 flow_divert_update_closed_state(fd_cb, how, TRUE);
2042
2043 how = flow_divert_tunnel_how_closed(fd_cb);
2044 if (how == SHUT_RDWR) {
2045 flow_divert_disconnect_socket(fd_cb->so);
2046 } else if (how == SHUT_RD) {
2047 socantrcvmore(fd_cb->so);
2048 } else if (how == SHUT_WR) {
2049 socantsendmore(fd_cb->so);
2050 }
2051
2052 socket_unlock(fd_cb->so, 0);
2053 }
2054 FDUNLOCK(fd_cb);
2055 }
2056
2057 static mbuf_t
2058 flow_divert_get_control_mbuf(struct flow_divert_pcb *fd_cb)
2059 {
2060 struct inpcb *inp = sotoinpcb(fd_cb->so);
2061 if ((inp->inp_vflag & INP_IPV4) && (inp->inp_flags & INP_RECVDSTADDR)) {
2062 struct in_addr ia = { };
2063
2064 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in)) {
2065 struct sockaddr_in *sin = (struct sockaddr_in *)(void *)fd_cb->local_address;
2066 bcopy(&sin->sin_addr, &ia, sizeof(struct in_addr));
2067 }
2068
2069 return sbcreatecontrol((caddr_t)&ia, sizeof(ia), IP_RECVDSTADDR, IPPROTO_IP);
2070 } else if ((inp->inp_vflag & INP_IPV6) && (inp->inp_flags & IN6P_PKTINFO)) {
2071 struct in6_pktinfo pi6;
2072 memset(&pi6, 0, sizeof(pi6));
2073
2074 if (fd_cb->local_address != NULL && fd_cb->local_address->sa_family == AF_INET6 && fd_cb->local_address->sa_len >= sizeof(struct sockaddr_in6)) {
2075 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(void *)fd_cb->local_address;
2076 bcopy(&sin6->sin6_addr, &pi6.ipi6_addr, sizeof(struct in6_addr));
2077 pi6.ipi6_ifindex = 0;
2078 }
2079
2080 return sbcreatecontrol((caddr_t)&pi6, sizeof(pi6), IPV6_PKTINFO, IPPROTO_IPV6);
2081 }
2082 return NULL;
2083 }
2084
2085 static void
2086 flow_divert_handle_data(struct flow_divert_pcb *fd_cb, mbuf_t packet, size_t offset)
2087 {
2088 FDLOCK(fd_cb);
2089 if (fd_cb->so != NULL) {
2090 int error = 0;
2091 mbuf_t data = NULL;
2092 size_t data_size;
2093 struct sockaddr_storage remote_address;
2094 boolean_t got_remote_sa = FALSE;
2095
2096 socket_lock(fd_cb->so, 0);
2097
2098 if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2099 uint32_t val_size = 0;
2100
2101 /* check if we got remote address with data */
2102 memset(&remote_address, 0, sizeof(remote_address));
2103 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, &val_size);
2104 if (error || val_size > sizeof(remote_address)) {
2105 FDLOG0(LOG_INFO, fd_cb, "No remote address provided");
2106 error = 0;
2107 } else {
2108 if (remote_address.ss_len > sizeof(remote_address)) {
2109 remote_address.ss_len = sizeof(remote_address);
2110 }
2111 /* validate the address */
2112 if (flow_divert_is_sockaddr_valid((struct sockaddr *)&remote_address)) {
2113 got_remote_sa = TRUE;
2114 }
2115 offset += (sizeof(uint8_t) + sizeof(uint32_t) + val_size);
2116 }
2117 }
2118
2119 data_size = (mbuf_pkthdr_len(packet) - offset);
2120
2121 FDLOG(LOG_DEBUG, fd_cb, "received %lu bytes of data", data_size);
2122
2123 error = mbuf_split(packet, offset, MBUF_DONTWAIT, &data);
2124 if (error || data == NULL) {
2125 FDLOG(LOG_ERR, fd_cb, "mbuf_split failed: %d", error);
2126 } else {
2127 if (flow_divert_check_no_cellular(fd_cb) ||
2128 flow_divert_check_no_expensive(fd_cb) ||
2129 flow_divert_check_no_constrained(fd_cb)) {
2130 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2131 flow_divert_send_close(fd_cb, SHUT_RDWR);
2132 flow_divert_disconnect_socket(fd_cb->so);
2133 } else if (!(fd_cb->so->so_state & SS_CANTRCVMORE)) {
2134 if (SOCK_TYPE(fd_cb->so) == SOCK_STREAM) {
2135 int appended = sbappendstream(&fd_cb->so->so_rcv, data);
2136 fd_cb->bytes_received += data_size;
2137 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2138 fd_cb->sb_size += data_size;
2139 if (appended) {
2140 sorwakeup(fd_cb->so);
2141 }
2142 data = NULL;
2143 } else if (SOCK_TYPE(fd_cb->so) == SOCK_DGRAM) {
2144 struct sockaddr *append_sa;
2145 mbuf_t mctl;
2146
2147 if (got_remote_sa == TRUE) {
2148 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2149 (struct sockaddr *)&remote_address, &append_sa);
2150 } else {
2151 error = flow_divert_dup_addr(fd_cb->so->so_proto->pr_domain->dom_family,
2152 fd_cb->remote_address, &append_sa);
2153 }
2154 if (error) {
2155 FDLOG0(LOG_ERR, fd_cb, "failed to dup the socket address.");
2156 }
2157
2158 mctl = flow_divert_get_control_mbuf(fd_cb);
2159 int append_error = 0;
2160 if (sbappendaddr(&fd_cb->so->so_rcv, append_sa, data, mctl, &append_error) || append_error == EJUSTRETURN) {
2161 fd_cb->bytes_received += data_size;
2162 flow_divert_add_data_statistics(fd_cb, data_size, FALSE);
2163 fd_cb->sb_size += data_size;
2164 if (append_error == 0) {
2165 sorwakeup(fd_cb->so);
2166 }
2167 data = NULL;
2168 }
2169 if (!error) {
2170 FREE(append_sa, M_TEMP);
2171 }
2172 }
2173 }
2174 }
2175 socket_unlock(fd_cb->so, 0);
2176 }
2177 FDUNLOCK(fd_cb);
2178 }
2179
2180 static void
2181 flow_divert_handle_read_notification(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2182 {
2183 uint32_t read_count;
2184 int error = 0;
2185
2186 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_READ_COUNT, sizeof(read_count), &read_count, NULL);
2187 if (error) {
2188 FDLOG(LOG_ERR, fd_cb, "failed to get the read count: %d", error);
2189 return;
2190 }
2191
2192 FDLOG(LOG_DEBUG, fd_cb, "received a read notification for %u bytes", ntohl(read_count));
2193
2194 FDLOCK(fd_cb);
2195 if (fd_cb->so != NULL) {
2196 socket_lock(fd_cb->so, 0);
2197 fd_cb->send_window += ntohl(read_count);
2198 flow_divert_send_buffered_data(fd_cb, FALSE);
2199 socket_unlock(fd_cb->so, 0);
2200 }
2201 FDUNLOCK(fd_cb);
2202 }
2203
2204 static void
2205 flow_divert_handle_group_init(struct flow_divert_group *group, mbuf_t packet, int offset)
2206 {
2207 int error = 0;
2208 uint32_t key_size = 0;
2209 int log_level;
2210 uint32_t flags = 0;
2211
2212 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, 0, NULL, &key_size);
2213 if (error) {
2214 FDLOG(LOG_ERR, &nil_pcb, "failed to get the key size: %d", error);
2215 return;
2216 }
2217
2218 if (key_size == 0 || key_size > FLOW_DIVERT_MAX_KEY_SIZE) {
2219 FDLOG(LOG_ERR, &nil_pcb, "Invalid key size: %u", key_size);
2220 return;
2221 }
2222
2223 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOG_LEVEL, sizeof(log_level), &log_level, NULL);
2224 if (!error) {
2225 nil_pcb.log_level = log_level;
2226 }
2227
2228 lck_rw_lock_exclusive(&group->lck);
2229
2230 if (group->token_key != NULL) {
2231 FREE(group->token_key, M_TEMP);
2232 group->token_key = NULL;
2233 }
2234
2235 MALLOC(group->token_key, uint8_t *, key_size, M_TEMP, M_WAITOK);
2236 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_TOKEN_KEY, key_size, group->token_key, NULL);
2237 if (error) {
2238 FDLOG(LOG_ERR, &nil_pcb, "failed to get the token key: %d", error);
2239 FREE(group->token_key, M_TEMP);
2240 group->token_key = NULL;
2241 lck_rw_done(&group->lck);
2242 return;
2243 }
2244
2245 group->token_key_size = key_size;
2246
2247 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_FLAGS, sizeof(flags), &flags, NULL);
2248 if (!error) {
2249 group->flags = flags;
2250 }
2251
2252 lck_rw_done(&group->lck);
2253 }
2254
2255 static void
2256 flow_divert_handle_properties_update(struct flow_divert_pcb *fd_cb, mbuf_t packet, int offset)
2257 {
2258 int error = 0;
2259 struct sockaddr_storage local_address;
2260 int out_if_index = 0;
2261 struct sockaddr_storage remote_address;
2262 uint32_t app_data_length = 0;
2263
2264 FDLOG0(LOG_INFO, fd_cb, "received a properties update");
2265
2266 memset(&local_address, 0, sizeof(local_address));
2267 memset(&remote_address, 0, sizeof(remote_address));
2268
2269 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_LOCAL_ADDR, sizeof(local_address), &local_address, NULL);
2270 if (error) {
2271 FDLOG0(LOG_INFO, fd_cb, "No local address provided in properties update");
2272 }
2273
2274 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_REMOTE_ADDR, sizeof(remote_address), &remote_address, NULL);
2275 if (error) {
2276 FDLOG0(LOG_INFO, fd_cb, "No remote address provided in properties update");
2277 }
2278
2279 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_OUT_IF_INDEX, sizeof(out_if_index), &out_if_index, NULL);
2280 if (error) {
2281 FDLOG0(LOG_INFO, fd_cb, "No output if index provided in properties update");
2282 }
2283
2284 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, 0, NULL, &app_data_length);
2285 if (error) {
2286 FDLOG0(LOG_INFO, fd_cb, "No application data provided in properties update");
2287 }
2288
2289 FDLOCK(fd_cb);
2290 if (fd_cb->so != NULL) {
2291 socket_lock(fd_cb->so, 0);
2292
2293 if (local_address.ss_family != 0) {
2294 if (local_address.ss_len > sizeof(local_address)) {
2295 local_address.ss_len = sizeof(local_address);
2296 }
2297 if (fd_cb->local_address != NULL) {
2298 FREE(fd_cb->local_address, M_SONAME);
2299 fd_cb->local_address = NULL;
2300 }
2301 fd_cb->local_address = dup_sockaddr((struct sockaddr *)&local_address, 1);
2302 }
2303
2304 if (remote_address.ss_family != 0) {
2305 if (remote_address.ss_len > sizeof(remote_address)) {
2306 remote_address.ss_len = sizeof(remote_address);
2307 }
2308 if (fd_cb->remote_address != NULL) {
2309 FREE(fd_cb->remote_address, M_SONAME);
2310 fd_cb->remote_address = NULL;
2311 }
2312 fd_cb->remote_address = dup_sockaddr((struct sockaddr *)&remote_address, 1);
2313 }
2314
2315 if (out_if_index > 0) {
2316 struct inpcb *inp = NULL;
2317 struct ifnet *ifp = NULL;
2318
2319 inp = sotoinpcb(fd_cb->so);
2320
2321 ifnet_head_lock_shared();
2322 if (out_if_index <= if_index) {
2323 ifp = ifindex2ifnet[out_if_index];
2324 }
2325
2326 if (ifp != NULL) {
2327 inp->inp_last_outifp = ifp;
2328 }
2329 ifnet_head_done();
2330 }
2331
2332 if (app_data_length > 0) {
2333 uint8_t *app_data = NULL;
2334 MALLOC(app_data, uint8_t *, app_data_length, M_TEMP, M_WAITOK);
2335 if (app_data != NULL) {
2336 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_APP_DATA, app_data_length, app_data, NULL);
2337 if (error == 0) {
2338 if (fd_cb->app_data != NULL) {
2339 FREE(fd_cb->app_data, M_TEMP);
2340 }
2341 fd_cb->app_data = app_data;
2342 fd_cb->app_data_length = app_data_length;
2343 } else {
2344 FDLOG(LOG_ERR, fd_cb, "Failed to copy %u bytes of application data from the properties update packet", app_data_length);
2345 FREE(app_data, M_TEMP);
2346 }
2347 } else {
2348 FDLOG(LOG_ERR, fd_cb, "Failed to allocate a buffer of size %u to hold the application data from the properties update", app_data_length);
2349 }
2350 }
2351
2352 socket_unlock(fd_cb->so, 0);
2353 }
2354 FDUNLOCK(fd_cb);
2355 }
2356
2357 static void
2358 flow_divert_handle_app_map_create(struct flow_divert_group *group, mbuf_t packet, int offset)
2359 {
2360 size_t bytes_mem_size;
2361 size_t child_maps_mem_size;
2362 int cursor;
2363 int error = 0;
2364 struct flow_divert_trie new_trie;
2365 int insert_error = 0;
2366 size_t nodes_mem_size;
2367 int prefix_count = -1;
2368 int signing_id_count = 0;
2369 size_t trie_memory_size = 0;
2370
2371 lck_rw_lock_exclusive(&group->lck);
2372
2373 /* Re-set the current trie */
2374 if (group->signing_id_trie.memory != NULL) {
2375 FREE(group->signing_id_trie.memory, M_TEMP);
2376 }
2377 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
2378 group->signing_id_trie.root = NULL_TRIE_IDX;
2379
2380 memset(&new_trie, 0, sizeof(new_trie));
2381
2382 /* Get the number of shared prefixes in the new set of signing ID strings */
2383 error = flow_divert_packet_get_tlv(packet, offset, FLOW_DIVERT_TLV_PREFIX_COUNT, sizeof(prefix_count), &prefix_count, NULL);
2384
2385 if (prefix_count < 0 || error) {
2386 FDLOG(LOG_ERR, &nil_pcb, "Invalid prefix count (%d) or an error occurred while reading the prefix count: %d", prefix_count, error);
2387 lck_rw_done(&group->lck);
2388 return;
2389 }
2390
2391 /* Compute the number of signing IDs and the total amount of bytes needed to store them */
2392 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2393 cursor >= 0;
2394 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2395 uint32_t sid_size = 0;
2396 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2397 if (error || sid_size == 0) {
2398 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d: %d", cursor, error);
2399 signing_id_count = 0;
2400 break;
2401 }
2402 new_trie.bytes_count += sid_size;
2403 signing_id_count++;
2404 }
2405
2406 if (signing_id_count == 0) {
2407 lck_rw_done(&group->lck);
2408 return;
2409 }
2410
2411 new_trie.nodes_count = (prefix_count + signing_id_count + 1); /* + 1 for the root node */
2412 new_trie.child_maps_count = (prefix_count + 1); /* + 1 for the root node */
2413
2414 FDLOG(LOG_INFO, &nil_pcb, "Nodes count = %lu, child maps count = %lu, bytes_count = %lu",
2415 new_trie.nodes_count, new_trie.child_maps_count, new_trie.bytes_count);
2416
2417 if (os_mul_overflow(sizeof(*new_trie.nodes), new_trie.nodes_count, &nodes_mem_size) ||
2418 os_mul3_overflow(sizeof(*new_trie.child_maps), CHILD_MAP_SIZE, new_trie.child_maps_count, &child_maps_mem_size) ||
2419 os_mul_overflow(sizeof(*new_trie.bytes), new_trie.bytes_count, &bytes_mem_size) ||
2420 os_add3_overflow(nodes_mem_size, child_maps_mem_size, bytes_mem_size, &trie_memory_size)) {
2421 FDLOG0(LOG_ERR, &nil_pcb, "Overflow while computing trie memory sizes");
2422 lck_rw_done(&group->lck);
2423 return;
2424 }
2425
2426 if (trie_memory_size > FLOW_DIVERT_MAX_TRIE_MEMORY) {
2427 FDLOG(LOG_ERR, &nil_pcb, "Trie memory size (%lu) is too big (maximum is %u)", trie_memory_size, FLOW_DIVERT_MAX_TRIE_MEMORY);
2428 lck_rw_done(&group->lck);
2429 return;
2430 }
2431
2432 MALLOC(new_trie.memory, void *, trie_memory_size, M_TEMP, M_WAITOK);
2433 if (new_trie.memory == NULL) {
2434 FDLOG(LOG_ERR, &nil_pcb, "Failed to allocate %lu bytes of memory for the signing ID trie",
2435 nodes_mem_size + child_maps_mem_size + bytes_mem_size);
2436 lck_rw_done(&group->lck);
2437 return;
2438 }
2439
2440 /* Initialize the free lists */
2441 new_trie.nodes = (struct flow_divert_trie_node *)new_trie.memory;
2442 new_trie.nodes_free_next = 0;
2443 memset(new_trie.nodes, 0, nodes_mem_size);
2444
2445 new_trie.child_maps = (uint16_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size);
2446 new_trie.child_maps_free_next = 0;
2447 memset(new_trie.child_maps, 0xff, child_maps_mem_size);
2448
2449 new_trie.bytes = (uint8_t *)(void *)((uint8_t *)new_trie.memory + nodes_mem_size + child_maps_mem_size);
2450 new_trie.bytes_free_next = 0;
2451 memset(new_trie.bytes, 0, bytes_mem_size);
2452
2453 /* The root is an empty node */
2454 new_trie.root = trie_node_alloc(&new_trie);
2455
2456 /* Add each signing ID to the trie */
2457 for (cursor = flow_divert_packet_find_tlv(packet, offset, FLOW_DIVERT_TLV_SIGNING_ID, &error, 0);
2458 cursor >= 0;
2459 cursor = flow_divert_packet_find_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, &error, 1)) {
2460 uint32_t sid_size = 0;
2461 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, 0, NULL, &sid_size);
2462 if (error || sid_size == 0) {
2463 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the length of the signing identifier at offset %d while building: %d", cursor, error);
2464 insert_error = EINVAL;
2465 break;
2466 }
2467 if (new_trie.bytes_free_next + sid_size <= new_trie.bytes_count) {
2468 uint16_t new_node_idx;
2469 error = flow_divert_packet_get_tlv(packet, cursor, FLOW_DIVERT_TLV_SIGNING_ID, sid_size, &TRIE_BYTE(&new_trie, new_trie.bytes_free_next), NULL);
2470 if (error) {
2471 FDLOG(LOG_ERR, &nil_pcb, "Failed to read the signing identifier at offset %d: %d", cursor, error);
2472 insert_error = EINVAL;
2473 break;
2474 }
2475 new_node_idx = flow_divert_trie_insert(&new_trie, new_trie.bytes_free_next, sid_size);
2476 if (new_node_idx == NULL_TRIE_IDX) {
2477 insert_error = EINVAL;
2478 break;
2479 }
2480 } else {
2481 FDLOG0(LOG_ERR, &nil_pcb, "No place to put signing ID for insertion");
2482 insert_error = ENOBUFS;
2483 break;
2484 }
2485 }
2486
2487 if (!insert_error) {
2488 group->signing_id_trie = new_trie;
2489 } else {
2490 FREE(new_trie.memory, M_TEMP);
2491 }
2492
2493 lck_rw_done(&group->lck);
2494 }
2495
2496 static int
2497 flow_divert_input(mbuf_t packet, struct flow_divert_group *group)
2498 {
2499 struct flow_divert_packet_header hdr;
2500 int error = 0;
2501 struct flow_divert_pcb *fd_cb;
2502
2503 if (mbuf_pkthdr_len(packet) < sizeof(hdr)) {
2504 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) < sizeof hdr (%lu)", mbuf_pkthdr_len(packet), sizeof(hdr));
2505 error = EINVAL;
2506 goto done;
2507 }
2508
2509 if (mbuf_pkthdr_len(packet) > FD_CTL_RCVBUFF_SIZE) {
2510 FDLOG(LOG_ERR, &nil_pcb, "got a bad packet, length (%lu) > %d", mbuf_pkthdr_len(packet), FD_CTL_RCVBUFF_SIZE);
2511 error = EINVAL;
2512 goto done;
2513 }
2514
2515 error = mbuf_copydata(packet, 0, sizeof(hdr), &hdr);
2516 if (error) {
2517 FDLOG(LOG_ERR, &nil_pcb, "mbuf_copydata failed for the header: %d", error);
2518 error = ENOBUFS;
2519 goto done;
2520 }
2521
2522 hdr.conn_id = ntohl(hdr.conn_id);
2523
2524 if (hdr.conn_id == 0) {
2525 switch (hdr.packet_type) {
2526 case FLOW_DIVERT_PKT_GROUP_INIT:
2527 flow_divert_handle_group_init(group, packet, sizeof(hdr));
2528 break;
2529 case FLOW_DIVERT_PKT_APP_MAP_CREATE:
2530 flow_divert_handle_app_map_create(group, packet, sizeof(hdr));
2531 break;
2532 default:
2533 FDLOG(LOG_WARNING, &nil_pcb, "got an unknown message type: %d", hdr.packet_type);
2534 break;
2535 }
2536 goto done;
2537 }
2538
2539 fd_cb = flow_divert_pcb_lookup(hdr.conn_id, group); /* This retains the PCB */
2540 if (fd_cb == NULL) {
2541 if (hdr.packet_type != FLOW_DIVERT_PKT_CLOSE && hdr.packet_type != FLOW_DIVERT_PKT_READ_NOTIFY) {
2542 FDLOG(LOG_NOTICE, &nil_pcb, "got a %s message from group %d for an unknown pcb: %u", flow_divert_packet_type2str(hdr.packet_type), group->ctl_unit, hdr.conn_id);
2543 }
2544 goto done;
2545 }
2546
2547 switch (hdr.packet_type) {
2548 case FLOW_DIVERT_PKT_CONNECT_RESULT:
2549 flow_divert_handle_connect_result(fd_cb, packet, sizeof(hdr));
2550 break;
2551 case FLOW_DIVERT_PKT_CLOSE:
2552 flow_divert_handle_close(fd_cb, packet, sizeof(hdr));
2553 break;
2554 case FLOW_DIVERT_PKT_DATA:
2555 flow_divert_handle_data(fd_cb, packet, sizeof(hdr));
2556 break;
2557 case FLOW_DIVERT_PKT_READ_NOTIFY:
2558 flow_divert_handle_read_notification(fd_cb, packet, sizeof(hdr));
2559 break;
2560 case FLOW_DIVERT_PKT_PROPERTIES_UPDATE:
2561 flow_divert_handle_properties_update(fd_cb, packet, sizeof(hdr));
2562 break;
2563 default:
2564 FDLOG(LOG_WARNING, fd_cb, "got an unknown message type: %d", hdr.packet_type);
2565 break;
2566 }
2567
2568 FDRELEASE(fd_cb);
2569
2570 done:
2571 mbuf_freem(packet);
2572 return error;
2573 }
2574
2575 static void
2576 flow_divert_close_all(struct flow_divert_group *group)
2577 {
2578 struct flow_divert_pcb *fd_cb;
2579 SLIST_HEAD(, flow_divert_pcb) tmp_list;
2580
2581 SLIST_INIT(&tmp_list);
2582
2583 lck_rw_lock_exclusive(&group->lck);
2584
2585 MBUFQ_DRAIN(&group->send_queue);
2586
2587 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
2588 FDRETAIN(fd_cb);
2589 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
2590 }
2591
2592 lck_rw_done(&group->lck);
2593
2594 while (!SLIST_EMPTY(&tmp_list)) {
2595 fd_cb = SLIST_FIRST(&tmp_list);
2596 FDLOCK(fd_cb);
2597 SLIST_REMOVE_HEAD(&tmp_list, tmp_list_entry);
2598 if (fd_cb->so != NULL) {
2599 socket_lock(fd_cb->so, 0);
2600 flow_divert_pcb_remove(fd_cb);
2601 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, TRUE);
2602 fd_cb->so->so_error = ECONNABORTED;
2603 flow_divert_disconnect_socket(fd_cb->so);
2604 socket_unlock(fd_cb->so, 0);
2605 }
2606 FDUNLOCK(fd_cb);
2607 FDRELEASE(fd_cb);
2608 }
2609 }
2610
2611 void
2612 flow_divert_detach(struct socket *so)
2613 {
2614 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2615
2616 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2617
2618 so->so_flags &= ~SOF_FLOW_DIVERT;
2619 so->so_fd_pcb = NULL;
2620
2621 FDLOG(LOG_INFO, fd_cb, "Detaching, ref count = %d", fd_cb->ref_count);
2622
2623 if (fd_cb->group != NULL) {
2624 /* Last-ditch effort to send any buffered data */
2625 flow_divert_send_buffered_data(fd_cb, TRUE);
2626
2627 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2628 flow_divert_send_close_if_needed(fd_cb);
2629 /* Remove from the group */
2630 flow_divert_pcb_remove(fd_cb);
2631 }
2632
2633 socket_unlock(so, 0);
2634 FDLOCK(fd_cb);
2635 fd_cb->so = NULL;
2636 FDUNLOCK(fd_cb);
2637 socket_lock(so, 0);
2638
2639 FDRELEASE(fd_cb); /* Release the socket's reference */
2640 }
2641
2642 static int
2643 flow_divert_close(struct socket *so)
2644 {
2645 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2646
2647 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2648
2649 FDLOG0(LOG_INFO, fd_cb, "Closing");
2650
2651 if (SOCK_TYPE(so) == SOCK_STREAM) {
2652 soisdisconnecting(so);
2653 sbflush(&so->so_rcv);
2654 }
2655
2656 flow_divert_send_buffered_data(fd_cb, TRUE);
2657 flow_divert_update_closed_state(fd_cb, SHUT_RDWR, FALSE);
2658 flow_divert_send_close_if_needed(fd_cb);
2659
2660 /* Remove from the group */
2661 flow_divert_pcb_remove(fd_cb);
2662
2663 return 0;
2664 }
2665
2666 static int
2667 flow_divert_disconnectx(struct socket *so, sae_associd_t aid,
2668 sae_connid_t cid __unused)
2669 {
2670 if (aid != SAE_ASSOCID_ANY && aid != SAE_ASSOCID_ALL) {
2671 return EINVAL;
2672 }
2673
2674 return flow_divert_close(so);
2675 }
2676
2677 static int
2678 flow_divert_shutdown(struct socket *so)
2679 {
2680 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2681
2682 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2683
2684 FDLOG0(LOG_INFO, fd_cb, "Can't send more");
2685
2686 socantsendmore(so);
2687
2688 flow_divert_update_closed_state(fd_cb, SHUT_WR, FALSE);
2689 flow_divert_send_close_if_needed(fd_cb);
2690
2691 return 0;
2692 }
2693
2694 static int
2695 flow_divert_rcvd(struct socket *so, int flags __unused)
2696 {
2697 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2698 uint32_t latest_sb_size;
2699 uint32_t read_count;
2700
2701 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2702
2703 latest_sb_size = fd_cb->so->so_rcv.sb_cc;
2704
2705 if (fd_cb->sb_size < latest_sb_size) {
2706 panic("flow divert rcvd event handler (%u): saved rcv buffer size (%u) is less than latest rcv buffer size (%u)",
2707 fd_cb->hash, fd_cb->sb_size, latest_sb_size);
2708 }
2709
2710 read_count = fd_cb->sb_size - latest_sb_size;
2711
2712 FDLOG(LOG_DEBUG, fd_cb, "app read %u bytes", read_count);
2713
2714 if (read_count > 0 && flow_divert_send_read_notification(fd_cb, read_count) == 0) {
2715 fd_cb->bytes_read_by_app += read_count;
2716 fd_cb->sb_size = latest_sb_size;
2717 }
2718
2719 return 0;
2720 }
2721
2722 static int
2723 flow_divert_append_target_endpoint_tlv(mbuf_t connect_packet, struct sockaddr *toaddr)
2724 {
2725 int error = 0;
2726 int port = 0;
2727
2728 if (!flow_divert_is_sockaddr_valid(toaddr)) {
2729 FDLOG(LOG_ERR, &nil_pcb, "Invalid target address, family = %u, length = %u", toaddr->sa_family, toaddr->sa_len);
2730 error = EINVAL;
2731 goto done;
2732 }
2733
2734 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_ADDRESS, toaddr->sa_len, toaddr);
2735 if (error) {
2736 goto done;
2737 }
2738
2739 if (toaddr->sa_family == AF_INET) {
2740 port = ntohs((satosin(toaddr))->sin_port);
2741 }
2742 #if INET6
2743 else {
2744 port = ntohs((satosin6(toaddr))->sin6_port);
2745 }
2746 #endif
2747
2748 error = flow_divert_packet_append_tlv(connect_packet, FLOW_DIVERT_TLV_TARGET_PORT, sizeof(port), &port);
2749 if (error) {
2750 goto done;
2751 }
2752
2753 done:
2754 return error;
2755 }
2756
2757 struct sockaddr *
2758 flow_divert_get_buffered_target_address(mbuf_t buffer)
2759 {
2760 if (buffer != NULL && buffer->m_type == MT_SONAME) {
2761 struct sockaddr *toaddr = mtod(buffer, struct sockaddr *);
2762 if (toaddr != NULL && flow_divert_is_sockaddr_valid(toaddr)) {
2763 return toaddr;
2764 }
2765 }
2766 return NULL;
2767 }
2768
2769 static boolean_t
2770 flow_divert_is_sockaddr_valid(struct sockaddr *addr)
2771 {
2772 switch (addr->sa_family) {
2773 case AF_INET:
2774 if (addr->sa_len < sizeof(struct sockaddr_in)) {
2775 return FALSE;
2776 }
2777 break;
2778 #if INET6
2779 case AF_INET6:
2780 if (addr->sa_len < sizeof(struct sockaddr_in6)) {
2781 return FALSE;
2782 }
2783 break;
2784 #endif /* INET6 */
2785 default:
2786 return FALSE;
2787 }
2788 return TRUE;
2789 }
2790
2791 static errno_t
2792 flow_divert_inp_to_sockaddr(const struct inpcb *inp, struct sockaddr **local_socket)
2793 {
2794 int error = 0;
2795 union sockaddr_in_4_6 sin46;
2796
2797 bzero(&sin46, sizeof(sin46));
2798 if (inp->inp_vflag & INP_IPV4) {
2799 struct sockaddr_in *sin = &sin46.sin;
2800
2801 sin->sin_family = AF_INET;
2802 sin->sin_len = sizeof(*sin);
2803 sin->sin_port = inp->inp_lport;
2804 sin->sin_addr = inp->inp_laddr;
2805 } else if (inp->inp_vflag & INP_IPV6) {
2806 struct sockaddr_in6 *sin6 = &sin46.sin6;
2807
2808 sin6->sin6_len = sizeof(*sin6);
2809 sin6->sin6_family = AF_INET6;
2810 sin6->sin6_port = inp->inp_lport;
2811 sin6->sin6_addr = inp->in6p_laddr;
2812 }
2813 *local_socket = dup_sockaddr((struct sockaddr *)&sin46, 1);
2814 if (*local_socket == NULL) {
2815 error = ENOBUFS;
2816 }
2817 return error;
2818 }
2819
2820 static boolean_t
2821 flow_divert_has_pcb_local_address(const struct inpcb *inp)
2822 {
2823 return inp->inp_lport != 0;
2824 }
2825
2826 static errno_t
2827 flow_divert_dup_addr(sa_family_t family, struct sockaddr *addr,
2828 struct sockaddr **dup)
2829 {
2830 int error = 0;
2831 struct sockaddr *result;
2832 struct sockaddr_storage ss;
2833
2834 if (addr != NULL) {
2835 result = addr;
2836 } else {
2837 memset(&ss, 0, sizeof(ss));
2838 ss.ss_family = family;
2839 if (ss.ss_family == AF_INET) {
2840 ss.ss_len = sizeof(struct sockaddr_in);
2841 }
2842 #if INET6
2843 else if (ss.ss_family == AF_INET6) {
2844 ss.ss_len = sizeof(struct sockaddr_in6);
2845 }
2846 #endif /* INET6 */
2847 else {
2848 error = EINVAL;
2849 }
2850 result = (struct sockaddr *)&ss;
2851 }
2852
2853 if (!error) {
2854 *dup = dup_sockaddr(result, 1);
2855 if (*dup == NULL) {
2856 error = ENOBUFS;
2857 }
2858 }
2859
2860 return error;
2861 }
2862
2863 static void
2864 flow_divert_disconnect_socket(struct socket *so)
2865 {
2866 soisdisconnected(so);
2867 if (SOCK_TYPE(so) == SOCK_DGRAM) {
2868 struct inpcb *inp = NULL;
2869
2870 inp = sotoinpcb(so);
2871 if (inp != NULL) {
2872 #if INET6
2873 if (SOCK_CHECK_DOM(so, PF_INET6)) {
2874 in6_pcbdetach(inp);
2875 } else
2876 #endif /* INET6 */
2877 in_pcbdetach(inp);
2878 }
2879 }
2880 }
2881
2882 static errno_t
2883 flow_divert_getpeername(struct socket *so, struct sockaddr **sa)
2884 {
2885 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2886
2887 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2888
2889 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2890 fd_cb->remote_address,
2891 sa);
2892 }
2893
2894 static errno_t
2895 flow_divert_getsockaddr(struct socket *so, struct sockaddr **sa)
2896 {
2897 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2898
2899 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2900
2901 return flow_divert_dup_addr(so->so_proto->pr_domain->dom_family,
2902 fd_cb->local_address,
2903 sa);
2904 }
2905
2906 static errno_t
2907 flow_divert_ctloutput(struct socket *so, struct sockopt *sopt)
2908 {
2909 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2910
2911 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2912
2913 if (sopt->sopt_name == SO_TRAFFIC_CLASS) {
2914 if (sopt->sopt_dir == SOPT_SET && fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) {
2915 flow_divert_send_traffic_class_update(fd_cb, so->so_traffic_class);
2916 }
2917 }
2918
2919 if (SOCK_DOM(so) == PF_INET) {
2920 return g_tcp_protosw->pr_ctloutput(so, sopt);
2921 }
2922 #if INET6
2923 else if (SOCK_DOM(so) == PF_INET6) {
2924 return g_tcp6_protosw->pr_ctloutput(so, sopt);
2925 }
2926 #endif
2927 return 0;
2928 }
2929
2930 errno_t
2931 flow_divert_connect_out(struct socket *so, struct sockaddr *to, proc_t p)
2932 {
2933 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
2934 int error = 0;
2935 struct inpcb *inp = sotoinpcb(so);
2936 struct sockaddr_in *sinp;
2937 mbuf_t connect_packet = NULL;
2938 int do_send = 1;
2939
2940 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
2941
2942 if (fd_cb->group == NULL) {
2943 error = ENETUNREACH;
2944 goto done;
2945 }
2946
2947 if (inp == NULL) {
2948 error = EINVAL;
2949 goto done;
2950 } else if (inp->inp_state == INPCB_STATE_DEAD) {
2951 if (so->so_error) {
2952 error = so->so_error;
2953 so->so_error = 0;
2954 } else {
2955 error = EINVAL;
2956 }
2957 goto done;
2958 }
2959
2960 if ((fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && !(fd_cb->flags & FLOW_DIVERT_TRANSFERRED)) {
2961 error = EALREADY;
2962 goto done;
2963 }
2964
2965 if (fd_cb->flags & FLOW_DIVERT_TRANSFERRED) {
2966 FDLOG0(LOG_INFO, fd_cb, "fully transferred");
2967 fd_cb->flags &= ~FLOW_DIVERT_TRANSFERRED;
2968 if (fd_cb->remote_address != NULL) {
2969 soisconnected(fd_cb->so);
2970 goto done;
2971 }
2972 }
2973
2974 FDLOG0(LOG_INFO, fd_cb, "Connecting");
2975
2976 if (fd_cb->connect_packet == NULL) {
2977 if (to == NULL) {
2978 FDLOG0(LOG_ERR, fd_cb, "No destination address available when creating connect packet");
2979 error = EINVAL;
2980 goto done;
2981 }
2982
2983 sinp = (struct sockaddr_in *)(void *)to;
2984 if (sinp->sin_family == AF_INET && IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
2985 error = EAFNOSUPPORT;
2986 goto done;
2987 }
2988
2989 error = flow_divert_create_connect_packet(fd_cb, to, so, p, &connect_packet);
2990 if (error) {
2991 goto done;
2992 }
2993
2994 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
2995 FDLOG0(LOG_INFO, fd_cb, "Delaying sending the connect packet until send or receive");
2996 do_send = 0;
2997 }
2998 } else {
2999 FDLOG0(LOG_INFO, fd_cb, "Sending saved connect packet");
3000 connect_packet = fd_cb->connect_packet;
3001 fd_cb->connect_packet = NULL;
3002 }
3003
3004 if (do_send) {
3005 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3006 if (error) {
3007 goto done;
3008 }
3009
3010 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3011 } else {
3012 fd_cb->connect_packet = connect_packet;
3013 connect_packet = NULL;
3014 }
3015
3016 soisconnecting(so);
3017
3018 done:
3019 if (error && connect_packet != NULL) {
3020 mbuf_freem(connect_packet);
3021 }
3022 return error;
3023 }
3024
3025 static int
3026 flow_divert_connectx_out_common(struct socket *so, struct sockaddr *dst,
3027 struct proc *p, sae_connid_t *pcid, struct uio *auio, user_ssize_t *bytes_written)
3028 {
3029 struct inpcb *inp = sotoinpcb(so);
3030 int error;
3031
3032 if (inp == NULL) {
3033 return EINVAL;
3034 }
3035
3036 VERIFY(dst != NULL);
3037
3038 error = flow_divert_connect_out(so, dst, p);
3039
3040 if (error != 0) {
3041 return error;
3042 }
3043
3044 /* if there is data, send it */
3045 if (auio != NULL) {
3046 user_ssize_t datalen = 0;
3047
3048 socket_unlock(so, 0);
3049
3050 VERIFY(bytes_written != NULL);
3051
3052 datalen = uio_resid(auio);
3053 error = so->so_proto->pr_usrreqs->pru_sosend(so, NULL, (uio_t)auio, NULL, NULL, 0);
3054 socket_lock(so, 0);
3055
3056 if (error == 0 || error == EWOULDBLOCK) {
3057 *bytes_written = datalen - uio_resid(auio);
3058 }
3059
3060 /*
3061 * sosend returns EWOULDBLOCK if it's a non-blocking
3062 * socket or a timeout occured (this allows to return
3063 * the amount of queued data through sendit()).
3064 *
3065 * However, connectx() returns EINPROGRESS in case of a
3066 * blocking socket. So we change the return value here.
3067 */
3068 if (error == EWOULDBLOCK) {
3069 error = EINPROGRESS;
3070 }
3071 }
3072
3073 if (error == 0 && pcid != NULL) {
3074 *pcid = 1; /* there is only 1 connection for a TCP */
3075 }
3076
3077 return error;
3078 }
3079
3080 static int
3081 flow_divert_connectx_out(struct socket *so, struct sockaddr *src __unused,
3082 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3083 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3084 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3085 {
3086 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3087 }
3088
3089 #if INET6
3090 static int
3091 flow_divert_connectx6_out(struct socket *so, struct sockaddr *src __unused,
3092 struct sockaddr *dst, struct proc *p, uint32_t ifscope __unused,
3093 sae_associd_t aid __unused, sae_connid_t *pcid, uint32_t flags __unused, void *arg __unused,
3094 uint32_t arglen __unused, struct uio *uio, user_ssize_t *bytes_written)
3095 {
3096 return flow_divert_connectx_out_common(so, dst, p, pcid, uio, bytes_written);
3097 }
3098 #endif /* INET6 */
3099
3100 static int
3101 flow_divert_getconninfo(struct socket *so, sae_connid_t cid, uint32_t *flags,
3102 uint32_t *ifindex, int32_t *soerror, user_addr_t src, socklen_t *src_len,
3103 user_addr_t dst, socklen_t *dst_len, uint32_t *aux_type,
3104 user_addr_t aux_data __unused, uint32_t *aux_len)
3105 {
3106 int error = 0;
3107 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3108 struct ifnet *ifp = NULL;
3109 struct inpcb *inp = sotoinpcb(so);
3110
3111 VERIFY((so->so_flags & SOF_FLOW_DIVERT));
3112
3113 if (so->so_fd_pcb == NULL || inp == NULL) {
3114 error = EINVAL;
3115 goto out;
3116 }
3117
3118 if (cid != SAE_CONNID_ANY && cid != SAE_CONNID_ALL && cid != 1) {
3119 error = EINVAL;
3120 goto out;
3121 }
3122
3123 ifp = inp->inp_last_outifp;
3124 *ifindex = ((ifp != NULL) ? ifp->if_index : 0);
3125 *soerror = so->so_error;
3126 *flags = 0;
3127
3128 if (so->so_state & SS_ISCONNECTED) {
3129 *flags |= (CIF_CONNECTED | CIF_PREFERRED);
3130 }
3131
3132 if (fd_cb->local_address == NULL) {
3133 struct sockaddr_in sin;
3134 bzero(&sin, sizeof(sin));
3135 sin.sin_len = sizeof(sin);
3136 sin.sin_family = AF_INET;
3137 *src_len = sin.sin_len;
3138 if (src != USER_ADDR_NULL) {
3139 error = copyout(&sin, src, sin.sin_len);
3140 if (error != 0) {
3141 goto out;
3142 }
3143 }
3144 } else {
3145 *src_len = fd_cb->local_address->sa_len;
3146 if (src != USER_ADDR_NULL) {
3147 error = copyout(fd_cb->local_address, src, fd_cb->local_address->sa_len);
3148 if (error != 0) {
3149 goto out;
3150 }
3151 }
3152 }
3153
3154 if (fd_cb->remote_address == NULL) {
3155 struct sockaddr_in sin;
3156 bzero(&sin, sizeof(sin));
3157 sin.sin_len = sizeof(sin);
3158 sin.sin_family = AF_INET;
3159 *dst_len = sin.sin_len;
3160 if (dst != USER_ADDR_NULL) {
3161 error = copyout(&sin, dst, sin.sin_len);
3162 if (error != 0) {
3163 goto out;
3164 }
3165 }
3166 } else {
3167 *dst_len = fd_cb->remote_address->sa_len;
3168 if (dst != USER_ADDR_NULL) {
3169 error = copyout(fd_cb->remote_address, dst, fd_cb->remote_address->sa_len);
3170 if (error != 0) {
3171 goto out;
3172 }
3173 }
3174 }
3175
3176 *aux_type = 0;
3177 *aux_len = 0;
3178
3179 out:
3180 return error;
3181 }
3182
3183 static int
3184 flow_divert_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp __unused, struct proc *p __unused)
3185 {
3186 int error = 0;
3187
3188 switch (cmd) {
3189 case SIOCGCONNINFO32: {
3190 struct so_cinforeq32 cifr;
3191 bcopy(data, &cifr, sizeof(cifr));
3192 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3193 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3194 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3195 &cifr.scir_aux_type, cifr.scir_aux_data,
3196 &cifr.scir_aux_len);
3197 if (error == 0) {
3198 bcopy(&cifr, data, sizeof(cifr));
3199 }
3200 break;
3201 }
3202
3203 case SIOCGCONNINFO64: {
3204 struct so_cinforeq64 cifr;
3205 bcopy(data, &cifr, sizeof(cifr));
3206 error = flow_divert_getconninfo(so, cifr.scir_cid, &cifr.scir_flags,
3207 &cifr.scir_ifindex, &cifr.scir_error, cifr.scir_src,
3208 &cifr.scir_src_len, cifr.scir_dst, &cifr.scir_dst_len,
3209 &cifr.scir_aux_type, cifr.scir_aux_data,
3210 &cifr.scir_aux_len);
3211 if (error == 0) {
3212 bcopy(&cifr, data, sizeof(cifr));
3213 }
3214 break;
3215 }
3216
3217 default:
3218 error = EOPNOTSUPP;
3219 }
3220
3221 return error;
3222 }
3223
3224 static int
3225 flow_divert_in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3226 {
3227 int error = flow_divert_control(so, cmd, data, ifp, p);
3228
3229 if (error == EOPNOTSUPP) {
3230 error = in_control(so, cmd, data, ifp, p);
3231 }
3232
3233 return error;
3234 }
3235
3236 static int
3237 flow_divert_in6_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp, struct proc *p)
3238 {
3239 int error = flow_divert_control(so, cmd, data, ifp, p);
3240
3241 if (error == EOPNOTSUPP) {
3242 error = in6_control(so, cmd, data, ifp, p);
3243 }
3244
3245 return error;
3246 }
3247
3248 static errno_t
3249 flow_divert_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3250 {
3251 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3252 int error = 0;
3253 struct inpcb *inp;
3254 #if CONTENT_FILTER
3255 struct m_tag *cfil_tag = NULL;
3256 #endif
3257
3258 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3259
3260 inp = sotoinpcb(so);
3261 if (inp == NULL || inp->inp_state == INPCB_STATE_DEAD) {
3262 error = ECONNRESET;
3263 goto done;
3264 }
3265
3266 if (control && mbuf_len(control) > 0) {
3267 error = EINVAL;
3268 goto done;
3269 }
3270
3271 if (flags & MSG_OOB) {
3272 error = EINVAL;
3273 goto done; /* We don't support OOB data */
3274 }
3275
3276 error = flow_divert_check_no_cellular(fd_cb) ||
3277 flow_divert_check_no_expensive(fd_cb) ||
3278 flow_divert_check_no_constrained(fd_cb);
3279 if (error) {
3280 goto done;
3281 }
3282
3283 /* Implicit connect */
3284 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED)) {
3285 FDLOG0(LOG_INFO, fd_cb, "implicit connect");
3286
3287 #if CONTENT_FILTER
3288 /*
3289 * If the socket is subject to a UDP Content Filter and no remote address is passed in,
3290 * retrieve the CFIL saved remote address from the mbuf and use it.
3291 */
3292 if (to == NULL && so->so_cfil_db) {
3293 struct sockaddr *cfil_faddr = NULL;
3294 cfil_tag = cfil_dgram_get_socket_state(data, NULL, NULL, &cfil_faddr, NULL);
3295 if (cfil_tag) {
3296 to = (struct sockaddr *)(void *)cfil_faddr;
3297 }
3298 FDLOG(LOG_INFO, fd_cb, "Using remote address from CFIL saved state: %p", to);
3299 }
3300 #endif
3301 error = flow_divert_connect_out(so, to, p);
3302 if (error) {
3303 goto done;
3304 }
3305
3306 if (so->so_flags1 & SOF1_DATA_IDEMPOTENT) {
3307 /* Open up the send window so that the data will get sent right away */
3308 fd_cb->send_window = mbuf_pkthdr_len(data);
3309 }
3310 }
3311
3312 FDLOG(LOG_DEBUG, fd_cb, "app wrote %lu bytes", mbuf_pkthdr_len(data));
3313
3314 fd_cb->bytes_written_by_app += mbuf_pkthdr_len(data);
3315 error = flow_divert_send_app_data(fd_cb, data, to);
3316 if (error) {
3317 goto done;
3318 }
3319
3320 data = NULL;
3321
3322 if (flags & PRUS_EOF) {
3323 flow_divert_shutdown(so);
3324 }
3325
3326 done:
3327 if (data) {
3328 mbuf_freem(data);
3329 }
3330 if (control) {
3331 mbuf_free(control);
3332 }
3333 #if CONTENT_FILTER
3334 if (cfil_tag) {
3335 m_tag_free(cfil_tag);
3336 }
3337 #endif
3338
3339 return error;
3340 }
3341
3342 static int
3343 flow_divert_preconnect(struct socket *so)
3344 {
3345 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3346 int error = 0;
3347
3348 if (!(fd_cb->flags & FLOW_DIVERT_CONNECT_STARTED) && fd_cb->connect_packet != NULL) {
3349 FDLOG0(LOG_INFO, fd_cb, "Pre-connect read: sending saved connect packet");
3350 mbuf_t connect_packet = fd_cb->connect_packet;
3351 fd_cb->connect_packet = NULL;
3352
3353 error = flow_divert_send_packet(fd_cb, connect_packet, TRUE);
3354 if (error) {
3355 mbuf_freem(connect_packet);
3356 }
3357
3358 fd_cb->flags |= FLOW_DIVERT_CONNECT_STARTED;
3359 }
3360
3361 soclearfastopen(so);
3362
3363 return error;
3364 }
3365
3366 static void
3367 flow_divert_set_protosw(struct socket *so)
3368 {
3369 so->so_flags |= SOF_FLOW_DIVERT;
3370 if (SOCK_DOM(so) == PF_INET) {
3371 so->so_proto = &g_flow_divert_in_protosw;
3372 }
3373 #if INET6
3374 else {
3375 so->so_proto = (struct protosw *)&g_flow_divert_in6_protosw;
3376 }
3377 #endif /* INET6 */
3378 }
3379
3380 static void
3381 flow_divert_set_udp_protosw(struct socket *so)
3382 {
3383 so->so_flags |= SOF_FLOW_DIVERT;
3384 if (SOCK_DOM(so) == PF_INET) {
3385 so->so_proto = &g_flow_divert_in_udp_protosw;
3386 }
3387 #if INET6
3388 else {
3389 so->so_proto = (struct protosw *)&g_flow_divert_in6_udp_protosw;
3390 }
3391 #endif /* INET6 */
3392 }
3393
3394 static errno_t
3395 flow_divert_attach(struct socket *so, uint32_t flow_id, uint32_t ctl_unit)
3396 {
3397 int error = 0;
3398 struct flow_divert_pcb *fd_cb = NULL;
3399 struct ifnet *ifp = NULL;
3400 struct inpcb *inp = NULL;
3401 struct socket *old_so;
3402 mbuf_t recv_data = NULL;
3403
3404 socket_unlock(so, 0);
3405
3406 FDLOG(LOG_INFO, &nil_pcb, "Attaching socket to flow %u", flow_id);
3407
3408 /* Find the flow divert control block */
3409 lck_rw_lock_shared(&g_flow_divert_group_lck);
3410 if (g_flow_divert_groups != NULL && g_active_group_count > 0) {
3411 struct flow_divert_group *group = g_flow_divert_groups[ctl_unit];
3412 if (group != NULL) {
3413 fd_cb = flow_divert_pcb_lookup(flow_id, group);
3414 }
3415 }
3416 lck_rw_done(&g_flow_divert_group_lck);
3417
3418 if (fd_cb == NULL) {
3419 error = ENOENT;
3420 goto done;
3421 }
3422
3423 FDLOCK(fd_cb);
3424
3425 /* Dis-associate the flow divert control block from its current socket */
3426 old_so = fd_cb->so;
3427
3428 inp = sotoinpcb(old_so);
3429
3430 VERIFY(inp != NULL);
3431
3432 socket_lock(old_so, 0);
3433 flow_divert_disconnect_socket(old_so);
3434 old_so->so_flags &= ~SOF_FLOW_DIVERT;
3435 old_so->so_fd_pcb = NULL;
3436 if (SOCK_TYPE(old_so) == SOCK_STREAM) {
3437 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_TCP, SOCK_STREAM);
3438 } else if (SOCK_TYPE(old_so) == SOCK_DGRAM) {
3439 old_so->so_proto = pffindproto(SOCK_DOM(old_so), IPPROTO_UDP, SOCK_DGRAM);
3440 }
3441 fd_cb->so = NULL;
3442 /* Save the output interface */
3443 ifp = inp->inp_last_outifp;
3444 if (old_so->so_rcv.sb_cc > 0) {
3445 error = mbuf_dup(old_so->so_rcv.sb_mb, MBUF_DONTWAIT, &recv_data);
3446 sbflush(&old_so->so_rcv);
3447 }
3448 socket_unlock(old_so, 0);
3449
3450 /* Associate the new socket with the flow divert control block */
3451 socket_lock(so, 0);
3452 so->so_fd_pcb = fd_cb;
3453 inp = sotoinpcb(so);
3454 inp->inp_last_outifp = ifp;
3455 if (recv_data != NULL) {
3456 if (sbappendstream(&so->so_rcv, recv_data)) {
3457 sorwakeup(so);
3458 }
3459 }
3460 if (SOCK_TYPE(so) == SOCK_STREAM) {
3461 flow_divert_set_protosw(so);
3462 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3463 flow_divert_set_udp_protosw(so);
3464 }
3465
3466 socket_unlock(so, 0);
3467
3468 fd_cb->so = so;
3469 fd_cb->flags |= FLOW_DIVERT_TRANSFERRED;
3470
3471 FDUNLOCK(fd_cb);
3472
3473 done:
3474 socket_lock(so, 0);
3475
3476 if (fd_cb != NULL) {
3477 FDRELEASE(fd_cb); /* Release the reference obtained via flow_divert_pcb_lookup */
3478 }
3479
3480 return error;
3481 }
3482
3483 errno_t
3484 flow_divert_implicit_data_out(struct socket *so, int flags, mbuf_t data, struct sockaddr *to, mbuf_t control, struct proc *p)
3485 {
3486 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3487 struct inpcb *inp;
3488 int error = 0;
3489
3490 inp = sotoinpcb(so);
3491 if (inp == NULL) {
3492 return EINVAL;
3493 }
3494
3495 if (fd_cb == NULL) {
3496 uint32_t fd_ctl_unit = necp_socket_get_flow_divert_control_unit(inp);
3497 if (fd_ctl_unit > 0) {
3498 error = flow_divert_pcb_init(so, fd_ctl_unit);
3499 fd_cb = so->so_fd_pcb;
3500 if (error != 0 || fd_cb == NULL) {
3501 goto done;
3502 }
3503 } else {
3504 error = ENETDOWN;
3505 goto done;
3506 }
3507 }
3508 return flow_divert_data_out(so, flags, data, to, control, p);
3509
3510 done:
3511 if (data) {
3512 mbuf_freem(data);
3513 }
3514 if (control) {
3515 mbuf_free(control);
3516 }
3517
3518 return error;
3519 }
3520
3521 errno_t
3522 flow_divert_pcb_init(struct socket *so, uint32_t ctl_unit)
3523 {
3524 errno_t error = 0;
3525 struct flow_divert_pcb *fd_cb;
3526
3527 if (so->so_flags & SOF_FLOW_DIVERT) {
3528 return EALREADY;
3529 }
3530
3531 fd_cb = flow_divert_pcb_create(so);
3532 if (fd_cb != NULL) {
3533 error = flow_divert_pcb_insert(fd_cb, ctl_unit);
3534 if (error) {
3535 FDLOG(LOG_ERR, fd_cb, "pcb insert failed: %d", error);
3536 FDRELEASE(fd_cb);
3537 } else {
3538 fd_cb->control_group_unit = ctl_unit;
3539 so->so_fd_pcb = fd_cb;
3540
3541 if (SOCK_TYPE(so) == SOCK_STREAM) {
3542 flow_divert_set_protosw(so);
3543 } else if (SOCK_TYPE(so) == SOCK_DGRAM) {
3544 flow_divert_set_udp_protosw(so);
3545 }
3546
3547 FDLOG0(LOG_INFO, fd_cb, "Created");
3548 }
3549 } else {
3550 error = ENOMEM;
3551 }
3552
3553 return error;
3554 }
3555
3556 errno_t
3557 flow_divert_token_set(struct socket *so, struct sockopt *sopt)
3558 {
3559 uint32_t ctl_unit = 0;
3560 uint32_t key_unit = 0;
3561 uint32_t flow_id = 0;
3562 int error = 0;
3563 int hmac_error = 0;
3564 mbuf_t token = NULL;
3565
3566 if (so->so_flags & SOF_FLOW_DIVERT) {
3567 error = EALREADY;
3568 goto done;
3569 }
3570
3571 if (g_init_result) {
3572 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_init failed (%d), cannot use flow divert", g_init_result);
3573 error = ENOPROTOOPT;
3574 goto done;
3575 }
3576
3577 if ((SOCK_TYPE(so) != SOCK_STREAM && SOCK_TYPE(so) != SOCK_DGRAM) ||
3578 (SOCK_PROTO(so) != IPPROTO_TCP && SOCK_PROTO(so) != IPPROTO_UDP) ||
3579 (SOCK_DOM(so) != PF_INET
3580 #if INET6
3581 && SOCK_DOM(so) != PF_INET6
3582 #endif
3583 )) {
3584 error = EINVAL;
3585 goto done;
3586 } else {
3587 if (SOCK_TYPE(so) == SOCK_STREAM && SOCK_PROTO(so) == IPPROTO_TCP) {
3588 struct tcpcb *tp = sototcpcb(so);
3589 if (tp == NULL || tp->t_state != TCPS_CLOSED) {
3590 error = EINVAL;
3591 goto done;
3592 }
3593 }
3594 }
3595
3596 error = soopt_getm(sopt, &token);
3597 if (error) {
3598 token = NULL;
3599 goto done;
3600 }
3601
3602 error = soopt_mcopyin(sopt, token);
3603 if (error) {
3604 token = NULL;
3605 goto done;
3606 }
3607
3608 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(key_unit), (void *)&key_unit, NULL);
3609 if (!error) {
3610 key_unit = ntohl(key_unit);
3611 if (key_unit >= GROUP_COUNT_MAX) {
3612 key_unit = 0;
3613 }
3614 } else if (error != ENOENT) {
3615 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the key unit from the token: %d", error);
3616 goto done;
3617 } else {
3618 key_unit = 0;
3619 }
3620
3621 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), (void *)&ctl_unit, NULL);
3622 if (error) {
3623 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the control socket unit from the token: %d", error);
3624 goto done;
3625 }
3626
3627 /* A valid kernel control unit is required */
3628 ctl_unit = ntohl(ctl_unit);
3629 if (ctl_unit == 0 || ctl_unit >= GROUP_COUNT_MAX) {
3630 FDLOG(LOG_ERR, &nil_pcb, "Got an invalid control socket unit: %u", ctl_unit);
3631 error = EINVAL;
3632 goto done;
3633 }
3634
3635 socket_unlock(so, 0);
3636 hmac_error = flow_divert_packet_verify_hmac(token, (key_unit != 0 ? key_unit : ctl_unit));
3637 socket_lock(so, 0);
3638
3639 if (hmac_error && hmac_error != ENOENT) {
3640 FDLOG(LOG_ERR, &nil_pcb, "HMAC verfication failed: %d", hmac_error);
3641 error = hmac_error;
3642 goto done;
3643 }
3644
3645 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_FLOW_ID, sizeof(flow_id), (void *)&flow_id, NULL);
3646 if (error && error != ENOENT) {
3647 FDLOG(LOG_ERR, &nil_pcb, "Failed to get the flow ID from the token: %d", error);
3648 goto done;
3649 }
3650
3651 if (flow_id == 0) {
3652 error = flow_divert_pcb_init(so, ctl_unit);
3653 if (error == 0) {
3654 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3655 int log_level = LOG_NOTICE;
3656
3657 error = flow_divert_packet_get_tlv(token, 0, FLOW_DIVERT_TLV_LOG_LEVEL,
3658 sizeof(log_level), &log_level, NULL);
3659 if (error == 0) {
3660 fd_cb->log_level = log_level;
3661 }
3662 error = 0;
3663
3664 fd_cb->connect_token = token;
3665 token = NULL;
3666 }
3667 } else {
3668 error = flow_divert_attach(so, flow_id, ctl_unit);
3669 }
3670
3671 if (hmac_error == 0) {
3672 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3673 if (fd_cb != NULL) {
3674 fd_cb->flags |= FLOW_DIVERT_HAS_HMAC;
3675 }
3676 }
3677
3678 done:
3679 if (token != NULL) {
3680 mbuf_freem(token);
3681 }
3682
3683 return error;
3684 }
3685
3686 errno_t
3687 flow_divert_token_get(struct socket *so, struct sockopt *sopt)
3688 {
3689 uint32_t ctl_unit;
3690 int error = 0;
3691 uint8_t hmac[SHA_DIGEST_LENGTH];
3692 struct flow_divert_pcb *fd_cb = so->so_fd_pcb;
3693 mbuf_t token = NULL;
3694 struct flow_divert_group *control_group = NULL;
3695
3696 if (!(so->so_flags & SOF_FLOW_DIVERT)) {
3697 error = EINVAL;
3698 goto done;
3699 }
3700
3701 VERIFY((so->so_flags & SOF_FLOW_DIVERT) && so->so_fd_pcb != NULL);
3702
3703 if (fd_cb->group == NULL) {
3704 error = EINVAL;
3705 goto done;
3706 }
3707
3708 error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &token);
3709 if (error) {
3710 FDLOG(LOG_ERR, fd_cb, "failed to allocate the header mbuf: %d", error);
3711 goto done;
3712 }
3713
3714 ctl_unit = htonl(fd_cb->group->ctl_unit);
3715
3716 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_CTL_UNIT, sizeof(ctl_unit), &ctl_unit);
3717 if (error) {
3718 goto done;
3719 }
3720
3721 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_FLOW_ID, sizeof(fd_cb->hash), &fd_cb->hash);
3722 if (error) {
3723 goto done;
3724 }
3725
3726 if (fd_cb->app_data != NULL) {
3727 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_APP_DATA, fd_cb->app_data_length, fd_cb->app_data);
3728 if (error) {
3729 goto done;
3730 }
3731 }
3732
3733 socket_unlock(so, 0);
3734 lck_rw_lock_shared(&g_flow_divert_group_lck);
3735
3736 if (g_flow_divert_groups != NULL && g_active_group_count > 0 &&
3737 fd_cb->control_group_unit > 0 && fd_cb->control_group_unit < GROUP_COUNT_MAX) {
3738 control_group = g_flow_divert_groups[fd_cb->control_group_unit];
3739 }
3740
3741 if (control_group != NULL) {
3742 lck_rw_lock_shared(&control_group->lck);
3743 ctl_unit = htonl(control_group->ctl_unit);
3744 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_KEY_UNIT, sizeof(ctl_unit), &ctl_unit);
3745 if (!error) {
3746 error = flow_divert_packet_compute_hmac(token, control_group, hmac);
3747 }
3748 lck_rw_done(&control_group->lck);
3749 } else {
3750 error = ENOPROTOOPT;
3751 }
3752
3753 lck_rw_done(&g_flow_divert_group_lck);
3754 socket_lock(so, 0);
3755
3756 if (error) {
3757 goto done;
3758 }
3759
3760 error = flow_divert_packet_append_tlv(token, FLOW_DIVERT_TLV_HMAC, sizeof(hmac), hmac);
3761 if (error) {
3762 goto done;
3763 }
3764
3765 if (sopt->sopt_val == USER_ADDR_NULL) {
3766 /* If the caller passed NULL to getsockopt, just set the size of the token and return */
3767 sopt->sopt_valsize = mbuf_pkthdr_len(token);
3768 goto done;
3769 }
3770
3771 error = soopt_mcopyout(sopt, token);
3772 if (error) {
3773 token = NULL; /* For some reason, soopt_mcopyout() frees the mbuf if it fails */
3774 goto done;
3775 }
3776
3777 done:
3778 if (token != NULL) {
3779 mbuf_freem(token);
3780 }
3781
3782 return error;
3783 }
3784
3785 static errno_t
3786 flow_divert_kctl_connect(kern_ctl_ref kctlref __unused, struct sockaddr_ctl *sac, void **unitinfo)
3787 {
3788 struct flow_divert_group *new_group = NULL;
3789 int error = 0;
3790
3791 if (sac->sc_unit >= GROUP_COUNT_MAX) {
3792 error = EINVAL;
3793 goto done;
3794 }
3795
3796 *unitinfo = NULL;
3797
3798 MALLOC_ZONE(new_group, struct flow_divert_group *, sizeof(*new_group), M_FLOW_DIVERT_GROUP, M_WAITOK);
3799 if (new_group == NULL) {
3800 error = ENOBUFS;
3801 goto done;
3802 }
3803
3804 memset(new_group, 0, sizeof(*new_group));
3805
3806 lck_rw_init(&new_group->lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
3807 RB_INIT(&new_group->pcb_tree);
3808 new_group->ctl_unit = sac->sc_unit;
3809 MBUFQ_INIT(&new_group->send_queue);
3810 new_group->signing_id_trie.root = NULL_TRIE_IDX;
3811
3812 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3813
3814 if (g_flow_divert_groups == NULL) {
3815 MALLOC(g_flow_divert_groups,
3816 struct flow_divert_group **,
3817 GROUP_COUNT_MAX * sizeof(struct flow_divert_group *),
3818 M_TEMP,
3819 M_WAITOK | M_ZERO);
3820 }
3821
3822 if (g_flow_divert_groups == NULL) {
3823 error = ENOBUFS;
3824 } else if (g_flow_divert_groups[sac->sc_unit] != NULL) {
3825 error = EALREADY;
3826 } else {
3827 g_flow_divert_groups[sac->sc_unit] = new_group;
3828 g_active_group_count++;
3829 }
3830
3831 lck_rw_done(&g_flow_divert_group_lck);
3832
3833 *unitinfo = new_group;
3834
3835 done:
3836 if (error != 0 && new_group != NULL) {
3837 FREE_ZONE(new_group, sizeof(*new_group), M_FLOW_DIVERT_GROUP);
3838 }
3839 return error;
3840 }
3841
3842 static errno_t
3843 flow_divert_kctl_disconnect(kern_ctl_ref kctlref __unused, uint32_t unit, void *unitinfo)
3844 {
3845 struct flow_divert_group *group = NULL;
3846 errno_t error = 0;
3847
3848 if (unit >= GROUP_COUNT_MAX) {
3849 return EINVAL;
3850 }
3851
3852 FDLOG(LOG_INFO, &nil_pcb, "disconnecting group %d", unit);
3853
3854 lck_rw_lock_exclusive(&g_flow_divert_group_lck);
3855
3856 if (g_flow_divert_groups == NULL || g_active_group_count == 0) {
3857 panic("flow divert group %u is disconnecting, but no groups are active (groups = %p, active count = %u", unit,
3858 g_flow_divert_groups, g_active_group_count);
3859 }
3860
3861 group = g_flow_divert_groups[unit];
3862
3863 if (group != (struct flow_divert_group *)unitinfo) {
3864 panic("group with unit %d (%p) != unit info (%p)", unit, group, unitinfo);
3865 }
3866
3867 g_flow_divert_groups[unit] = NULL;
3868 g_active_group_count--;
3869
3870 if (g_active_group_count == 0) {
3871 FREE(g_flow_divert_groups, M_TEMP);
3872 g_flow_divert_groups = NULL;
3873 }
3874
3875 lck_rw_done(&g_flow_divert_group_lck);
3876
3877 if (group != NULL) {
3878 flow_divert_close_all(group);
3879
3880 lck_rw_lock_exclusive(&group->lck);
3881
3882 if (group->token_key != NULL) {
3883 memset(group->token_key, 0, group->token_key_size);
3884 FREE(group->token_key, M_TEMP);
3885 group->token_key = NULL;
3886 group->token_key_size = 0;
3887 }
3888
3889 /* Re-set the current trie */
3890 if (group->signing_id_trie.memory != NULL) {
3891 FREE(group->signing_id_trie.memory, M_TEMP);
3892 }
3893 memset(&group->signing_id_trie, 0, sizeof(group->signing_id_trie));
3894 group->signing_id_trie.root = NULL_TRIE_IDX;
3895
3896 lck_rw_done(&group->lck);
3897
3898 FREE_ZONE(group, sizeof(*group), M_FLOW_DIVERT_GROUP);
3899 } else {
3900 error = EINVAL;
3901 }
3902
3903 return error;
3904 }
3905
3906 static errno_t
3907 flow_divert_kctl_send(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, mbuf_t m, int flags __unused)
3908 {
3909 return flow_divert_input(m, (struct flow_divert_group *)unitinfo);
3910 }
3911
3912 static void
3913 flow_divert_kctl_rcvd(kern_ctl_ref kctlref __unused, uint32_t unit __unused, void *unitinfo, int flags __unused)
3914 {
3915 struct flow_divert_group *group = (struct flow_divert_group *)unitinfo;
3916
3917 if (!OSTestAndClear(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits)) {
3918 struct flow_divert_pcb *fd_cb;
3919 SLIST_HEAD(, flow_divert_pcb) tmp_list;
3920
3921 lck_rw_lock_shared(&g_flow_divert_group_lck);
3922 lck_rw_lock_exclusive(&group->lck);
3923
3924 while (!MBUFQ_EMPTY(&group->send_queue)) {
3925 mbuf_t next_packet;
3926 FDLOG0(LOG_DEBUG, &nil_pcb, "trying ctl_enqueuembuf again");
3927 next_packet = MBUFQ_FIRST(&group->send_queue);
3928 int error = ctl_enqueuembuf(g_flow_divert_kctl_ref, group->ctl_unit, next_packet, CTL_DATA_EOR);
3929 if (error) {
3930 FDLOG(LOG_DEBUG, &nil_pcb, "ctl_enqueuembuf returned an error: %d", error);
3931 OSTestAndSet(GROUP_BIT_CTL_ENQUEUE_BLOCKED, &group->atomic_bits);
3932 lck_rw_done(&group->lck);
3933 lck_rw_done(&g_flow_divert_group_lck);
3934 return;
3935 }
3936 MBUFQ_DEQUEUE(&group->send_queue, next_packet);
3937 }
3938
3939 SLIST_INIT(&tmp_list);
3940
3941 RB_FOREACH(fd_cb, fd_pcb_tree, &group->pcb_tree) {
3942 FDRETAIN(fd_cb);
3943 SLIST_INSERT_HEAD(&tmp_list, fd_cb, tmp_list_entry);
3944 }
3945
3946 lck_rw_done(&group->lck);
3947
3948 SLIST_FOREACH(fd_cb, &tmp_list, tmp_list_entry) {
3949 FDLOCK(fd_cb);
3950 if (fd_cb->so != NULL) {
3951 socket_lock(fd_cb->so, 0);
3952 if (fd_cb->group != NULL) {
3953 flow_divert_send_buffered_data(fd_cb, FALSE);
3954 }
3955 socket_unlock(fd_cb->so, 0);
3956 }
3957 FDUNLOCK(fd_cb);
3958 FDRELEASE(fd_cb);
3959 }
3960
3961 lck_rw_done(&g_flow_divert_group_lck);
3962 }
3963 }
3964
3965 static int
3966 flow_divert_kctl_init(void)
3967 {
3968 struct kern_ctl_reg ctl_reg;
3969 int result;
3970
3971 memset(&ctl_reg, 0, sizeof(ctl_reg));
3972
3973 strlcpy(ctl_reg.ctl_name, FLOW_DIVERT_CONTROL_NAME, sizeof(ctl_reg.ctl_name));
3974 ctl_reg.ctl_name[sizeof(ctl_reg.ctl_name) - 1] = '\0';
3975 ctl_reg.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED;
3976 ctl_reg.ctl_sendsize = FD_CTL_SENDBUFF_SIZE;
3977 ctl_reg.ctl_recvsize = FD_CTL_RCVBUFF_SIZE;
3978
3979 ctl_reg.ctl_connect = flow_divert_kctl_connect;
3980 ctl_reg.ctl_disconnect = flow_divert_kctl_disconnect;
3981 ctl_reg.ctl_send = flow_divert_kctl_send;
3982 ctl_reg.ctl_rcvd = flow_divert_kctl_rcvd;
3983
3984 result = ctl_register(&ctl_reg, &g_flow_divert_kctl_ref);
3985
3986 if (result) {
3987 FDLOG(LOG_ERR, &nil_pcb, "flow_divert_kctl_init - ctl_register failed: %d\n", result);
3988 return result;
3989 }
3990
3991 return 0;
3992 }
3993
3994 void
3995 flow_divert_init(void)
3996 {
3997 memset(&nil_pcb, 0, sizeof(nil_pcb));
3998 nil_pcb.log_level = LOG_NOTICE;
3999
4000 g_tcp_protosw = pffindproto(AF_INET, IPPROTO_TCP, SOCK_STREAM);
4001
4002 VERIFY(g_tcp_protosw != NULL);
4003
4004 memcpy(&g_flow_divert_in_protosw, g_tcp_protosw, sizeof(g_flow_divert_in_protosw));
4005 memcpy(&g_flow_divert_in_usrreqs, g_tcp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_usrreqs));
4006
4007 g_flow_divert_in_usrreqs.pru_connect = flow_divert_connect_out;
4008 g_flow_divert_in_usrreqs.pru_connectx = flow_divert_connectx_out;
4009 g_flow_divert_in_usrreqs.pru_control = flow_divert_in_control;
4010 g_flow_divert_in_usrreqs.pru_disconnect = flow_divert_close;
4011 g_flow_divert_in_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4012 g_flow_divert_in_usrreqs.pru_peeraddr = flow_divert_getpeername;
4013 g_flow_divert_in_usrreqs.pru_rcvd = flow_divert_rcvd;
4014 g_flow_divert_in_usrreqs.pru_send = flow_divert_data_out;
4015 g_flow_divert_in_usrreqs.pru_shutdown = flow_divert_shutdown;
4016 g_flow_divert_in_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4017 g_flow_divert_in_usrreqs.pru_preconnect = flow_divert_preconnect;
4018
4019 g_flow_divert_in_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4020 g_flow_divert_in_protosw.pr_ctloutput = flow_divert_ctloutput;
4021
4022 /*
4023 * Socket filters shouldn't attach/detach to/from this protosw
4024 * since pr_protosw is to be used instead, which points to the
4025 * real protocol; if they do, it is a bug and we should panic.
4026 */
4027 g_flow_divert_in_protosw.pr_filter_head.tqh_first =
4028 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4029 g_flow_divert_in_protosw.pr_filter_head.tqh_last =
4030 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4031
4032 /* UDP */
4033 g_udp_protosw = pffindproto(AF_INET, IPPROTO_UDP, SOCK_DGRAM);
4034 VERIFY(g_udp_protosw != NULL);
4035
4036 memcpy(&g_flow_divert_in_udp_protosw, g_udp_protosw, sizeof(g_flow_divert_in_udp_protosw));
4037 memcpy(&g_flow_divert_in_udp_usrreqs, g_udp_protosw->pr_usrreqs, sizeof(g_flow_divert_in_udp_usrreqs));
4038
4039 g_flow_divert_in_udp_usrreqs.pru_connect = flow_divert_connect_out;
4040 g_flow_divert_in_udp_usrreqs.pru_connectx = flow_divert_connectx_out;
4041 g_flow_divert_in_udp_usrreqs.pru_control = flow_divert_in_control;
4042 g_flow_divert_in_udp_usrreqs.pru_disconnect = flow_divert_close;
4043 g_flow_divert_in_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4044 g_flow_divert_in_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
4045 g_flow_divert_in_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4046 g_flow_divert_in_udp_usrreqs.pru_send = flow_divert_data_out;
4047 g_flow_divert_in_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4048 g_flow_divert_in_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4049 g_flow_divert_in_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4050 g_flow_divert_in_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4051 g_flow_divert_in_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4052
4053 g_flow_divert_in_udp_protosw.pr_usrreqs = &g_flow_divert_in_usrreqs;
4054 g_flow_divert_in_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4055
4056 /*
4057 * Socket filters shouldn't attach/detach to/from this protosw
4058 * since pr_protosw is to be used instead, which points to the
4059 * real protocol; if they do, it is a bug and we should panic.
4060 */
4061 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_first =
4062 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4063 g_flow_divert_in_udp_protosw.pr_filter_head.tqh_last =
4064 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4065
4066 #if INET6
4067 g_tcp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_TCP, SOCK_STREAM);
4068
4069 VERIFY(g_tcp6_protosw != NULL);
4070
4071 memcpy(&g_flow_divert_in6_protosw, g_tcp6_protosw, sizeof(g_flow_divert_in6_protosw));
4072 memcpy(&g_flow_divert_in6_usrreqs, g_tcp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_usrreqs));
4073
4074 g_flow_divert_in6_usrreqs.pru_connect = flow_divert_connect_out;
4075 g_flow_divert_in6_usrreqs.pru_connectx = flow_divert_connectx6_out;
4076 g_flow_divert_in6_usrreqs.pru_control = flow_divert_in6_control;
4077 g_flow_divert_in6_usrreqs.pru_disconnect = flow_divert_close;
4078 g_flow_divert_in6_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4079 g_flow_divert_in6_usrreqs.pru_peeraddr = flow_divert_getpeername;
4080 g_flow_divert_in6_usrreqs.pru_rcvd = flow_divert_rcvd;
4081 g_flow_divert_in6_usrreqs.pru_send = flow_divert_data_out;
4082 g_flow_divert_in6_usrreqs.pru_shutdown = flow_divert_shutdown;
4083 g_flow_divert_in6_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4084 g_flow_divert_in6_usrreqs.pru_preconnect = flow_divert_preconnect;
4085
4086 g_flow_divert_in6_protosw.pr_usrreqs = &g_flow_divert_in6_usrreqs;
4087 g_flow_divert_in6_protosw.pr_ctloutput = flow_divert_ctloutput;
4088 /*
4089 * Socket filters shouldn't attach/detach to/from this protosw
4090 * since pr_protosw is to be used instead, which points to the
4091 * real protocol; if they do, it is a bug and we should panic.
4092 */
4093 g_flow_divert_in6_protosw.pr_filter_head.tqh_first =
4094 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4095 g_flow_divert_in6_protosw.pr_filter_head.tqh_last =
4096 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4097
4098 /* UDP6 */
4099 g_udp6_protosw = (struct ip6protosw *)pffindproto(AF_INET6, IPPROTO_UDP, SOCK_DGRAM);
4100
4101 VERIFY(g_udp6_protosw != NULL);
4102
4103 memcpy(&g_flow_divert_in6_udp_protosw, g_udp6_protosw, sizeof(g_flow_divert_in6_udp_protosw));
4104 memcpy(&g_flow_divert_in6_udp_usrreqs, g_udp6_protosw->pr_usrreqs, sizeof(g_flow_divert_in6_udp_usrreqs));
4105
4106 g_flow_divert_in6_udp_usrreqs.pru_connect = flow_divert_connect_out;
4107 g_flow_divert_in6_udp_usrreqs.pru_connectx = flow_divert_connectx6_out;
4108 g_flow_divert_in6_udp_usrreqs.pru_control = flow_divert_in6_control;
4109 g_flow_divert_in6_udp_usrreqs.pru_disconnect = flow_divert_close;
4110 g_flow_divert_in6_udp_usrreqs.pru_disconnectx = flow_divert_disconnectx;
4111 g_flow_divert_in6_udp_usrreqs.pru_peeraddr = flow_divert_getpeername;
4112 g_flow_divert_in6_udp_usrreqs.pru_rcvd = flow_divert_rcvd;
4113 g_flow_divert_in6_udp_usrreqs.pru_send = flow_divert_data_out;
4114 g_flow_divert_in6_udp_usrreqs.pru_shutdown = flow_divert_shutdown;
4115 g_flow_divert_in6_udp_usrreqs.pru_sockaddr = flow_divert_getsockaddr;
4116 g_flow_divert_in6_udp_usrreqs.pru_sosend_list = pru_sosend_list_notsupp;
4117 g_flow_divert_in6_udp_usrreqs.pru_soreceive_list = pru_soreceive_list_notsupp;
4118 g_flow_divert_in6_udp_usrreqs.pru_preconnect = flow_divert_preconnect;
4119
4120 g_flow_divert_in6_udp_protosw.pr_usrreqs = &g_flow_divert_in6_udp_usrreqs;
4121 g_flow_divert_in6_udp_protosw.pr_ctloutput = flow_divert_ctloutput;
4122 /*
4123 * Socket filters shouldn't attach/detach to/from this protosw
4124 * since pr_protosw is to be used instead, which points to the
4125 * real protocol; if they do, it is a bug and we should panic.
4126 */
4127 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_first =
4128 (struct socket_filter *)(uintptr_t)0xdeadbeefdeadbeef;
4129 g_flow_divert_in6_udp_protosw.pr_filter_head.tqh_last =
4130 (struct socket_filter **)(uintptr_t)0xdeadbeefdeadbeef;
4131 #endif /* INET6 */
4132
4133 flow_divert_grp_attr = lck_grp_attr_alloc_init();
4134 if (flow_divert_grp_attr == NULL) {
4135 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_attr_alloc_init failed");
4136 g_init_result = ENOMEM;
4137 goto done;
4138 }
4139
4140 flow_divert_mtx_grp = lck_grp_alloc_init(FLOW_DIVERT_CONTROL_NAME, flow_divert_grp_attr);
4141 if (flow_divert_mtx_grp == NULL) {
4142 FDLOG0(LOG_ERR, &nil_pcb, "lck_grp_alloc_init failed");
4143 g_init_result = ENOMEM;
4144 goto done;
4145 }
4146
4147 flow_divert_mtx_attr = lck_attr_alloc_init();
4148 if (flow_divert_mtx_attr == NULL) {
4149 FDLOG0(LOG_ERR, &nil_pcb, "lck_attr_alloc_init failed");
4150 g_init_result = ENOMEM;
4151 goto done;
4152 }
4153
4154 g_init_result = flow_divert_kctl_init();
4155 if (g_init_result) {
4156 goto done;
4157 }
4158
4159 lck_rw_init(&g_flow_divert_group_lck, flow_divert_mtx_grp, flow_divert_mtx_attr);
4160
4161 done:
4162 if (g_init_result != 0) {
4163 if (flow_divert_mtx_attr != NULL) {
4164 lck_attr_free(flow_divert_mtx_attr);
4165 flow_divert_mtx_attr = NULL;
4166 }
4167 if (flow_divert_mtx_grp != NULL) {
4168 lck_grp_free(flow_divert_mtx_grp);
4169 flow_divert_mtx_grp = NULL;
4170 }
4171 if (flow_divert_grp_attr != NULL) {
4172 lck_grp_attr_free(flow_divert_grp_attr);
4173 flow_divert_grp_attr = NULL;
4174 }
4175
4176 if (g_flow_divert_kctl_ref != NULL) {
4177 ctl_deregister(g_flow_divert_kctl_ref);
4178 g_flow_divert_kctl_ref = NULL;
4179 }
4180 }
4181 }
mirror of XNU