]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/tcp_output.c
projects / apple / xnu.git / blob
? search:


84fe091ccec8c585712c7a74fb687d2ea2f112c0
[apple/xnu.git] / bsd / netinet / tcp_output.c
1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #define _IP_VHL
71
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/protosw.h>
80 #include <sys/socket.h>
81 #include <sys/socketvar.h>
82
83 #include <net/route.h>
84 #include <net/ntstat.h>
85 #include <net/if_var.h>
86 #include <net/if.h>
87 #include <net/if_types.h>
88 #include <net/dlil.h>
89
90 #include <netinet/in.h>
91 #include <netinet/in_systm.h>
92 #include <netinet/in_var.h>
93 #include <netinet/in_tclass.h>
94 #include <netinet/ip.h>
95 #include <netinet/in_pcb.h>
96 #include <netinet/ip_var.h>
97 #include <mach/sdt.h>
98 #include <netinet6/in6_pcb.h>
99 #include <netinet/ip6.h>
100 #include <netinet6/ip6_var.h>
101 #include <netinet/tcp.h>
102 #define TCPOUTFLAGS
103 #include <netinet/tcp_cache.h>
104 #include <netinet/tcp_fsm.h>
105 #include <netinet/tcp_seq.h>
106 #include <netinet/tcp_timer.h>
107 #include <netinet/tcp_var.h>
108 #include <netinet/tcpip.h>
109 #include <netinet/tcp_cc.h>
110 #if TCPDEBUG
111 #include <netinet/tcp_debug.h>
112 #endif
113 #include <netinet/tcp_log.h>
114 #include <sys/kdebug.h>
115 #include <mach/sdt.h>
116
117 #if IPSEC
118 #include <netinet6/ipsec.h>
119 #endif /*IPSEC*/
120
121 #if MPTCP
122 #include <netinet/mptcp_var.h>
123 #include <netinet/mptcp.h>
124 #include <netinet/mptcp_opt.h>
125 #include <netinet/mptcp_seq.h>
126 #endif
127
128 #include <corecrypto/ccaes.h>
129
130 #define DBG_LAYER_BEG NETDBG_CODE(DBG_NETTCP, 1)
131 #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3)
132 #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
133
134 SYSCTL_SKMEM_TCP_INT(OID_AUTO, path_mtu_discovery,
135 CTLFLAG_RW | CTLFLAG_LOCKED, int, path_mtu_discovery, 1,
136 "Enable Path MTU Discovery");
137
138 SYSCTL_SKMEM_TCP_INT(OID_AUTO, local_slowstart_flightsize,
139 CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz_local, 8,
140 "Slow start flight size for local networks");
141
142 int tcp_do_tso = 1;
143 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED,
144 &tcp_do_tso, 0, "Enable TCP Segmentation Offload");
145
146 SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_setup_percentage,
147 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_ecn_setup_percentage, 100,
148 "Max ECN setup percentage");
149
150 SYSCTL_SKMEM_TCP_INT(OID_AUTO, do_ack_compression,
151 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_ack_compression, 1,
152 "Enable TCP ACK compression (on (cell only): 1, off: 0, on (all interfaces): 2)");
153
154 SYSCTL_SKMEM_TCP_INT(OID_AUTO, ack_compression_rate,
155 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_ack_compression_rate, TCP_COMP_CHANGE_RATE,
156 "Rate at which we force sending new ACKs (in ms)");
157
158 static int
159 sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS
160 {
161 #pragma unused(oidp, arg1, arg2)
162 int i, err = 0, changed = 0;
163 struct ifnet *ifp;
164
165 err = sysctl_io_number(req, tcp_ecn_outbound, sizeof(int32_t),
166 &i, &changed);
167 if (err != 0 || req->newptr == USER_ADDR_NULL) {
168 return err;
169 }
170
171 if (changed) {
172 if ((tcp_ecn_outbound == 0 || tcp_ecn_outbound == 1) &&
173 (i == 0 || i == 1)) {
174 tcp_ecn_outbound = i;
175 SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound);
176 return err;
177 }
178 if (tcp_ecn_outbound == 2 && (i == 0 || i == 1)) {
179 /*
180 * Reset ECN enable flags on non-cellular
181 * interfaces so that the system default will take
182 * over
183 */
184 ifnet_head_lock_shared();
185 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
186 if (!IFNET_IS_CELLULAR(ifp)) {
187 if_clear_eflags(ifp,
188 IFEF_ECN_ENABLE |
189 IFEF_ECN_DISABLE);
190 }
191 }
192 ifnet_head_done();
193 } else {
194 /*
195 * Set ECN enable flags on non-cellular
196 * interfaces
197 */
198 ifnet_head_lock_shared();
199 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
200 if (!IFNET_IS_CELLULAR(ifp)) {
201 if_set_eflags(ifp, IFEF_ECN_ENABLE);
202 if_clear_eflags(ifp, IFEF_ECN_DISABLE);
203 }
204 }
205 ifnet_head_done();
206 }
207 tcp_ecn_outbound = i;
208 SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound);
209 }
210 /* Change the other one too as the work is done */
211 if (i == 2 || tcp_ecn_inbound == 2) {
212 tcp_ecn_inbound = i;
213 SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_negotiate_in, tcp_ecn_inbound);
214 }
215 return err;
216 }
217
218 int tcp_ecn_outbound = 2;
219 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, ecn_initiate_out,
220 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_outbound, 0,
221 sysctl_change_ecn_setting, "IU",
222 "Initiate ECN for outbound connections");
223
224 int tcp_ecn_inbound = 2;
225 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, ecn_negotiate_in,
226 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_inbound, 0,
227 sysctl_change_ecn_setting, "IU",
228 "Initiate ECN for inbound connections");
229
230 SYSCTL_SKMEM_TCP_INT(OID_AUTO, packetchain,
231 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_packet_chaining, 50,
232 "Enable TCP output packet chaining");
233
234 SYSCTL_SKMEM_TCP_INT(OID_AUTO, socket_unlocked_on_output,
235 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_output_unlocked, 1,
236 "Unlock TCP when sending packets down to IP");
237
238 SYSCTL_SKMEM_TCP_INT(OID_AUTO, min_iaj_win,
239 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_min_iaj_win, MIN_IAJ_WIN,
240 "Minimum recv win based on inter-packet arrival jitter");
241
242 SYSCTL_SKMEM_TCP_INT(OID_AUTO, acc_iaj_react_limit,
243 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_acc_iaj_react_limit,
244 ACC_IAJ_REACT_LIMIT, "Accumulated IAJ when receiver starts to react");
245
246 SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufinc,
247 CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_inc,
248 8 * 1024, "Increment in send socket bufffer size");
249
250 SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufmax,
251 CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_max, 2 * 1024 * 1024,
252 "Maximum send socket buffer size");
253
254 SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_recvbg,
255 CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_use_rtt_recvbg, 1,
256 "Use RTT for bg recv algorithm");
257
258 SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_throttle_minwin,
259 CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_recv_throttle_minwin, 16 * 1024,
260 "Minimum recv win for throttling");
261
262 SYSCTL_SKMEM_TCP_INT(OID_AUTO, enable_tlp,
263 CTLFLAG_RW | CTLFLAG_LOCKED,
264 int32_t, tcp_enable_tlp, 1, "Enable Tail loss probe");
265
266 static int32_t packchain_newlist = 0;
267 static int32_t packchain_looped = 0;
268 static int32_t packchain_sent = 0;
269
270 /* temporary: for testing */
271 #if IPSEC
272 extern int ipsec_bypass;
273 #endif
274
275 extern int slowlink_wsize; /* window correction for slow links */
276
277 extern u_int32_t dlil_filter_disable_tso_count;
278 extern u_int32_t kipf_count;
279
280 static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *,
281 int, struct mbuf *, int, int, boolean_t);
282 static int tcp_recv_throttle(struct tcpcb *tp);
283
284 static int32_t
285 tcp_tfo_check(struct tcpcb *tp, int32_t len)
286 {
287 struct socket *so = tp->t_inpcb->inp_socket;
288 unsigned int optlen = 0;
289 unsigned int cookie_len;
290
291 if (tp->t_flags & TF_NOOPT) {
292 goto fallback;
293 }
294
295 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
296 !tcp_heuristic_do_tfo(tp)) {
297 tp->t_tfo_stats |= TFO_S_HEURISTICS_DISABLE;
298 tcpstat.tcps_tfo_heuristics_disable++;
299 goto fallback;
300 }
301
302 if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) {
303 return len;
304 }
305
306 optlen += TCPOLEN_MAXSEG;
307
308 if (tp->t_flags & TF_REQ_SCALE) {
309 optlen += 4;
310 }
311
312 #if MPTCP
313 if ((so->so_flags & SOF_MP_SUBFLOW) && mptcp_enable &&
314 (tp->t_rxtshift <= mptcp_mpcap_retries ||
315 (tptomptp(tp)->mpt_mpte->mpte_flags & MPTE_FORCE_ENABLE))) {
316 optlen += sizeof(struct mptcp_mpcapable_opt_common) + sizeof(mptcp_key_t);
317 }
318 #endif /* MPTCP */
319
320 if (tp->t_flags & TF_REQ_TSTMP) {
321 optlen += TCPOLEN_TSTAMP_APPA;
322 }
323
324 if (SACK_ENABLED(tp)) {
325 optlen += TCPOLEN_SACK_PERMITTED;
326 }
327
328 /* Now, decide whether to use TFO or not */
329
330 /* Don't even bother trying if there is no space at all... */
331 if (MAX_TCPOPTLEN - optlen < TCPOLEN_FASTOPEN_REQ) {
332 goto fallback;
333 }
334
335 cookie_len = tcp_cache_get_cookie_len(tp);
336 if (cookie_len == 0) {
337 /* No cookie, so we request one */
338 return 0;
339 }
340
341 /* There is not enough space for the cookie, so we cannot do TFO */
342 if (MAX_TCPOPTLEN - optlen < cookie_len) {
343 goto fallback;
344 }
345
346 /* Do not send SYN+data if there is more in the queue than MSS */
347 if (so->so_snd.sb_cc > (tp->t_maxopd - MAX_TCPOPTLEN)) {
348 goto fallback;
349 }
350
351 /* Ok, everything looks good. We can go on and do TFO */
352 return len;
353
354 fallback:
355 tcp_disable_tfo(tp);
356 return 0;
357 }
358
359 /* Returns the number of bytes written to the TCP option-space */
360 static unsigned int
361 tcp_tfo_write_cookie_rep(struct tcpcb *tp, unsigned int optlen, u_char *opt)
362 {
363 u_char out[CCAES_BLOCK_SIZE];
364 unsigned ret = 0;
365 u_char *bp;
366
367 if ((MAX_TCPOPTLEN - optlen) <
368 (TCPOLEN_FASTOPEN_REQ + TFO_COOKIE_LEN_DEFAULT)) {
369 return ret;
370 }
371
372 tcp_tfo_gen_cookie(tp->t_inpcb, out, sizeof(out));
373
374 bp = opt + optlen;
375
376 *bp++ = TCPOPT_FASTOPEN;
377 *bp++ = 2 + TFO_COOKIE_LEN_DEFAULT;
378 memcpy(bp, out, TFO_COOKIE_LEN_DEFAULT);
379 ret += 2 + TFO_COOKIE_LEN_DEFAULT;
380
381 tp->t_tfo_stats |= TFO_S_COOKIE_SENT;
382 tcpstat.tcps_tfo_cookie_sent++;
383
384 return ret;
385 }
386
387 static unsigned int
388 tcp_tfo_write_cookie(struct tcpcb *tp, unsigned int optlen, int32_t len,
389 u_char *opt)
390 {
391 uint8_t tfo_len;
392 struct socket *so = tp->t_inpcb->inp_socket;
393 unsigned ret = 0;
394 int res;
395 u_char *bp;
396
397 if (TCPOLEN_FASTOPEN_REQ > MAX_TCPOPTLEN - optlen) {
398 return 0;
399 }
400 tfo_len = (uint8_t)(MAX_TCPOPTLEN - optlen - TCPOLEN_FASTOPEN_REQ);
401
402 if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) {
403 /* If there is some data, let's track it */
404 if (len > 0) {
405 tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT;
406 tcpstat.tcps_tfo_syn_data_sent++;
407 }
408
409 return 0;
410 }
411
412 bp = opt + optlen;
413
414 /*
415 * The cookie will be copied in the appropriate place within the
416 * TCP-option space. That way we avoid the need for an intermediate
417 * variable.
418 */
419 res = tcp_cache_get_cookie(tp, bp + TCPOLEN_FASTOPEN_REQ, &tfo_len);
420 if (res == 0) {
421 *bp++ = TCPOPT_FASTOPEN;
422 *bp++ = TCPOLEN_FASTOPEN_REQ;
423 ret += TCPOLEN_FASTOPEN_REQ;
424
425 tp->t_tfo_flags |= TFO_F_COOKIE_REQ;
426
427 tp->t_tfo_stats |= TFO_S_COOKIE_REQ;
428 tcpstat.tcps_tfo_cookie_req++;
429 } else {
430 *bp++ = TCPOPT_FASTOPEN;
431 *bp++ = TCPOLEN_FASTOPEN_REQ + tfo_len;
432
433 ret += TCPOLEN_FASTOPEN_REQ + tfo_len;
434
435 tp->t_tfo_flags |= TFO_F_COOKIE_SENT;
436
437 /* If there is some data, let's track it */
438 if (len > 0) {
439 tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT;
440 tcpstat.tcps_tfo_syn_data_sent++;
441 }
442 }
443
444 return ret;
445 }
446
447 static inline bool
448 tcp_send_ecn_flags_on_syn(struct tcpcb *tp)
449 {
450 return !(tp->ecn_flags & TE_SETUPSENT);
451 }
452
453 void
454 tcp_set_ecn(struct tcpcb *tp, struct ifnet *ifp)
455 {
456 boolean_t inbound;
457
458 /*
459 * Socket option has precedence
460 */
461 if (tp->ecn_flags & TE_ECN_MODE_ENABLE) {
462 tp->ecn_flags |= TE_ENABLE_ECN;
463 goto check_heuristic;
464 }
465
466 if (tp->ecn_flags & TE_ECN_MODE_DISABLE) {
467 tp->ecn_flags &= ~TE_ENABLE_ECN;
468 return;
469 }
470 /*
471 * Per interface setting comes next
472 */
473 if (ifp != NULL) {
474 if (ifp->if_eflags & IFEF_ECN_ENABLE) {
475 tp->ecn_flags |= TE_ENABLE_ECN;
476 goto check_heuristic;
477 }
478
479 if (ifp->if_eflags & IFEF_ECN_DISABLE) {
480 tp->ecn_flags &= ~TE_ENABLE_ECN;
481 return;
482 }
483 }
484 /*
485 * System wide settings come last
486 */
487 inbound = (tp->t_inpcb->inp_socket->so_head != NULL);
488 if ((inbound && tcp_ecn_inbound == 1) ||
489 (!inbound && tcp_ecn_outbound == 1)) {
490 tp->ecn_flags |= TE_ENABLE_ECN;
491 goto check_heuristic;
492 } else {
493 tp->ecn_flags &= ~TE_ENABLE_ECN;
494 }
495
496 return;
497
498 check_heuristic:
499 if (!tcp_heuristic_do_ecn(tp)) {
500 tp->ecn_flags &= ~TE_ENABLE_ECN;
501 }
502
503 /*
504 * If the interface setting, system-level setting and heuristics
505 * allow to enable ECN, randomly select 5% of connections to
506 * enable it
507 */
508 if ((tp->ecn_flags & (TE_ECN_MODE_ENABLE | TE_ECN_MODE_DISABLE
509 | TE_ENABLE_ECN)) == TE_ENABLE_ECN) {
510 /*
511 * Use the random value in iss for randomizing
512 * this selection
513 */
514 if ((tp->iss % 100) >= tcp_ecn_setup_percentage) {
515 tp->ecn_flags &= ~TE_ENABLE_ECN;
516 }
517 }
518 }
519
520 int
521 tcp_flight_size(struct tcpcb *tp)
522 {
523 int ret;
524
525 VERIFY(tp->sackhint.sack_bytes_acked >= 0);
526 VERIFY(tp->sackhint.sack_bytes_rexmit >= 0);
527
528 /*
529 * RFC6675, SetPipe (), SACK'd bytes are discounted. All the rest is still in-flight.
530 */
531 ret = tp->snd_nxt - tp->snd_una - tp->sackhint.sack_bytes_acked;
532
533 if (ret < 0) {
534 /*
535 * This happens when the RTO-timer fires because snd_nxt gets artificially
536 * decreased. If we then receive some SACK-blogs, sack_bytes_acked is
537 * going to be high.
538 */
539 ret = 0;
540 }
541
542 return ret;
543 }
544
545 /*
546 * Tcp output routine: figure out what should be sent and send it.
547 *
548 * Returns: 0 Success
549 * EADDRNOTAVAIL
550 * ENOBUFS
551 * EMSGSIZE
552 * EHOSTUNREACH
553 * ENETDOWN
554 * ip_output_list:ENOMEM
555 * ip_output_list:EADDRNOTAVAIL
556 * ip_output_list:ENETUNREACH
557 * ip_output_list:EHOSTUNREACH
558 * ip_output_list:EACCES
559 * ip_output_list:EMSGSIZE
560 * ip_output_list:ENOBUFS
561 * ip_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
562 * ip6_output_list:EINVAL
563 * ip6_output_list:EOPNOTSUPP
564 * ip6_output_list:EHOSTUNREACH
565 * ip6_output_list:EADDRNOTAVAIL
566 * ip6_output_list:ENETUNREACH
567 * ip6_output_list:EMSGSIZE
568 * ip6_output_list:ENOBUFS
569 * ip6_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
570 */
571 int
572 tcp_output(struct tcpcb *tp)
573 {
574 struct inpcb *inp = tp->t_inpcb;
575 struct socket *so = inp->inp_socket;
576 int32_t len, recwin, sendwin, off;
577 uint8_t flags;
578 int error;
579 struct mbuf *m;
580 struct ip *ip = NULL;
581 struct ip6_hdr *ip6 = NULL;
582 struct tcphdr *th;
583 u_char opt[TCP_MAXOLEN];
584 unsigned int ipoptlen, optlen, hdrlen;
585 int idle, sendalot, lost = 0;
586 int i, sack_rxmit;
587 int tso = 0;
588 int sack_bytes_rxmt;
589 tcp_seq old_snd_nxt = 0;
590 struct sackhole *p;
591 #if IPSEC
592 unsigned int ipsec_optlen = 0;
593 #endif /* IPSEC */
594 int idle_time = 0;
595 struct mbuf *packetlist = NULL;
596 struct mbuf *tp_inp_options = inp->inp_depend4.inp4_options;
597 int isipv6 = inp->inp_vflag & INP_IPV6;
598 int packchain_listadd = 0;
599 int so_options = so->so_options;
600 struct rtentry *rt;
601 u_int32_t svc_flags = 0, allocated_len;
602 unsigned int sackoptlen = 0;
603 #if MPTCP
604 boolean_t mptcp_acknow;
605 #endif /* MPTCP */
606 boolean_t cell = FALSE;
607 boolean_t wifi = FALSE;
608 boolean_t wired = FALSE;
609 boolean_t sack_rescue_rxt = FALSE;
610 int sotc = so->so_traffic_class;
611 boolean_t do_not_compress = FALSE;
612 boolean_t sack_rxmted = FALSE;
613
614 /*
615 * Determine length of data that should be transmitted,
616 * and flags that will be used.
617 * If there is some data or critical controls (SYN, RST)
618 * to send, then transmit; otherwise, investigate further.
619 */
620 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
621
622 /* Since idle_time is signed integer, the following integer subtraction
623 * will take care of wrap around of tcp_now
624 */
625 idle_time = tcp_now - tp->t_rcvtime;
626 if (idle && idle_time >= TCP_IDLETIMEOUT(tp)) {
627 if (CC_ALGO(tp)->after_idle != NULL &&
628 (tp->tcp_cc_index != TCP_CC_ALGO_CUBIC_INDEX ||
629 idle_time >= TCP_CC_CWND_NONVALIDATED_PERIOD)) {
630 CC_ALGO(tp)->after_idle(tp);
631 tcp_ccdbg_trace(tp, NULL, TCP_CC_IDLE_TIMEOUT);
632 }
633
634 /*
635 * Do some other tasks that need to be done after
636 * idle time
637 */
638 if (!SLIST_EMPTY(&tp->t_rxt_segments)) {
639 tcp_rxtseg_clean(tp);
640 }
641
642 /* If stretch ack was auto-disabled, re-evaluate it */
643 tcp_cc_after_idle_stretchack(tp);
644 tp->t_forced_acks = TCP_FORCED_ACKS_COUNT;
645 }
646 tp->t_flags &= ~TF_LASTIDLE;
647 if (idle) {
648 if (tp->t_flags & TF_MORETOCOME) {
649 tp->t_flags |= TF_LASTIDLE;
650 idle = 0;
651 }
652 }
653 #if MPTCP
654 if (tp->t_mpflags & TMPF_RESET) {
655 tcp_check_timer_state(tp);
656 /*
657 * Once a RST has been sent for an MPTCP subflow,
658 * the subflow socket stays around until deleted.
659 * No packets such as FINs must be sent after RST.
660 */
661 return 0;
662 }
663 #endif /* MPTCP */
664
665 again:
666 #if MPTCP
667 mptcp_acknow = FALSE;
668 #endif
669 do_not_compress = FALSE;
670
671 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
672
673 if (isipv6) {
674 KERNEL_DEBUG(DBG_LAYER_BEG,
675 ((inp->inp_fport << 16) | inp->inp_lport),
676 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
677 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
678 sendalot, 0, 0);
679 } else {
680 KERNEL_DEBUG(DBG_LAYER_BEG,
681 ((inp->inp_fport << 16) | inp->inp_lport),
682 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
683 (inp->inp_faddr.s_addr & 0xffff)),
684 sendalot, 0, 0);
685 }
686 /*
687 * If the route generation id changed, we need to check that our
688 * local (source) IP address is still valid. If it isn't either
689 * return error or silently do nothing (assuming the address will
690 * come back before the TCP connection times out).
691 */
692 rt = inp->inp_route.ro_rt;
693 if (rt != NULL && ROUTE_UNUSABLE(&tp->t_inpcb->inp_route)) {
694 struct ifnet *ifp;
695 struct in_ifaddr *ia = NULL;
696 struct in6_ifaddr *ia6 = NULL;
697 int found_srcaddr = 0;
698
699 /* disable multipages at the socket */
700 somultipages(so, FALSE);
701
702 /* Disable TSO for the socket until we know more */
703 tp->t_flags &= ~TF_TSO;
704
705 soif2kcl(so, FALSE);
706
707 if (isipv6) {
708 ia6 = ifa_foraddr6(&inp->in6p_laddr);
709 if (ia6 != NULL) {
710 found_srcaddr = 1;
711 }
712 } else {
713 ia = ifa_foraddr(inp->inp_laddr.s_addr);
714 if (ia != NULL) {
715 found_srcaddr = 1;
716 }
717 }
718
719 /* check that the source address is still valid */
720 if (found_srcaddr == 0) {
721 soevent(so,
722 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR));
723
724 if (tp->t_state >= TCPS_CLOSE_WAIT) {
725 tcp_drop(tp, EADDRNOTAVAIL);
726 return EADDRNOTAVAIL;
727 }
728
729 /*
730 * Set retransmit timer if it wasn't set,
731 * reset Persist timer and shift register as the
732 * advertised peer window may not be valid anymore
733 */
734 if (tp->t_timer[TCPT_REXMT] == 0) {
735 tp->t_timer[TCPT_REXMT] =
736 OFFSET_FROM_START(tp, tp->t_rxtcur);
737 if (tp->t_timer[TCPT_PERSIST] != 0) {
738 tp->t_timer[TCPT_PERSIST] = 0;
739 tp->t_persist_stop = 0;
740 TCP_RESET_REXMT_STATE(tp);
741 }
742 }
743
744 if (tp->t_pktlist_head != NULL) {
745 m_freem_list(tp->t_pktlist_head);
746 }
747 TCP_PKTLIST_CLEAR(tp);
748
749 /* drop connection if source address isn't available */
750 if (so->so_flags & SOF_NOADDRAVAIL) {
751 tcp_drop(tp, EADDRNOTAVAIL);
752 return EADDRNOTAVAIL;
753 } else {
754 tcp_check_timer_state(tp);
755 return 0; /* silently ignore, keep data in socket: address may be back */
756 }
757 }
758 if (ia != NULL) {
759 IFA_REMREF(&ia->ia_ifa);
760 }
761
762 if (ia6 != NULL) {
763 IFA_REMREF(&ia6->ia_ifa);
764 }
765
766 /*
767 * Address is still valid; check for multipages capability
768 * again in case the outgoing interface has changed.
769 */
770 RT_LOCK(rt);
771 if ((ifp = rt->rt_ifp) != NULL) {
772 somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
773 tcp_set_tso(tp, ifp);
774 soif2kcl(so, (ifp->if_eflags & IFEF_2KCL));
775 tcp_set_ecn(tp, ifp);
776 }
777 if (rt->rt_flags & RTF_UP) {
778 RT_GENID_SYNC(rt);
779 }
780 /*
781 * See if we should do MTU discovery. Don't do it if:
782 * 1) it is disabled via the sysctl
783 * 2) the route isn't up
784 * 3) the MTU is locked (if it is, then discovery
785 * has been disabled)
786 */
787
788 if (!path_mtu_discovery || ((rt != NULL) &&
789 (!(rt->rt_flags & RTF_UP) ||
790 (rt->rt_rmx.rmx_locks & RTV_MTU)))) {
791 tp->t_flags &= ~TF_PMTUD;
792 } else {
793 tp->t_flags |= TF_PMTUD;
794 }
795
796 RT_UNLOCK(rt);
797 }
798
799 if (rt != NULL) {
800 cell = IFNET_IS_CELLULAR(rt->rt_ifp);
801 wifi = (!cell && IFNET_IS_WIFI(rt->rt_ifp));
802 wired = (!wifi && IFNET_IS_WIRED(rt->rt_ifp));
803 }
804
805 /*
806 * If we've recently taken a timeout, snd_max will be greater than
807 * snd_nxt. There may be SACK information that allows us to avoid
808 * resending already delivered data. Adjust snd_nxt accordingly.
809 */
810 if (SACK_ENABLED(tp) && SEQ_LT(tp->snd_nxt, tp->snd_max)) {
811 tcp_sack_adjust(tp);
812 }
813 sendalot = 0;
814 off = tp->snd_nxt - tp->snd_una;
815 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
816
817 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) {
818 sendwin = min(sendwin, slowlink_wsize);
819 }
820
821 flags = tcp_outflags[tp->t_state];
822 /*
823 * Send any SACK-generated retransmissions. If we're explicitly
824 * trying to send out new data (when sendalot is 1), bypass this
825 * function. If we retransmit in fast recovery mode, decrement
826 * snd_cwnd, since we're replacing a (future) new transmission
827 * with a retransmission now, and we previously incremented
828 * snd_cwnd in tcp_input().
829 */
830 /*
831 * Still in sack recovery , reset rxmit flag to zero.
832 */
833 sack_rxmit = 0;
834 sack_bytes_rxmt = 0;
835 len = 0;
836 p = NULL;
837 if (SACK_ENABLED(tp) && IN_FASTRECOVERY(tp) &&
838 (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
839 int32_t cwin;
840
841 if (tcp_do_better_lr) {
842 cwin = min(tp->snd_wnd, tp->snd_cwnd) - tcp_flight_size(tp);
843 if (cwin <= 0 && sack_rxmted == FALSE) {
844 /* Allow to clock out at least on per period */
845 cwin = tp->t_maxseg;
846 }
847
848 sack_rxmted = TRUE;
849 } else {
850 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
851 }
852 if (cwin < 0) {
853 cwin = 0;
854 }
855 /* Do not retransmit SACK segments beyond snd_recover */
856 if (SEQ_GT(p->end, tp->snd_recover)) {
857 /*
858 * (At least) part of sack hole extends beyond
859 * snd_recover. Check to see if we can rexmit data
860 * for this hole.
861 */
862 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
863 /*
864 * Can't rexmit any more data for this hole.
865 * That data will be rexmitted in the next
866 * sack recovery episode, when snd_recover
867 * moves past p->rxmit.
868 */
869 p = NULL;
870 goto after_sack_rexmit;
871 } else {
872 /* Can rexmit part of the current hole */
873 len = ((int32_t)min(cwin,
874 tp->snd_recover - p->rxmit));
875 }
876 } else {
877 len = ((int32_t)min(cwin, p->end - p->rxmit));
878 }
879 if (len > 0) {
880 off = p->rxmit - tp->snd_una;
881 sack_rxmit = 1;
882 sendalot = 1;
883 /* Everything sent after snd_nxt will allow us to account for fast-retransmit of the retransmitted segment */
884 tp->send_highest_sack = tp->snd_nxt;
885 tp->t_new_dupacks = 0;
886 tcpstat.tcps_sack_rexmits++;
887 tcpstat.tcps_sack_rexmit_bytes +=
888 min(len, tp->t_maxseg);
889 } else {
890 len = 0;
891 }
892 }
893 after_sack_rexmit:
894 /*
895 * Get standard flags, and add SYN or FIN if requested by 'hidden'
896 * state flags.
897 */
898 if (tp->t_flags & TF_NEEDFIN) {
899 flags |= TH_FIN;
900 }
901
902 /*
903 * If in persist timeout with window of 0, send 1 byte.
904 * Otherwise, if window is small but nonzero
905 * and timer expired, we will send what we can
906 * and go to transmit state.
907 */
908 if (tp->t_flagsext & TF_FORCE) {
909 if (sendwin == 0) {
910 /*
911 * If we still have some data to send, then
912 * clear the FIN bit. Usually this would
913 * happen below when it realizes that we
914 * aren't sending all the data. However,
915 * if we have exactly 1 byte of unsent data,
916 * then it won't clear the FIN bit below,
917 * and if we are in persist state, we wind
918 * up sending the packet without recording
919 * that we sent the FIN bit.
920 *
921 * We can't just blindly clear the FIN bit,
922 * because if we don't have any more data
923 * to send then the probe will be the FIN
924 * itself.
925 */
926 if (off < so->so_snd.sb_cc) {
927 flags &= ~TH_FIN;
928 }
929 sendwin = 1;
930 } else {
931 tp->t_timer[TCPT_PERSIST] = 0;
932 tp->t_persist_stop = 0;
933 TCP_RESET_REXMT_STATE(tp);
934 }
935 }
936
937 /*
938 * If snd_nxt == snd_max and we have transmitted a FIN, the
939 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
940 * a negative length. This can also occur when TCP opens up
941 * its congestion window while receiving additional duplicate
942 * acks after fast-retransmit because TCP will reset snd_nxt
943 * to snd_max after the fast-retransmit.
944 *
945 * In the normal retransmit-FIN-only case, however, snd_nxt will
946 * be set to snd_una, the offset will be 0, and the length may
947 * wind up 0.
948 *
949 * If sack_rxmit is true we are retransmitting from the scoreboard
950 * in which case len is already set.
951 */
952 if (sack_rxmit == 0) {
953 if (sack_bytes_rxmt == 0) {
954 len = min(so->so_snd.sb_cc, sendwin) - off;
955 } else {
956 int32_t cwin;
957
958 if (tcp_do_better_lr) {
959 cwin = tp->snd_cwnd - tcp_flight_size(tp);
960 } else {
961 cwin = tp->snd_cwnd -
962 (tp->snd_nxt - tp->sack_newdata) -
963 sack_bytes_rxmt;
964 }
965 if (cwin < 0) {
966 cwin = 0;
967 }
968 /*
969 * We are inside of a SACK recovery episode and are
970 * sending new data, having retransmitted all the
971 * data possible in the scoreboard.
972 */
973 len = min(so->so_snd.sb_cc, tp->snd_wnd) - off;
974 /*
975 * Don't remove this (len > 0) check !
976 * We explicitly check for len > 0 here (although it
977 * isn't really necessary), to work around a gcc
978 * optimization issue - to force gcc to compute
979 * len above. Without this check, the computation
980 * of len is bungled by the optimizer.
981 */
982 if (len > 0) {
983 len = imin(len, cwin);
984 } else {
985 len = 0;
986 }
987 /*
988 * At this point SACK recovery can not send any
989 * data from scoreboard or any new data. Check
990 * if we can do a rescue retransmit towards the
991 * tail end of recovery window.
992 */
993 if (len == 0 && cwin > 0 &&
994 SEQ_LT(tp->snd_fack, tp->snd_recover) &&
995 !(tp->t_flagsext & TF_RESCUE_RXT)) {
996 len = min((tp->snd_recover - tp->snd_fack),
997 tp->t_maxseg);
998 len = imin(len, cwin);
999 old_snd_nxt = tp->snd_nxt;
1000 sack_rescue_rxt = TRUE;
1001 tp->snd_nxt = tp->snd_recover - len;
1002 /*
1003 * If FIN has been sent, snd_max
1004 * must have been advanced to cover it.
1005 */
1006 if ((tp->t_flags & TF_SENTFIN) &&
1007 tp->snd_max == tp->snd_recover) {
1008 tp->snd_nxt--;
1009 }
1010
1011 off = tp->snd_nxt - tp->snd_una;
1012 sendalot = 0;
1013 tp->t_flagsext |= TF_RESCUE_RXT;
1014 }
1015 }
1016 }
1017
1018 /*
1019 * Lop off SYN bit if it has already been sent. However, if this
1020 * is SYN-SENT state and if segment contains data and if we don't
1021 * know that foreign host supports TAO, suppress sending segment.
1022 */
1023 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
1024 if (tp->t_state == TCPS_SYN_RECEIVED && tfo_enabled(tp) && tp->snd_nxt == tp->snd_una + 1) {
1025 /* We are sending the SYN again! */
1026 off--;
1027 len++;
1028 } else {
1029 if (tp->t_state != TCPS_SYN_RECEIVED || tfo_enabled(tp)) {
1030 flags &= ~TH_SYN;
1031 }
1032
1033 off--;
1034 len++;
1035 if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
1036 while (inp->inp_sndinprog_cnt == 0 &&
1037 tp->t_pktlist_head != NULL) {
1038 packetlist = tp->t_pktlist_head;
1039 packchain_listadd = tp->t_lastchain;
1040 packchain_sent++;
1041 TCP_PKTLIST_CLEAR(tp);
1042
1043 error = tcp_ip_output(so, tp, packetlist,
1044 packchain_listadd, tp_inp_options,
1045 (so_options & SO_DONTROUTE),
1046 (sack_rxmit || (sack_bytes_rxmt != 0)),
1047 isipv6);
1048 }
1049
1050 /*
1051 * tcp was closed while we were in ip,
1052 * resume close
1053 */
1054 if (inp->inp_sndinprog_cnt == 0 &&
1055 (tp->t_flags & TF_CLOSING)) {
1056 tp->t_flags &= ~TF_CLOSING;
1057 (void) tcp_close(tp);
1058 } else {
1059 tcp_check_timer_state(tp);
1060 }
1061 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
1062 0, 0, 0, 0, 0);
1063 return 0;
1064 }
1065 }
1066 }
1067
1068 /*
1069 * Be careful not to send data and/or FIN on SYN segments.
1070 * This measure is needed to prevent interoperability problems
1071 * with not fully conformant TCP implementations.
1072 *
1073 * In case of TFO, we handle the setting of the len in
1074 * tcp_tfo_check. In case TFO is not enabled, never ever send
1075 * SYN+data.
1076 */
1077 if ((flags & TH_SYN) && !tfo_enabled(tp)) {
1078 len = 0;
1079 flags &= ~TH_FIN;
1080 }
1081
1082 /*
1083 * Don't send a RST with data.
1084 */
1085 if (flags & TH_RST) {
1086 len = 0;
1087 }
1088
1089 if ((flags & TH_SYN) && tp->t_state <= TCPS_SYN_SENT && tfo_enabled(tp)) {
1090 len = tcp_tfo_check(tp, len);
1091 }
1092
1093 /*
1094 * The check here used to be (len < 0). Some times len is zero
1095 * when the congestion window is closed and we need to check
1096 * if persist timer has to be set in that case. But don't set
1097 * persist until connection is established.
1098 */
1099 if (len <= 0 && !(flags & TH_SYN)) {
1100 /*
1101 * If FIN has been sent but not acked,
1102 * but we haven't been called to retransmit,
1103 * len will be < 0. Otherwise, window shrank
1104 * after we sent into it. If window shrank to 0,
1105 * cancel pending retransmit, pull snd_nxt back
1106 * to (closed) window, and set the persist timer
1107 * if it isn't already going. If the window didn't
1108 * close completely, just wait for an ACK.
1109 */
1110 len = 0;
1111 if (sendwin == 0) {
1112 tp->t_timer[TCPT_REXMT] = 0;
1113 tp->t_timer[TCPT_PTO] = 0;
1114 TCP_RESET_REXMT_STATE(tp);
1115 tp->snd_nxt = tp->snd_una;
1116 off = 0;
1117 if (tp->t_timer[TCPT_PERSIST] == 0) {
1118 tcp_setpersist(tp);
1119 }
1120 }
1121 }
1122
1123 /*
1124 * Automatic sizing of send socket buffer. Increase the send
1125 * socket buffer size if all of the following criteria are met
1126 * 1. the receiver has enough buffer space for this data
1127 * 2. send buffer is filled to 7/8th with data (so we actually
1128 * have data to make use of it);
1129 * 3. our send window (slow start and congestion controlled) is
1130 * larger than sent but unacknowledged data in send buffer.
1131 */
1132 if (!INP_WAIT_FOR_IF_FEEDBACK(inp) && !IN_FASTRECOVERY(tp) &&
1133 (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
1134 tcp_cansbgrow(&so->so_snd)) {
1135 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
1136 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
1137 sendwin >= (so->so_snd.sb_cc - (tp->snd_nxt - tp->snd_una))) {
1138 if (sbreserve(&so->so_snd,
1139 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
1140 tcp_autosndbuf_max)) == 1) {
1141 so->so_snd.sb_idealsize = so->so_snd.sb_hiwat;
1142 }
1143 }
1144 }
1145
1146 /*
1147 * Truncate to the maximum segment length or enable TCP Segmentation
1148 * Offloading (if supported by hardware) and ensure that FIN is removed
1149 * if the length no longer contains the last data byte.
1150 *
1151 * TSO may only be used if we are in a pure bulk sending state.
1152 * The presence of TCP-MD5, SACK retransmits, SACK advertizements,
1153 * filters and IP options, as well as disabling hardware checksum
1154 * offload prevent using TSO. With TSO the TCP header is the same
1155 * (except for the sequence number) for all generated packets. This
1156 * makes it impossible to transmit any options which vary per generated
1157 * segment or packet.
1158 *
1159 * The length of TSO bursts is limited to TCP_MAXWIN. That limit and
1160 * removal of FIN (if not already catched here) are handled later after
1161 * the exact length of the TCP options are known.
1162 */
1163 #if IPSEC
1164 /*
1165 * Pre-calculate here as we save another lookup into the darknesses
1166 * of IPsec that way and can actually decide if TSO is ok.
1167 */
1168 if (ipsec_bypass == 0) {
1169 ipsec_optlen = (unsigned int)ipsec_hdrsiz_tcp(tp);
1170 }
1171 #endif
1172 if (len > tp->t_maxseg) {
1173 if ((tp->t_flags & TF_TSO) && tcp_do_tso && hwcksum_tx &&
1174 ip_use_randomid && kipf_count == 0 &&
1175 dlil_filter_disable_tso_count == 0 &&
1176 tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
1177 sack_bytes_rxmt == 0 &&
1178 inp->inp_options == NULL &&
1179 inp->in6p_options == NULL
1180 #if IPSEC
1181 && ipsec_optlen == 0
1182 #endif
1183 ) {
1184 tso = 1;
1185 sendalot = 0;
1186 } else {
1187 len = tp->t_maxseg;
1188 sendalot = 1;
1189 tso = 0;
1190 }
1191 } else {
1192 tso = 0;
1193 }
1194
1195 /* Send one segment or less as a tail loss probe */
1196 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1197 len = min(len, tp->t_maxseg);
1198 sendalot = 0;
1199 tso = 0;
1200 }
1201
1202 #if MPTCP
1203 if (so->so_flags & SOF_MP_SUBFLOW && off < 0) {
1204 os_log_error(mptcp_log_handle, "%s - %lx: offset is negative! len %d off %d\n",
1205 __func__, (unsigned long)VM_KERNEL_ADDRPERM(tp->t_mpsub->mpts_mpte),
1206 len, off);
1207 }
1208
1209 if ((so->so_flags & SOF_MP_SUBFLOW) &&
1210 !(tp->t_mpflags & TMPF_TCP_FALLBACK)) {
1211 int newlen = len;
1212 if (tp->t_state >= TCPS_ESTABLISHED &&
1213 (tp->t_mpflags & TMPF_SND_MPPRIO ||
1214 tp->t_mpflags & TMPF_SND_REM_ADDR ||
1215 tp->t_mpflags & TMPF_SND_MPFAIL ||
1216 tp->t_mpflags & TMPF_SND_KEYS ||
1217 tp->t_mpflags & TMPF_SND_JACK)) {
1218 if (len > 0) {
1219 len = 0;
1220 tso = 0;
1221 }
1222 /*
1223 * On a new subflow, don't try to send again, because
1224 * we are still waiting for the fourth ack.
1225 */
1226 if (!(tp->t_mpflags & TMPF_PREESTABLISHED)) {
1227 sendalot = 1;
1228 }
1229 mptcp_acknow = TRUE;
1230 } else {
1231 mptcp_acknow = FALSE;
1232 }
1233 /*
1234 * The contiguous bytes in the subflow socket buffer can be
1235 * discontiguous at the MPTCP level. Since only one DSS
1236 * option can be sent in one packet, reduce length to match
1237 * the contiguous MPTCP level. Set sendalot to send remainder.
1238 */
1239 if (len > 0 && off >= 0) {
1240 newlen = mptcp_adj_sendlen(so, off);
1241 }
1242
1243 if (newlen < len) {
1244 len = newlen;
1245 if (len <= tp->t_maxseg) {
1246 tso = 0;
1247 }
1248 }
1249 }
1250 #endif /* MPTCP */
1251
1252 if (sack_rxmit) {
1253 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc)) {
1254 flags &= ~TH_FIN;
1255 }
1256 } else {
1257 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc)) {
1258 flags &= ~TH_FIN;
1259 }
1260 }
1261 /*
1262 * Compare available window to amount of window
1263 * known to peer (as advertised window less
1264 * next expected input). If the difference is at least two
1265 * max size segments, or at least 25% of the maximum possible
1266 * window, then want to send a window update to peer.
1267 */
1268 recwin = tcp_sbspace(tp);
1269
1270 if (!(so->so_flags & SOF_MP_SUBFLOW)) {
1271 if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) &&
1272 recwin < (int)tp->t_maxseg) {
1273 recwin = 0;
1274 }
1275 } else {
1276 struct mptcb *mp_tp = tptomptp(tp);
1277 struct socket *mp_so = mptetoso(mp_tp->mpt_mpte);
1278
1279 if (recwin < (int32_t)(mp_so->so_rcv.sb_hiwat / 4) &&
1280 recwin < (int)tp->t_maxseg) {
1281 recwin = 0;
1282 }
1283 }
1284
1285 #if TRAFFIC_MGT
1286 if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) {
1287 if (recwin > 0 && tcp_recv_throttle(tp)) {
1288 uint32_t min_iaj_win = tcp_min_iaj_win * tp->t_maxseg;
1289 uint32_t bg_rwintop = tp->rcv_adv;
1290 if (SEQ_LT(bg_rwintop, tp->rcv_nxt + min_iaj_win)) {
1291 bg_rwintop = tp->rcv_nxt + min_iaj_win;
1292 }
1293 recwin = imin((int32_t)(bg_rwintop - tp->rcv_nxt),
1294 recwin);
1295 if (recwin < 0) {
1296 recwin = 0;
1297 }
1298 }
1299 }
1300 #endif /* TRAFFIC_MGT */
1301
1302 if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale)) {
1303 recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
1304 }
1305
1306 if (!(so->so_flags & SOF_MP_SUBFLOW)) {
1307 if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) {
1308 recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt);
1309 }
1310 } else {
1311 struct mptcb *mp_tp = tptomptp(tp);
1312 int64_t recwin_announced = (int64_t)(mp_tp->mpt_rcvadv - mp_tp->mpt_rcvnxt);
1313
1314 /* Don't remove what we announced at the MPTCP-layer */
1315 VERIFY(recwin_announced < INT32_MAX && recwin_announced > INT32_MIN);
1316 if (recwin < (int32_t)recwin_announced) {
1317 recwin = (int32_t)recwin_announced;
1318 }
1319 }
1320
1321 /*
1322 * Sender silly window avoidance. We transmit under the following
1323 * conditions when len is non-zero:
1324 *
1325 * - we've timed out (e.g. persist timer)
1326 * - we need to retransmit
1327 * - We have a full segment (or more with TSO)
1328 * - This is the last buffer in a write()/send() and we are
1329 * either idle or running NODELAY
1330 * - we have more then 1/2 the maximum send window's worth of
1331 * data (receiver may be limited the window size)
1332 */
1333 if (len) {
1334 if (tp->t_flagsext & TF_FORCE) {
1335 goto send;
1336 }
1337 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
1338 goto send;
1339 }
1340 if (sack_rxmit) {
1341 goto send;
1342 }
1343
1344 /*
1345 * If this here is the first segment after SYN/ACK and TFO
1346 * is being used, then we always send it, regardless of Nagle,...
1347 */
1348 if (tp->t_state == TCPS_SYN_RECEIVED &&
1349 tfo_enabled(tp) &&
1350 (tp->t_tfo_flags & TFO_F_COOKIE_VALID) &&
1351 tp->snd_nxt == tp->iss + 1) {
1352 goto send;
1353 }
1354
1355 /*
1356 * Send new data on the connection only if it is
1357 * not flow controlled
1358 */
1359 if (!INP_WAIT_FOR_IF_FEEDBACK(inp) ||
1360 tp->t_state != TCPS_ESTABLISHED) {
1361 if (len >= tp->t_maxseg) {
1362 goto send;
1363 }
1364
1365 if (!(tp->t_flags & TF_MORETOCOME) &&
1366 (idle || tp->t_flags & TF_NODELAY ||
1367 (tp->t_flags & TF_MAXSEGSNT) ||
1368 ALLOW_LIMITED_TRANSMIT(tp)) &&
1369 (tp->t_flags & TF_NOPUSH) == 0 &&
1370 (len + off >= so->so_snd.sb_cc ||
1371 /*
1372 * MPTCP needs to respect the DSS-mappings. So, it
1373 * may be sending data that *could* have been
1374 * coalesced, but cannot because of
1375 * mptcp_adj_sendlen().
1376 */
1377 so->so_flags & SOF_MP_SUBFLOW)) {
1378 goto send;
1379 }
1380 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
1381 goto send;
1382 }
1383 } else {
1384 tcpstat.tcps_fcholdpacket++;
1385 }
1386 }
1387
1388 if (recwin > 0) {
1389 /*
1390 * "adv" is the amount we can increase the window,
1391 * taking into account that we are limited by
1392 * TCP_MAXWIN << tp->rcv_scale.
1393 */
1394 int32_t adv, oldwin = 0;
1395 adv = imin(recwin, (int)TCP_MAXWIN << tp->rcv_scale) -
1396 (tp->rcv_adv - tp->rcv_nxt);
1397
1398 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
1399 oldwin = tp->rcv_adv - tp->rcv_nxt;
1400 }
1401
1402 if (tcp_ack_strategy == TCP_ACK_STRATEGY_LEGACY) {
1403 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
1404 /*
1405 * Update only if the resulting scaled value of
1406 * the window changed, or if there is a change in
1407 * the sequence since the last ack. This avoids
1408 * what appears as dupe ACKS (see rdar://5640997)
1409 *
1410 * If streaming is detected avoid sending too many
1411 * window updates. We will depend on the delack
1412 * timer to send a window update when needed.
1413 *
1414 * If there is more data to read, don't send an ACK.
1415 * Otherwise we will end up sending many ACKs if the
1416 * application is doing micro-reads.
1417 */
1418 if (!(tp->t_flags & TF_STRETCHACK) &&
1419 (tp->last_ack_sent != tp->rcv_nxt ||
1420 ((oldwin + adv) >> tp->rcv_scale) >
1421 (oldwin >> tp->rcv_scale))) {
1422 goto send;
1423 }
1424 }
1425 } else {
1426 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
1427 /*
1428 * ACK every second full-sized segment, if the
1429 * ACK is advancing or the window becomes bigger
1430 */
1431 if (so->so_rcv.sb_cc < so->so_rcv.sb_lowat &&
1432 (tp->last_ack_sent != tp->rcv_nxt ||
1433 ((oldwin + adv) >> tp->rcv_scale) >
1434 (oldwin >> tp->rcv_scale))) {
1435 goto send;
1436 }
1437 } else if (tp->t_flags & TF_DELACK) {
1438 /*
1439 * If we delayed the ACK and the window
1440 * is not advancing by a lot (< 2MSS), ACK
1441 * immediately if the last incoming packet had
1442 * the push flag set and we emptied the buffer.
1443 *
1444 * This takes care of a sender doing small
1445 * repeated writes with Nagle enabled.
1446 */
1447 if (so->so_rcv.sb_cc == 0 &&
1448 tp->last_ack_sent != tp->rcv_nxt &&
1449 (tp->t_flagsext & TF_LAST_IS_PSH)) {
1450 goto send;
1451 }
1452 }
1453 }
1454 if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat) {
1455 goto send;
1456 }
1457
1458 /*
1459 * Make sure that the delayed ack timer is set if
1460 * we delayed sending a window update because of
1461 * streaming detection.
1462 */
1463 if (tcp_ack_strategy == TCP_ACK_STRATEGY_LEGACY &&
1464 (tp->t_flags & TF_STRETCHACK) &&
1465 !(tp->t_flags & TF_DELACK)) {
1466 tp->t_flags |= TF_DELACK;
1467 tp->t_timer[TCPT_DELACK] =
1468 OFFSET_FROM_START(tp, tcp_delack);
1469 }
1470 }
1471
1472 /*
1473 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
1474 * is also a catch-all for the retransmit timer timeout case.
1475 */
1476 if (tp->t_flags & TF_ACKNOW) {
1477 if (tp->t_forced_acks > 0) {
1478 tp->t_forced_acks--;
1479 }
1480 goto send;
1481 }
1482 if ((flags & TH_RST) || (flags & TH_SYN)) {
1483 goto send;
1484 }
1485 if (SEQ_GT(tp->snd_up, tp->snd_una)) {
1486 goto send;
1487 }
1488 #if MPTCP
1489 if (mptcp_acknow) {
1490 goto send;
1491 }
1492 #endif /* MPTCP */
1493 /*
1494 * If our state indicates that FIN should be sent
1495 * and we have not yet done so, then we need to send.
1496 */
1497 if ((flags & TH_FIN) &&
1498 (!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una)) {
1499 goto send;
1500 }
1501 /*
1502 * In SACK, it is possible for tcp_output to fail to send a segment
1503 * after the retransmission timer has been turned off. Make sure
1504 * that the retransmission timer is set.
1505 */
1506 if (SACK_ENABLED(tp) && (tp->t_state >= TCPS_ESTABLISHED) &&
1507 SEQ_GT(tp->snd_max, tp->snd_una) &&
1508 tp->t_timer[TCPT_REXMT] == 0 &&
1509 tp->t_timer[TCPT_PERSIST] == 0) {
1510 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp,
1511 tp->t_rxtcur);
1512 goto just_return;
1513 }
1514 /*
1515 * TCP window updates are not reliable, rather a polling protocol
1516 * using ``persist'' packets is used to insure receipt of window
1517 * updates. The three ``states'' for the output side are:
1518 * idle not doing retransmits or persists
1519 * persisting to move a small or zero window
1520 * (re)transmitting and thereby not persisting
1521 *
1522 * tp->t_timer[TCPT_PERSIST]
1523 * is set when we are in persist state.
1524 * tp->t_force
1525 * is set when we are called to send a persist packet.
1526 * tp->t_timer[TCPT_REXMT]
1527 * is set when we are retransmitting
1528 * The output side is idle when both timers are zero.
1529 *
1530 * If send window is too small, there is data to transmit, and no
1531 * retransmit or persist is pending, then go to persist state.
1532 * If nothing happens soon, send when timer expires:
1533 * if window is nonzero, transmit what we can,
1534 * otherwise force out a byte.
1535 */
1536 if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 &&
1537 tp->t_timer[TCPT_PERSIST] == 0) {
1538 TCP_RESET_REXMT_STATE(tp);
1539 tcp_setpersist(tp);
1540 }
1541 just_return:
1542 /*
1543 * If there is no reason to send a segment, just return.
1544 * but if there is some packets left in the packet list, send them now.
1545 */
1546 while (inp->inp_sndinprog_cnt == 0 &&
1547 tp->t_pktlist_head != NULL) {
1548 packetlist = tp->t_pktlist_head;
1549 packchain_listadd = tp->t_lastchain;
1550 packchain_sent++;
1551 TCP_PKTLIST_CLEAR(tp);
1552
1553 error = tcp_ip_output(so, tp, packetlist,
1554 packchain_listadd,
1555 tp_inp_options, (so_options & SO_DONTROUTE),
1556 (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6);
1557 }
1558 /* tcp was closed while we were in ip; resume close */
1559 if (inp->inp_sndinprog_cnt == 0 &&
1560 (tp->t_flags & TF_CLOSING)) {
1561 tp->t_flags &= ~TF_CLOSING;
1562 (void) tcp_close(tp);
1563 } else {
1564 tcp_check_timer_state(tp);
1565 }
1566 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
1567 return 0;
1568
1569 send:
1570 /*
1571 * Set TF_MAXSEGSNT flag if the segment size is greater than
1572 * the max segment size.
1573 */
1574 if (len > 0) {
1575 do_not_compress = TRUE;
1576
1577 if (len >= tp->t_maxseg) {
1578 tp->t_flags |= TF_MAXSEGSNT;
1579 } else {
1580 tp->t_flags &= ~TF_MAXSEGSNT;
1581 }
1582 }
1583 /*
1584 * Before ESTABLISHED, force sending of initial options
1585 * unless TCP set not to do any options.
1586 * NOTE: we assume that the IP/TCP header plus TCP options
1587 * always fit in a single mbuf, leaving room for a maximum
1588 * link header, i.e.
1589 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1590 */
1591 optlen = 0;
1592 if (isipv6) {
1593 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
1594 } else {
1595 hdrlen = sizeof(struct tcpiphdr);
1596 }
1597 if (flags & TH_SYN) {
1598 tp->snd_nxt = tp->iss;
1599 if ((tp->t_flags & TF_NOOPT) == 0) {
1600 u_short mss;
1601
1602 opt[0] = TCPOPT_MAXSEG;
1603 opt[1] = TCPOLEN_MAXSEG;
1604 mss = htons((u_short) tcp_mssopt(tp));
1605 (void)memcpy(opt + 2, &mss, sizeof(mss));
1606 optlen = TCPOLEN_MAXSEG;
1607
1608 if ((tp->t_flags & TF_REQ_SCALE) &&
1609 ((flags & TH_ACK) == 0 ||
1610 (tp->t_flags & TF_RCVD_SCALE))) {
1611 *((u_int32_t *)(void *)(opt + optlen)) = htonl(
1612 TCPOPT_NOP << 24 |
1613 TCPOPT_WINDOW << 16 |
1614 TCPOLEN_WINDOW << 8 |
1615 tp->request_r_scale);
1616 optlen += 4;
1617 }
1618 #if MPTCP
1619 if (mptcp_enable && (so->so_flags & SOF_MP_SUBFLOW)) {
1620 optlen = mptcp_setup_syn_opts(so, opt, optlen);
1621 }
1622 #endif /* MPTCP */
1623 }
1624 }
1625
1626 /*
1627 * Send a timestamp and echo-reply if this is a SYN and our side
1628 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1629 * and our peer have sent timestamps in our SYN's.
1630 */
1631 if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
1632 (flags & TH_RST) == 0 &&
1633 ((flags & TH_ACK) == 0 ||
1634 (tp->t_flags & TF_RCVD_TSTMP))) {
1635 u_int32_t *lp = (u_int32_t *)(void *)(opt + optlen);
1636
1637 /* Form timestamp option as shown in appendix A of RFC 1323. */
1638 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1639 *lp++ = htonl(tcp_now);
1640 *lp = htonl(tp->ts_recent);
1641 optlen += TCPOLEN_TSTAMP_APPA;
1642 }
1643
1644 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1645 /*
1646 * Tack on the SACK permitted option *last*.
1647 * And do padding of options after tacking this on.
1648 * This is because of MSS, TS, WinScale and Signatures are
1649 * all present, we have just 2 bytes left for the SACK
1650 * permitted option, which is just enough.
1651 */
1652 /*
1653 * If this is the first SYN of connection (not a SYN
1654 * ACK), include SACK permitted option. If this is a
1655 * SYN ACK, include SACK permitted option if peer has
1656 * already done so. This is only for active connect,
1657 * since the syncache takes care of the passive connect.
1658 */
1659 if ((flags & TH_SYN) &&
1660 (!(flags & TH_ACK) || (tp->t_flags & TF_SACK_PERMIT))) {
1661 u_char *bp;
1662 bp = (u_char *)opt + optlen;
1663
1664 *bp++ = TCPOPT_SACK_PERMITTED;
1665 *bp++ = TCPOLEN_SACK_PERMITTED;
1666 optlen += TCPOLEN_SACK_PERMITTED;
1667 }
1668 }
1669 #if MPTCP
1670 if (so->so_flags & SOF_MP_SUBFLOW) {
1671 /*
1672 * Its important to piggyback acks with data as ack only packets
1673 * may get lost and data packets that don't send Data ACKs
1674 * still advance the subflow level ACK and therefore make it
1675 * hard for the remote end to recover in low cwnd situations.
1676 */
1677 if (len != 0) {
1678 tp->t_mpflags |= (TMPF_SEND_DSN |
1679 TMPF_MPTCP_ACKNOW);
1680 } else {
1681 tp->t_mpflags |= TMPF_MPTCP_ACKNOW;
1682 }
1683 optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags,
1684 len, &mptcp_acknow, &do_not_compress);
1685 tp->t_mpflags &= ~TMPF_SEND_DSN;
1686 }
1687 #endif /* MPTCP */
1688
1689 if (tfo_enabled(tp) && !(tp->t_flags & TF_NOOPT) &&
1690 (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
1691 optlen += tcp_tfo_write_cookie(tp, optlen, len, opt);
1692 }
1693
1694 if (tfo_enabled(tp) &&
1695 (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK) &&
1696 (tp->t_tfo_flags & TFO_F_OFFER_COOKIE)) {
1697 optlen += tcp_tfo_write_cookie_rep(tp, optlen, opt);
1698 }
1699
1700 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1701 /*
1702 * Send SACKs if necessary. This should be the last
1703 * option processed. Only as many SACKs are sent as
1704 * are permitted by the maximum options size.
1705 *
1706 * In general, SACK blocks consume 8*n+2 bytes.
1707 * So a full size SACK blocks option is 34 bytes
1708 * (to generate 4 SACK blocks). At a minimum,
1709 * we need 10 bytes (to generate 1 SACK block).
1710 * If TCP Timestamps (12 bytes) and TCP Signatures
1711 * (18 bytes) are both present, we'll just have
1712 * 10 bytes for SACK options 40 - (12 + 18).
1713 */
1714 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1715 (tp->t_flags & TF_SACK_PERMIT) &&
1716 (tp->rcv_numsacks > 0 || TCP_SEND_DSACK_OPT(tp)) &&
1717 MAX_TCPOPTLEN - optlen - 2 >= TCPOLEN_SACK) {
1718 int nsack, padlen;
1719 u_char *bp = (u_char *)opt + optlen;
1720 u_int32_t *lp;
1721
1722 nsack = (MAX_TCPOPTLEN - optlen - 2) / TCPOLEN_SACK;
1723 nsack = min(nsack, (tp->rcv_numsacks +
1724 (TCP_SEND_DSACK_OPT(tp) ? 1 : 0)));
1725 sackoptlen = (2 + nsack * TCPOLEN_SACK);
1726 VERIFY(sackoptlen < UINT8_MAX);
1727
1728 /*
1729 * First we need to pad options so that the
1730 * SACK blocks can start at a 4-byte boundary
1731 * (sack option and length are at a 2 byte offset).
1732 */
1733 padlen = (MAX_TCPOPTLEN - optlen - sackoptlen) % 4;
1734 optlen += padlen;
1735 while (padlen-- > 0) {
1736 *bp++ = TCPOPT_NOP;
1737 }
1738
1739 tcpstat.tcps_sack_send_blocks++;
1740 *bp++ = TCPOPT_SACK;
1741 *bp++ = (uint8_t)sackoptlen;
1742 lp = (u_int32_t *)(void *)bp;
1743
1744 /*
1745 * First block of SACK option should represent
1746 * DSACK. Prefer to send SACK information if there
1747 * is space for only one SACK block. This will
1748 * allow for faster recovery.
1749 */
1750 if (TCP_SEND_DSACK_OPT(tp) && nsack > 0 &&
1751 (tp->rcv_numsacks == 0 || nsack > 1)) {
1752 *lp++ = htonl(tp->t_dsack_lseq);
1753 *lp++ = htonl(tp->t_dsack_rseq);
1754 tcpstat.tcps_dsack_sent++;
1755 tp->t_dsack_sent++;
1756 nsack--;
1757 }
1758 VERIFY(nsack == 0 || tp->rcv_numsacks >= nsack);
1759 for (i = 0; i < nsack; i++) {
1760 struct sackblk sack = tp->sackblks[i];
1761 *lp++ = htonl(sack.start);
1762 *lp++ = htonl(sack.end);
1763 }
1764 optlen += sackoptlen;
1765 }
1766 }
1767
1768 /* Pad TCP options to a 4 byte boundary */
1769 if (optlen < MAX_TCPOPTLEN && (optlen % sizeof(u_int32_t))) {
1770 int pad = sizeof(u_int32_t) - (optlen % sizeof(u_int32_t));
1771 u_char *bp = (u_char *)opt + optlen;
1772
1773 optlen += pad;
1774 while (pad) {
1775 *bp++ = TCPOPT_EOL;
1776 pad--;
1777 }
1778 }
1779
1780 /*
1781 * RFC 3168 states that:
1782 * - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
1783 * to handle the TCP ECE flag, even if you also later send a
1784 * non-ECN-setup SYN/SYN-ACK.
1785 * - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
1786 * the ip ECT flag.
1787 *
1788 * It is not clear how the ECE flag would ever be set if you never
1789 * set the IP ECT flag on outbound packets. All the same, we use
1790 * the TE_SETUPSENT to indicate that we have committed to handling
1791 * the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
1792 * whether or not we should set the IP ECT flag on outbound packet
1793 *
1794 * For a SYN-ACK, send an ECN setup SYN-ACK
1795 */
1796 if ((flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK) &&
1797 (tp->ecn_flags & TE_ENABLE_ECN)) {
1798 if (tp->ecn_flags & TE_SETUPRECEIVED) {
1799 if (tcp_send_ecn_flags_on_syn(tp)) {
1800 /*
1801 * Setting TH_ECE makes this an ECN-setup
1802 * SYN-ACK
1803 */
1804 flags |= TH_ECE;
1805
1806 /*
1807 * Record that we sent the ECN-setup and
1808 * default to setting IP ECT.
1809 */
1810 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1811 tcpstat.tcps_ecn_server_setup++;
1812 tcpstat.tcps_ecn_server_success++;
1813 } else {
1814 /*
1815 * We sent an ECN-setup SYN-ACK but it was
1816 * dropped. Fallback to non-ECN-setup
1817 * SYN-ACK and clear flag to indicate that
1818 * we should not send data with IP ECT set
1819 *
1820 * Pretend we didn't receive an
1821 * ECN-setup SYN.
1822 *
1823 * We already incremented the counter
1824 * assuming that the ECN setup will
1825 * succeed. Decrementing here
1826 * tcps_ecn_server_success to correct it.
1827 */
1828 if (tp->ecn_flags & TE_SETUPSENT) {
1829 tcpstat.tcps_ecn_lost_synack++;
1830 tcpstat.tcps_ecn_server_success--;
1831 tp->ecn_flags |= TE_LOST_SYNACK;
1832 }
1833
1834 tp->ecn_flags &=
1835 ~(TE_SETUPRECEIVED | TE_SENDIPECT |
1836 TE_SENDCWR);
1837 }
1838 }
1839 } else if ((flags & (TH_SYN | TH_ACK)) == TH_SYN &&
1840 (tp->ecn_flags & TE_ENABLE_ECN)) {
1841 if (tcp_send_ecn_flags_on_syn(tp)) {
1842 /*
1843 * Setting TH_ECE and TH_CWR makes this an
1844 * ECN-setup SYN
1845 */
1846 flags |= (TH_ECE | TH_CWR);
1847 tcpstat.tcps_ecn_client_setup++;
1848 tp->ecn_flags |= TE_CLIENT_SETUP;
1849
1850 /*
1851 * Record that we sent the ECN-setup and default to
1852 * setting IP ECT.
1853 */
1854 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1855 } else {
1856 /*
1857 * We sent an ECN-setup SYN but it was dropped.
1858 * Fall back to non-ECN and clear flag indicating
1859 * we should send data with IP ECT set.
1860 */
1861 if (tp->ecn_flags & TE_SETUPSENT) {
1862 tcpstat.tcps_ecn_lost_syn++;
1863 tp->ecn_flags |= TE_LOST_SYN;
1864 }
1865 tp->ecn_flags &= ~TE_SENDIPECT;
1866 }
1867 }
1868
1869 /*
1870 * Check if we should set the TCP CWR flag.
1871 * CWR flag is sent when we reduced the congestion window because
1872 * we received a TCP ECE or we performed a fast retransmit. We
1873 * never set the CWR flag on retransmitted packets. We only set
1874 * the CWR flag on data packets. Pure acks don't have this set.
1875 */
1876 if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
1877 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1878 flags |= TH_CWR;
1879 tp->ecn_flags &= ~TE_SENDCWR;
1880 }
1881
1882 /*
1883 * Check if we should set the TCP ECE flag.
1884 */
1885 if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
1886 flags |= TH_ECE;
1887 tcpstat.tcps_ecn_sent_ece++;
1888 }
1889
1890
1891 hdrlen += optlen;
1892
1893 /* Reset DSACK sequence numbers */
1894 tp->t_dsack_lseq = 0;
1895 tp->t_dsack_rseq = 0;
1896
1897 if (isipv6) {
1898 ipoptlen = ip6_optlen(inp);
1899 } else {
1900 if (tp_inp_options) {
1901 ipoptlen = tp_inp_options->m_len -
1902 offsetof(struct ipoption, ipopt_list);
1903 } else {
1904 ipoptlen = 0;
1905 }
1906 }
1907 #if IPSEC
1908 ipoptlen += ipsec_optlen;
1909 #endif
1910
1911 /*
1912 * Adjust data length if insertion of options will
1913 * bump the packet length beyond the t_maxopd length.
1914 * Clear the FIN bit because we cut off the tail of
1915 * the segment.
1916 *
1917 * When doing TSO limit a burst to TCP_MAXWIN minus the
1918 * IP, TCP and Options length to keep ip->ip_len from
1919 * overflowing. Prevent the last segment from being
1920 * fractional thus making them all equal sized and set
1921 * the flag to continue sending. TSO is disabled when
1922 * IP options or IPSEC are present.
1923 */
1924 if (len + optlen + ipoptlen > tp->t_maxopd) {
1925 /*
1926 * If there is still more to send,
1927 * don't close the connection.
1928 */
1929 flags &= ~TH_FIN;
1930 if (tso) {
1931 int32_t tso_maxlen;
1932
1933 tso_maxlen = tp->tso_max_segment_size ?
1934 tp->tso_max_segment_size : TCP_MAXWIN;
1935
1936 if (len > tso_maxlen - hdrlen - optlen) {
1937 len = tso_maxlen - hdrlen - optlen;
1938 sendalot = 1;
1939 } else if (tp->t_flags & TF_NEEDFIN) {
1940 sendalot = 1;
1941 }
1942
1943 if (len % (tp->t_maxopd - optlen) != 0) {
1944 len = len - (len % (tp->t_maxopd - optlen));
1945 sendalot = 1;
1946 }
1947 } else {
1948 len = tp->t_maxopd - optlen - ipoptlen;
1949 sendalot = 1;
1950 }
1951 }
1952
1953 if (max_linkhdr + hdrlen > MCLBYTES) {
1954 panic("tcphdr too big");
1955 }
1956
1957 /* Check if there is enough data in the send socket
1958 * buffer to start measuring bandwidth
1959 */
1960 if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 &&
1961 (tp->t_bwmeas != NULL) &&
1962 (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0) {
1963 tp->t_bwmeas->bw_size = min(min(
1964 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)),
1965 tp->snd_cwnd), tp->snd_wnd);
1966 if (tp->t_bwmeas->bw_minsize > 0 &&
1967 tp->t_bwmeas->bw_size < tp->t_bwmeas->bw_minsize) {
1968 tp->t_bwmeas->bw_size = 0;
1969 }
1970 if (tp->t_bwmeas->bw_maxsize > 0) {
1971 tp->t_bwmeas->bw_size = min(tp->t_bwmeas->bw_size,
1972 tp->t_bwmeas->bw_maxsize);
1973 }
1974 if (tp->t_bwmeas->bw_size > 0) {
1975 tp->t_flagsext |= TF_BWMEAS_INPROGRESS;
1976 tp->t_bwmeas->bw_start = tp->snd_max;
1977 tp->t_bwmeas->bw_ts = tcp_now;
1978 }
1979 }
1980
1981 VERIFY(inp->inp_flowhash != 0);
1982 /*
1983 * Grab a header mbuf, attaching a copy of data to
1984 * be transmitted, and initialize the header from
1985 * the template for sends on this connection.
1986 */
1987 if (len) {
1988 /* Remember what the last head-of-line packet-size was */
1989 if (tp->t_pmtud_lastseg_size == 0 && tp->snd_nxt == tp->snd_una) {
1990 ASSERT(len + optlen + ipoptlen <= IP_MAXPACKET);
1991 tp->t_pmtud_lastseg_size = (uint16_t)(len + optlen + ipoptlen);
1992 }
1993 if ((tp->t_flagsext & TF_FORCE) && len == 1) {
1994 tcpstat.tcps_sndprobe++;
1995 } else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1996 tcpstat.tcps_sndrexmitpack++;
1997 tcpstat.tcps_sndrexmitbyte += len;
1998 if (nstat_collect) {
1999 nstat_route_tx(inp->inp_route.ro_rt, 1,
2000 len, NSTAT_TX_FLAG_RETRANSMIT);
2001 INP_ADD_STAT(inp, cell, wifi, wired,
2002 txpackets, 1);
2003 INP_ADD_STAT(inp, cell, wifi, wired,
2004 txbytes, len);
2005 tp->t_stat.txretransmitbytes += len;
2006 tp->t_stat.rxmitpkts++;
2007 }
2008 } else {
2009 tcpstat.tcps_sndpack++;
2010 tcpstat.tcps_sndbyte += len;
2011
2012 if (nstat_collect) {
2013 INP_ADD_STAT(inp, cell, wifi, wired,
2014 txpackets, 1);
2015 INP_ADD_STAT(inp, cell, wifi, wired,
2016 txbytes, len);
2017 }
2018 inp_decr_sndbytes_unsent(so, len);
2019 }
2020 inp_set_activity_bitmap(inp);
2021 #if MPTCP
2022 if (tp->t_mpflags & TMPF_MPTCP_TRUE) {
2023 tcpstat.tcps_mp_sndpacks++;
2024 tcpstat.tcps_mp_sndbytes += len;
2025 }
2026 #endif /* MPTCP */
2027 /*
2028 * try to use the new interface that allocates all
2029 * the necessary mbuf hdrs under 1 mbuf lock and
2030 * avoids rescanning the socket mbuf list if
2031 * certain conditions are met. This routine can't
2032 * be used in the following cases...
2033 * 1) the protocol headers exceed the capacity of
2034 * of a single mbuf header's data area (no cluster attached)
2035 * 2) the length of the data being transmitted plus
2036 * the protocol headers fits into a single mbuf header's
2037 * data area (no cluster attached)
2038 */
2039 m = NULL;
2040
2041 /* minimum length we are going to allocate */
2042 allocated_len = MHLEN;
2043 if (MHLEN < hdrlen + max_linkhdr) {
2044 MGETHDR(m, M_DONTWAIT, MT_HEADER);
2045 if (m == NULL) {
2046 error = ENOBUFS;
2047 goto out;
2048 }
2049 MCLGET(m, M_DONTWAIT);
2050 if ((m->m_flags & M_EXT) == 0) {
2051 m_freem(m);
2052 error = ENOBUFS;
2053 goto out;
2054 }
2055 m->m_data += max_linkhdr;
2056 m->m_len = hdrlen;
2057 allocated_len = MCLBYTES;
2058 }
2059 if (len <= allocated_len - hdrlen - max_linkhdr) {
2060 if (m == NULL) {
2061 VERIFY(allocated_len <= MHLEN);
2062 MGETHDR(m, M_DONTWAIT, MT_HEADER);
2063 if (m == NULL) {
2064 error = ENOBUFS;
2065 goto out;
2066 }
2067 m->m_data += max_linkhdr;
2068 m->m_len = hdrlen;
2069 }
2070 /* makes sure we still have data left to be sent at this point */
2071 if (so->so_snd.sb_mb == NULL || off < 0) {
2072 if (m != NULL) {
2073 m_freem(m);
2074 }
2075 error = 0; /* should we return an error? */
2076 goto out;
2077 }
2078 m_copydata(so->so_snd.sb_mb, off, (int) len,
2079 mtod(m, caddr_t) + hdrlen);
2080 m->m_len += len;
2081 } else {
2082 uint32_t copymode;
2083 /*
2084 * Retain packet header metadata at the socket
2085 * buffer if this is is an MPTCP subflow,
2086 * otherwise move it.
2087 */
2088 copymode = M_COPYM_MOVE_HDR;
2089 #if MPTCP
2090 if (so->so_flags & SOF_MP_SUBFLOW) {
2091 copymode = M_COPYM_NOOP_HDR;
2092 }
2093 #endif /* MPTCP */
2094 if (m != NULL) {
2095 m->m_next = m_copym_mode(so->so_snd.sb_mb,
2096 off, (int)len, M_DONTWAIT, copymode);
2097 if (m->m_next == NULL) {
2098 (void) m_free(m);
2099 error = ENOBUFS;
2100 goto out;
2101 }
2102 } else {
2103 /*
2104 * make sure we still have data left
2105 * to be sent at this point
2106 */
2107 if (so->so_snd.sb_mb == NULL) {
2108 error = 0; /* should we return an error? */
2109 goto out;
2110 }
2111
2112 /*
2113 * m_copym_with_hdrs will always return the
2114 * last mbuf pointer and the offset into it that
2115 * it acted on to fullfill the current request,
2116 * whether a valid 'hint' was passed in or not.
2117 */
2118 if ((m = m_copym_with_hdrs(so->so_snd.sb_mb,
2119 off, len, M_DONTWAIT, NULL, NULL,
2120 copymode)) == NULL) {
2121 error = ENOBUFS;
2122 goto out;
2123 }
2124 m->m_data += max_linkhdr;
2125 m->m_len = hdrlen;
2126 }
2127 }
2128 /*
2129 * If we're sending everything we've got, set PUSH.
2130 * (This will keep happy those implementations which only
2131 * give data to the user when a buffer fills or
2132 * a PUSH comes in.)
2133 *
2134 * On SYN-segments we should not add the PUSH-flag.
2135 */
2136 if (off + len == so->so_snd.sb_cc && !(flags & TH_SYN)) {
2137 flags |= TH_PUSH;
2138 }
2139 } else {
2140 if (tp->t_flags & TF_ACKNOW) {
2141 tcpstat.tcps_sndacks++;
2142 } else if (flags & (TH_SYN | TH_FIN | TH_RST)) {
2143 tcpstat.tcps_sndctrl++;
2144 } else if (SEQ_GT(tp->snd_up, tp->snd_una)) {
2145 tcpstat.tcps_sndurg++;
2146 } else {
2147 tcpstat.tcps_sndwinup++;
2148 }
2149
2150 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
2151 if (m == NULL) {
2152 error = ENOBUFS;
2153 goto out;
2154 }
2155 if (MHLEN < (hdrlen + max_linkhdr)) {
2156 MCLGET(m, M_DONTWAIT);
2157 if ((m->m_flags & M_EXT) == 0) {
2158 m_freem(m);
2159 error = ENOBUFS;
2160 goto out;
2161 }
2162 }
2163 m->m_data += max_linkhdr;
2164 m->m_len = hdrlen;
2165 }
2166 m->m_pkthdr.rcvif = 0;
2167
2168 /* Any flag other than pure-ACK: Do not compress! */
2169 if (flags & ~(TH_ACK)) {
2170 do_not_compress = TRUE;
2171 }
2172
2173 if (tp->rcv_scale == 0) {
2174 do_not_compress = TRUE;
2175 }
2176
2177 if (do_not_compress || (tcp_do_ack_compression == 1 && !cell) || __improbable(!tcp_do_ack_compression)) {
2178 m->m_pkthdr.comp_gencnt = 0;
2179 } else {
2180 if (TSTMP_LT(tp->t_comp_lastinc + tcp_ack_compression_rate, tcp_now)) {
2181 tp->t_comp_gencnt++;
2182 /* 0 means no compression, thus jump this */
2183 if (tp->t_comp_gencnt <= TCP_ACK_COMPRESSION_DUMMY) {
2184 tp->t_comp_gencnt = TCP_ACK_COMPRESSION_DUMMY + 1;
2185 }
2186 tp->t_comp_lastinc = tcp_now;
2187 }
2188 m->m_pkthdr.comp_gencnt = tp->t_comp_gencnt;
2189 }
2190
2191 if (isipv6) {
2192 ip6 = mtod(m, struct ip6_hdr *);
2193 th = (struct tcphdr *)(void *)(ip6 + 1);
2194 tcp_fillheaders(tp, ip6, th);
2195 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
2196 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
2197 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
2198 }
2199 svc_flags |= PKT_SCF_IPV6;
2200 #if PF_ECN
2201 m_pftag(m)->pftag_hdr = (void *)ip6;
2202 m_pftag(m)->pftag_flags |= PF_TAG_HDR_INET6;
2203 #endif /* PF_ECN */
2204 } else {
2205 ip = mtod(m, struct ip *);
2206 th = (struct tcphdr *)(void *)(ip + 1);
2207 /* this picks up the pseudo header (w/o the length) */
2208 tcp_fillheaders(tp, ip, th);
2209 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
2210 !SEQ_LT(tp->snd_nxt, tp->snd_max) &&
2211 !sack_rxmit && !(flags & TH_SYN)) {
2212 ip->ip_tos |= IPTOS_ECN_ECT0;
2213 }
2214 #if PF_ECN
2215 m_pftag(m)->pftag_hdr = (void *)ip;
2216 m_pftag(m)->pftag_flags |= PF_TAG_HDR_INET;
2217 #endif /* PF_ECN */
2218 }
2219
2220 /*
2221 * Fill in fields, remembering maximum advertised
2222 * window for use in delaying messages about window sizes.
2223 * If resending a FIN, be sure not to use a new sequence number.
2224 */
2225 if ((flags & TH_FIN) && (tp->t_flags & TF_SENTFIN) &&
2226 tp->snd_nxt == tp->snd_max) {
2227 tp->snd_nxt--;
2228 }
2229 /*
2230 * If we are doing retransmissions, then snd_nxt will
2231 * not reflect the first unsent octet. For ACK only
2232 * packets, we do not want the sequence number of the
2233 * retransmitted packet, we want the sequence number
2234 * of the next unsent octet. So, if there is no data
2235 * (and no SYN or FIN), use snd_max instead of snd_nxt
2236 * when filling in ti_seq. But if we are in persist
2237 * state, snd_max might reflect one byte beyond the
2238 * right edge of the window, so use snd_nxt in that
2239 * case, since we know we aren't doing a retransmission.
2240 * (retransmit and persist are mutually exclusive...)
2241 *
2242 * Note the state of this retransmit segment to detect spurious
2243 * retransmissions.
2244 */
2245 if (sack_rxmit == 0) {
2246 if (len || (flags & (TH_SYN | TH_FIN)) ||
2247 tp->t_timer[TCPT_PERSIST]) {
2248 th->th_seq = htonl(tp->snd_nxt);
2249 if (len > 0) {
2250 m->m_pkthdr.tx_start_seq = tp->snd_nxt;
2251 m->m_pkthdr.pkt_flags |= PKTF_START_SEQ;
2252 }
2253 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
2254 if (SACK_ENABLED(tp) && len > 1) {
2255 tcp_rxtseg_insert(tp, tp->snd_nxt,
2256 (tp->snd_nxt + len - 1));
2257 }
2258 if (len > 0) {
2259 m->m_pkthdr.pkt_flags |=
2260 PKTF_TCP_REXMT;
2261 }
2262 }
2263 } else {
2264 th->th_seq = htonl(tp->snd_max);
2265 }
2266 } else {
2267 th->th_seq = htonl(p->rxmit);
2268 if (len > 0) {
2269 m->m_pkthdr.pkt_flags |=
2270 (PKTF_TCP_REXMT | PKTF_START_SEQ);
2271 m->m_pkthdr.tx_start_seq = p->rxmit;
2272 }
2273 tcp_rxtseg_insert(tp, p->rxmit, (p->rxmit + len - 1));
2274 p->rxmit += len;
2275 tp->sackhint.sack_bytes_rexmit += len;
2276 }
2277 th->th_ack = htonl(tp->rcv_nxt);
2278 tp->last_ack_sent = tp->rcv_nxt;
2279 if (optlen) {
2280 bcopy(opt, th + 1, optlen);
2281 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
2282 }
2283 th->th_flags = flags;
2284 th->th_win = htons((u_short) (recwin >> tp->rcv_scale));
2285 tp->t_last_recwin = recwin;
2286 if (!(so->so_flags & SOF_MP_SUBFLOW)) {
2287 if (recwin > 0 && SEQ_LT(tp->rcv_adv, tp->rcv_nxt + recwin)) {
2288 tp->rcv_adv = tp->rcv_nxt + recwin;
2289 }
2290 } else {
2291 struct mptcb *mp_tp = tptomptp(tp);
2292 if (recwin > 0) {
2293 tp->rcv_adv = tp->rcv_nxt + recwin;
2294 }
2295
2296 if (recwin > 0 && MPTCP_SEQ_LT(mp_tp->mpt_rcvadv, mp_tp->mpt_rcvnxt + recwin)) {
2297 mp_tp->mpt_rcvadv = mp_tp->mpt_rcvnxt + recwin;
2298 }
2299 }
2300
2301 /*
2302 * Adjust the RXWIN0SENT flag - indicate that we have advertised
2303 * a 0 window. This may cause the remote transmitter to stall. This
2304 * flag tells soreceive() to disable delayed acknowledgements when
2305 * draining the buffer. This can occur if the receiver is attempting
2306 * to read more data then can be buffered prior to transmitting on
2307 * the connection.
2308 */
2309 if (th->th_win == 0) {
2310 tp->t_flags |= TF_RXWIN0SENT;
2311 } else {
2312 tp->t_flags &= ~TF_RXWIN0SENT;
2313 }
2314
2315 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
2316 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
2317 th->th_flags |= TH_URG;
2318 } else {
2319 /*
2320 * If no urgent pointer to send, then we pull
2321 * the urgent pointer to the left edge of the send window
2322 * so that it doesn't drift into the send window on sequence
2323 * number wraparound.
2324 */
2325 tp->snd_up = tp->snd_una; /* drag it along */
2326 }
2327
2328 /*
2329 * Put TCP length in extended header, and then
2330 * checksum extended header and data.
2331 */
2332 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
2333
2334 /*
2335 * If this is potentially the last packet on the stream, then mark
2336 * it in order to enable some optimizations in the underlying
2337 * layers
2338 */
2339 if (tp->t_state != TCPS_ESTABLISHED &&
2340 (tp->t_state == TCPS_CLOSING || tp->t_state == TCPS_TIME_WAIT
2341 || tp->t_state == TCPS_LAST_ACK || (th->th_flags & TH_RST))) {
2342 m->m_pkthdr.pkt_flags |= PKTF_LAST_PKT;
2343 }
2344
2345 if (isipv6) {
2346 /*
2347 * ip6_plen is not need to be filled now, and will be filled
2348 * in ip6_output.
2349 */
2350 m->m_pkthdr.csum_flags = CSUM_TCPIPV6;
2351 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2352 if (len + optlen) {
2353 th->th_sum = in_addword(th->th_sum,
2354 htons((u_short)(optlen + len)));
2355 }
2356 } else {
2357 m->m_pkthdr.csum_flags = CSUM_TCP;
2358 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2359 if (len + optlen) {
2360 th->th_sum = in_addword(th->th_sum,
2361 htons((u_short)(optlen + len)));
2362 }
2363 }
2364
2365 /*
2366 * Enable TSO and specify the size of the segments.
2367 * The TCP pseudo header checksum is always provided.
2368 */
2369 if (tso) {
2370 if (isipv6) {
2371 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV6;
2372 } else {
2373 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV4;
2374 }
2375
2376 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
2377 } else {
2378 m->m_pkthdr.tso_segsz = 0;
2379 }
2380
2381 /*
2382 * In transmit state, time the transmission and arrange for
2383 * the retransmit. In persist state, just set snd_max.
2384 */
2385 if (!(tp->t_flagsext & TF_FORCE)
2386 || tp->t_timer[TCPT_PERSIST] == 0) {
2387 tcp_seq startseq = tp->snd_nxt;
2388
2389 /*
2390 * Advance snd_nxt over sequence space of this segment.
2391 */
2392 if (flags & (TH_SYN | TH_FIN)) {
2393 if (flags & TH_SYN) {
2394 tp->snd_nxt++;
2395 }
2396 if ((flags & TH_FIN) &&
2397 !(tp->t_flags & TF_SENTFIN)) {
2398 tp->snd_nxt++;
2399 tp->t_flags |= TF_SENTFIN;
2400 }
2401 }
2402 if (sack_rxmit) {
2403 goto timer;
2404 }
2405 if (sack_rescue_rxt == TRUE) {
2406 tp->snd_nxt = old_snd_nxt;
2407 sack_rescue_rxt = FALSE;
2408 tcpstat.tcps_pto_in_recovery++;
2409 } else {
2410 tp->snd_nxt += len;
2411 }
2412 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
2413 tp->snd_max = tp->snd_nxt;
2414 tp->t_sndtime = tcp_now;
2415 /*
2416 * Time this transmission if not a retransmission and
2417 * not currently timing anything.
2418 */
2419 if (tp->t_rtttime == 0) {
2420 tp->t_rtttime = tcp_now;
2421 tp->t_rtseq = startseq;
2422 tcpstat.tcps_segstimed++;
2423
2424 /* update variables related to pipe ack */
2425 tp->t_pipeack_lastuna = tp->snd_una;
2426 }
2427 }
2428
2429 /*
2430 * Set retransmit timer if not currently set,
2431 * and not doing an ack or a keep-alive probe.
2432 */
2433 timer:
2434 if (tp->t_timer[TCPT_REXMT] == 0 &&
2435 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
2436 tp->snd_nxt != tp->snd_una || (flags & TH_FIN))) {
2437 if (tp->t_timer[TCPT_PERSIST]) {
2438 tp->t_timer[TCPT_PERSIST] = 0;
2439 tp->t_persist_stop = 0;
2440 TCP_RESET_REXMT_STATE(tp);
2441 }
2442 tp->t_timer[TCPT_REXMT] =
2443 OFFSET_FROM_START(tp, tp->t_rxtcur);
2444 }
2445
2446 /*
2447 * Set tail loss probe timeout if new data is being
2448 * transmitted. This will be supported only when
2449 * SACK option is enabled on a connection.
2450 *
2451 * Every time new data is sent PTO will get reset.
2452 */
2453 if (tcp_enable_tlp && len != 0 && tp->t_state == TCPS_ESTABLISHED &&
2454 SACK_ENABLED(tp) && !IN_FASTRECOVERY(tp) &&
2455 tp->snd_nxt == tp->snd_max &&
2456 SEQ_GT(tp->snd_nxt, tp->snd_una) &&
2457 tp->t_rxtshift == 0 &&
2458 (tp->t_flagsext & (TF_SENT_TLPROBE | TF_PKTS_REORDERED)) == 0) {
2459 uint32_t pto, srtt;
2460
2461 if (tcp_do_better_lr) {
2462 srtt = tp->t_srtt >> TCP_RTT_SHIFT;
2463 pto = 2 * srtt;
2464 if ((tp->snd_max - tp->snd_una) <= tp->t_maxseg) {
2465 pto += tcp_delack;
2466 } else {
2467 pto += 2;
2468 }
2469 } else {
2470 /*
2471 * Using SRTT alone to set PTO can cause spurious
2472 * retransmissions on wireless networks where there
2473 * is a lot of variance in RTT. Taking variance
2474 * into account will avoid this.
2475 */
2476 srtt = tp->t_srtt >> TCP_RTT_SHIFT;
2477 pto = ((TCP_REXMTVAL(tp)) * 3) >> 1;
2478 pto = max(2 * srtt, pto);
2479 if ((tp->snd_max - tp->snd_una) == tp->t_maxseg) {
2480 pto = max(pto,
2481 (((3 * pto) >> 2) + tcp_delack * 2));
2482 } else {
2483 pto = max(10, pto);
2484 }
2485 }
2486
2487 /* if RTO is less than PTO, choose RTO instead */
2488 if (tp->t_rxtcur < pto) {
2489 pto = tp->t_rxtcur;
2490 }
2491
2492 tp->t_timer[TCPT_PTO] = OFFSET_FROM_START(tp, pto);
2493 }
2494 } else {
2495 /*
2496 * Persist case, update snd_max but since we are in
2497 * persist mode (no window) we do not update snd_nxt.
2498 */
2499 int xlen = len;
2500 if (flags & TH_SYN) {
2501 ++xlen;
2502 }
2503 if ((flags & TH_FIN) &&
2504 !(tp->t_flags & TF_SENTFIN)) {
2505 ++xlen;
2506 tp->t_flags |= TF_SENTFIN;
2507 }
2508 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) {
2509 tp->snd_max = tp->snd_nxt + len;
2510 tp->t_sndtime = tcp_now;
2511 }
2512 }
2513
2514 #if TCPDEBUG
2515 /*
2516 * Trace.
2517 */
2518 if (so_options & SO_DEBUG) {
2519 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
2520 }
2521 #endif
2522
2523 /*
2524 * Fill in IP length and desired time to live and
2525 * send to IP level. There should be a better way
2526 * to handle ttl and tos; we could keep them in
2527 * the template, but need a way to checksum without them.
2528 */
2529 /*
2530 * m->m_pkthdr.len should have been set before cksum calcuration,
2531 * because in6_cksum() need it.
2532 */
2533 if (isipv6) {
2534 /*
2535 * we separately set hoplimit for every segment, since the
2536 * user might want to change the value via setsockopt.
2537 * Also, desired default hop limit might be changed via
2538 * Neighbor Discovery.
2539 */
2540 ip6->ip6_hlim = in6_selecthlim(inp, inp->in6p_route.ro_rt ?
2541 inp->in6p_route.ro_rt->rt_ifp : NULL);
2542
2543 /* TODO: IPv6 IP6TOS_ECT bit on */
2544 KERNEL_DEBUG(DBG_LAYER_BEG,
2545 ((inp->inp_fport << 16) | inp->inp_lport),
2546 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
2547 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
2548 sendalot, 0, 0);
2549 } else {
2550 ASSERT(m->m_pkthdr.len <= IP_MAXPACKET);
2551 ip->ip_len = (u_short)m->m_pkthdr.len;
2552 ip->ip_ttl = inp->inp_ip_ttl; /* XXX */
2553 ip->ip_tos |= (inp->inp_ip_tos & ~IPTOS_ECN_MASK);/* XXX */
2554 KERNEL_DEBUG(DBG_LAYER_BEG,
2555 ((inp->inp_fport << 16) | inp->inp_lport),
2556 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
2557 (inp->inp_faddr.s_addr & 0xffff)), 0, 0, 0);
2558 }
2559
2560 /*
2561 * See if we should do MTU discovery.
2562 * Look at the flag updated on the following criterias:
2563 * 1) Path MTU discovery is authorized by the sysctl
2564 * 2) The route isn't set yet (unlikely but could happen)
2565 * 3) The route is up
2566 * 4) the MTU is not locked (if it is, then discovery has been
2567 * disabled for that route)
2568 */
2569 if (!isipv6) {
2570 if (path_mtu_discovery && (tp->t_flags & TF_PMTUD)) {
2571 ip->ip_off |= IP_DF;
2572 }
2573 }
2574
2575 #if NECP
2576 {
2577 necp_kernel_policy_id policy_id;
2578 necp_kernel_policy_id skip_policy_id;
2579 u_int32_t route_rule_id;
2580 u_int32_t pass_flags;
2581 if (!necp_socket_is_allowed_to_send_recv(inp, NULL, 0, &policy_id, &route_rule_id, &skip_policy_id, &pass_flags)) {
2582 TCP_LOG_DROP_NECP(isipv6 ? (void *)ip6 : (void *)ip, th, tp, true);
2583 m_freem(m);
2584 error = EHOSTUNREACH;
2585 goto out;
2586 }
2587 necp_mark_packet_from_socket(m, inp, policy_id, route_rule_id, skip_policy_id, pass_flags);
2588
2589 if (net_qos_policy_restricted != 0) {
2590 necp_socket_update_qos_marking(inp, inp->inp_route.ro_rt, route_rule_id);
2591 }
2592 }
2593 #endif /* NECP */
2594
2595 #if IPSEC
2596 if (inp->inp_sp != NULL) {
2597 ipsec_setsocket(m, so);
2598 }
2599 #endif /*IPSEC*/
2600
2601 /*
2602 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
2603 */
2604 lost = 0;
2605
2606 /*
2607 * Embed the flow hash in pkt hdr and mark the packet as
2608 * capable of flow controlling
2609 */
2610 m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB;
2611 m->m_pkthdr.pkt_flowid = inp->inp_flowhash;
2612 m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV);
2613 m->m_pkthdr.pkt_proto = IPPROTO_TCP;
2614 m->m_pkthdr.tx_tcp_pid = so->last_pid;
2615 if (so->so_flags & SOF_DELEGATED) {
2616 m->m_pkthdr.tx_tcp_e_pid = so->e_pid;
2617 } else {
2618 m->m_pkthdr.tx_tcp_e_pid = 0;
2619 }
2620
2621 m->m_nextpkt = NULL;
2622
2623 if (inp->inp_last_outifp != NULL &&
2624 !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) {
2625 /* Hint to prioritize this packet if
2626 * 1. if the packet has no data
2627 * 2. the interface supports transmit-start model and did
2628 * not disable ACK prioritization.
2629 * 3. Only ACK flag is set.
2630 * 4. there is no outstanding data on this connection.
2631 */
2632 if (len == 0 && (inp->inp_last_outifp->if_eflags & (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART) {
2633 if (th->th_flags == TH_ACK &&
2634 tp->snd_una == tp->snd_max &&
2635 tp->t_timer[TCPT_REXMT] == 0) {
2636 svc_flags |= PKT_SCF_TCP_ACK;
2637 }
2638 if (th->th_flags & TH_SYN) {
2639 svc_flags |= PKT_SCF_TCP_SYN;
2640 }
2641 }
2642 set_packet_service_class(m, so, sotc, svc_flags);
2643 } else {
2644 /*
2645 * Optimization for loopback just set the mbuf
2646 * service class
2647 */
2648 (void) m_set_service_class(m, so_tc2msc(sotc));
2649 }
2650
2651 TCP_LOG_TH_FLAGS(isipv6 ? (void *)ip6 : (void *)ip, th, tp, true,
2652 inp->inp_last_outifp != NULL ? inp->inp_last_outifp :
2653 inp->inp_boundifp);
2654
2655 tp->t_pktlist_sentlen += len;
2656 tp->t_lastchain++;
2657
2658 if (isipv6) {
2659 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
2660 struct ip6 *, ip6, struct tcpcb *, tp, struct tcphdr *,
2661 th);
2662 } else {
2663 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
2664 struct ip *, ip, struct tcpcb *, tp, struct tcphdr *, th);
2665 }
2666
2667 if (tp->t_pktlist_head != NULL) {
2668 tp->t_pktlist_tail->m_nextpkt = m;
2669 tp->t_pktlist_tail = m;
2670 } else {
2671 packchain_newlist++;
2672 tp->t_pktlist_head = tp->t_pktlist_tail = m;
2673 }
2674
2675 if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
2676 (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
2677 (tp->t_flags & TF_ACKNOW) ||
2678 (tp->t_flagsext & TF_FORCE) ||
2679 tp->t_lastchain >= tcp_packet_chaining) {
2680 error = 0;
2681 while (inp->inp_sndinprog_cnt == 0 &&
2682 tp->t_pktlist_head != NULL) {
2683 packetlist = tp->t_pktlist_head;
2684 packchain_listadd = tp->t_lastchain;
2685 packchain_sent++;
2686 lost = tp->t_pktlist_sentlen;
2687 TCP_PKTLIST_CLEAR(tp);
2688
2689 error = tcp_ip_output(so, tp, packetlist,
2690 packchain_listadd, tp_inp_options,
2691 (so_options & SO_DONTROUTE),
2692 (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6);
2693 if (error) {
2694 /*
2695 * Take into account the rest of unsent
2696 * packets in the packet list for this tcp
2697 * into "lost", since we're about to free
2698 * the whole list below.
2699 */
2700 lost += tp->t_pktlist_sentlen;
2701 break;
2702 } else {
2703 lost = 0;
2704 }
2705 }
2706 /* tcp was closed while we were in ip; resume close */
2707 if (inp->inp_sndinprog_cnt == 0 &&
2708 (tp->t_flags & TF_CLOSING)) {
2709 tp->t_flags &= ~TF_CLOSING;
2710 (void) tcp_close(tp);
2711 return 0;
2712 }
2713 } else {
2714 error = 0;
2715 packchain_looped++;
2716 tcpstat.tcps_sndtotal++;
2717
2718 goto again;
2719 }
2720 if (error) {
2721 /*
2722 * Assume that the packets were lost, so back out the
2723 * sequence number advance, if any. Note that the "lost"
2724 * variable represents the amount of user data sent during
2725 * the recent call to ip_output_list() plus the amount of
2726 * user data in the packet list for this tcp at the moment.
2727 */
2728 if (!(tp->t_flagsext & TF_FORCE)
2729 || tp->t_timer[TCPT_PERSIST] == 0) {
2730 /*
2731 * No need to check for TH_FIN here because
2732 * the TF_SENTFIN flag handles that case.
2733 */
2734 if ((flags & TH_SYN) == 0) {
2735 if (sack_rxmit) {
2736 if (SEQ_GT((p->rxmit - lost),
2737 tp->snd_una)) {
2738 p->rxmit -= lost;
2739
2740 if (SEQ_LT(p->rxmit, p->start)) {
2741 p->rxmit = p->start;
2742 }
2743 } else {
2744 lost = p->rxmit - tp->snd_una;
2745 p->rxmit = tp->snd_una;
2746
2747 if (SEQ_LT(p->rxmit, p->start)) {
2748 p->rxmit = p->start;
2749 }
2750 }
2751 tp->sackhint.sack_bytes_rexmit -= lost;
2752 if (tp->sackhint.sack_bytes_rexmit < 0) {
2753 tp->sackhint.sack_bytes_rexmit = 0;
2754 }
2755 } else {
2756 if (SEQ_GT((tp->snd_nxt - lost),
2757 tp->snd_una)) {
2758 tp->snd_nxt -= lost;
2759 } else {
2760 tp->snd_nxt = tp->snd_una;
2761 }
2762 }
2763 }
2764 }
2765 out:
2766 if (tp->t_pktlist_head != NULL) {
2767 m_freem_list(tp->t_pktlist_head);
2768 }
2769 TCP_PKTLIST_CLEAR(tp);
2770
2771 if (error == ENOBUFS) {
2772 /*
2773 * Set retransmit timer if not currently set
2774 * when we failed to send a segment that can be
2775 * retransmitted (i.e. not pure ack or rst)
2776 */
2777 if (tp->t_timer[TCPT_REXMT] == 0 &&
2778 tp->t_timer[TCPT_PERSIST] == 0 &&
2779 (len != 0 || (flags & (TH_SYN | TH_FIN)) != 0 ||
2780 so->so_snd.sb_cc > 0)) {
2781 tp->t_timer[TCPT_REXMT] =
2782 OFFSET_FROM_START(tp, tp->t_rxtcur);
2783 }
2784 tp->snd_cwnd = tp->t_maxseg;
2785 tp->t_bytes_acked = 0;
2786 tcp_check_timer_state(tp);
2787 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
2788
2789 tcp_ccdbg_trace(tp, NULL, TCP_CC_OUTPUT_ERROR);
2790 return 0;
2791 }
2792 if (error == EMSGSIZE) {
2793 /*
2794 * ip_output() will have already fixed the route
2795 * for us. tcp_mtudisc() will, as its last action,
2796 * initiate retransmission, so it is important to
2797 * not do so here.
2798 *
2799 * If TSO was active we either got an interface
2800 * without TSO capabilits or TSO was turned off.
2801 * Disable it for this connection as too and
2802 * immediatly retry with MSS sized segments generated
2803 * by this function.
2804 */
2805 if (tso) {
2806 tp->t_flags &= ~TF_TSO;
2807 }
2808
2809 tcp_mtudisc(inp, 0);
2810 tcp_check_timer_state(tp);
2811
2812 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
2813 return 0;
2814 }
2815 /*
2816 * Unless this is due to interface restriction policy,
2817 * treat EHOSTUNREACH/ENETDOWN as a soft error.
2818 */
2819 if ((error == EHOSTUNREACH || error == ENETDOWN) &&
2820 TCPS_HAVERCVDSYN(tp->t_state) &&
2821 !inp_restricted_send(inp, inp->inp_last_outifp)) {
2822 tp->t_softerror = error;
2823 error = 0;
2824 }
2825 tcp_check_timer_state(tp);
2826 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
2827 return error;
2828 }
2829
2830 tcpstat.tcps_sndtotal++;
2831
2832 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
2833 if (sendalot) {
2834 goto again;
2835 }
2836
2837 tcp_check_timer_state(tp);
2838
2839 return 0;
2840 }
2841
2842 static int
2843 tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
2844 int cnt, struct mbuf *opt, int flags, int sack_in_progress, boolean_t isipv6)
2845 {
2846 int error = 0;
2847 boolean_t chain;
2848 boolean_t unlocked = FALSE;
2849 boolean_t ifdenied = FALSE;
2850 struct inpcb *inp = tp->t_inpcb;
2851 struct ip_out_args ipoa;
2852 struct route ro;
2853 struct ifnet *outif = NULL;
2854 bool check_qos_marking_again = (so->so_flags1 & SOF1_QOSMARKING_POLICY_OVERRIDE) ? FALSE : TRUE;
2855
2856 bzero(&ipoa, sizeof(ipoa));
2857 ipoa.ipoa_boundif = IFSCOPE_NONE;
2858 ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR;
2859 ipoa.ipoa_sotc = SO_TC_UNSPEC;
2860 ipoa.ipoa_netsvctype = _NET_SERVICE_TYPE_UNSPEC;
2861 struct ip6_out_args ip6oa;
2862 struct route_in6 ro6;
2863
2864 bzero(&ip6oa, sizeof(ip6oa));
2865 ip6oa.ip6oa_boundif = IFSCOPE_NONE;
2866 ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR;
2867 ip6oa.ip6oa_sotc = SO_TC_UNSPEC;
2868 ip6oa.ip6oa_netsvctype = _NET_SERVICE_TYPE_UNSPEC;
2869
2870 struct flowadv *adv =
2871 (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv);
2872
2873 /* If socket was bound to an ifindex, tell ip_output about it */
2874 if (inp->inp_flags & INP_BOUND_IF) {
2875 if (isipv6) {
2876 ip6oa.ip6oa_boundif = inp->inp_boundifp->if_index;
2877 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
2878 } else {
2879 ipoa.ipoa_boundif = inp->inp_boundifp->if_index;
2880 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
2881 }
2882 }
2883
2884 if (INP_NO_CELLULAR(inp)) {
2885 if (isipv6) {
2886 ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR;
2887 } else {
2888 ipoa.ipoa_flags |= IPOAF_NO_CELLULAR;
2889 }
2890 }
2891 if (INP_NO_EXPENSIVE(inp)) {
2892 if (isipv6) {
2893 ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE;
2894 } else {
2895 ipoa.ipoa_flags |= IPOAF_NO_EXPENSIVE;
2896 }
2897 }
2898 if (INP_NO_CONSTRAINED(inp)) {
2899 if (isipv6) {
2900 ip6oa.ip6oa_flags |= IP6OAF_NO_CONSTRAINED;
2901 } else {
2902 ipoa.ipoa_flags |= IPOAF_NO_CONSTRAINED;
2903 }
2904 }
2905 if (INP_AWDL_UNRESTRICTED(inp)) {
2906 if (isipv6) {
2907 ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED;
2908 } else {
2909 ipoa.ipoa_flags |= IPOAF_AWDL_UNRESTRICTED;
2910 }
2911 }
2912 if (INP_INTCOPROC_ALLOWED(inp) && isipv6) {
2913 ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED;
2914 }
2915 if (isipv6) {
2916 ip6oa.ip6oa_sotc = so->so_traffic_class;
2917 ip6oa.ip6oa_netsvctype = so->so_netsvctype;
2918 ip6oa.qos_marking_gencount = inp->inp_policyresult.results.qos_marking_gencount;
2919 } else {
2920 ipoa.ipoa_sotc = so->so_traffic_class;
2921 ipoa.ipoa_netsvctype = so->so_netsvctype;
2922 ipoa.qos_marking_gencount = inp->inp_policyresult.results.qos_marking_gencount;
2923 }
2924 if ((so->so_flags1 & SOF1_QOSMARKING_ALLOWED)) {
2925 if (isipv6) {
2926 ip6oa.ip6oa_flags |= IP6OAF_QOSMARKING_ALLOWED;
2927 } else {
2928 ipoa.ipoa_flags |= IPOAF_QOSMARKING_ALLOWED;
2929 }
2930 }
2931 if (check_qos_marking_again) {
2932 if (isipv6) {
2933 ip6oa.ip6oa_flags |= IP6OAF_REDO_QOSMARKING_POLICY;
2934 } else {
2935 ipoa.ipoa_flags |= IPOAF_REDO_QOSMARKING_POLICY;
2936 }
2937 }
2938 if (isipv6) {
2939 flags |= IPV6_OUTARGS;
2940 } else {
2941 flags |= IP_OUTARGS;
2942 }
2943
2944 /* Copy the cached route and take an extra reference */
2945 if (isipv6) {
2946 in6p_route_copyout(inp, &ro6);
2947 } else {
2948 inp_route_copyout(inp, &ro);
2949 }
2950
2951 /*
2952 * Make sure ACK/DELACK conditions are cleared before
2953 * we unlock the socket.
2954 */
2955 tp->last_ack_sent = tp->rcv_nxt;
2956 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
2957 tp->t_timer[TCPT_DELACK] = 0;
2958 tp->t_unacksegs = 0;
2959
2960 /* Increment the count of outstanding send operations */
2961 inp->inp_sndinprog_cnt++;
2962
2963 /*
2964 * If allowed, unlock TCP socket while in IP
2965 * but only if the connection is established and
2966 * in a normal mode where reentrancy on the tcpcb won't be
2967 * an issue:
2968 * - there is no SACK episode
2969 * - we're not in Fast Recovery mode
2970 * - if we're not sending from an upcall.
2971 */
2972 if (tcp_output_unlocked && !so->so_upcallusecount &&
2973 (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) &&
2974 !IN_FASTRECOVERY(tp) && !(so->so_flags & SOF_MP_SUBFLOW)) {
2975 unlocked = TRUE;
2976 socket_unlock(so, 0);
2977 }
2978
2979 /*
2980 * Don't send down a chain of packets when:
2981 * - TCP chaining is disabled
2982 * - there is an IPsec rule set
2983 * - there is a non default rule set for the firewall
2984 */
2985
2986 chain = tcp_packet_chaining > 1
2987 #if IPSEC
2988 && ipsec_bypass
2989 #endif
2990 ; // I'm important, not extraneous
2991
2992 while (pkt != NULL) {
2993 struct mbuf *npkt = pkt->m_nextpkt;
2994
2995 if (!chain) {
2996 pkt->m_nextpkt = NULL;
2997 /*
2998 * If we are not chaining, make sure to set the packet
2999 * list count to 0 so that IP takes the right path;
3000 * this is important for cases such as IPsec where a
3001 * single mbuf might result in multiple mbufs as part
3002 * of the encapsulation. If a non-zero count is passed
3003 * down to IP, the head of the chain might change and
3004 * we could end up skipping it (thus generating bogus
3005 * packets). Fixing it in IP would be desirable, but
3006 * for now this would do it.
3007 */
3008 cnt = 0;
3009 }
3010 if (isipv6) {
3011 error = ip6_output_list(pkt, cnt,
3012 inp->in6p_outputopts, &ro6, flags, NULL, NULL,
3013 &ip6oa);
3014 ifdenied = (ip6oa.ip6oa_retflags & IP6OARF_IFDENIED);
3015 } else {
3016 error = ip_output_list(pkt, cnt, opt, &ro, flags, NULL,
3017 &ipoa);
3018 ifdenied = (ipoa.ipoa_retflags & IPOARF_IFDENIED);
3019 }
3020
3021 if (chain || error) {
3022 /*
3023 * If we sent down a chain then we are done since
3024 * the callee had taken care of everything; else
3025 * we need to free the rest of the chain ourselves.
3026 */
3027 if (!chain) {
3028 m_freem_list(npkt);
3029 }
3030 break;
3031 }
3032 pkt = npkt;
3033 }
3034
3035 if (unlocked) {
3036 socket_lock(so, 0);
3037 }
3038
3039 /*
3040 * Enter flow controlled state if the connection is established
3041 * and is not in recovery. Flow control is allowed only if there
3042 * is outstanding data.
3043 *
3044 * A connection will enter suspended state even if it is in
3045 * recovery.
3046 */
3047 if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) ||
3048 adv->code == FADV_SUSPENDED) &&
3049 !(tp->t_flags & TF_CLOSING) &&
3050 tp->t_state == TCPS_ESTABLISHED &&
3051 SEQ_GT(tp->snd_max, tp->snd_una)) {
3052 int rc;
3053 rc = inp_set_fc_state(inp, adv->code);
3054
3055 if (rc == 1) {
3056 tcp_ccdbg_trace(tp, NULL,
3057 ((adv->code == FADV_FLOW_CONTROLLED) ?
3058 TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND));
3059 }
3060 }
3061
3062 /*
3063 * When an interface queue gets suspended, some of the
3064 * packets are dropped. Return ENOBUFS, to update the
3065 * pcb state.
3066 */
3067 if (adv->code == FADV_SUSPENDED) {
3068 error = ENOBUFS;
3069 }
3070
3071 VERIFY(inp->inp_sndinprog_cnt > 0);
3072 if (--inp->inp_sndinprog_cnt == 0) {
3073 inp->inp_flags &= ~(INP_FC_FEEDBACK);
3074 if (inp->inp_sndingprog_waiters > 0) {
3075 wakeup(&inp->inp_sndinprog_cnt);
3076 }
3077 }
3078
3079 if (isipv6) {
3080 /*
3081 * When an NECP IP tunnel policy forces the outbound interface,
3082 * ip6_output_list() informs the transport layer what is the actual
3083 * outgoing interface
3084 */
3085 if (ip6oa.ip6oa_flags & IP6OAF_BOUND_IF) {
3086 outif = ifindex2ifnet[ip6oa.ip6oa_boundif];
3087 } else if (ro6.ro_rt != NULL) {
3088 outif = ro6.ro_rt->rt_ifp;
3089 }
3090 } else {
3091 if (ro.ro_rt != NULL) {
3092 outif = ro.ro_rt->rt_ifp;
3093 }
3094 }
3095 if (check_qos_marking_again) {
3096 uint32_t qos_marking_gencount;
3097 bool allow_qos_marking;
3098 if (isipv6) {
3099 qos_marking_gencount = ip6oa.qos_marking_gencount;
3100 allow_qos_marking = ip6oa.ip6oa_flags & IP6OAF_QOSMARKING_ALLOWED ? TRUE : FALSE;
3101 } else {
3102 qos_marking_gencount = ipoa.qos_marking_gencount;
3103 allow_qos_marking = ipoa.ipoa_flags & IPOAF_QOSMARKING_ALLOWED ? TRUE : FALSE;
3104 }
3105 inp->inp_policyresult.results.qos_marking_gencount = qos_marking_gencount;
3106 if (allow_qos_marking == TRUE) {
3107 inp->inp_socket->so_flags1 |= SOF1_QOSMARKING_ALLOWED;
3108 } else {
3109 inp->inp_socket->so_flags1 &= ~SOF1_QOSMARKING_ALLOWED;
3110 }
3111 }
3112
3113 if (outif != NULL && outif != inp->inp_last_outifp) {
3114 /* Update the send byte count */
3115 if (so->so_snd.sb_cc > 0 && so->so_snd.sb_flags & SB_SNDBYTE_CNT) {
3116 inp_decr_sndbytes_total(so, so->so_snd.sb_cc);
3117 inp_decr_sndbytes_allunsent(so, tp->snd_una);
3118 so->so_snd.sb_flags &= ~SB_SNDBYTE_CNT;
3119 }
3120 inp->inp_last_outifp = outif;
3121 }
3122
3123 if (error != 0 && ifdenied &&
3124 (INP_NO_CELLULAR(inp) || INP_NO_EXPENSIVE(inp) || INP_NO_CONSTRAINED(inp))) {
3125 soevent(so,
3126 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_IFDENIED));
3127 }
3128
3129 /* Synchronize cached PCB route & options */
3130 if (isipv6) {
3131 in6p_route_copyin(inp, &ro6);
3132 } else {
3133 inp_route_copyin(inp, &ro);
3134 }
3135
3136 if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 &&
3137 tp->t_inpcb->inp_route.ro_rt != NULL) {
3138 /* If we found the route and there is an rtt on it
3139 * reset the retransmit timer
3140 */
3141 tcp_getrt_rtt(tp, tp->t_inpcb->in6p_route.ro_rt);
3142 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
3143 }
3144 return error;
3145 }
3146
3147 int tcptv_persmin_val = TCPTV_PERSMIN;
3148
3149 void
3150 tcp_setpersist(struct tcpcb *tp)
3151 {
3152 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
3153
3154 /* If a PERSIST_TIMER option was set we will limit the
3155 * time the persist timer will be active for that connection
3156 * in order to avoid DOS by using zero window probes.
3157 * see rdar://5805356
3158 */
3159
3160 if (tp->t_persist_timeout != 0 &&
3161 tp->t_timer[TCPT_PERSIST] == 0 &&
3162 tp->t_persist_stop == 0) {
3163 tp->t_persist_stop = tcp_now + tp->t_persist_timeout;
3164 }
3165
3166 /*
3167 * Start/restart persistance timer.
3168 */
3169 TCPT_RANGESET(tp->t_timer[TCPT_PERSIST],
3170 t * tcp_backoff[tp->t_rxtshift],
3171 tcptv_persmin_val, TCPTV_PERSMAX, 0);
3172 tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]);
3173
3174 if (tp->t_rxtshift < TCP_MAXRXTSHIFT) {
3175 tp->t_rxtshift++;
3176 }
3177 }
3178
3179 static int
3180 tcp_recv_throttle(struct tcpcb *tp)
3181 {
3182 uint32_t base_rtt, newsize;
3183 struct sockbuf *sbrcv = &tp->t_inpcb->inp_socket->so_rcv;
3184
3185 if (tcp_use_rtt_recvbg == 1 &&
3186 TSTMP_SUPPORTED(tp)) {
3187 /*
3188 * Timestamps are supported on this connection. Use
3189 * RTT to look for an increase in latency.
3190 */
3191
3192 /*
3193 * If the connection is already being throttled, leave it
3194 * in that state until rtt comes closer to base rtt
3195 */
3196 if (tp->t_flagsext & TF_RECV_THROTTLE) {
3197 return 1;
3198 }
3199
3200 base_rtt = get_base_rtt(tp);
3201
3202 if (base_rtt != 0 && tp->t_rttcur != 0) {
3203 /*
3204 * if latency increased on a background flow,
3205 * return 1 to start throttling.
3206 */
3207 if (tp->t_rttcur > (base_rtt + target_qdelay)) {
3208 tp->t_flagsext |= TF_RECV_THROTTLE;
3209 if (tp->t_recv_throttle_ts == 0) {
3210 tp->t_recv_throttle_ts = tcp_now;
3211 }
3212 /*
3213 * Reduce the recv socket buffer size to
3214 * minimize latecy.
3215 */
3216 if (sbrcv->sb_idealsize >
3217 tcp_recv_throttle_minwin) {
3218 newsize = sbrcv->sb_idealsize >> 1;
3219 /* Set a minimum of 16 K */
3220 newsize =
3221 max(newsize,
3222 tcp_recv_throttle_minwin);
3223 sbrcv->sb_idealsize = newsize;
3224 }
3225 return 1;
3226 } else {
3227 return 0;
3228 }
3229 }
3230 }
3231
3232 /*
3233 * Timestamps are not supported or there is no good RTT
3234 * measurement. Use IPDV in this case.
3235 */
3236 if (tp->acc_iaj > tcp_acc_iaj_react_limit) {
3237 return 1;
3238 }
3239
3240 return 0;
3241 }
mirror of XNU