]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/tcp_output.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3247.1.106.tar.gz
[apple/xnu.git] / bsd / netinet / tcp_output.c
1 /*
2 * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #define _IP_VHL
71
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/protosw.h>
80 #include <sys/socket.h>
81 #include <sys/socketvar.h>
82
83 #include <net/route.h>
84 #include <net/ntstat.h>
85 #include <net/if_var.h>
86 #include <net/if.h>
87 #include <net/if_types.h>
88 #include <net/dlil.h>
89
90 #include <netinet/in.h>
91 #include <netinet/in_systm.h>
92 #include <netinet/in_var.h>
93 #include <netinet/ip.h>
94 #include <netinet/in_pcb.h>
95 #include <netinet/ip_var.h>
96 #include <mach/sdt.h>
97 #if INET6
98 #include <netinet6/in6_pcb.h>
99 #include <netinet/ip6.h>
100 #include <netinet6/ip6_var.h>
101 #endif
102 #include <netinet/tcp.h>
103 #define TCPOUTFLAGS
104 #include <netinet/tcp_cache.h>
105 #include <netinet/tcp_fsm.h>
106 #include <netinet/tcp_seq.h>
107 #include <netinet/tcp_timer.h>
108 #include <netinet/tcp_var.h>
109 #include <netinet/tcpip.h>
110 #include <netinet/tcp_cc.h>
111 #if TCPDEBUG
112 #include <netinet/tcp_debug.h>
113 #endif
114 #include <sys/kdebug.h>
115 #include <mach/sdt.h>
116
117 #if IPSEC
118 #include <netinet6/ipsec.h>
119 #endif /*IPSEC*/
120
121 #if CONFIG_MACF_NET
122 #include <security/mac_framework.h>
123 #endif /* MAC_SOCKET */
124
125 #include <netinet/lro_ext.h>
126 #if MPTCP
127 #include <netinet/mptcp_var.h>
128 #include <netinet/mptcp.h>
129 #include <netinet/mptcp_opt.h>
130 #endif
131
132 #include <corecrypto/ccaes.h>
133
134 #define DBG_LAYER_BEG NETDBG_CODE(DBG_NETTCP, 1)
135 #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3)
136 #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
137
138 int path_mtu_discovery = 1;
139 SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery,
140 CTLFLAG_RW | CTLFLAG_LOCKED, &path_mtu_discovery, 1,
141 "Enable Path MTU Discovery");
142
143 int ss_fltsz = 1;
144 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize,
145 CTLFLAG_RW | CTLFLAG_LOCKED,&ss_fltsz, 1,
146 "Slow start flight size");
147
148 int ss_fltsz_local = 8; /* starts with eight segments max */
149 SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize,
150 CTLFLAG_RW | CTLFLAG_LOCKED, &ss_fltsz_local, 1,
151 "Slow start flight size for local networks");
152
153 int tcp_do_tso = 1;
154 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED,
155 &tcp_do_tso, 0, "Enable TCP Segmentation Offload");
156
157 int tcp_ecn_outbound = 0;
158 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_initiate_out,
159 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_outbound, 0,
160 "Initiate ECN for outbound connections");
161
162 int tcp_ecn_inbound = 0;
163 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_negotiate_in,
164 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_inbound, 0,
165 "Allow ECN negotiation for inbound connections");
166
167 int tcp_packet_chaining = 50;
168 SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain,
169 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining, 0,
170 "Enable TCP output packet chaining");
171
172 int tcp_output_unlocked = 1;
173 SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output,
174 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked, 0,
175 "Unlock TCP when sending packets down to IP");
176
177 int tcp_do_rfc3390 = 1;
178 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390,
179 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3390, 1,
180 "Calculate intial slowstart cwnd depending on MSS");
181
182 int tcp_min_iaj_win = MIN_IAJ_WIN;
183 SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win,
184 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_min_iaj_win, 1,
185 "Minimum recv win based on inter-packet arrival jitter");
186
187 int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT;
188 SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit,
189 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_acc_iaj_react_limit, 1,
190 "Accumulated IAJ when receiver starts to react");
191
192 uint32_t tcp_do_autosendbuf = 1;
193 SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf,
194 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autosendbuf, 1,
195 "Enable send socket buffer auto-tuning");
196
197 uint32_t tcp_autosndbuf_inc = 8 * 1024;
198 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc,
199 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_inc, 1,
200 "Increment in send socket bufffer size");
201
202 uint32_t tcp_autosndbuf_max = 512 * 1024;
203 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax,
204 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_max, 1,
205 "Maximum send socket buffer size");
206
207 uint32_t tcp_prioritize_acks = 1;
208 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize,
209 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_prioritize_acks, 1,
210 "Prioritize pure acks");
211
212 uint32_t tcp_use_rtt_recvbg = 1;
213 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_recvbg,
214 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_rtt_recvbg, 1,
215 "Use RTT for bg recv algorithm");
216
217 uint32_t tcp_recv_throttle_minwin = 16 * 1024;
218 SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_throttle_minwin,
219 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recv_throttle_minwin, 1,
220 "Minimum recv win for throttling");
221
222 int32_t tcp_enable_tlp = 1;
223 SYSCTL_INT(_net_inet_tcp, OID_AUTO, enable_tlp,
224 CTLFLAG_RW | CTLFLAG_LOCKED,
225 &tcp_enable_tlp, 1, "Enable Tail loss probe");
226
227 static int32_t packchain_newlist = 0;
228 static int32_t packchain_looped = 0;
229 static int32_t packchain_sent = 0;
230
231 /* temporary: for testing */
232 #if IPSEC
233 extern int ipsec_bypass;
234 #endif
235
236 extern int slowlink_wsize; /* window correction for slow links */
237 #if IPFIREWALL
238 extern int fw_enable; /* firewall check for packet chaining */
239 extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */
240 #endif /* IPFIREWALL */
241
242 extern u_int32_t dlil_filter_disable_tso_count;
243 extern u_int32_t kipf_count;
244 extern int tcp_recv_bg;
245
246 static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int,
247 struct mbuf *, int, int, int32_t, boolean_t);
248 static struct mbuf* tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th);
249 static int tcp_recv_throttle(struct tcpcb *tp);
250
251 static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len)
252 {
253 struct socket *so = tp->t_inpcb->inp_socket;
254 unsigned int optlen = 0;
255 unsigned int cookie_len;
256
257 if (tp->t_flags & TF_NOOPT)
258 goto fallback;
259
260 if (!tcp_heuristic_do_tfo(tp))
261 goto fallback;
262
263 optlen += TCPOLEN_MAXSEG;
264
265 if (tp->t_flags & TF_REQ_SCALE)
266 optlen += 4;
267
268 #if MPTCP
269 if ((so->so_flags & SOF_MP_SUBFLOW) && mptcp_enable &&
270 tp->t_rxtshift <= mptcp_mpcap_retries)
271 optlen += sizeof(struct mptcp_mpcapable_opt_common) + sizeof(mptcp_key_t);
272 #endif /* MPTCP */
273
274 if (tp->t_flags & TF_REQ_TSTMP)
275 optlen += TCPOLEN_TSTAMP_APPA;
276
277 if (SACK_ENABLED(tp))
278 optlen += TCPOLEN_SACK_PERMITTED;
279
280 /* Now, decide whether to use TFO or not */
281
282 /* Don't even bother trying if there is no space at all... */
283 if (MAX_TCPOPTLEN - optlen < TCPOLEN_FASTOPEN_REQ)
284 goto fallback;
285
286 cookie_len = tcp_cache_get_cookie_len(tp);
287 if (cookie_len == 0)
288 /* No cookie, so we request one */
289 return (0);
290
291 /* Do not send SYN+data if there is more in the queue than MSS */
292 if (so->so_snd.sb_cc > (tp->t_maxopd - MAX_TCPOPTLEN))
293 goto fallback;
294
295 /* Ok, everything looks good. We can go on and do TFO */
296 return (len);
297
298 fallback:
299 tp->t_flagsext &= ~TF_FASTOPEN;
300 return (0);
301 }
302
303 /* Returns the number of bytes written to the TCP option-space */
304 static unsigned
305 tcp_tfo_write_cookie_rep(struct tcpcb *tp, unsigned optlen, u_char *opt)
306 {
307 u_char out[CCAES_BLOCK_SIZE];
308 unsigned ret = 0;
309 u_char *bp;
310
311 if ((MAX_TCPOPTLEN - optlen) <
312 (TCPOLEN_FASTOPEN_REQ + TFO_COOKIE_LEN_DEFAULT))
313 return (ret);
314
315 tcp_tfo_gen_cookie(tp->t_inpcb, out, sizeof(out));
316
317 bp = opt + optlen;
318
319 *bp++ = TCPOPT_FASTOPEN;
320 *bp++ = 2 + TFO_COOKIE_LEN_DEFAULT;
321 memcpy(bp, out, TFO_COOKIE_LEN_DEFAULT);
322 ret += 2 + TFO_COOKIE_LEN_DEFAULT;
323
324 tp->t_tfo_stats |= TFO_S_COOKIE_SENT;
325 tcpstat.tcps_tfo_cookie_sent++;
326
327 return (ret);
328 }
329
330 static unsigned
331 tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t *len,
332 u_char *opt)
333 {
334 u_int8_t tfo_len = MAX_TCPOPTLEN - optlen - TCPOLEN_FASTOPEN_REQ;
335 unsigned ret = 0;
336 int res;
337 u_char *bp;
338
339 bp = opt + optlen;
340
341 /*
342 * The cookie will be copied in the appropriate place within the
343 * TCP-option space. That way we avoid the need for an intermediate
344 * variable.
345 */
346 res = tcp_cache_get_cookie(tp, bp + TCPOLEN_FASTOPEN_REQ, &tfo_len);
347 if (res == 0) {
348 *bp++ = TCPOPT_FASTOPEN;
349 *bp++ = TCPOLEN_FASTOPEN_REQ;
350 ret += TCPOLEN_FASTOPEN_REQ;
351
352 tp->t_tfo_flags |= TFO_F_COOKIE_REQ;
353
354 tp->t_tfo_stats |= TFO_S_COOKIE_REQ;
355 tcpstat.tcps_tfo_cookie_req++;
356 } else {
357 *bp++ = TCPOPT_FASTOPEN;
358 *bp++ = TCPOLEN_FASTOPEN_REQ + tfo_len;
359
360 ret += TCPOLEN_FASTOPEN_REQ + tfo_len;
361
362 tp->t_tfo_flags |= TFO_F_COOKIE_SENT;
363
364 /* If there is some data, let's track it */
365 if (*len) {
366 tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT;
367 tcpstat.tcps_tfo_syn_data_sent++;
368 }
369 }
370
371 return (ret);
372 }
373
374 static inline bool
375 tcp_send_ecn_flags_on_syn(struct tcpcb *tp, struct socket *so)
376 {
377 return(!((tp->ecn_flags & TE_SETUPSENT) ||
378 (so->so_flags & SOF_MP_SUBFLOW) ||
379 (tp->t_flagsext & TF_FASTOPEN)));
380 }
381
382 /*
383 * Tcp output routine: figure out what should be sent and send it.
384 *
385 * Returns: 0 Success
386 * EADDRNOTAVAIL
387 * ENOBUFS
388 * EMSGSIZE
389 * EHOSTUNREACH
390 * ENETDOWN
391 * ip_output_list:ENOMEM
392 * ip_output_list:EADDRNOTAVAIL
393 * ip_output_list:ENETUNREACH
394 * ip_output_list:EHOSTUNREACH
395 * ip_output_list:EACCES
396 * ip_output_list:EMSGSIZE
397 * ip_output_list:ENOBUFS
398 * ip_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
399 * ip6_output_list:EINVAL
400 * ip6_output_list:EOPNOTSUPP
401 * ip6_output_list:EHOSTUNREACH
402 * ip6_output_list:EADDRNOTAVAIL
403 * ip6_output_list:ENETUNREACH
404 * ip6_output_list:EMSGSIZE
405 * ip6_output_list:ENOBUFS
406 * ip6_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
407 */
408 int
409 tcp_output(struct tcpcb *tp)
410 {
411 struct inpcb *inp = tp->t_inpcb;
412 struct socket *so = inp->inp_socket;
413 int32_t len, recwin, sendwin, off;
414 int flags, error;
415 struct mbuf *m;
416 struct ip *ip = NULL;
417 struct ipovly *ipov = NULL;
418 #if INET6
419 struct ip6_hdr *ip6 = NULL;
420 #endif /* INET6 */
421 struct tcphdr *th;
422 u_char opt[TCP_MAXOLEN];
423 unsigned ipoptlen, optlen, hdrlen;
424 int idle, sendalot, lost = 0;
425 int i, sack_rxmit;
426 int tso = 0;
427 int sack_bytes_rxmt;
428 tcp_seq old_snd_nxt = 0;
429 struct sackhole *p;
430 #if IPSEC
431 unsigned ipsec_optlen = 0;
432 #endif /* IPSEC */
433 int idle_time = 0;
434 struct mbuf *packetlist = NULL;
435 struct mbuf *tp_inp_options = inp->inp_depend4.inp4_options;
436 #if INET6
437 int isipv6 = inp->inp_vflag & INP_IPV6 ;
438 #endif
439 short packchain_listadd = 0;
440 int so_options = so->so_options;
441 struct rtentry *rt;
442 u_int32_t basertt, svc_flags = 0, allocated_len;
443 u_int32_t lro_ackmore = (tp->t_lropktlen != 0) ? 1 : 0;
444 struct mbuf *mnext = NULL;
445 int sackoptlen = 0;
446 #if MPTCP
447 unsigned int *dlenp = NULL;
448 u_int8_t *finp = NULL;
449 u_int32_t *sseqp = NULL;
450 u_int64_t dss_val = 0;
451 boolean_t mptcp_acknow = FALSE;
452 boolean_t early_data_sent = FALSE;
453 #endif /* MPTCP */
454 boolean_t cell = FALSE;
455 boolean_t wifi = FALSE;
456 boolean_t wired = FALSE;
457 boolean_t sack_rescue_rxt = FALSE;
458
459 /*
460 * Determine length of data that should be transmitted,
461 * and flags that will be used.
462 * If there is some data or critical controls (SYN, RST)
463 * to send, then transmit; otherwise, investigate further.
464 */
465 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
466
467 /* Since idle_time is signed integer, the following integer subtraction
468 * will take care of wrap around of tcp_now
469 */
470 idle_time = tcp_now - tp->t_rcvtime;
471 if (idle && idle_time >= TCP_IDLETIMEOUT(tp)) {
472 if (CC_ALGO(tp)->after_idle != NULL &&
473 (tp->tcp_cc_index != TCP_CC_ALGO_CUBIC_INDEX ||
474 idle_time >= TCP_CC_CWND_NONVALIDATED_PERIOD)) {
475 CC_ALGO(tp)->after_idle(tp);
476 tcp_ccdbg_trace(tp, NULL, TCP_CC_IDLE_TIMEOUT);
477 }
478
479 /*
480 * Do some other tasks that need to be done after
481 * idle time
482 */
483 if (!SLIST_EMPTY(&tp->t_rxt_segments))
484 tcp_rxtseg_clean(tp);
485
486 /* If stretch ack was auto-disabled, re-evaluate it */
487 tcp_cc_after_idle_stretchack(tp);
488 }
489 tp->t_flags &= ~TF_LASTIDLE;
490 if (idle) {
491 if (tp->t_flags & TF_MORETOCOME) {
492 tp->t_flags |= TF_LASTIDLE;
493 idle = 0;
494 }
495 }
496 #if MPTCP
497 if (tp->t_mpflags & TMPF_RESET) {
498 tcp_check_timer_state(tp);
499 /*
500 * Once a RST has been sent for an MPTCP subflow,
501 * the subflow socket stays around until deleted.
502 * No packets such as FINs must be sent after RST.
503 */
504 return (0);
505 }
506 #endif /* MPTCP */
507
508 again:
509 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
510
511 #if INET6
512 if (isipv6) {
513 KERNEL_DEBUG(DBG_LAYER_BEG,
514 ((inp->inp_fport << 16) | inp->inp_lport),
515 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
516 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
517 sendalot,0,0);
518 } else
519 #endif
520
521 {
522 KERNEL_DEBUG(DBG_LAYER_BEG,
523 ((inp->inp_fport << 16) | inp->inp_lport),
524 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
525 (inp->inp_faddr.s_addr & 0xffff)),
526 sendalot,0,0);
527 }
528 /*
529 * If the route generation id changed, we need to check that our
530 * local (source) IP address is still valid. If it isn't either
531 * return error or silently do nothing (assuming the address will
532 * come back before the TCP connection times out).
533 */
534 rt = inp->inp_route.ro_rt;
535 if (rt != NULL && ROUTE_UNUSABLE(&tp->t_inpcb->inp_route)) {
536 struct ifnet *ifp;
537 struct in_ifaddr *ia = NULL;
538 struct in6_ifaddr *ia6 = NULL;
539 int found_srcaddr = 0;
540
541 /* disable multipages at the socket */
542 somultipages(so, FALSE);
543
544 /* Disable TSO for the socket until we know more */
545 tp->t_flags &= ~TF_TSO;
546
547 soif2kcl(so, FALSE);
548
549 if (isipv6) {
550 ia6 = ifa_foraddr6(&inp->in6p_laddr);
551 if (ia6 != NULL)
552 found_srcaddr = 1;
553 } else {
554 ia = ifa_foraddr(inp->inp_laddr.s_addr);
555 if (ia != NULL)
556 found_srcaddr = 1;
557 }
558
559 /* check that the source address is still valid */
560 if (found_srcaddr == 0) {
561 soevent(so,
562 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR));
563
564 if (tp->t_state >= TCPS_CLOSE_WAIT) {
565 tcp_drop(tp, EADDRNOTAVAIL);
566 return(EADDRNOTAVAIL);
567 }
568
569 /* Set retransmit timer if it wasn't set,
570 * reset Persist timer and shift register as the
571 * advertised peer window may not be valid anymore
572 */
573
574 if (!tp->t_timer[TCPT_REXMT]) {
575 tp->t_timer[TCPT_REXMT] =
576 OFFSET_FROM_START(tp, tp->t_rxtcur);
577 if (tp->t_timer[TCPT_PERSIST]) {
578 tp->t_timer[TCPT_PERSIST] = 0;
579 tp->t_rxtshift = 0;
580 tp->t_persist_stop = 0;
581 tp->t_rxtstart = 0;
582 }
583 }
584
585 if (tp->t_pktlist_head != NULL)
586 m_freem_list(tp->t_pktlist_head);
587 TCP_PKTLIST_CLEAR(tp);
588
589 /* drop connection if source address isn't available */
590 if (so->so_flags & SOF_NOADDRAVAIL) {
591 tcp_drop(tp, EADDRNOTAVAIL);
592 return(EADDRNOTAVAIL);
593 } else {
594 tcp_check_timer_state(tp);
595 return(0); /* silently ignore, keep data in socket: address may be back */
596 }
597 }
598 if (ia != NULL)
599 IFA_REMREF(&ia->ia_ifa);
600
601 if (ia6 != NULL)
602 IFA_REMREF(&ia6->ia_ifa);
603
604 /*
605 * Address is still valid; check for multipages capability
606 * again in case the outgoing interface has changed.
607 */
608 RT_LOCK(rt);
609 if ((ifp = rt->rt_ifp) != NULL) {
610 somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
611 tcp_set_tso(tp, ifp);
612 soif2kcl(so,
613 (ifp->if_eflags & IFEF_2KCL));
614 }
615 if (rt->rt_flags & RTF_UP)
616 RT_GENID_SYNC(rt);
617 /*
618 * See if we should do MTU discovery. Don't do it if:
619 * 1) it is disabled via the sysctl
620 * 2) the route isn't up
621 * 3) the MTU is locked (if it is, then discovery
622 * has been disabled)
623 */
624
625 if (!path_mtu_discovery || ((rt != NULL) &&
626 (!(rt->rt_flags & RTF_UP) ||
627 (rt->rt_rmx.rmx_locks & RTV_MTU))))
628 tp->t_flags &= ~TF_PMTUD;
629 else
630 tp->t_flags |= TF_PMTUD;
631
632 RT_UNLOCK(rt);
633 }
634
635 if (rt != NULL) {
636 cell = IFNET_IS_CELLULAR(rt->rt_ifp);
637 wifi = (!cell && IFNET_IS_WIFI(rt->rt_ifp));
638 wired = (!wifi && IFNET_IS_WIRED(rt->rt_ifp));
639 }
640
641 /*
642 * If we've recently taken a timeout, snd_max will be greater than
643 * snd_nxt. There may be SACK information that allows us to avoid
644 * resending already delivered data. Adjust snd_nxt accordingly.
645 */
646 if (SACK_ENABLED(tp) && SEQ_LT(tp->snd_nxt, tp->snd_max))
647 tcp_sack_adjust(tp);
648 sendalot = 0;
649 off = tp->snd_nxt - tp->snd_una;
650 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
651
652 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0)
653 sendwin = min(sendwin, slowlink_wsize);
654
655 flags = tcp_outflags[tp->t_state];
656 /*
657 * Send any SACK-generated retransmissions. If we're explicitly
658 * trying to send out new data (when sendalot is 1), bypass this
659 * function. If we retransmit in fast recovery mode, decrement
660 * snd_cwnd, since we're replacing a (future) new transmission
661 * with a retransmission now, and we previously incremented
662 * snd_cwnd in tcp_input().
663 */
664 /*
665 * Still in sack recovery , reset rxmit flag to zero.
666 */
667 sack_rxmit = 0;
668 sack_bytes_rxmt = 0;
669 len = 0;
670 p = NULL;
671 if (SACK_ENABLED(tp) && IN_FASTRECOVERY(tp) &&
672 (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
673 int32_t cwin;
674
675 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
676 if (cwin < 0)
677 cwin = 0;
678 /* Do not retransmit SACK segments beyond snd_recover */
679 if (SEQ_GT(p->end, tp->snd_recover)) {
680 /*
681 * (At least) part of sack hole extends beyond
682 * snd_recover. Check to see if we can rexmit data
683 * for this hole.
684 */
685 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
686 /*
687 * Can't rexmit any more data for this hole.
688 * That data will be rexmitted in the next
689 * sack recovery episode, when snd_recover
690 * moves past p->rxmit.
691 */
692 p = NULL;
693 goto after_sack_rexmit;
694 } else
695 /* Can rexmit part of the current hole */
696 len = ((int32_t)min(cwin,
697 tp->snd_recover - p->rxmit));
698 } else {
699 len = ((int32_t)min(cwin, p->end - p->rxmit));
700 }
701 if (len > 0) {
702 off = p->rxmit - tp->snd_una;
703 sack_rxmit = 1;
704 sendalot = 1;
705 tcpstat.tcps_sack_rexmits++;
706 tcpstat.tcps_sack_rexmit_bytes +=
707 min(len, tp->t_maxseg);
708 if (nstat_collect) {
709 nstat_route_tx(inp->inp_route.ro_rt, 1,
710 min(len, tp->t_maxseg),
711 NSTAT_TX_FLAG_RETRANSMIT);
712 INP_ADD_STAT(inp, cell, wifi, wired,
713 txpackets, 1);
714 INP_ADD_STAT(inp, cell, wifi, wired,
715 txbytes, min(len, tp->t_maxseg));
716 tp->t_stat.txretransmitbytes += min(len, tp->t_maxseg);
717 }
718 } else {
719 len = 0;
720 }
721 }
722 after_sack_rexmit:
723 /*
724 * Get standard flags, and add SYN or FIN if requested by 'hidden'
725 * state flags.
726 */
727 if (tp->t_flags & TF_NEEDFIN)
728 flags |= TH_FIN;
729 if (tp->t_flags & TF_NEEDSYN)
730 flags |= TH_SYN;
731
732 /*
733 * If in persist timeout with window of 0, send 1 byte.
734 * Otherwise, if window is small but nonzero
735 * and timer expired, we will send what we can
736 * and go to transmit state.
737 */
738 if (tp->t_flagsext & TF_FORCE) {
739 if (sendwin == 0) {
740 /*
741 * If we still have some data to send, then
742 * clear the FIN bit. Usually this would
743 * happen below when it realizes that we
744 * aren't sending all the data. However,
745 * if we have exactly 1 byte of unsent data,
746 * then it won't clear the FIN bit below,
747 * and if we are in persist state, we wind
748 * up sending the packet without recording
749 * that we sent the FIN bit.
750 *
751 * We can't just blindly clear the FIN bit,
752 * because if we don't have any more data
753 * to send then the probe will be the FIN
754 * itself.
755 */
756 if (off < so->so_snd.sb_cc)
757 flags &= ~TH_FIN;
758 sendwin = 1;
759 } else {
760 tp->t_timer[TCPT_PERSIST] = 0;
761 tp->t_rxtshift = 0;
762 tp->t_rxtstart = 0;
763 tp->t_persist_stop = 0;
764 }
765 }
766
767 /*
768 * If snd_nxt == snd_max and we have transmitted a FIN, the
769 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
770 * a negative length. This can also occur when TCP opens up
771 * its congestion window while receiving additional duplicate
772 * acks after fast-retransmit because TCP will reset snd_nxt
773 * to snd_max after the fast-retransmit.
774 *
775 * In the normal retransmit-FIN-only case, however, snd_nxt will
776 * be set to snd_una, the offset will be 0, and the length may
777 * wind up 0.
778 *
779 * If sack_rxmit is true we are retransmitting from the scoreboard
780 * in which case len is already set.
781 */
782 if (sack_rxmit == 0) {
783 if (sack_bytes_rxmt == 0) {
784 len = min(so->so_snd.sb_cc, sendwin) - off;
785 } else {
786 int32_t cwin;
787
788 cwin = tp->snd_cwnd -
789 (tp->snd_nxt - tp->sack_newdata) -
790 sack_bytes_rxmt;
791 if (cwin < 0)
792 cwin = 0;
793 /*
794 * We are inside of a SACK recovery episode and are
795 * sending new data, having retransmitted all the
796 * data possible in the scoreboard.
797 */
798 len = min(so->so_snd.sb_cc, tp->snd_wnd)
799 - off;
800 /*
801 * Don't remove this (len > 0) check !
802 * We explicitly check for len > 0 here (although it
803 * isn't really necessary), to work around a gcc
804 * optimization issue - to force gcc to compute
805 * len above. Without this check, the computation
806 * of len is bungled by the optimizer.
807 */
808 if (len > 0) {
809 len = imin(len, cwin);
810 } else {
811 len = 0;
812 }
813 /*
814 * At this point SACK recovery can not send any
815 * data from scoreboard or any new data. Check
816 * if we can do a rescue retransmit towards the
817 * tail end of recovery window.
818 */
819 if (len == 0 && cwin > 0 &&
820 SEQ_LT(tp->snd_fack, tp->snd_recover) &&
821 !(tp->t_flagsext & TF_RESCUE_RXT)) {
822 len = min((tp->snd_recover - tp->snd_fack),
823 tp->t_maxseg);
824 len = imin(len, cwin);
825 old_snd_nxt = tp->snd_nxt;
826 sack_rescue_rxt = TRUE;
827 tp->snd_nxt = tp->snd_recover - len;
828 /*
829 * If FIN has been sent, snd_max
830 * must have been advanced to cover it.
831 */
832 if ((tp->t_flags & TF_SENTFIN) &&
833 tp->snd_max == tp->snd_recover)
834 tp->snd_nxt--;
835
836 off = tp->snd_nxt - tp->snd_una;
837 sendalot = 0;
838 tp->t_flagsext |= TF_RESCUE_RXT;
839 }
840 }
841 }
842
843 #if MPTCP
844 if ((tp->t_mpflags & TMPF_FASTJOIN_SEND) &&
845 (tp->t_state == TCPS_SYN_SENT) &&
846 (!(tp->t_flags & TF_CLOSING)) &&
847 (so->so_snd.sb_cc != 0) &&
848 (tp->t_rxtshift == 0)) {
849 flags &= ~TH_SYN;
850 flags |= TH_ACK;
851 off = 0;
852 len = min(so->so_snd.sb_cc, tp->t_maxseg);
853 early_data_sent = TRUE;
854 } else if (early_data_sent) {
855 /* for now, we allow only one data segment to be sent */
856 return (0);
857 }
858 #endif /* MPTCP */
859 /*
860 * Lop off SYN bit if it has already been sent. However, if this
861 * is SYN-SENT state and if segment contains data and if we don't
862 * know that foreign host supports TAO, suppress sending segment.
863 */
864 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
865 if (tp->t_state != TCPS_SYN_RECEIVED || tfo_enabled(tp))
866 flags &= ~TH_SYN;
867 off--, len++;
868 if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
869 while (inp->inp_sndinprog_cnt == 0 &&
870 tp->t_pktlist_head != NULL) {
871 packetlist = tp->t_pktlist_head;
872 packchain_listadd = tp->t_lastchain;
873 packchain_sent++;
874 TCP_PKTLIST_CLEAR(tp);
875
876 error = tcp_ip_output(so, tp, packetlist,
877 packchain_listadd, tp_inp_options,
878 (so_options & SO_DONTROUTE),
879 (sack_rxmit | (sack_bytes_rxmt != 0)), 0,
880 #if INET6
881 isipv6);
882 #else /* INET6 */
883 0);
884 #endif /* !INET6 */
885
886
887 }
888
889 /*
890 * tcp was closed while we were in ip,
891 * resume close
892 */
893 if (inp->inp_sndinprog_cnt == 0 &&
894 (tp->t_flags & TF_CLOSING)) {
895 tp->t_flags &= ~TF_CLOSING;
896 (void) tcp_close(tp);
897 } else {
898 tcp_check_timer_state(tp);
899 }
900 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
901 0,0,0,0,0);
902 return(0);
903 }
904 }
905
906 /*
907 * Be careful not to send data and/or FIN on SYN segments.
908 * This measure is needed to prevent interoperability problems
909 * with not fully conformant TCP implementations.
910 *
911 * In case of TFO, we handle the setting of the len in
912 * tcp_tfo_check. In case TFO is not enabled, never ever send
913 * SYN+data.
914 */
915 if ((flags & TH_SYN) && !tfo_enabled(tp)) {
916 len = 0;
917 flags &= ~TH_FIN;
918 }
919
920 if ((flags & TH_SYN) && tp->t_state <= TCPS_SYN_SENT && tfo_enabled(tp))
921 len = tcp_tfo_check(tp, len);
922
923 /*
924 * The check here used to be (len < 0). Some times len is zero
925 * when the congestion window is closed and we need to check
926 * if persist timer has to be set in that case. But don't set
927 * persist until connection is established.
928 */
929 if (len <= 0 && !(flags & TH_SYN)) {
930 /*
931 * If FIN has been sent but not acked,
932 * but we haven't been called to retransmit,
933 * len will be < 0. Otherwise, window shrank
934 * after we sent into it. If window shrank to 0,
935 * cancel pending retransmit, pull snd_nxt back
936 * to (closed) window, and set the persist timer
937 * if it isn't already going. If the window didn't
938 * close completely, just wait for an ACK.
939 */
940 len = 0;
941 if (sendwin == 0) {
942 tp->t_timer[TCPT_REXMT] = 0;
943 tp->t_timer[TCPT_PTO] = 0;
944 tp->t_rxtshift = 0;
945 tp->t_rxtstart = 0;
946 tp->snd_nxt = tp->snd_una;
947 off = 0;
948 if (tp->t_timer[TCPT_PERSIST] == 0)
949 tcp_setpersist(tp);
950 }
951 }
952
953 /*
954 * Automatic sizing of send socket buffer. Increase the send
955 * socket buffer size if all of the following criteria are met
956 * 1. the receiver has enough buffer space for this data
957 * 2. send buffer is filled to 7/8th with data (so we actually
958 * have data to make use of it);
959 * 3. our send window (slow start and congestion controlled) is
960 * larger than sent but unacknowledged data in send buffer.
961 */
962 basertt = get_base_rtt(tp);
963 if (tcp_do_autosendbuf == 1 &&
964 !INP_WAIT_FOR_IF_FEEDBACK(inp) && !IN_FASTRECOVERY(tp) &&
965 (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
966 tcp_cansbgrow(&so->so_snd)) {
967 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
968 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
969 sendwin >= (so->so_snd.sb_cc -
970 (tp->snd_nxt - tp->snd_una))) {
971 /* Also increase the send buffer only if the
972 * round-trip time is not increasing because we do
973 * not want to contribute to latency by filling
974 * buffers.
975 * We also do not want to hold onto application's
976 * old data for too long. Interactive applications
977 * would rather discard old data.
978 */
979 if (tp->t_rttcur <= (basertt + 25)) {
980 if (sbreserve(&so->so_snd,
981 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
982 tcp_autosndbuf_max)) == 1) {
983 so->so_snd.sb_idealsize = so->so_snd.sb_hiwat;
984 }
985 } else {
986 so->so_snd.sb_idealsize =
987 max(tcp_sendspace, so->so_snd.sb_hiwat -
988 (2 * tcp_autosndbuf_inc));
989 so->so_snd.sb_flags |= SB_TRIM;
990 }
991 }
992 }
993
994 /*
995 * Truncate to the maximum segment length or enable TCP Segmentation
996 * Offloading (if supported by hardware) and ensure that FIN is removed
997 * if the length no longer contains the last data byte.
998 *
999 * TSO may only be used if we are in a pure bulk sending state.
1000 * The presence of TCP-MD5, SACK retransmits, SACK advertizements,
1001 * ipfw rules and IP options, as well as disabling hardware checksum
1002 * offload prevent using TSO. With TSO the TCP header is the same
1003 * (except for the sequence number) for all generated packets. This
1004 * makes it impossible to transmit any options which vary per generated
1005 * segment or packet.
1006 *
1007 * The length of TSO bursts is limited to TCP_MAXWIN. That limit and
1008 * removal of FIN (if not already catched here) are handled later after
1009 * the exact length of the TCP options are known.
1010 */
1011 #if IPSEC
1012 /*
1013 * Pre-calculate here as we save another lookup into the darknesses
1014 * of IPsec that way and can actually decide if TSO is ok.
1015 */
1016 if (ipsec_bypass == 0)
1017 ipsec_optlen = ipsec_hdrsiz_tcp(tp);
1018 #endif
1019 if (len > tp->t_maxseg) {
1020 if ((tp->t_flags & TF_TSO) && tcp_do_tso && hwcksum_tx &&
1021 ip_use_randomid && kipf_count == 0 &&
1022 dlil_filter_disable_tso_count == 0 &&
1023 tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
1024 sack_bytes_rxmt == 0 &&
1025 inp->inp_options == NULL &&
1026 inp->in6p_options == NULL
1027 #if IPSEC
1028 && ipsec_optlen == 0
1029 #endif
1030 #if IPFIREWALL
1031 && (fw_enable == 0 || fw_bypass)
1032 #endif
1033 ) {
1034 tso = 1;
1035 sendalot = 0;
1036 } else {
1037 len = tp->t_maxseg;
1038 sendalot = 1;
1039 tso = 0;
1040 }
1041 }
1042
1043 /* Send one segment or less as a tail loss probe */
1044 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1045 len = min(len, tp->t_maxseg);
1046 sendalot = 0;
1047 tso = 0;
1048 }
1049
1050 #if MPTCP
1051 if ((so->so_flags & SOF_MP_SUBFLOW) &&
1052 !(tp->t_mpflags & TMPF_TCP_FALLBACK)) {
1053 int newlen = len;
1054 if (!(tp->t_mpflags & TMPF_PREESTABLISHED) &&
1055 (tp->t_state > TCPS_CLOSED) &&
1056 ((tp->t_mpflags & TMPF_SND_MPPRIO) ||
1057 (tp->t_mpflags & TMPF_SND_REM_ADDR) ||
1058 (tp->t_mpflags & TMPF_SND_MPFAIL) ||
1059 (tp->t_mpflags & TMPF_MPCAP_RETRANSMIT))) {
1060 if (len > 0) {
1061 len = 0;
1062 }
1063 sendalot = 1;
1064 mptcp_acknow = TRUE;
1065 } else {
1066 mptcp_acknow = FALSE;
1067 }
1068 /*
1069 * The contiguous bytes in the subflow socket buffer can be
1070 * discontiguous at the MPTCP level. Since only one DSS
1071 * option can be sent in one packet, reduce length to match
1072 * the contiguous MPTCP level. Set sendalot to send remainder.
1073 */
1074 if (len > 0)
1075 newlen = mptcp_adj_sendlen(so, off, len);
1076 if (newlen < len) {
1077 len = newlen;
1078 sendalot = 1;
1079 }
1080 }
1081 #endif /* MPTCP */
1082
1083 /*
1084 * If the socket is capable of doing unordered send,
1085 * pull the amount of data that can be sent from the
1086 * unordered priority queues to the serial queue in
1087 * the socket buffer. If bytes are not yet available
1088 * in the highest priority message, we may not be able
1089 * to send any new data.
1090 */
1091 if (so->so_flags & SOF_ENABLE_MSGS) {
1092 if ((off + len) >
1093 so->so_msg_state->msg_serial_bytes) {
1094 sbpull_unordered_data(so, off, len);
1095
1096 /* check if len needs to be modified */
1097 if ((off + len) >
1098 so->so_msg_state->msg_serial_bytes) {
1099 len = so->so_msg_state->msg_serial_bytes - off;
1100 if (len <= 0) {
1101 len = 0;
1102 tcpstat.tcps_msg_sndwaithipri++;
1103 }
1104 }
1105 }
1106 }
1107
1108 if (sack_rxmit) {
1109 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
1110 flags &= ~TH_FIN;
1111 } else {
1112 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
1113 flags &= ~TH_FIN;
1114 }
1115
1116 recwin = tcp_sbspace(tp);
1117
1118 /*
1119 * Sender silly window avoidance. We transmit under the following
1120 * conditions when len is non-zero:
1121 *
1122 * - we've timed out (e.g. persist timer)
1123 * - we need to retransmit
1124 * - We have a full segment (or more with TSO)
1125 * - This is the last buffer in a write()/send() and we are
1126 * either idle or running NODELAY
1127 * - we have more then 1/2 the maximum send window's worth of
1128 * data (receiver may be limited the window size)
1129 */
1130 if (len) {
1131 if (tp->t_flagsext & TF_FORCE)
1132 goto send;
1133 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
1134 goto send;
1135 if (sack_rxmit)
1136 goto send;
1137
1138 /*
1139 * Send new data on the connection only if it is
1140 * not flow controlled
1141 */
1142 if (!INP_WAIT_FOR_IF_FEEDBACK(inp) ||
1143 tp->t_state != TCPS_ESTABLISHED) {
1144 if (len >= tp->t_maxseg)
1145 goto send;
1146 if (!(tp->t_flags & TF_MORETOCOME) &&
1147 (idle || tp->t_flags & TF_NODELAY ||
1148 tp->t_flags & TF_MAXSEGSNT ||
1149 ALLOW_LIMITED_TRANSMIT(tp)) &&
1150 (tp->t_flags & TF_NOPUSH) == 0 &&
1151 len + off >= so->so_snd.sb_cc)
1152 goto send;
1153 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
1154 goto send;
1155 } else {
1156 tcpstat.tcps_fcholdpacket++;
1157 }
1158 }
1159
1160 /*
1161 * Compare available window to amount of window
1162 * known to peer (as advertised window less
1163 * next expected input). If the difference is at least two
1164 * max size segments, or at least 25% of the maximum possible
1165 * window, then want to send a window update to peer.
1166 * Skip this if the connection is in T/TCP half-open state.
1167 */
1168 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) {
1169 /*
1170 * "adv" is the amount we can increase the window,
1171 * taking into account that we are limited by
1172 * TCP_MAXWIN << tp->rcv_scale.
1173 */
1174 int32_t adv, oldwin = 0;
1175 adv = imin(recwin, (int)TCP_MAXWIN << tp->rcv_scale) -
1176 (tp->rcv_adv - tp->rcv_nxt);
1177
1178 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt))
1179 oldwin = tp->rcv_adv - tp->rcv_nxt;
1180
1181 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
1182 /*
1183 * Update only if the resulting scaled value of
1184 * the window changed, or if there is a change in
1185 * the sequence since the last ack. This avoids
1186 * what appears as dupe ACKS (see rdar://5640997)
1187 *
1188 * If streaming is detected avoid sending too many
1189 * window updates. We will depend on the delack
1190 * timer to send a window update when needed.
1191 */
1192 if (!(tp->t_flags & TF_STRETCHACK) &&
1193 (tp->last_ack_sent != tp->rcv_nxt ||
1194 ((oldwin + adv) >> tp->rcv_scale) >
1195 (oldwin >> tp->rcv_scale))) {
1196 goto send;
1197 }
1198
1199 /*
1200 * Make sure that the delayed ack timer is set if
1201 * we delayed sending a window update because of
1202 * streaming detection.
1203 */
1204 if ((tp->t_flags & TF_STRETCHACK) &&
1205 !(tp->t_flags & TF_DELACK)) {
1206 tp->t_flags |= TF_DELACK;
1207 tp->t_timer[TCPT_DELACK] =
1208 OFFSET_FROM_START(tp, tcp_delack);
1209 }
1210 }
1211 if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat)
1212 goto send;
1213 }
1214
1215 /*
1216 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
1217 * is also a catch-all for the retransmit timer timeout case.
1218 */
1219 if (tp->t_flags & TF_ACKNOW)
1220 goto send;
1221 if ((flags & TH_RST) ||
1222 ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
1223 goto send;
1224 if (SEQ_GT(tp->snd_up, tp->snd_una))
1225 goto send;
1226 #if MPTCP
1227 if (mptcp_acknow)
1228 goto send;
1229 #endif /* MPTCP */
1230 /*
1231 * If our state indicates that FIN should be sent
1232 * and we have not yet done so, then we need to send.
1233 */
1234 if ((flags & TH_FIN) &&
1235 (!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
1236 goto send;
1237 /*
1238 * In SACK, it is possible for tcp_output to fail to send a segment
1239 * after the retransmission timer has been turned off. Make sure
1240 * that the retransmission timer is set.
1241 */
1242 if (SACK_ENABLED(tp) && (tp->t_state >= TCPS_ESTABLISHED) &&
1243 SEQ_GT(tp->snd_max, tp->snd_una) &&
1244 tp->t_timer[TCPT_REXMT] == 0 &&
1245 tp->t_timer[TCPT_PERSIST] == 0) {
1246 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp,
1247 tp->t_rxtcur);
1248 goto just_return;
1249 }
1250 /*
1251 * TCP window updates are not reliable, rather a polling protocol
1252 * using ``persist'' packets is used to insure receipt of window
1253 * updates. The three ``states'' for the output side are:
1254 * idle not doing retransmits or persists
1255 * persisting to move a small or zero window
1256 * (re)transmitting and thereby not persisting
1257 *
1258 * tp->t_timer[TCPT_PERSIST]
1259 * is set when we are in persist state.
1260 * tp->t_force
1261 * is set when we are called to send a persist packet.
1262 * tp->t_timer[TCPT_REXMT]
1263 * is set when we are retransmitting
1264 * The output side is idle when both timers are zero.
1265 *
1266 * If send window is too small, there is data to transmit, and no
1267 * retransmit or persist is pending, then go to persist state.
1268 * If nothing happens soon, send when timer expires:
1269 * if window is nonzero, transmit what we can,
1270 * otherwise force out a byte.
1271 */
1272 if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 &&
1273 tp->t_timer[TCPT_PERSIST] == 0) {
1274 tp->t_rxtshift = 0;
1275 tp->t_rxtstart = 0;
1276 tcp_setpersist(tp);
1277 }
1278 just_return:
1279 /*
1280 * If there is no reason to send a segment, just return.
1281 * but if there is some packets left in the packet list, send them now.
1282 */
1283 while (inp->inp_sndinprog_cnt == 0 &&
1284 tp->t_pktlist_head != NULL) {
1285 packetlist = tp->t_pktlist_head;
1286 packchain_listadd = tp->t_lastchain;
1287 packchain_sent++;
1288 TCP_PKTLIST_CLEAR(tp);
1289
1290 error = tcp_ip_output(so, tp, packetlist,
1291 packchain_listadd,
1292 tp_inp_options, (so_options & SO_DONTROUTE),
1293 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
1294 #if INET6
1295 isipv6);
1296 #else /* INET6 */
1297 0);
1298 #endif /* !INET6 */
1299 }
1300 /* tcp was closed while we were in ip; resume close */
1301 if (inp->inp_sndinprog_cnt == 0 &&
1302 (tp->t_flags & TF_CLOSING)) {
1303 tp->t_flags &= ~TF_CLOSING;
1304 (void) tcp_close(tp);
1305 } else {
1306 tcp_check_timer_state(tp);
1307 }
1308 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1309 return (0);
1310
1311 send:
1312 /*
1313 * Set TF_MAXSEGSNT flag if the segment size is greater than
1314 * the max segment size.
1315 */
1316 if (len > 0) {
1317 if (len >= tp->t_maxseg)
1318 tp->t_flags |= TF_MAXSEGSNT;
1319 else
1320 tp->t_flags &= ~TF_MAXSEGSNT;
1321 }
1322 /*
1323 * Before ESTABLISHED, force sending of initial options
1324 * unless TCP set not to do any options.
1325 * NOTE: we assume that the IP/TCP header plus TCP options
1326 * always fit in a single mbuf, leaving room for a maximum
1327 * link header, i.e.
1328 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1329 */
1330 optlen = 0;
1331 #if INET6
1332 if (isipv6)
1333 hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1334 else
1335 #endif
1336 hdrlen = sizeof (struct tcpiphdr);
1337 if (flags & TH_SYN) {
1338 tp->snd_nxt = tp->iss;
1339 if ((tp->t_flags & TF_NOOPT) == 0) {
1340 u_short mss;
1341
1342 opt[0] = TCPOPT_MAXSEG;
1343 opt[1] = TCPOLEN_MAXSEG;
1344 mss = htons((u_short) tcp_mssopt(tp));
1345 (void)memcpy(opt + 2, &mss, sizeof(mss));
1346 optlen = TCPOLEN_MAXSEG;
1347
1348 if ((tp->t_flags & TF_REQ_SCALE) &&
1349 ((flags & TH_ACK) == 0 ||
1350 (tp->t_flags & TF_RCVD_SCALE))) {
1351 *((u_int32_t *)(void *)(opt + optlen)) = htonl(
1352 TCPOPT_NOP << 24 |
1353 TCPOPT_WINDOW << 16 |
1354 TCPOLEN_WINDOW << 8 |
1355 tp->request_r_scale);
1356 optlen += 4;
1357 }
1358 #if MPTCP
1359 if (mptcp_enable) {
1360 optlen = mptcp_setup_syn_opts(so, flags, opt,
1361 optlen);
1362 }
1363 #endif /* MPTCP */
1364 }
1365 }
1366
1367 /*
1368 * Send a timestamp and echo-reply if this is a SYN and our side
1369 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1370 * and our peer have sent timestamps in our SYN's.
1371 */
1372 if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
1373 (flags & TH_RST) == 0 &&
1374 ((flags & TH_ACK) == 0 ||
1375 (tp->t_flags & TF_RCVD_TSTMP))) {
1376 u_int32_t *lp = (u_int32_t *)(void *)(opt + optlen);
1377
1378 /* Form timestamp option as shown in appendix A of RFC 1323. */
1379 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1380 *lp++ = htonl(tcp_now);
1381 *lp = htonl(tp->ts_recent);
1382 optlen += TCPOLEN_TSTAMP_APPA;
1383 }
1384
1385 /* Note the timestamp for receive buffer autosizing */
1386 if (tp->rfbuf_ts == 0 && (so->so_rcv.sb_flags & SB_AUTOSIZE))
1387 tp->rfbuf_ts = tcp_now;
1388
1389 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1390 /*
1391 * Tack on the SACK permitted option *last*.
1392 * And do padding of options after tacking this on.
1393 * This is because of MSS, TS, WinScale and Signatures are
1394 * all present, we have just 2 bytes left for the SACK
1395 * permitted option, which is just enough.
1396 */
1397 /*
1398 * If this is the first SYN of connection (not a SYN
1399 * ACK), include SACK permitted option. If this is a
1400 * SYN ACK, include SACK permitted option if peer has
1401 * already done so. This is only for active connect,
1402 * since the syncache takes care of the passive connect.
1403 */
1404 if ((flags & TH_SYN) &&
1405 (!(flags & TH_ACK) || (tp->t_flags & TF_SACK_PERMIT))) {
1406 u_char *bp;
1407 bp = (u_char *)opt + optlen;
1408
1409 *bp++ = TCPOPT_SACK_PERMITTED;
1410 *bp++ = TCPOLEN_SACK_PERMITTED;
1411 optlen += TCPOLEN_SACK_PERMITTED;
1412 }
1413 }
1414 #if MPTCP
1415 if (so->so_flags & SOF_MP_SUBFLOW) {
1416 /*
1417 * Its important to piggyback acks with data as ack only packets
1418 * may get lost and data packets that don't send Data ACKs
1419 * still advance the subflow level ACK and therefore make it
1420 * hard for the remote end to recover in low cwnd situations.
1421 */
1422 if (len != 0) {
1423 tp->t_mpflags |= (TMPF_SEND_DSN |
1424 TMPF_MPTCP_ACKNOW);
1425 } else {
1426 tp->t_mpflags |= TMPF_MPTCP_ACKNOW;
1427 }
1428 optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags,
1429 len, &dlenp, &finp, &dss_val, &sseqp, &mptcp_acknow);
1430 tp->t_mpflags &= ~TMPF_SEND_DSN;
1431 }
1432 #endif /* MPTCP */
1433
1434 if (tfo_enabled(tp) && !(tp->t_flags & TF_NOOPT) &&
1435 (flags & (TH_SYN | TH_ACK)) == TH_SYN)
1436 optlen += tcp_tfo_write_cookie(tp, optlen, &len, opt);
1437
1438 if (tfo_enabled(tp) &&
1439 (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK) &&
1440 (tp->t_tfo_flags & TFO_F_OFFER_COOKIE))
1441 optlen += tcp_tfo_write_cookie_rep(tp, optlen, opt);
1442
1443 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1444 /*
1445 * Send SACKs if necessary. This should be the last
1446 * option processed. Only as many SACKs are sent as
1447 * are permitted by the maximum options size.
1448 *
1449 * In general, SACK blocks consume 8*n+2 bytes.
1450 * So a full size SACK blocks option is 34 bytes
1451 * (to generate 4 SACK blocks). At a minimum,
1452 * we need 10 bytes (to generate 1 SACK block).
1453 * If TCP Timestamps (12 bytes) and TCP Signatures
1454 * (18 bytes) are both present, we'll just have
1455 * 10 bytes for SACK options 40 - (12 + 18).
1456 */
1457 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1458 (tp->t_flags & TF_SACK_PERMIT) &&
1459 (tp->rcv_numsacks > 0 || TCP_SEND_DSACK_OPT(tp)) &&
1460 MAX_TCPOPTLEN - optlen - 2 >= TCPOLEN_SACK) {
1461 int nsack, padlen;
1462 u_char *bp = (u_char *)opt + optlen;
1463 u_int32_t *lp;
1464
1465 nsack = (MAX_TCPOPTLEN - optlen - 2) / TCPOLEN_SACK;
1466 nsack = min(nsack, (tp->rcv_numsacks +
1467 (TCP_SEND_DSACK_OPT(tp) ? 1 : 0)));
1468 sackoptlen = (2 + nsack * TCPOLEN_SACK);
1469
1470 /*
1471 * First we need to pad options so that the
1472 * SACK blocks can start at a 4-byte boundary
1473 * (sack option and length are at a 2 byte offset).
1474 */
1475 padlen = (MAX_TCPOPTLEN - optlen - sackoptlen) % 4;
1476 optlen += padlen;
1477 while (padlen-- > 0)
1478 *bp++ = TCPOPT_NOP;
1479
1480 tcpstat.tcps_sack_send_blocks++;
1481 *bp++ = TCPOPT_SACK;
1482 *bp++ = sackoptlen;
1483 lp = (u_int32_t *)(void *)bp;
1484
1485 /*
1486 * First block of SACK option should represent
1487 * DSACK. Prefer to send SACK information if there
1488 * is space for only one SACK block. This will
1489 * allow for faster recovery.
1490 */
1491 if (TCP_SEND_DSACK_OPT(tp) && nsack > 0 &&
1492 (tp->rcv_numsacks == 0 || nsack > 1)) {
1493 *lp++ = htonl(tp->t_dsack_lseq);
1494 *lp++ = htonl(tp->t_dsack_rseq);
1495 tcpstat.tcps_dsack_sent++;
1496 nsack--;
1497 }
1498 VERIFY(nsack == 0 || tp->rcv_numsacks >= nsack);
1499 for (i = 0; i < nsack; i++) {
1500 struct sackblk sack = tp->sackblks[i];
1501 *lp++ = htonl(sack.start);
1502 *lp++ = htonl(sack.end);
1503 }
1504 optlen += sackoptlen;
1505 }
1506 }
1507
1508 /* Pad TCP options to a 4 byte boundary */
1509 if (optlen < MAX_TCPOPTLEN && (optlen % sizeof(u_int32_t))) {
1510 int pad = sizeof(u_int32_t) - (optlen % sizeof(u_int32_t));
1511 u_char *bp = (u_char *)opt + optlen;
1512
1513 optlen += pad;
1514 while (pad) {
1515 *bp++ = TCPOPT_EOL;
1516 pad--;
1517 }
1518 }
1519
1520 /*
1521 * RFC 3168 states that:
1522 * - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
1523 * to handle the TCP ECE flag, even if you also later send a
1524 * non-ECN-setup SYN/SYN-ACK.
1525 * - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
1526 * the ip ECT flag.
1527 *
1528 * It is not clear how the ECE flag would ever be set if you never
1529 * set the IP ECT flag on outbound packets. All the same, we use
1530 * the TE_SETUPSENT to indicate that we have committed to handling
1531 * the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
1532 * whether or not we should set the IP ECT flag on outbound packet
1533 *
1534 * For a SYN-ACK, send an ECN setup SYN-ACK
1535 */
1536 if ((tcp_ecn_inbound || (tp->t_flags & TF_ENABLE_ECN))
1537 && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
1538 if (tp->ecn_flags & TE_SETUPRECEIVED) {
1539 if (tcp_send_ecn_flags_on_syn(tp, so)) {
1540 /*
1541 * Setting TH_ECE makes this an ECN-setup
1542 * SYN-ACK
1543 */
1544 flags |= TH_ECE;
1545
1546 /*
1547 * Record that we sent the ECN-setup and
1548 * default to setting IP ECT.
1549 */
1550 tp->ecn_flags |= (TE_SETUPSENT|TE_SENDIPECT);
1551 tcpstat.tcps_ecn_server_setup++;
1552 tcpstat.tcps_ecn_server_success++;
1553 } else {
1554 /*
1555 * We sent an ECN-setup SYN-ACK but it was
1556 * dropped. Fallback to non-ECN-setup
1557 * SYN-ACK and clear flag to indicate that
1558 * we should not send data with IP ECT set
1559 *
1560 * Pretend we didn't receive an
1561 * ECN-setup SYN.
1562 *
1563 * We already incremented the counter
1564 * assuming that the ECN setup will
1565 * succeed. Decrementing here
1566 * tcps_ecn_server_success to correct it.
1567 */
1568 if (tp->ecn_flags & TE_SETUPSENT) {
1569 tcpstat.tcps_ecn_lost_synack++;
1570 tcpstat.tcps_ecn_server_success--;
1571 }
1572
1573 tp->ecn_flags &=
1574 ~(TE_SETUPRECEIVED | TE_SENDIPECT |
1575 TE_SENDCWR);
1576 }
1577 }
1578 } else if ((tcp_ecn_outbound || (tp->t_flags & TF_ENABLE_ECN))
1579 && (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
1580 if (tcp_send_ecn_flags_on_syn(tp, so)) {
1581 /*
1582 * Setting TH_ECE and TH_CWR makes this an
1583 * ECN-setup SYN
1584 */
1585 flags |= (TH_ECE | TH_CWR);
1586 tcpstat.tcps_ecn_client_setup++;
1587
1588 /*
1589 * Record that we sent the ECN-setup and default to
1590 * setting IP ECT.
1591 */
1592 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1593 } else {
1594 /*
1595 * We sent an ECN-setup SYN but it was dropped.
1596 * Fall back to non-ECN and clear flag indicating
1597 * we should send data with IP ECT set.
1598 */
1599 if (tp->ecn_flags & TE_SETUPSENT)
1600 tcpstat.tcps_ecn_lost_syn++;
1601 tp->ecn_flags &= ~TE_SENDIPECT;
1602 }
1603 }
1604
1605 /*
1606 * Check if we should set the TCP CWR flag.
1607 * CWR flag is sent when we reduced the congestion window because
1608 * we received a TCP ECE or we performed a fast retransmit. We
1609 * never set the CWR flag on retransmitted packets. We only set
1610 * the CWR flag on data packets. Pure acks don't have this set.
1611 */
1612 if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
1613 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1614 flags |= TH_CWR;
1615 tp->ecn_flags &= ~TE_SENDCWR;
1616 }
1617
1618 /*
1619 * Check if we should set the TCP ECE flag.
1620 */
1621 if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
1622 flags |= TH_ECE;
1623 tcpstat.tcps_ecn_sent_ece++;
1624 }
1625
1626
1627 hdrlen += optlen;
1628
1629 /* Reset DSACK sequence numbers */
1630 tp->t_dsack_lseq = 0;
1631 tp->t_dsack_rseq = 0;
1632
1633 #if INET6
1634 if (isipv6)
1635 ipoptlen = ip6_optlen(inp);
1636 else
1637 #endif
1638 {
1639 if (tp_inp_options) {
1640 ipoptlen = tp_inp_options->m_len -
1641 offsetof(struct ipoption, ipopt_list);
1642 } else {
1643 ipoptlen = 0;
1644 }
1645 }
1646 #if IPSEC
1647 ipoptlen += ipsec_optlen;
1648 #endif
1649
1650 /*
1651 * Adjust data length if insertion of options will
1652 * bump the packet length beyond the t_maxopd length.
1653 * Clear the FIN bit because we cut off the tail of
1654 * the segment.
1655 *
1656 * When doing TSO limit a burst to TCP_MAXWIN minus the
1657 * IP, TCP and Options length to keep ip->ip_len from
1658 * overflowing. Prevent the last segment from being
1659 * fractional thus making them all equal sized and set
1660 * the flag to continue sending. TSO is disabled when
1661 * IP options or IPSEC are present.
1662 */
1663 if (len + optlen + ipoptlen > tp->t_maxopd) {
1664 /*
1665 * If there is still more to send,
1666 * don't close the connection.
1667 */
1668 flags &= ~TH_FIN;
1669 if (tso) {
1670 int32_t tso_maxlen;
1671
1672 tso_maxlen = tp->tso_max_segment_size ?
1673 tp->tso_max_segment_size : TCP_MAXWIN;
1674
1675 if (len > tso_maxlen - hdrlen - optlen) {
1676 len = tso_maxlen - hdrlen - optlen;
1677 len = len - (len % (tp->t_maxopd - optlen));
1678 sendalot = 1;
1679 } else if (tp->t_flags & TF_NEEDFIN) {
1680 sendalot = 1;
1681 }
1682 } else {
1683 len = tp->t_maxopd - optlen - ipoptlen;
1684 sendalot = 1;
1685 }
1686 }
1687 #if MPTCP
1688 /* Adjust the length in the DSS option, if it is lesser than len */
1689 if (dlenp) {
1690 /*
1691 * To test this path without SACK, artificially
1692 * decrement len with something like
1693 * if (len > 10)
1694 len -= 10;
1695 */
1696 if (ntohs(*dlenp) > len) {
1697 *dlenp = htons(len);
1698 /* Unset the FIN flag, if len was adjusted */
1699 if (finp) {
1700 *finp &= ~MDSS_F;
1701 }
1702 sendalot = 1;
1703 }
1704 }
1705 #endif /* MPTCP */
1706
1707 if (max_linkhdr + hdrlen > MCLBYTES)
1708 panic("tcphdr too big");
1709
1710 /* Check if there is enough data in the send socket
1711 * buffer to start measuring bw
1712 */
1713 if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 &&
1714 (tp->t_bwmeas != NULL) &&
1715 (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 &&
1716 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >=
1717 tp->t_bwmeas->bw_minsize) {
1718 tp->t_bwmeas->bw_size = min(
1719 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)),
1720 tp->t_bwmeas->bw_maxsize);
1721 tp->t_flagsext |= TF_BWMEAS_INPROGRESS;
1722 tp->t_bwmeas->bw_start = tp->snd_max;
1723 tp->t_bwmeas->bw_ts = tcp_now;
1724 }
1725
1726 VERIFY(inp->inp_flowhash != 0);
1727 /*
1728 * Grab a header mbuf, attaching a copy of data to
1729 * be transmitted, and initialize the header from
1730 * the template for sends on this connection.
1731 */
1732 if (len) {
1733 tp->t_pmtud_lastseg_size = len + optlen + ipoptlen;
1734 if ((tp->t_flagsext & TF_FORCE) && len == 1)
1735 tcpstat.tcps_sndprobe++;
1736 else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1737 tcpstat.tcps_sndrexmitpack++;
1738 tcpstat.tcps_sndrexmitbyte += len;
1739 if (nstat_collect) {
1740 nstat_route_tx(inp->inp_route.ro_rt, 1,
1741 len, NSTAT_TX_FLAG_RETRANSMIT);
1742 INP_ADD_STAT(inp, cell, wifi, wired,
1743 txpackets, 1);
1744 INP_ADD_STAT(inp, cell, wifi, wired,
1745 txbytes, len);
1746 tp->t_stat.txretransmitbytes += len;
1747 }
1748 } else {
1749 tcpstat.tcps_sndpack++;
1750 tcpstat.tcps_sndbyte += len;
1751
1752 if (nstat_collect) {
1753 INP_ADD_STAT(inp, cell, wifi, wired,
1754 txpackets, 1);
1755 INP_ADD_STAT(inp, cell, wifi, wired,
1756 txbytes, len);
1757 }
1758 }
1759 #if MPTCP
1760 if (tp->t_mpflags & TMPF_MPTCP_TRUE) {
1761 tcpstat.tcps_mp_sndpacks++;
1762 tcpstat.tcps_mp_sndbytes += len;
1763 }
1764 #endif /* MPTCP */
1765 /*
1766 * try to use the new interface that allocates all
1767 * the necessary mbuf hdrs under 1 mbuf lock and
1768 * avoids rescanning the socket mbuf list if
1769 * certain conditions are met. This routine can't
1770 * be used in the following cases...
1771 * 1) the protocol headers exceed the capacity of
1772 * of a single mbuf header's data area (no cluster attached)
1773 * 2) the length of the data being transmitted plus
1774 * the protocol headers fits into a single mbuf header's
1775 * data area (no cluster attached)
1776 */
1777 m = NULL;
1778
1779 /* minimum length we are going to allocate */
1780 allocated_len = MHLEN;
1781 if (MHLEN < hdrlen + max_linkhdr) {
1782 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1783 if (m == NULL) {
1784 error = ENOBUFS;
1785 goto out;
1786 }
1787 MCLGET(m, M_DONTWAIT);
1788 if ((m->m_flags & M_EXT) == 0) {
1789 m_freem(m);
1790 error = ENOBUFS;
1791 goto out;
1792 }
1793 m->m_data += max_linkhdr;
1794 m->m_len = hdrlen;
1795 allocated_len = MCLBYTES;
1796 }
1797 if (len <= allocated_len - hdrlen - max_linkhdr) {
1798 if (m == NULL) {
1799 VERIFY(allocated_len <= MHLEN);
1800 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1801 if (m == NULL) {
1802 error = ENOBUFS;
1803 goto out;
1804 }
1805 m->m_data += max_linkhdr;
1806 m->m_len = hdrlen;
1807 }
1808 /* makes sure we still have data left to be sent at this point */
1809 if (so->so_snd.sb_mb == NULL || off < 0) {
1810 if (m != NULL) m_freem(m);
1811 error = 0; /* should we return an error? */
1812 goto out;
1813 }
1814 m_copydata(so->so_snd.sb_mb, off, (int) len,
1815 mtod(m, caddr_t) + hdrlen);
1816 m->m_len += len;
1817 } else {
1818 uint32_t copymode;
1819 /*
1820 * Retain packet header metadata at the socket
1821 * buffer if this is is an MPTCP subflow,
1822 * otherwise move it.
1823 */
1824 copymode = M_COPYM_MOVE_HDR;
1825 #if MPTCP
1826 if (so->so_flags & SOF_MP_SUBFLOW) {
1827 copymode = M_COPYM_NOOP_HDR;
1828 }
1829 #endif /* MPTCP */
1830 if (m != NULL) {
1831 m->m_next = m_copym_mode(so->so_snd.sb_mb,
1832 off, (int)len, M_DONTWAIT, copymode);
1833 if (m->m_next == NULL) {
1834 (void) m_free(m);
1835 error = ENOBUFS;
1836 goto out;
1837 }
1838 } else {
1839 /*
1840 * make sure we still have data left
1841 * to be sent at this point
1842 */
1843 if (so->so_snd.sb_mb == NULL) {
1844 error = 0; /* should we return an error? */
1845 goto out;
1846 }
1847
1848 /*
1849 * m_copym_with_hdrs will always return the
1850 * last mbuf pointer and the offset into it that
1851 * it acted on to fullfill the current request,
1852 * whether a valid 'hint' was passed in or not.
1853 */
1854 if ((m = m_copym_with_hdrs(so->so_snd.sb_mb,
1855 off, len, M_DONTWAIT, NULL, NULL,
1856 copymode)) == NULL) {
1857 error = ENOBUFS;
1858 goto out;
1859 }
1860 m->m_data += max_linkhdr;
1861 m->m_len = hdrlen;
1862 }
1863 }
1864 /*
1865 * If we're sending everything we've got, set PUSH.
1866 * (This will keep happy those implementations which only
1867 * give data to the user when a buffer fills or
1868 * a PUSH comes in.)
1869 *
1870 * On SYN-segments we should not add the PUSH-flag.
1871 */
1872 if (off + len == so->so_snd.sb_cc && !(flags & TH_SYN))
1873 flags |= TH_PUSH;
1874 } else {
1875 if (tp->t_flags & TF_ACKNOW)
1876 tcpstat.tcps_sndacks++;
1877 else if (flags & (TH_SYN|TH_FIN|TH_RST))
1878 tcpstat.tcps_sndctrl++;
1879 else if (SEQ_GT(tp->snd_up, tp->snd_una))
1880 tcpstat.tcps_sndurg++;
1881 else
1882 tcpstat.tcps_sndwinup++;
1883
1884 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1885 if (m == NULL) {
1886 error = ENOBUFS;
1887 goto out;
1888 }
1889 if (MHLEN < (hdrlen + max_linkhdr)) {
1890 MCLGET(m, M_DONTWAIT);
1891 if ((m->m_flags & M_EXT) == 0) {
1892 m_freem(m);
1893 error = ENOBUFS;
1894 goto out;
1895 }
1896 }
1897 m->m_data += max_linkhdr;
1898 m->m_len = hdrlen;
1899 }
1900 m->m_pkthdr.rcvif = 0;
1901 #if MPTCP
1902 /* Before opt is copied to the mbuf, set the csum field */
1903 mptcp_output_csum(tp, m, len, hdrlen, dss_val, sseqp);
1904 #endif /* MPTCP */
1905 #if CONFIG_MACF_NET
1906 mac_mbuf_label_associate_inpcb(inp, m);
1907 #endif
1908 #if INET6
1909 if (isipv6) {
1910 ip6 = mtod(m, struct ip6_hdr *);
1911 th = (struct tcphdr *)(void *)(ip6 + 1);
1912 tcp_fillheaders(tp, ip6, th);
1913 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1914 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1915 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
1916 }
1917 svc_flags |= PKT_SCF_IPV6;
1918 #if PF_ECN
1919 m->m_pkthdr.pf_mtag.pftag_hdr = (void *)ip6;
1920 m->m_pkthdr.pf_mtag.pftag_flags |= PF_TAG_HDR_INET6;
1921 #endif /* PF_ECN */
1922 } else
1923 #endif /* INET6 */
1924 {
1925 ip = mtod(m, struct ip *);
1926 ipov = (struct ipovly *)ip;
1927 th = (struct tcphdr *)(void *)(ip + 1);
1928 /* this picks up the pseudo header (w/o the length) */
1929 tcp_fillheaders(tp, ip, th);
1930 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1931 !SEQ_LT(tp->snd_nxt, tp->snd_max) &&
1932 !sack_rxmit && !(flags & TH_SYN)) {
1933 ip->ip_tos |= IPTOS_ECN_ECT0;
1934 }
1935 #if PF_ECN
1936 m->m_pkthdr.pf_mtag.pftag_hdr = (void *)ip;
1937 m->m_pkthdr.pf_mtag.pftag_flags |= PF_TAG_HDR_INET;
1938 #endif /* PF_ECN */
1939 }
1940
1941 /*
1942 * Fill in fields, remembering maximum advertised
1943 * window for use in delaying messages about window sizes.
1944 * If resending a FIN, be sure not to use a new sequence number.
1945 */
1946 if ((flags & TH_FIN) && (tp->t_flags & TF_SENTFIN) &&
1947 tp->snd_nxt == tp->snd_max)
1948 tp->snd_nxt--;
1949 /*
1950 * If we are doing retransmissions, then snd_nxt will
1951 * not reflect the first unsent octet. For ACK only
1952 * packets, we do not want the sequence number of the
1953 * retransmitted packet, we want the sequence number
1954 * of the next unsent octet. So, if there is no data
1955 * (and no SYN or FIN), use snd_max instead of snd_nxt
1956 * when filling in ti_seq. But if we are in persist
1957 * state, snd_max might reflect one byte beyond the
1958 * right edge of the window, so use snd_nxt in that
1959 * case, since we know we aren't doing a retransmission.
1960 * (retransmit and persist are mutually exclusive...)
1961 *
1962 * Note the state of this retransmit segment to detect spurious
1963 * retransmissions.
1964 */
1965 if (sack_rxmit == 0) {
1966 if (len || (flags & (TH_SYN|TH_FIN)) ||
1967 tp->t_timer[TCPT_PERSIST]) {
1968 th->th_seq = htonl(tp->snd_nxt);
1969 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
1970 if (SACK_ENABLED(tp) && len > 1) {
1971 tcp_rxtseg_insert(tp, tp->snd_nxt,
1972 (tp->snd_nxt + len - 1));
1973 }
1974 m->m_pkthdr.pkt_flags |= PKTF_TCP_REXMT;
1975 }
1976 } else {
1977 th->th_seq = htonl(tp->snd_max);
1978 }
1979 } else {
1980 th->th_seq = htonl(p->rxmit);
1981 tcp_rxtseg_insert(tp, p->rxmit, (p->rxmit + len - 1));
1982 p->rxmit += len;
1983 tp->sackhint.sack_bytes_rexmit += len;
1984 m->m_pkthdr.pkt_flags |= PKTF_TCP_REXMT;
1985 }
1986 th->th_ack = htonl(tp->rcv_nxt);
1987 tp->last_ack_sent = tp->rcv_nxt;
1988 #if MPTCP
1989 /* Initialize the ACK field to a value as 0 ack fields are dropped */
1990 if (early_data_sent) {
1991 th->th_ack = th->th_seq + 1;
1992 }
1993 #endif /* MPTCP */
1994 if (optlen) {
1995 bcopy(opt, th + 1, optlen);
1996 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1997 }
1998 th->th_flags = flags;
1999 /*
2000 * Calculate receive window. Don't shrink window,
2001 * but avoid silly window syndrome.
2002 */
2003 if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && recwin < (int)tp->t_maxseg)
2004 recwin = 0;
2005 if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt))
2006 recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt);
2007 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) {
2008 if (recwin > (int32_t)slowlink_wsize)
2009 recwin = slowlink_wsize;
2010 }
2011
2012 #if TRAFFIC_MGT
2013 if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) {
2014 if (tcp_recv_throttle(tp)) {
2015 uint32_t min_iaj_win =
2016 tcp_min_iaj_win * tp->t_maxseg;
2017 if (tp->iaj_rwintop == 0 ||
2018 SEQ_LT(tp->iaj_rwintop, tp->rcv_adv))
2019 tp->iaj_rwintop = tp->rcv_adv;
2020 if (SEQ_LT(tp->iaj_rwintop,
2021 tp->rcv_nxt + min_iaj_win))
2022 tp->iaj_rwintop = tp->rcv_nxt + min_iaj_win;
2023 recwin = min(tp->iaj_rwintop - tp->rcv_nxt, recwin);
2024 }
2025 }
2026 #endif /* TRAFFIC_MGT */
2027
2028 if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
2029 recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
2030 th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
2031
2032 /*
2033 * Adjust the RXWIN0SENT flag - indicate that we have advertised
2034 * a 0 window. This may cause the remote transmitter to stall. This
2035 * flag tells soreceive() to disable delayed acknowledgements when
2036 * draining the buffer. This can occur if the receiver is attempting
2037 * to read more data then can be buffered prior to transmitting on
2038 * the connection.
2039 */
2040 if (th->th_win == 0)
2041 tp->t_flags |= TF_RXWIN0SENT;
2042 else
2043 tp->t_flags &= ~TF_RXWIN0SENT;
2044 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
2045 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
2046 th->th_flags |= TH_URG;
2047 } else {
2048 /*
2049 * If no urgent pointer to send, then we pull
2050 * the urgent pointer to the left edge of the send window
2051 * so that it doesn't drift into the send window on sequence
2052 * number wraparound.
2053 */
2054 tp->snd_up = tp->snd_una; /* drag it along */
2055 }
2056
2057 /*
2058 * Put TCP length in extended header, and then
2059 * checksum extended header and data.
2060 */
2061 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
2062 #if INET6
2063 if (isipv6) {
2064 /*
2065 * ip6_plen is not need to be filled now, and will be filled
2066 * in ip6_output.
2067 */
2068 m->m_pkthdr.csum_flags = CSUM_TCPIPV6;
2069 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2070 if (len + optlen)
2071 th->th_sum = in_addword(th->th_sum,
2072 htons((u_short)(optlen + len)));
2073 }
2074 else
2075 #endif /* INET6 */
2076 {
2077 m->m_pkthdr.csum_flags = CSUM_TCP;
2078 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2079 if (len + optlen)
2080 th->th_sum = in_addword(th->th_sum,
2081 htons((u_short)(optlen + len)));
2082 }
2083
2084 /*
2085 * Enable TSO and specify the size of the segments.
2086 * The TCP pseudo header checksum is always provided.
2087 */
2088 if (tso) {
2089 #if INET6
2090 if (isipv6)
2091 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV6;
2092 else
2093 #endif /* INET6 */
2094 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV4;
2095
2096 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
2097 } else {
2098 m->m_pkthdr.tso_segsz = 0;
2099 }
2100
2101 /*
2102 * In transmit state, time the transmission and arrange for
2103 * the retransmit. In persist state, just set snd_max.
2104 */
2105 if (!(tp->t_flagsext & TF_FORCE)
2106 || tp->t_timer[TCPT_PERSIST] == 0) {
2107 tcp_seq startseq = tp->snd_nxt;
2108
2109 /*
2110 * Advance snd_nxt over sequence space of this segment.
2111 */
2112 if (flags & (TH_SYN|TH_FIN)) {
2113 if (flags & TH_SYN)
2114 tp->snd_nxt++;
2115 if ((flags & TH_FIN) &&
2116 !(tp->t_flags & TF_SENTFIN)) {
2117 tp->snd_nxt++;
2118 tp->t_flags |= TF_SENTFIN;
2119 }
2120 }
2121 if (sack_rxmit)
2122 goto timer;
2123 if (sack_rescue_rxt == TRUE) {
2124 tp->snd_nxt = old_snd_nxt;
2125 sack_rescue_rxt = FALSE;
2126 tcpstat.tcps_pto_in_recovery++;
2127 } else {
2128 tp->snd_nxt += len;
2129 }
2130 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
2131 tp->snd_max = tp->snd_nxt;
2132 /*
2133 * Time this transmission if not a retransmission and
2134 * not currently timing anything.
2135 */
2136 if (tp->t_rtttime == 0) {
2137 tp->t_rtttime = tcp_now;
2138 tp->t_rtseq = startseq;
2139 tcpstat.tcps_segstimed++;
2140
2141 /* update variables related to pipe ack */
2142 tp->t_pipeack_lastuna = tp->snd_una;
2143 }
2144 }
2145
2146 /*
2147 * Set retransmit timer if not currently set,
2148 * and not doing an ack or a keep-alive probe.
2149 */
2150 timer:
2151 if (tp->t_timer[TCPT_REXMT] == 0 &&
2152 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
2153 tp->snd_nxt != tp->snd_una || (flags & TH_FIN))) {
2154 if (tp->t_timer[TCPT_PERSIST]) {
2155 tp->t_timer[TCPT_PERSIST] = 0;
2156 tp->t_rxtshift = 0;
2157 tp->t_rxtstart = 0;
2158 tp->t_persist_stop = 0;
2159 }
2160 tp->t_timer[TCPT_REXMT] =
2161 OFFSET_FROM_START(tp, tp->t_rxtcur);
2162 }
2163
2164 /*
2165 * Set tail loss probe timeout if new data is being
2166 * transmitted. This will be supported only when
2167 * SACK option is enabled on a connection.
2168 *
2169 * Every time new data is sent PTO will get reset.
2170 */
2171 if (tcp_enable_tlp && tp->t_state == TCPS_ESTABLISHED &&
2172 SACK_ENABLED(tp) && !IN_FASTRECOVERY(tp)
2173 && tp->snd_nxt == tp->snd_max
2174 && SEQ_GT(tp->snd_nxt, tp->snd_una)
2175 && tp->t_rxtshift == 0
2176 && (tp->t_flagsext & (TF_SENT_TLPROBE|TF_PKTS_REORDERED)) == 0) {
2177 u_int32_t pto, srtt, new_rto = 0;
2178
2179 /*
2180 * Using SRTT alone to set PTO can cause spurious
2181 * retransmissions on wireless networks where there
2182 * is a lot of variance in RTT. Taking variance
2183 * into account will avoid this.
2184 */
2185 srtt = tp->t_srtt >> TCP_RTT_SHIFT;
2186 pto = ((TCP_REXMTVAL(tp)) * 3) >> 1;
2187 pto = max (2 * srtt, pto);
2188 if ((tp->snd_max - tp->snd_una) == tp->t_maxseg)
2189 pto = max(pto,
2190 (((3 * pto) >> 2) + tcp_delack * 2));
2191 else
2192 pto = max(10, pto);
2193
2194 /* if RTO is less than PTO, choose RTO instead */
2195 if (tp->t_rxtcur < pto) {
2196 /*
2197 * Schedule PTO instead of RTO in favor of
2198 * fast recovery.
2199 */
2200 pto = tp->t_rxtcur;
2201
2202 /* Reset the next RTO to be after PTO. */
2203 TCPT_RANGESET(new_rto,
2204 (pto + TCP_REXMTVAL(tp)),
2205 max(tp->t_rttmin, tp->t_rttcur + 2),
2206 TCPTV_REXMTMAX, 0);
2207 tp->t_timer[TCPT_REXMT] =
2208 OFFSET_FROM_START(tp, new_rto);
2209 }
2210 tp->t_timer[TCPT_PTO] = OFFSET_FROM_START(tp, pto);
2211 }
2212 } else {
2213 /*
2214 * Persist case, update snd_max but since we are in
2215 * persist mode (no window) we do not update snd_nxt.
2216 */
2217 int xlen = len;
2218 if (flags & TH_SYN)
2219 ++xlen;
2220 if ((flags & TH_FIN) &&
2221 !(tp->t_flags & TF_SENTFIN)) {
2222 ++xlen;
2223 tp->t_flags |= TF_SENTFIN;
2224 }
2225 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
2226 tp->snd_max = tp->snd_nxt + len;
2227 }
2228
2229 #if TCPDEBUG
2230 /*
2231 * Trace.
2232 */
2233 if (so_options & SO_DEBUG)
2234 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
2235 #endif
2236
2237 /*
2238 * Fill in IP length and desired time to live and
2239 * send to IP level. There should be a better way
2240 * to handle ttl and tos; we could keep them in
2241 * the template, but need a way to checksum without them.
2242 */
2243 #if INET6
2244 /*
2245 * m->m_pkthdr.len should have been set before cksum calcuration,
2246 * because in6_cksum() need it.
2247 */
2248 if (isipv6) {
2249 /*
2250 * we separately set hoplimit for every segment, since the
2251 * user might want to change the value via setsockopt.
2252 * Also, desired default hop limit might be changed via
2253 * Neighbor Discovery.
2254 */
2255 ip6->ip6_hlim = in6_selecthlim(inp, inp->in6p_route.ro_rt ?
2256 inp->in6p_route.ro_rt->rt_ifp : NULL);
2257
2258 /* TODO: IPv6 IP6TOS_ECT bit on */
2259 KERNEL_DEBUG(DBG_LAYER_BEG,
2260 ((inp->inp_fport << 16) | inp->inp_lport),
2261 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
2262 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
2263 sendalot,0,0);
2264 } else
2265 #endif /* INET6 */
2266 {
2267 ip->ip_len = m->m_pkthdr.len;
2268 ip->ip_ttl = inp->inp_ip_ttl; /* XXX */
2269 ip->ip_tos |= (inp->inp_ip_tos & ~IPTOS_ECN_MASK);/* XXX */
2270 KERNEL_DEBUG(DBG_LAYER_BEG,
2271 ((inp->inp_fport << 16) | inp->inp_lport),
2272 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
2273 (inp->inp_faddr.s_addr & 0xffff)), 0,0,0);
2274 }
2275
2276 /*
2277 * See if we should do MTU discovery.
2278 * Look at the flag updated on the following criterias:
2279 * 1) Path MTU discovery is authorized by the sysctl
2280 * 2) The route isn't set yet (unlikely but could happen)
2281 * 3) The route is up
2282 * 4) the MTU is not locked (if it is, then discovery has been
2283 * disabled for that route)
2284 */
2285 #if INET6
2286 if (!isipv6)
2287 #endif /* INET6 */
2288 if (path_mtu_discovery && (tp->t_flags & TF_PMTUD))
2289 ip->ip_off |= IP_DF;
2290
2291 #if NECP
2292 {
2293 necp_kernel_policy_id policy_id;
2294 u_int32_t route_rule_id;
2295 if (!necp_socket_is_allowed_to_send_recv(inp, &policy_id, &route_rule_id)) {
2296 m_freem(m);
2297 error = EHOSTUNREACH;
2298 goto out;
2299 }
2300
2301 necp_mark_packet_from_socket(m, inp, policy_id, route_rule_id);
2302 }
2303 #endif /* NECP */
2304
2305 #if IPSEC
2306 if (inp->inp_sp != NULL)
2307 ipsec_setsocket(m, so);
2308 #endif /*IPSEC*/
2309
2310 /*
2311 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
2312 */
2313 lost = 0;
2314
2315 /*
2316 * Embed the flow hash in pkt hdr and mark the packet as
2317 * capable of flow controlling
2318 */
2319 m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB;
2320 m->m_pkthdr.pkt_flowid = inp->inp_flowhash;
2321 m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC;
2322 #if MPTCP
2323 /* Disable flow advisory when using MPTCP. */
2324 if (!(tp->t_mpflags & TMPF_MPTCP_TRUE))
2325 #endif /* MPTCP */
2326 m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV;
2327 m->m_pkthdr.pkt_proto = IPPROTO_TCP;
2328
2329 m->m_nextpkt = NULL;
2330
2331 if (inp->inp_last_outifp != NULL &&
2332 !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) {
2333 /* Hint to prioritize this packet if
2334 * 1. if the packet has no data
2335 * 2. the interface supports transmit-start model and did
2336 * not disable ACK prioritization.
2337 * 3. Only ACK flag is set.
2338 * 4. there is no outstanding data on this connection.
2339 */
2340 if (tcp_prioritize_acks != 0 && len == 0 &&
2341 (inp->inp_last_outifp->if_eflags &
2342 (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART &&
2343 th->th_flags == TH_ACK && tp->snd_una == tp->snd_max &&
2344 tp->t_timer[TCPT_REXMT] == 0) {
2345 svc_flags |= PKT_SCF_TCP_ACK;
2346 }
2347 set_packet_service_class(m, so, MBUF_SC_UNSPEC, svc_flags);
2348 }
2349
2350 tp->t_pktlist_sentlen += len;
2351 tp->t_lastchain++;
2352
2353 #if INET6
2354 if (isipv6) {
2355 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
2356 struct ip6 *, ip6, struct tcpcb *, tp, struct tcphdr *,
2357 th);
2358 } else
2359 #endif /* INET6 */
2360 {
2361 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
2362 struct ip *, ip, struct tcpcb *, tp, struct tcphdr *, th);
2363 }
2364
2365 if (tp->t_pktlist_head != NULL) {
2366 tp->t_pktlist_tail->m_nextpkt = m;
2367 tp->t_pktlist_tail = m;
2368 } else {
2369 packchain_newlist++;
2370 tp->t_pktlist_head = tp->t_pktlist_tail = m;
2371 }
2372
2373 if ((lro_ackmore) && (!sackoptlen) && (!tp->t_timer[TCPT_PERSIST]) &&
2374 ((th->th_flags & TH_ACK) == TH_ACK) && (!len) &&
2375 (tp->t_state == TCPS_ESTABLISHED)) {
2376 /* For a pure ACK, see if you need to send more of them */
2377 mnext = tcp_send_lroacks(tp, m, th);
2378 if (mnext) {
2379 tp->t_pktlist_tail->m_nextpkt = mnext;
2380 if (mnext->m_nextpkt == NULL) {
2381 tp->t_pktlist_tail = mnext;
2382 tp->t_lastchain++;
2383 } else {
2384 struct mbuf *tail, *next;
2385 next = mnext->m_nextpkt;
2386 tail = next->m_nextpkt;
2387 while (tail) {
2388 next = tail;
2389 tail = tail->m_nextpkt;
2390 tp->t_lastchain++;
2391 }
2392 tp->t_pktlist_tail = next;
2393 }
2394 }
2395 }
2396
2397 if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
2398 (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
2399 (tp->t_flags & (TH_PUSH | TF_ACKNOW)) ||
2400 (tp->t_flagsext & TF_FORCE) ||
2401 tp->t_lastchain >= tcp_packet_chaining) {
2402 error = 0;
2403 while (inp->inp_sndinprog_cnt == 0 &&
2404 tp->t_pktlist_head != NULL) {
2405 packetlist = tp->t_pktlist_head;
2406 packchain_listadd = tp->t_lastchain;
2407 packchain_sent++;
2408 lost = tp->t_pktlist_sentlen;
2409 TCP_PKTLIST_CLEAR(tp);
2410
2411 error = tcp_ip_output(so, tp, packetlist,
2412 packchain_listadd, tp_inp_options,
2413 (so_options & SO_DONTROUTE),
2414 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
2415 #if INET6
2416 isipv6);
2417 #else /* INET6 */
2418 0);
2419 #endif /* !INET6 */
2420 if (error) {
2421 /*
2422 * Take into account the rest of unsent
2423 * packets in the packet list for this tcp
2424 * into "lost", since we're about to free
2425 * the whole list below.
2426 */
2427 lost += tp->t_pktlist_sentlen;
2428 break;
2429 } else {
2430 lost = 0;
2431 }
2432 }
2433 /* tcp was closed while we were in ip; resume close */
2434 if (inp->inp_sndinprog_cnt == 0 &&
2435 (tp->t_flags & TF_CLOSING)) {
2436 tp->t_flags &= ~TF_CLOSING;
2437 (void) tcp_close(tp);
2438 return (0);
2439 }
2440 } else {
2441 error = 0;
2442 packchain_looped++;
2443 tcpstat.tcps_sndtotal++;
2444
2445 goto again;
2446 }
2447 if (error) {
2448 /*
2449 * Assume that the packets were lost, so back out the
2450 * sequence number advance, if any. Note that the "lost"
2451 * variable represents the amount of user data sent during
2452 * the recent call to ip_output_list() plus the amount of
2453 * user data in the packet list for this tcp at the moment.
2454 */
2455 if (!(tp->t_flagsext & TF_FORCE)
2456 || tp->t_timer[TCPT_PERSIST] == 0) {
2457 /*
2458 * No need to check for TH_FIN here because
2459 * the TF_SENTFIN flag handles that case.
2460 */
2461 if ((flags & TH_SYN) == 0) {
2462 if (sack_rxmit) {
2463 if (SEQ_GT((p->rxmit - lost),
2464 tp->snd_una)) {
2465 p->rxmit -= lost;
2466 } else {
2467 lost = p->rxmit - tp->snd_una;
2468 p->rxmit = tp->snd_una;
2469 }
2470 tp->sackhint.sack_bytes_rexmit -= lost;
2471 } else {
2472 if (SEQ_GT((tp->snd_nxt - lost),
2473 tp->snd_una))
2474 tp->snd_nxt -= lost;
2475 else
2476 tp->snd_nxt = tp->snd_una;
2477 }
2478 }
2479 }
2480 out:
2481 if (tp->t_pktlist_head != NULL)
2482 m_freem_list(tp->t_pktlist_head);
2483 TCP_PKTLIST_CLEAR(tp);
2484
2485 if (error == ENOBUFS) {
2486 if (!tp->t_timer[TCPT_REXMT] &&
2487 !tp->t_timer[TCPT_PERSIST])
2488 tp->t_timer[TCPT_REXMT] =
2489 OFFSET_FROM_START(tp, tp->t_rxtcur);
2490 tp->snd_cwnd = tp->t_maxseg;
2491 tp->t_bytes_acked = 0;
2492 tcp_check_timer_state(tp);
2493 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2494
2495 tcp_ccdbg_trace(tp, NULL, TCP_CC_OUTPUT_ERROR);
2496 return (0);
2497 }
2498 if (error == EMSGSIZE) {
2499 /*
2500 * ip_output() will have already fixed the route
2501 * for us. tcp_mtudisc() will, as its last action,
2502 * initiate retransmission, so it is important to
2503 * not do so here.
2504 *
2505 * If TSO was active we either got an interface
2506 * without TSO capabilits or TSO was turned off.
2507 * Disable it for this connection as too and
2508 * immediatly retry with MSS sized segments generated
2509 * by this function.
2510 */
2511 if (tso)
2512 tp->t_flags &= ~TF_TSO;
2513
2514 tcp_mtudisc(inp, 0);
2515 tcp_check_timer_state(tp);
2516
2517 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2518 return 0;
2519 }
2520 /*
2521 * Unless this is due to interface restriction policy,
2522 * treat EHOSTUNREACH/ENETDOWN as a soft error.
2523 */
2524 if ((error == EHOSTUNREACH || error == ENETDOWN) &&
2525 TCPS_HAVERCVDSYN(tp->t_state) &&
2526 !inp_restricted_send(inp, inp->inp_last_outifp)) {
2527 tp->t_softerror = error;
2528 error = 0;
2529 }
2530 tcp_check_timer_state(tp);
2531 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2532 return (error);
2533 }
2534
2535 tcpstat.tcps_sndtotal++;
2536
2537 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
2538 if (sendalot)
2539 goto again;
2540
2541 tcp_check_timer_state(tp);
2542 return (0);
2543 }
2544
2545 static int
2546 tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
2547 int cnt, struct mbuf *opt, int flags, int sack_in_progress, int recwin,
2548 boolean_t isipv6)
2549 {
2550 int error = 0;
2551 boolean_t chain;
2552 boolean_t unlocked = FALSE;
2553 boolean_t ifdenied = FALSE;
2554 struct inpcb *inp = tp->t_inpcb;
2555 struct ip_out_args ipoa =
2556 { IFSCOPE_NONE, { 0 }, IPOAF_SELECT_SRCIF|IPOAF_BOUND_SRCADDR, 0 };
2557 struct route ro;
2558 struct ifnet *outif = NULL;
2559 #if INET6
2560 struct ip6_out_args ip6oa =
2561 { IFSCOPE_NONE, { 0 }, IP6OAF_SELECT_SRCIF|IP6OAF_BOUND_SRCADDR, 0 };
2562 struct route_in6 ro6;
2563 struct flowadv *adv =
2564 (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv);
2565 #else /* INET6 */
2566 struct flowadv *adv = &ipoa.ipoa_flowadv;
2567 #endif /* !INET6 */
2568
2569 /* If socket was bound to an ifindex, tell ip_output about it */
2570 if (inp->inp_flags & INP_BOUND_IF) {
2571 #if INET6
2572 if (isipv6) {
2573 ip6oa.ip6oa_boundif = inp->inp_boundifp->if_index;
2574 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
2575 } else
2576 #endif /* INET6 */
2577 {
2578 ipoa.ipoa_boundif = inp->inp_boundifp->if_index;
2579 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
2580 }
2581 }
2582
2583 if (INP_NO_CELLULAR(inp)) {
2584 #if INET6
2585 if (isipv6)
2586 ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR;
2587 else
2588 #endif /* INET6 */
2589 ipoa.ipoa_flags |= IPOAF_NO_CELLULAR;
2590 }
2591 if (INP_NO_EXPENSIVE(inp)) {
2592 #if INET6
2593 if (isipv6)
2594 ip6oa.ip6oa_flags |= IP6OAF_NO_EXPENSIVE;
2595 else
2596 #endif /* INET6 */
2597 ipoa.ipoa_flags |= IPOAF_NO_EXPENSIVE;
2598
2599 }
2600 if (INP_AWDL_UNRESTRICTED(inp)) {
2601 #if INET6
2602 if (isipv6)
2603 ip6oa.ip6oa_flags |= IP6OAF_AWDL_UNRESTRICTED;
2604 else
2605 #endif /* INET6 */
2606 ipoa.ipoa_flags |= IPOAF_AWDL_UNRESTRICTED;
2607
2608 }
2609 #if INET6
2610 if (isipv6)
2611 flags |= IPV6_OUTARGS;
2612 else
2613 #endif /* INET6 */
2614 flags |= IP_OUTARGS;
2615
2616 /* Copy the cached route and take an extra reference */
2617 #if INET6
2618 if (isipv6)
2619 in6p_route_copyout(inp, &ro6);
2620 else
2621 #endif /* INET6 */
2622 inp_route_copyout(inp, &ro);
2623
2624 /*
2625 * Data sent (as far as we can tell).
2626 * If this advertises a larger window than any other segment,
2627 * then remember the size of the advertised window.
2628 * Make sure ACK/DELACK conditions are cleared before
2629 * we unlock the socket.
2630 */
2631 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
2632 tp->rcv_adv = tp->rcv_nxt + recwin;
2633 tp->last_ack_sent = tp->rcv_nxt;
2634 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
2635 tp->t_timer[TCPT_DELACK] = 0;
2636 tp->t_unacksegs = 0;
2637
2638 /* Increment the count of outstanding send operations */
2639 inp->inp_sndinprog_cnt++;
2640
2641 /*
2642 * If allowed, unlock TCP socket while in IP
2643 * but only if the connection is established and
2644 * in a normal mode where reentrancy on the tcpcb won't be
2645 * an issue:
2646 * - there is no SACK episode
2647 * - we're not in Fast Recovery mode
2648 * - if we're not sending from an upcall.
2649 */
2650 if (tcp_output_unlocked && !so->so_upcallusecount &&
2651 (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) &&
2652 !IN_FASTRECOVERY(tp)) {
2653
2654 unlocked = TRUE;
2655 socket_unlock(so, 0);
2656 }
2657
2658 /*
2659 * Don't send down a chain of packets when:
2660 * - TCP chaining is disabled
2661 * - there is an IPsec rule set
2662 * - there is a non default rule set for the firewall
2663 */
2664
2665 chain = tcp_packet_chaining > 1
2666 #if IPSEC
2667 && ipsec_bypass
2668 #endif
2669 #if IPFIREWALL
2670 && (fw_enable == 0 || fw_bypass)
2671 #endif
2672 ; // I'm important, not extraneous
2673
2674
2675 while (pkt != NULL) {
2676 struct mbuf *npkt = pkt->m_nextpkt;
2677
2678 if (!chain) {
2679 pkt->m_nextpkt = NULL;
2680 /*
2681 * If we are not chaining, make sure to set the packet
2682 * list count to 0 so that IP takes the right path;
2683 * this is important for cases such as IPSec where a
2684 * single mbuf might result in multiple mbufs as part
2685 * of the encapsulation. If a non-zero count is passed
2686 * down to IP, the head of the chain might change and
2687 * we could end up skipping it (thus generating bogus
2688 * packets). Fixing it in IP would be desirable, but
2689 * for now this would do it.
2690 */
2691 cnt = 0;
2692 }
2693 #if INET6
2694 if (isipv6) {
2695 error = ip6_output_list(pkt, cnt,
2696 inp->in6p_outputopts, &ro6, flags, NULL, NULL,
2697 &ip6oa);
2698 ifdenied = (ip6oa.ip6oa_retflags & IP6OARF_IFDENIED);
2699 } else {
2700 #endif /* INET6 */
2701 error = ip_output_list(pkt, cnt, opt, &ro, flags, NULL,
2702 &ipoa);
2703 ifdenied = (ipoa.ipoa_retflags & IPOARF_IFDENIED);
2704 }
2705
2706 if (chain || error) {
2707 /*
2708 * If we sent down a chain then we are done since
2709 * the callee had taken care of everything; else
2710 * we need to free the rest of the chain ourselves.
2711 */
2712 if (!chain)
2713 m_freem_list(npkt);
2714 break;
2715 }
2716 pkt = npkt;
2717 }
2718
2719 if (unlocked)
2720 socket_lock(so, 0);
2721
2722 /*
2723 * Enter flow controlled state if the connection is established
2724 * and is not in recovery.
2725 *
2726 * A connection will enter suspended state even if it is in
2727 * recovery.
2728 */
2729 if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) ||
2730 adv->code == FADV_SUSPENDED) &&
2731 !(tp->t_flags & TF_CLOSING) &&
2732 tp->t_state == TCPS_ESTABLISHED) {
2733 int rc;
2734 rc = inp_set_fc_state(inp, adv->code);
2735
2736 if (rc == 1)
2737 tcp_ccdbg_trace(tp, NULL,
2738 ((adv->code == FADV_FLOW_CONTROLLED) ?
2739 TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND));
2740 }
2741
2742 /*
2743 * When an interface queue gets suspended, some of the
2744 * packets are dropped. Return ENOBUFS, to update the
2745 * pcb state.
2746 */
2747 if (adv->code == FADV_SUSPENDED)
2748 error = ENOBUFS;
2749
2750 VERIFY(inp->inp_sndinprog_cnt > 0);
2751 if ( --inp->inp_sndinprog_cnt == 0)
2752 inp->inp_flags &= ~(INP_FC_FEEDBACK);
2753
2754 #if INET6
2755 if (isipv6) {
2756 if (ro6.ro_rt != NULL && (outif = ro6.ro_rt->rt_ifp) !=
2757 inp->in6p_last_outifp)
2758 inp->in6p_last_outifp = outif;
2759 } else
2760 #endif /* INET6 */
2761 if (ro.ro_rt != NULL && (outif = ro.ro_rt->rt_ifp) !=
2762 inp->inp_last_outifp)
2763 inp->inp_last_outifp = outif;
2764
2765 if (error != 0 && ifdenied &&
2766 (INP_NO_CELLULAR(inp) || INP_NO_EXPENSIVE(inp)))
2767 soevent(inp->inp_socket,
2768 (SO_FILT_HINT_LOCKED|SO_FILT_HINT_IFDENIED));
2769
2770 /* Synchronize cached PCB route & options */
2771 #if INET6
2772 if (isipv6)
2773 in6p_route_copyin(inp, &ro6);
2774 else
2775 #endif /* INET6 */
2776 inp_route_copyin(inp, &ro);
2777
2778 if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 &&
2779 tp->t_inpcb->inp_route.ro_rt != NULL) {
2780 /* If we found the route and there is an rtt on it
2781 * reset the retransmit timer
2782 */
2783 tcp_getrt_rtt(tp, tp->t_inpcb->in6p_route.ro_rt);
2784 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
2785 }
2786 return (error);
2787 }
2788
2789 void
2790 tcp_setpersist(tp)
2791 register struct tcpcb *tp;
2792 {
2793 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
2794
2795 /* If a PERSIST_TIMER option was set we will limit the
2796 * time the persist timer will be active for that connection
2797 * in order to avoid DOS by using zero window probes.
2798 * see rdar://5805356
2799 */
2800
2801 if ((tp->t_persist_timeout != 0) &&
2802 (tp->t_timer[TCPT_PERSIST] == 0) &&
2803 (tp->t_persist_stop == 0)) {
2804 tp->t_persist_stop = tcp_now + tp->t_persist_timeout;
2805 }
2806
2807 /*
2808 * Start/restart persistance timer.
2809 */
2810 TCPT_RANGESET(tp->t_timer[TCPT_PERSIST],
2811 t * tcp_backoff[tp->t_rxtshift],
2812 TCPTV_PERSMIN, TCPTV_PERSMAX, 0);
2813 tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]);
2814
2815 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
2816 tp->t_rxtshift++;
2817 }
2818
2819 /*
2820 * Send as many acks as data coalesced. Every other packet when stretch
2821 * ACK is not enabled. Every 8 packets, if stretch ACK is enabled.
2822 */
2823 static struct mbuf*
2824 tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th)
2825 {
2826 struct mbuf *mnext = NULL, *ack_chain = NULL, *tail = NULL;
2827 int count = 0;
2828 tcp_seq org_ack = ntohl(th->th_ack);
2829 tcp_seq prev_ack = 0;
2830 int tack_offset = 28; /* XXX IPv6 and IP options not supported */
2831 int twin_offset = 34; /* XXX IPv6 and IP options not supported */
2832 int ack_size = (tp->t_flags & TF_STRETCHACK) ?
2833 (maxseg_unacked * tp->t_maxseg) : (tp->t_maxseg << 1);
2834 int segs_acked = (tp->t_flags & TF_STRETCHACK) ? maxseg_unacked : 2;
2835 struct mbuf *prev_ack_pkt = NULL;
2836 struct socket *so = tp->t_inpcb->inp_socket;
2837 unsigned short winsz = ntohs(th->th_win);
2838 unsigned int scaled_win = winsz<<tp->rcv_scale;
2839 tcp_seq win_rtedge = org_ack + scaled_win;
2840
2841 count = tp->t_lropktlen/tp->t_maxseg;
2842
2843 prev_ack = (org_ack - tp->t_lropktlen) + ack_size;
2844 if (prev_ack < org_ack) {
2845 ack_chain = m_dup(m, M_DONTWAIT);
2846 if (ack_chain) {
2847 th->th_ack = htonl(prev_ack);
2848 /* Keep adv window constant for duplicated ACK packets */
2849 scaled_win = win_rtedge - prev_ack;
2850 if (scaled_win > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
2851 scaled_win = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
2852 th->th_win = htons(scaled_win>>tp->rcv_scale);
2853 if (lrodebug == 5) {
2854 printf("%s: win = %d winsz = %d sc = %d"
2855 " lro_len %d %d\n",
2856 __func__, scaled_win>>tp->rcv_scale, winsz,
2857 tp->rcv_scale, tp->t_lropktlen, count);
2858 }
2859 tail = ack_chain;
2860 count -= segs_acked; /* accounts for prev_ack packet */
2861 count = (count <= segs_acked) ? 0 : count - segs_acked;
2862 tcpstat.tcps_sndacks++;
2863 so_tc_update_stats(m, so, m_get_service_class(m));
2864 } else {
2865 return NULL;
2866 }
2867 }
2868 else {
2869 tp->t_lropktlen = 0;
2870 return NULL;
2871 }
2872
2873 prev_ack_pkt = ack_chain;
2874
2875 while (count > 0) {
2876 if ((prev_ack + ack_size) < org_ack) {
2877 prev_ack += ack_size;
2878 } else {
2879 /*
2880 * The last ACK sent must have the ACK number that TCP
2881 * thinks is the last sent ACK number.
2882 */
2883 prev_ack = org_ack;
2884 }
2885 mnext = m_dup(prev_ack_pkt, M_DONTWAIT);
2886 if (mnext) {
2887 /* Keep adv window constant for duplicated ACK packets */
2888 scaled_win = win_rtedge - prev_ack;
2889 if (scaled_win > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
2890 scaled_win = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
2891 winsz = htons(scaled_win>>tp->rcv_scale);
2892 if (lrodebug == 5) {
2893 printf("%s: winsz = %d ack %x count %d\n",
2894 __func__, scaled_win>>tp->rcv_scale,
2895 prev_ack, count);
2896 }
2897 bcopy(&winsz, mtod(prev_ack_pkt, caddr_t) + twin_offset, 2);
2898 HTONL(prev_ack);
2899 bcopy(&prev_ack, mtod(prev_ack_pkt, caddr_t) + tack_offset, 4);
2900 NTOHL(prev_ack);
2901 tail->m_nextpkt = mnext;
2902 tail = mnext;
2903 count -= segs_acked;
2904 tcpstat.tcps_sndacks++;
2905 so_tc_update_stats(m, so, m_get_service_class(m));
2906 } else {
2907 if (lrodebug == 5) {
2908 printf("%s: failed to alloc mbuf.\n", __func__);
2909 }
2910 break;
2911 }
2912 prev_ack_pkt = mnext;
2913 }
2914 tp->t_lropktlen = 0;
2915 return ack_chain;
2916 }
2917
2918 static int
2919 tcp_recv_throttle (struct tcpcb *tp)
2920 {
2921 uint32_t base_rtt, newsize;
2922 int32_t qdelay;
2923 struct sockbuf *sbrcv = &tp->t_inpcb->inp_socket->so_rcv;
2924
2925 if (tcp_use_rtt_recvbg == 1 &&
2926 TSTMP_SUPPORTED(tp)) {
2927 /*
2928 * Timestamps are supported on this connection. Use
2929 * RTT to look for an increase in latency.
2930 */
2931
2932 /*
2933 * If the connection is already being throttled, leave it
2934 * in that state until rtt comes closer to base rtt
2935 */
2936 if (tp->t_flagsext & TF_RECV_THROTTLE)
2937 return (1);
2938
2939 base_rtt = get_base_rtt(tp);
2940
2941 if (base_rtt != 0 && tp->t_rttcur != 0) {
2942 qdelay = tp->t_rttcur - base_rtt;
2943 /*
2944 * if latency increased on a background flow,
2945 * return 1 to start throttling.
2946 */
2947 if (qdelay > target_qdelay) {
2948 tp->t_flagsext |= TF_RECV_THROTTLE;
2949
2950 /*
2951 * Reduce the recv socket buffer size to
2952 * minimize latecy.
2953 */
2954 if (sbrcv->sb_idealsize >
2955 tcp_recv_throttle_minwin) {
2956 newsize = sbrcv->sb_idealsize >> 1;
2957 /* Set a minimum of 16 K */
2958 newsize =
2959 max(newsize,
2960 tcp_recv_throttle_minwin);
2961 sbrcv->sb_idealsize = newsize;
2962 }
2963 return (1);
2964 } else {
2965 return (0);
2966 }
2967 }
2968 }
2969
2970 /*
2971 * Timestamps are not supported or there is no good RTT
2972 * measurement. Use IPDV in this case.
2973 */
2974 if (tp->acc_iaj > tcp_acc_iaj_react_limit)
2975 return (1);
2976
2977 return (0);
2978 }
mirror of XNU