]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/tcp_output.c
projects / apple / xnu.git / blob
? search:


fa6e5348c0337aeb7847d37e17a534a230bba4c5
[apple/xnu.git] / bsd / netinet / tcp_output.c
1 /*
2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #define _IP_VHL
71
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/protosw.h>
80 #include <sys/socket.h>
81 #include <sys/socketvar.h>
82
83 #include <net/route.h>
84 #include <net/ntstat.h>
85 #include <net/if_var.h>
86 #include <net/if.h>
87 #include <net/if_types.h>
88 #include <net/dlil.h>
89
90 #include <netinet/in.h>
91 #include <netinet/in_systm.h>
92 #include <netinet/in_var.h>
93 #include <netinet/ip.h>
94 #include <netinet/in_pcb.h>
95 #include <netinet/ip_var.h>
96 #include <mach/sdt.h>
97 #if INET6
98 #include <netinet6/in6_pcb.h>
99 #include <netinet/ip6.h>
100 #include <netinet6/ip6_var.h>
101 #endif
102 #include <netinet/tcp.h>
103 #define TCPOUTFLAGS
104 #include <netinet/tcp_fsm.h>
105 #include <netinet/tcp_seq.h>
106 #include <netinet/tcp_timer.h>
107 #include <netinet/tcp_var.h>
108 #include <netinet/tcpip.h>
109 #include <netinet/tcp_cc.h>
110 #if TCPDEBUG
111 #include <netinet/tcp_debug.h>
112 #endif
113 #include <sys/kdebug.h>
114 #include <mach/sdt.h>
115
116 #if IPSEC
117 #include <netinet6/ipsec.h>
118 #endif /*IPSEC*/
119
120 #if CONFIG_MACF_NET
121 #include <security/mac_framework.h>
122 #endif /* MAC_SOCKET */
123
124 #include <netinet/lro_ext.h>
125 #if MPTCP
126 #include <netinet/mptcp_var.h>
127 #include <netinet/mptcp.h>
128 #include <netinet/mptcp_opt.h>
129 #endif
130
131 #define DBG_LAYER_BEG NETDBG_CODE(DBG_NETTCP, 1)
132 #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3)
133 #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
134
135 int path_mtu_discovery = 1;
136 SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW | CTLFLAG_LOCKED,
137 &path_mtu_discovery, 1, "Enable Path MTU Discovery");
138
139 int ss_fltsz = 1;
140 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
141 &ss_fltsz, 1, "Slow start flight size");
142
143 int ss_fltsz_local = 8; /* starts with eight segments max */
144 SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
145 &ss_fltsz_local, 1, "Slow start flight size for local networks");
146
147 int tcp_do_tso = 1;
148 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED,
149 &tcp_do_tso, 0, "Enable TCP Segmentation Offload");
150
151
152 int tcp_ecn_outbound = 0;
153 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_initiate_out, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_outbound,
154 0, "Initiate ECN for outbound connections");
155
156 int tcp_ecn_inbound = 0;
157 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_inbound,
158 0, "Allow ECN negotiation for inbound connections");
159
160 int tcp_packet_chaining = 50;
161 SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining,
162 0, "Enable TCP output packet chaining");
163
164 int tcp_output_unlocked = 1;
165 SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked,
166 0, "Unlock TCP when sending packets down to IP");
167
168 int tcp_do_rfc3390 = 1;
169 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, CTLFLAG_RW | CTLFLAG_LOCKED,
170 &tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS");
171
172 int tcp_min_iaj_win = MIN_IAJ_WIN;
173 SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, CTLFLAG_RW | CTLFLAG_LOCKED,
174 &tcp_min_iaj_win, 1, "Minimum recv win based on inter-packet arrival jitter");
175
176 int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT;
177 SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, CTLFLAG_RW | CTLFLAG_LOCKED,
178 &tcp_acc_iaj_react_limit, 1, "Accumulated IAJ when receiver starts to react");
179
180 uint32_t tcp_do_autosendbuf = 1;
181 SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, CTLFLAG_RW | CTLFLAG_LOCKED,
182 &tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning");
183
184 uint32_t tcp_autosndbuf_inc = 8 * 1024;
185 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, CTLFLAG_RW | CTLFLAG_LOCKED,
186 &tcp_autosndbuf_inc, 1, "Increment in send socket bufffer size");
187
188 uint32_t tcp_autosndbuf_max = 512 * 1024;
189 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, CTLFLAG_RW | CTLFLAG_LOCKED,
190 &tcp_autosndbuf_max, 1, "Maximum send socket buffer size");
191
192 uint32_t tcp_prioritize_acks = 1;
193 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, CTLFLAG_RW | CTLFLAG_LOCKED,
194 &tcp_prioritize_acks, 1, "Prioritize pure acks");
195
196 uint32_t tcp_use_rtt_recvbg = 1;
197 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_recvbg,
198 CTLFLAG_RW | CTLFLAG_LOCKED,
199 &tcp_use_rtt_recvbg, 1, "Use RTT for bg recv algorithm");
200
201 uint32_t tcp_recv_throttle_minwin = 16 * 1024;
202 SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_throttle_minwin,
203 CTLFLAG_RW | CTLFLAG_LOCKED,
204 &tcp_recv_throttle_minwin, 1, "Minimum recv win for throttling");
205
206
207 static int32_t packchain_newlist = 0;
208 static int32_t packchain_looped = 0;
209 static int32_t packchain_sent = 0;
210
211 /* temporary: for testing */
212 #if IPSEC
213 extern int ipsec_bypass;
214 #endif
215
216 extern int slowlink_wsize; /* window correction for slow links */
217 #if IPFIREWALL
218 extern int fw_enable; /* firewall check for packet chaining */
219 extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */
220 #endif /* IPFIREWALL */
221
222 extern u_int32_t dlil_filter_disable_tso_count;
223 extern u_int32_t kipf_count;
224 extern int tcp_recv_bg;
225 extern int maxseg_unacked;
226
227 static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int,
228 struct mbuf *, int, int, int32_t, boolean_t);
229
230 extern uint32_t get_base_rtt(struct tcpcb *tp);
231 static struct mbuf* tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th);
232 static int tcp_recv_throttle(struct tcpcb *tp);
233
234 /*
235 * Tcp output routine: figure out what should be sent and send it.
236 *
237 * Returns: 0 Success
238 * EADDRNOTAVAIL
239 * ENOBUFS
240 * EMSGSIZE
241 * EHOSTUNREACH
242 * ENETDOWN
243 * ip_output_list:ENOMEM
244 * ip_output_list:EADDRNOTAVAIL
245 * ip_output_list:ENETUNREACH
246 * ip_output_list:EHOSTUNREACH
247 * ip_output_list:EACCES
248 * ip_output_list:EMSGSIZE
249 * ip_output_list:ENOBUFS
250 * ip_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
251 * ip6_output_list:EINVAL
252 * ip6_output_list:EOPNOTSUPP
253 * ip6_output_list:EHOSTUNREACH
254 * ip6_output_list:EADDRNOTAVAIL
255 * ip6_output_list:ENETUNREACH
256 * ip6_output_list:EMSGSIZE
257 * ip6_output_list:ENOBUFS
258 * ip6_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
259 */
260 int
261 tcp_output(struct tcpcb *tp)
262 {
263 struct inpcb *inp = tp->t_inpcb;
264 struct socket *so = inp->inp_socket;
265 int32_t len, recwin, sendwin, off;
266 int flags, error;
267 register struct mbuf *m;
268 struct ip *ip = NULL;
269 register struct ipovly *ipov = NULL;
270 #if INET6
271 struct ip6_hdr *ip6 = NULL;
272 #endif /* INET6 */
273 register struct tcphdr *th;
274 u_char opt[TCP_MAXOLEN];
275 unsigned ipoptlen, optlen, hdrlen;
276 int idle, sendalot, lost = 0;
277 int i, sack_rxmit;
278 int tso = 0;
279 int sack_bytes_rxmt;
280 struct sackhole *p;
281 #if IPSEC
282 unsigned ipsec_optlen = 0;
283 #endif /* IPSEC */
284 int last_off = 0;
285 int m_off = 0;
286 int idle_time = 0;
287 struct mbuf *m_lastm = NULL;
288 struct mbuf *m_head = NULL;
289 struct mbuf *packetlist = NULL;
290 struct mbuf *tp_inp_options = inp->inp_depend4.inp4_options;
291 #if INET6
292 int isipv6 = inp->inp_vflag & INP_IPV6 ;
293 #endif
294 short packchain_listadd = 0;
295 int so_options = so->so_options;
296 struct rtentry *rt;
297 u_int32_t basertt, svc_flags = 0, allocated_len;
298 u_int32_t lro_ackmore = (tp->t_lropktlen != 0) ? 1 : 0;
299 struct mbuf *mnext = NULL;
300 int sackoptlen = 0;
301 #if MPTCP
302 unsigned int *dlenp = NULL;
303 u_int8_t *finp = NULL;
304 u_int32_t *sseqp = NULL;
305 u_int64_t dss_val = 0;
306 int mptcp_acknow = 0;
307 #endif /* MPTCP */
308 boolean_t cell = FALSE;
309 boolean_t wifi = FALSE;
310
311 /*
312 * Determine length of data that should be transmitted,
313 * and flags that will be used.
314 * If there is some data or critical controls (SYN, RST)
315 * to send, then transmit; otherwise, investigate further.
316 */
317 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
318
319 /* Since idle_time is signed integer, the following integer subtraction
320 * will take care of wrap around of tcp_now
321 */
322 idle_time = tcp_now - tp->t_rcvtime;
323 if (idle && idle_time >= TCP_IDLETIMEOUT(tp)) {
324 if (CC_ALGO(tp)->after_idle != NULL)
325 CC_ALGO(tp)->after_idle(tp);
326 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
327 struct tcpcb *, tp, struct tcphdr *, NULL,
328 int32_t, TCP_CC_IDLE_TIMEOUT);
329 }
330 tp->t_flags &= ~TF_LASTIDLE;
331 if (idle) {
332 if (tp->t_flags & TF_MORETOCOME) {
333 tp->t_flags |= TF_LASTIDLE;
334 idle = 0;
335 }
336 }
337 #if MPTCP
338 if (tp->t_mpflags & TMPF_RESET) {
339 tcp_check_timer_state(tp);
340 /*
341 * Once a RST has been sent for an MPTCP subflow,
342 * the subflow socket stays around until deleted.
343 * No packets such as FINs must be sent after RST.
344 */
345 return (0);
346 }
347 #endif /* MPTCP */
348
349 again:
350 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
351
352 #if INET6
353 if (isipv6) {
354 KERNEL_DEBUG(DBG_LAYER_BEG,
355 ((inp->inp_fport << 16) | inp->inp_lport),
356 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
357 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
358 sendalot,0,0);
359 } else
360 #endif
361
362 {
363 KERNEL_DEBUG(DBG_LAYER_BEG,
364 ((inp->inp_fport << 16) | inp->inp_lport),
365 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
366 (inp->inp_faddr.s_addr & 0xffff)),
367 sendalot,0,0);
368 }
369 /*
370 * If the route generation id changed, we need to check that our
371 * local (source) IP address is still valid. If it isn't either
372 * return error or silently do nothing (assuming the address will
373 * come back before the TCP connection times out).
374 */
375 rt = inp->inp_route.ro_rt;
376 if (rt != NULL && ROUTE_UNUSABLE(&tp->t_inpcb->inp_route)) {
377 struct ifnet *ifp;
378 struct in_ifaddr *ia = NULL;
379 struct in6_ifaddr *ia6 = NULL;
380 int found_srcaddr = 0;
381
382 /* disable multipages at the socket */
383 somultipages(so, FALSE);
384
385 /* Disable TSO for the socket until we know more */
386 tp->t_flags &= ~TF_TSO;
387
388 if (isipv6) {
389 ia6 = ifa_foraddr6(&inp->in6p_laddr);
390 if (ia6 != NULL)
391 found_srcaddr = 1;
392 } else {
393 ia = ifa_foraddr(inp->inp_laddr.s_addr);
394 if (ia != NULL)
395 found_srcaddr = 1;
396 }
397
398 /* check that the source address is still valid */
399 if (found_srcaddr == 0) {
400 soevent(so,
401 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR));
402
403 if (tp->t_state >= TCPS_CLOSE_WAIT) {
404 tcp_drop(tp, EADDRNOTAVAIL);
405 return(EADDRNOTAVAIL);
406 }
407
408 /* set Retransmit timer if it wasn't set
409 * reset Persist timer and shift register as the
410 * advertised peer window may not be valid anymore
411 */
412
413 if (!tp->t_timer[TCPT_REXMT]) {
414 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
415 if (tp->t_timer[TCPT_PERSIST]) {
416 tp->t_timer[TCPT_PERSIST] = 0;
417 tp->t_rxtshift = 0;
418 tp->t_persist_stop = 0;
419 tp->t_rxtstart = 0;
420 }
421 }
422
423 if (tp->t_pktlist_head != NULL)
424 m_freem_list(tp->t_pktlist_head);
425 TCP_PKTLIST_CLEAR(tp);
426
427 /* drop connection if source address isn't available */
428 if (so->so_flags & SOF_NOADDRAVAIL) {
429 tcp_drop(tp, EADDRNOTAVAIL);
430 return(EADDRNOTAVAIL);
431 }
432 else {
433 tcp_check_timer_state(tp);
434 return(0); /* silently ignore, keep data in socket: address may be back */
435 }
436 }
437 if (ia != NULL)
438 IFA_REMREF(&ia->ia_ifa);
439
440 if (ia6 != NULL)
441 IFA_REMREF(&ia6->ia_ifa);
442
443 /*
444 * Address is still valid; check for multipages capability
445 * again in case the outgoing interface has changed.
446 */
447 RT_LOCK(rt);
448 if ((ifp = rt->rt_ifp) != NULL) {
449 somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
450 tcp_set_tso(tp, ifp);
451 }
452 if (rt->rt_flags & RTF_UP)
453 RT_GENID_SYNC(rt);
454 /*
455 * See if we should do MTU discovery. Don't do it if:
456 * 1) it is disabled via the sysctl
457 * 2) the route isn't up
458 * 3) the MTU is locked (if it is, then discovery has been
459 * disabled)
460 */
461
462 if (!path_mtu_discovery || ((rt != NULL) &&
463 (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU))))
464 tp->t_flags &= ~TF_PMTUD;
465 else
466 tp->t_flags |= TF_PMTUD;
467
468 RT_UNLOCK(rt);
469 }
470
471 if (rt != NULL) {
472 cell = IFNET_IS_CELLULAR(rt->rt_ifp);
473 wifi = (!cell && IFNET_IS_WIFI(rt->rt_ifp));
474 }
475
476 /*
477 * If we've recently taken a timeout, snd_max will be greater than
478 * snd_nxt. There may be SACK information that allows us to avoid
479 * resending already delivered data. Adjust snd_nxt accordingly.
480 */
481 if (SACK_ENABLED(tp) && SEQ_LT(tp->snd_nxt, tp->snd_max))
482 tcp_sack_adjust(tp);
483 sendalot = 0;
484 off = tp->snd_nxt - tp->snd_una;
485 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
486
487 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0)
488 sendwin = min(sendwin, slowlink_wsize);
489
490 flags = tcp_outflags[tp->t_state];
491 /*
492 * Send any SACK-generated retransmissions. If we're explicitly trying
493 * to send out new data (when sendalot is 1), bypass this function.
494 * If we retransmit in fast recovery mode, decrement snd_cwnd, since
495 * we're replacing a (future) new transmission with a retransmission
496 * now, and we previously incremented snd_cwnd in tcp_input().
497 */
498 /*
499 * Still in sack recovery , reset rxmit flag to zero.
500 */
501 sack_rxmit = 0;
502 sack_bytes_rxmt = 0;
503 len = 0;
504 p = NULL;
505 if (SACK_ENABLED(tp) && IN_FASTRECOVERY(tp) &&
506 (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
507 int32_t cwin;
508
509 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
510 if (cwin < 0)
511 cwin = 0;
512 /* Do not retransmit SACK segments beyond snd_recover */
513 if (SEQ_GT(p->end, tp->snd_recover)) {
514 /*
515 * (At least) part of sack hole extends beyond
516 * snd_recover. Check to see if we can rexmit data
517 * for this hole.
518 */
519 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
520 /*
521 * Can't rexmit any more data for this hole.
522 * That data will be rexmitted in the next
523 * sack recovery episode, when snd_recover
524 * moves past p->rxmit.
525 */
526 p = NULL;
527 goto after_sack_rexmit;
528 } else
529 /* Can rexmit part of the current hole */
530 len = ((int32_t)min(cwin,
531 tp->snd_recover - p->rxmit));
532 } else {
533 len = ((int32_t)min(cwin, p->end - p->rxmit));
534 }
535 if (len > 0) {
536 off = p->rxmit - tp->snd_una; /* update off only if we really transmit SACK data */
537 sack_rxmit = 1;
538 sendalot = 1;
539 tcpstat.tcps_sack_rexmits++;
540 tcpstat.tcps_sack_rexmit_bytes +=
541 min(len, tp->t_maxseg);
542 if (nstat_collect) {
543 nstat_route_tx(inp->inp_route.ro_rt, 1,
544 min(len, tp->t_maxseg),
545 NSTAT_TX_FLAG_RETRANSMIT);
546 INP_ADD_STAT(inp, cell, wifi, txpackets, 1);
547 INP_ADD_STAT(inp, cell, wifi, txbytes,
548 min(len, tp->t_maxseg));
549 tp->t_stat.txretransmitbytes += min(len, tp->t_maxseg);
550 }
551 } else {
552 len = 0;
553 }
554 }
555 after_sack_rexmit:
556 /*
557 * Get standard flags, and add SYN or FIN if requested by 'hidden'
558 * state flags.
559 */
560 if (tp->t_flags & TF_NEEDFIN)
561 flags |= TH_FIN;
562 if (tp->t_flags & TF_NEEDSYN)
563 flags |= TH_SYN;
564
565 /*
566 * If in persist timeout with window of 0, send 1 byte.
567 * Otherwise, if window is small but nonzero
568 * and timer expired, we will send what we can
569 * and go to transmit state.
570 */
571 if (tp->t_force) {
572 if (sendwin == 0) {
573 /*
574 * If we still have some data to send, then
575 * clear the FIN bit. Usually this would
576 * happen below when it realizes that we
577 * aren't sending all the data. However,
578 * if we have exactly 1 byte of unsent data,
579 * then it won't clear the FIN bit below,
580 * and if we are in persist state, we wind
581 * up sending the packet without recording
582 * that we sent the FIN bit.
583 *
584 * We can't just blindly clear the FIN bit,
585 * because if we don't have any more data
586 * to send then the probe will be the FIN
587 * itself.
588 */
589 if (off < so->so_snd.sb_cc)
590 flags &= ~TH_FIN;
591 sendwin = 1;
592 } else {
593 tp->t_timer[TCPT_PERSIST] = 0;
594 tp->t_rxtshift = 0;
595 tp->t_rxtstart = 0;
596 tp->t_persist_stop = 0;
597 }
598 }
599
600 /*
601 * If snd_nxt == snd_max and we have transmitted a FIN, the
602 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
603 * a negative length. This can also occur when TCP opens up
604 * its congestion window while receiving additional duplicate
605 * acks after fast-retransmit because TCP will reset snd_nxt
606 * to snd_max after the fast-retransmit.
607 *
608 * In the normal retransmit-FIN-only case, however, snd_nxt will
609 * be set to snd_una, the offset will be 0, and the length may
610 * wind up 0.
611 *
612 * If sack_rxmit is true we are retransmitting from the scoreboard
613 * in which case len is already set.
614 */
615 if (sack_rxmit == 0) {
616 if (sack_bytes_rxmt == 0)
617 len = min(so->so_snd.sb_cc, sendwin) - off;
618 else {
619 int32_t cwin;
620
621 /*
622 * We are inside of a SACK recovery episode and are
623 * sending new data, having retransmitted all the
624 * data possible in the scoreboard.
625 */
626 len = min(so->so_snd.sb_cc, tp->snd_wnd)
627 - off;
628 /*
629 * Don't remove this (len > 0) check !
630 * We explicitly check for len > 0 here (although it
631 * isn't really necessary), to work around a gcc
632 * optimization issue - to force gcc to compute
633 * len above. Without this check, the computation
634 * of len is bungled by the optimizer.
635 */
636 if (len > 0) {
637 cwin = tp->snd_cwnd -
638 (tp->snd_nxt - tp->sack_newdata) -
639 sack_bytes_rxmt;
640 if (cwin < 0)
641 cwin = 0;
642 len = imin(len, cwin);
643 }
644 else
645 len = 0;
646 }
647 }
648
649 /*
650 * Lop off SYN bit if it has already been sent. However, if this
651 * is SYN-SENT state and if segment contains data and if we don't
652 * know that foreign host supports TAO, suppress sending segment.
653 */
654 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
655 if (tp->t_state != TCPS_SYN_RECEIVED)
656 flags &= ~TH_SYN;
657 off--, len++;
658 if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
659 while (inp->inp_sndinprog_cnt == 0 &&
660 tp->t_pktlist_head != NULL) {
661 packetlist = tp->t_pktlist_head;
662 packchain_listadd = tp->t_lastchain;
663 packchain_sent++;
664 TCP_PKTLIST_CLEAR(tp);
665
666 error = tcp_ip_output(so, tp, packetlist,
667 packchain_listadd, tp_inp_options,
668 (so_options & SO_DONTROUTE),
669 (sack_rxmit | (sack_bytes_rxmt != 0)), 0,
670 #if INET6
671 isipv6);
672 #else /* INET6 */
673 0);
674 #endif /* !INET6 */
675
676
677 }
678 /*
679 * tcp was closed while we were in ip,
680 * resume close
681 */
682 if (inp->inp_sndinprog_cnt == 0 &&
683 (tp->t_flags & TF_CLOSING)) {
684 tp->t_flags &= ~TF_CLOSING;
685 (void) tcp_close(tp);
686 } else {
687 tcp_check_timer_state(tp);
688 }
689 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
690 0,0,0,0,0);
691 return(0);
692 }
693 }
694
695 /*
696 * Be careful not to send data and/or FIN on SYN segments.
697 * This measure is needed to prevent interoperability problems
698 * with not fully conformant TCP implementations.
699 */
700 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
701 len = 0;
702 flags &= ~TH_FIN;
703 }
704
705 /* The check here used to be (len < 0). Some times len is zero when
706 * the congestion window is closed and we need to check if persist timer
707 * has to be set in that case. But don't set persist until connection
708 * is established.
709 */
710 if (len <= 0 && !(flags & TH_SYN)) {
711 /*
712 * If FIN has been sent but not acked,
713 * but we haven't been called to retransmit,
714 * len will be < 0. Otherwise, window shrank
715 * after we sent into it. If window shrank to 0,
716 * cancel pending retransmit, pull snd_nxt back
717 * to (closed) window, and set the persist timer
718 * if it isn't already going. If the window didn't
719 * close completely, just wait for an ACK.
720 */
721 len = 0;
722 if (sendwin == 0) {
723 tp->t_timer[TCPT_REXMT] = 0;
724 tp->t_rxtshift = 0;
725 tp->t_rxtstart = 0;
726 tp->snd_nxt = tp->snd_una;
727 if (tp->t_timer[TCPT_PERSIST] == 0)
728 tcp_setpersist(tp);
729 }
730 }
731
732 /* Automatic sizing of send socket buffer. Increase the send socket buffer
733 * size if all of the following criteria are met
734 * 1. the receiver has enough buffer space for this data
735 * 2. send buffer is filled to 7/8th with data (so we actually
736 * have data to make use of it);
737 * 3. our send window (slow start and congestion controlled) is
738 * larger than sent but unacknowledged data in send buffer.
739 */
740 basertt = get_base_rtt(tp);
741 if (tcp_do_autosendbuf == 1 &&
742 !INP_WAIT_FOR_IF_FEEDBACK(inp) && !IN_FASTRECOVERY(tp) &&
743 (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
744 tcp_cansbgrow(&so->so_snd)) {
745 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
746 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
747 sendwin >= (so->so_snd.sb_cc -
748 (tp->snd_nxt - tp->snd_una))) {
749 /* Also increase the send buffer only if the
750 * round-trip time is not increasing because we do
751 * not want to contribute to latency by filling buffers.
752 * We also do not want to hold onto application's
753 * old data for too long. Interactive applications would
754 * rather discard old data.
755 */
756 if (tp->t_rttcur <=
757 (basertt + 25)) {
758 if (sbreserve(&so->so_snd,
759 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
760 tcp_autosndbuf_max)) == 1) {
761 so->so_snd.sb_idealsize = so->so_snd.sb_hiwat;
762 }
763 } else {
764 so->so_snd.sb_idealsize =
765 max(tcp_sendspace, so->so_snd.sb_hiwat -
766 (2 * tcp_autosndbuf_inc));
767 so->so_snd.sb_flags |= SB_TRIM;
768 }
769 }
770 }
771
772 /*
773 * Truncate to the maximum segment length or enable TCP Segmentation
774 * Offloading (if supported by hardware) and ensure that FIN is removed
775 * if the length no longer contains the last data byte.
776 *
777 * TSO may only be used if we are in a pure bulk sending state.
778 * The presence of TCP-MD5, SACK retransmits, SACK advertizements,
779 * ipfw rules and IP options, as well as disabling hardware checksum
780 * offload prevent using TSO. With TSO the TCP header is the same
781 * (except for the sequence number) for all generated packets. This
782 * makes it impossible to transmit any options which vary per generated
783 * segment or packet.
784 *
785 * The length of TSO bursts is limited to TCP_MAXWIN. That limit and
786 * removal of FIN (if not already catched here) are handled later after
787 * the exact length of the TCP options are known.
788 */
789 #if IPSEC
790 /*
791 * Pre-calculate here as we save another lookup into the darknesses
792 * of IPsec that way and can actually decide if TSO is ok.
793 */
794 if (ipsec_bypass == 0)
795 ipsec_optlen = ipsec_hdrsiz_tcp(tp);
796 #endif
797
798 if (len > tp->t_maxseg) {
799 if ((tp->t_flags & TF_TSO) && tcp_do_tso && hwcksum_tx &&
800 ip_use_randomid && kipf_count == 0 && dlil_filter_disable_tso_count == 0 &&
801 tp->rcv_numsacks == 0 && sack_rxmit == 0 && sack_bytes_rxmt == 0 &&
802 inp->inp_options == NULL &&
803 inp->in6p_options == NULL
804 #if IPSEC
805 && ipsec_optlen == 0
806 #endif
807 #if IPFIREWALL
808 && (fw_enable == 0 || fw_bypass)
809 #endif
810 ) {
811 tso = 1;
812 sendalot = 0;
813 } else {
814 len = tp->t_maxseg;
815 sendalot = 1;
816 tso = 0;
817 }
818 }
819 #if MPTCP
820 if (so->so_flags & SOF_MP_SUBFLOW) {
821 int newlen = len;
822 if ((tp->t_mpflags & TMPF_SND_MPPRIO) ||
823 (tp->t_mpflags & TMPF_SND_REM_ADDR) ||
824 (tp->t_mpflags & TMPF_SND_MPFAIL)) {
825 if (len > 0) {
826 len = 0;
827 }
828 sendalot = 1;
829 mptcp_acknow = 1;
830 } else {
831 mptcp_acknow = 0;
832 }
833 /*
834 * The contiguous bytes in the subflow socket buffer can be
835 * discontiguous at the MPTCP level. Since only one DSS
836 * option can be sent in one packet, reduce length to match
837 * the contiguous MPTCP level. Set sendalot to send remainder.
838 */
839 if (len > 0)
840 newlen = mptcp_adj_sendlen(so, off, len);
841 if (newlen < len) {
842 len = newlen;
843 sendalot = 1;
844 }
845 }
846 #endif /* MPTCP */
847 if (sack_rxmit) {
848 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
849 flags &= ~TH_FIN;
850 } else {
851 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
852 flags &= ~TH_FIN;
853 }
854
855 recwin = tcp_sbspace(tp);
856
857
858 /*
859 * If the socket is capable of doing unordered send,
860 * pull the amount of data that can be sent from the
861 * unordered priority queues to the serial queue in
862 * the socket buffer. If bytes are not yet available
863 * in the highest priority message, we may not be able
864 * to send any new data.
865 */
866 if (so->so_flags & SOF_ENABLE_MSGS) {
867 if ((off + len) >
868 so->so_msg_state->msg_serial_bytes) {
869 sbpull_unordered_data(so, off, len);
870
871 /* check if len needs to be modified */
872 if ((off + len) >
873 so->so_msg_state->msg_serial_bytes) {
874 len = so->so_msg_state->msg_serial_bytes - off;
875 if (len <= 0) {
876 len = 0;
877 tcpstat.tcps_msg_sndwaithipri++;
878 }
879 }
880 }
881 }
882
883 /*
884 * Sender silly window avoidance. We transmit under the following
885 * conditions when len is non-zero:
886 *
887 * - we've timed out (e.g. persist timer)
888 * - we need to retransmit
889 * - We have a full segment (or more with TSO)
890 * - This is the last buffer in a write()/send() and we are
891 * either idle or running NODELAY
892 * - we have more then 1/2 the maximum send window's worth of
893 * data (receiver may be limited the window size)
894 */
895 if (len) {
896 if (tp->t_force)
897 goto send;
898 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
899 goto send;
900 if (sack_rxmit)
901 goto send;
902
903 /*
904 * Send new data on the connection only if it is
905 * not flow controlled
906 */
907 if (!INP_WAIT_FOR_IF_FEEDBACK(inp) ||
908 tp->t_state != TCPS_ESTABLISHED) {
909 if (len >= tp->t_maxseg)
910 goto send;
911 if (!(tp->t_flags & TF_MORETOCOME) &&
912 (idle || tp->t_flags & TF_NODELAY ||
913 tp->t_flags & TF_MAXSEGSNT ||
914 ALLOW_LIMITED_TRANSMIT(tp)) &&
915 (tp->t_flags & TF_NOPUSH) == 0 &&
916 len + off >= so->so_snd.sb_cc)
917 goto send;
918 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
919 goto send;
920 } else {
921 tcpstat.tcps_fcholdpacket++;
922 }
923 }
924
925 /*
926 * Compare available window to amount of window
927 * known to peer (as advertised window less
928 * next expected input). If the difference is at least two
929 * max size segments, or at least 25% of the maximum possible
930 * window, then want to send a window update to peer.
931 * Skip this if the connection is in T/TCP half-open state.
932 */
933 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) {
934 /*
935 * "adv" is the amount we can increase the window,
936 * taking into account that we are limited by
937 * TCP_MAXWIN << tp->rcv_scale.
938 */
939 int32_t adv, oldwin = 0;
940 adv = imin(recwin, (int)TCP_MAXWIN << tp->rcv_scale) -
941 (tp->rcv_adv - tp->rcv_nxt);
942
943 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt))
944 oldwin = tp->rcv_adv - tp->rcv_nxt;
945
946 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
947 /* Update only if the resulting scaled value of the window changed, or
948 * if there is a change in the sequence since the last ack.
949 * This avoids what appears as dupe ACKS (see rdar://5640997)
950 *
951 * If streaming is detected avoid sending too many window updates.
952 * We will depend on the delack timer to send a window update
953 * when needed.
954 */
955 if (!(tp->t_flags & TF_STRETCHACK) &&
956 (tp->last_ack_sent != tp->rcv_nxt ||
957 ((oldwin + adv) >> tp->rcv_scale) >
958 (oldwin >> tp->rcv_scale))) {
959 goto send;
960 }
961
962 /* Make sure that the delayed ack timer is set if we
963 * delayed sending a window update because of streaming
964 * detection.
965 */
966 if ((tp->t_flags & TF_STRETCHACK) &&
967 !(tp->t_flags & TF_DELACK)) {
968 tp->t_flags |= TF_DELACK;
969 tp->t_timer[TCPT_DELACK] =
970 OFFSET_FROM_START(tp, tcp_delack);
971 }
972 }
973 if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat)
974 goto send;
975 }
976
977 /*
978 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
979 * is also a catch-all for the retransmit timer timeout case.
980 */
981 if (tp->t_flags & TF_ACKNOW)
982 goto send;
983 if ((flags & TH_RST) ||
984 ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
985 goto send;
986 if (SEQ_GT(tp->snd_up, tp->snd_una))
987 goto send;
988 #if MPTCP
989 if (mptcp_acknow)
990 goto send;
991 #endif /* MPTCP */
992 /*
993 * If our state indicates that FIN should be sent
994 * and we have not yet done so, then we need to send.
995 */
996 if ((flags & TH_FIN) &&
997 (!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
998 goto send;
999 /*
1000 * In SACK, it is possible for tcp_output to fail to send a segment
1001 * after the retransmission timer has been turned off. Make sure
1002 * that the retransmission timer is set.
1003 */
1004 if (SACK_ENABLED(tp) && (tp->t_state >= TCPS_ESTABLISHED) &&
1005 SEQ_GT(tp->snd_max, tp->snd_una) &&
1006 tp->t_timer[TCPT_REXMT] == 0 &&
1007 tp->t_timer[TCPT_PERSIST] == 0) {
1008 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1009 goto just_return;
1010 }
1011 /*
1012 * TCP window updates are not reliable, rather a polling protocol
1013 * using ``persist'' packets is used to insure receipt of window
1014 * updates. The three ``states'' for the output side are:
1015 * idle not doing retransmits or persists
1016 * persisting to move a small or zero window
1017 * (re)transmitting and thereby not persisting
1018 *
1019 * tp->t_timer[TCPT_PERSIST]
1020 * is set when we are in persist state.
1021 * tp->t_force
1022 * is set when we are called to send a persist packet.
1023 * tp->t_timer[TCPT_REXMT]
1024 * is set when we are retransmitting
1025 * The output side is idle when both timers are zero.
1026 *
1027 * If send window is too small, there is data to transmit, and no
1028 * retransmit or persist is pending, then go to persist state.
1029 * If nothing happens soon, send when timer expires:
1030 * if window is nonzero, transmit what we can,
1031 * otherwise force out a byte.
1032 */
1033 if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 &&
1034 tp->t_timer[TCPT_PERSIST] == 0) {
1035 tp->t_rxtshift = 0;
1036 tp->t_rxtstart = 0;
1037 tcp_setpersist(tp);
1038 }
1039 just_return:
1040 /*
1041 * If there is no reason to send a segment, just return.
1042 * but if there is some packets left in the packet list, send them now.
1043 */
1044 while (inp->inp_sndinprog_cnt == 0 &&
1045 tp->t_pktlist_head != NULL) {
1046 packetlist = tp->t_pktlist_head;
1047 packchain_listadd = tp->t_lastchain;
1048 packchain_sent++;
1049 TCP_PKTLIST_CLEAR(tp);
1050
1051 error = tcp_ip_output(so, tp, packetlist, packchain_listadd,
1052 tp_inp_options, (so_options & SO_DONTROUTE),
1053 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
1054 #if INET6
1055 isipv6);
1056 #else /* INET6 */
1057 0);
1058 #endif /* !INET6 */
1059 }
1060 /* tcp was closed while we were in ip; resume close */
1061 if (inp->inp_sndinprog_cnt == 0 &&
1062 (tp->t_flags & TF_CLOSING)) {
1063 tp->t_flags &= ~TF_CLOSING;
1064 (void) tcp_close(tp);
1065 } else {
1066 tcp_check_timer_state(tp);
1067 }
1068 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1069 return (0);
1070
1071 send:
1072 /*
1073 * Set TF_MAXSEGSNT flag if the segment size is greater than
1074 * the max segment size.
1075 */
1076 if (len > 0) {
1077 if (len >= tp->t_maxseg)
1078 tp->t_flags |= TF_MAXSEGSNT;
1079 else
1080 tp->t_flags &= ~TF_MAXSEGSNT;
1081 }
1082 /*
1083 * Before ESTABLISHED, force sending of initial options
1084 * unless TCP set not to do any options.
1085 * NOTE: we assume that the IP/TCP header plus TCP options
1086 * always fit in a single mbuf, leaving room for a maximum
1087 * link header, i.e.
1088 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1089 */
1090 optlen = 0;
1091 #if INET6
1092 if (isipv6)
1093 hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1094 else
1095 #endif
1096 hdrlen = sizeof (struct tcpiphdr);
1097 if (flags & TH_SYN) {
1098 tp->snd_nxt = tp->iss;
1099 if ((tp->t_flags & TF_NOOPT) == 0) {
1100 u_short mss;
1101
1102 opt[0] = TCPOPT_MAXSEG;
1103 opt[1] = TCPOLEN_MAXSEG;
1104 mss = htons((u_short) tcp_mssopt(tp));
1105 (void)memcpy(opt + 2, &mss, sizeof(mss));
1106 optlen = TCPOLEN_MAXSEG;
1107
1108 if ((tp->t_flags & TF_REQ_SCALE) &&
1109 ((flags & TH_ACK) == 0 ||
1110 (tp->t_flags & TF_RCVD_SCALE))) {
1111 *((u_int32_t *)(void *)(opt + optlen)) = htonl(
1112 TCPOPT_NOP << 24 |
1113 TCPOPT_WINDOW << 16 |
1114 TCPOLEN_WINDOW << 8 |
1115 tp->request_r_scale);
1116 optlen += 4;
1117 }
1118 #if MPTCP
1119 if (mptcp_enable) {
1120 optlen = mptcp_setup_syn_opts(so, flags, opt,
1121 optlen);
1122 }
1123 #endif /* MPTCP */
1124 }
1125 }
1126
1127 /*
1128 RFC 3168 states that:
1129 - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
1130 to handle the TCP ECE flag, even if you also later send a
1131 non-ECN-setup SYN/SYN-ACK.
1132 - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
1133 the ip ECT flag.
1134
1135 It is not clear how the ECE flag would ever be set if you never
1136 set the IP ECT flag on outbound packets. All the same, we use
1137 the TE_SETUPSENT to indicate that we have committed to handling
1138 the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
1139 whether or not we should set the IP ECT flag on outbound packets.
1140 */
1141 /*
1142 * For a SYN-ACK, send an ECN setup SYN-ACK
1143 */
1144 if (tcp_ecn_inbound && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
1145 if ((tp->ecn_flags & TE_SETUPRECEIVED) != 0) {
1146 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1147 /* Setting TH_ECE makes this an ECN-setup SYN-ACK */
1148 flags |= TH_ECE;
1149
1150 /*
1151 * Record that we sent the ECN-setup and default to
1152 * setting IP ECT.
1153 */
1154 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1155 }
1156 else {
1157 /*
1158 * We sent an ECN-setup SYN-ACK but it was dropped.
1159 * Fallback to non-ECN-setup SYN-ACK and clear flag
1160 * that to indicate we should not send data with IP ECT set.
1161 *
1162 * Pretend we didn't receive an ECN-setup SYN.
1163 */
1164 tp->ecn_flags &= ~TE_SETUPRECEIVED;
1165 }
1166 }
1167 }
1168 else if (tcp_ecn_outbound && (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
1169 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1170 /* Setting TH_ECE and TH_CWR makes this an ECN-setup SYN */
1171 flags |= (TH_ECE | TH_CWR);
1172
1173 /*
1174 * Record that we sent the ECN-setup and default to
1175 * setting IP ECT.
1176 */
1177 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1178 }
1179 else {
1180 /*
1181 * We sent an ECN-setup SYN but it was dropped.
1182 * Fall back to no ECN and clear flag indicating
1183 * we should send data with IP ECT set.
1184 */
1185 tp->ecn_flags &= ~TE_SENDIPECT;
1186 }
1187 }
1188
1189 /*
1190 * Check if we should set the TCP CWR flag.
1191 * CWR flag is sent when we reduced the congestion window because
1192 * we received a TCP ECE or we performed a fast retransmit. We
1193 * never set the CWR flag on retransmitted packets. We only set
1194 * the CWR flag on data packets. Pure acks don't have this set.
1195 */
1196 if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
1197 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1198 flags |= TH_CWR;
1199 tp->ecn_flags &= ~TE_SENDCWR;
1200 }
1201
1202 /*
1203 * Check if we should set the TCP ECE flag.
1204 */
1205 if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
1206 flags |= TH_ECE;
1207 }
1208
1209 /*
1210 * Send a timestamp and echo-reply if this is a SYN and our side
1211 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1212 * and our peer have sent timestamps in our SYN's.
1213 */
1214 if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
1215 (flags & TH_RST) == 0 &&
1216 ((flags & TH_ACK) == 0 ||
1217 (tp->t_flags & TF_RCVD_TSTMP))) {
1218 u_int32_t *lp = (u_int32_t *)(void *)(opt + optlen);
1219
1220 /* Form timestamp option as shown in appendix A of RFC 1323. */
1221 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1222 *lp++ = htonl(tcp_now);
1223 *lp = htonl(tp->ts_recent);
1224 optlen += TCPOLEN_TSTAMP_APPA;
1225 }
1226
1227 /* Note the timestamp for receive buffer autosizing */
1228 if (tp->rfbuf_ts == 0 && (so->so_rcv.sb_flags & SB_AUTOSIZE))
1229 tp->rfbuf_ts = tcp_now;
1230
1231 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1232 /*
1233 * Tack on the SACK permitted option *last*.
1234 * And do padding of options after tacking this on.
1235 * This is because of MSS, TS, WinScale and Signatures are
1236 * all present, we have just 2 bytes left for the SACK
1237 * permitted option, which is just enough.
1238 */
1239 /*
1240 * If this is the first SYN of connection (not a SYN
1241 * ACK), include SACK permitted option. If this is a
1242 * SYN ACK, include SACK permitted option if peer has
1243 * already done so. This is only for active connect,
1244 * since the syncache takes care of the passive connect.
1245 */
1246 if ((flags & TH_SYN) &&
1247 (!(flags & TH_ACK) || (tp->t_flags & TF_SACK_PERMIT))) {
1248 u_char *bp;
1249 bp = (u_char *)opt + optlen;
1250
1251 *bp++ = TCPOPT_SACK_PERMITTED;
1252 *bp++ = TCPOLEN_SACK_PERMITTED;
1253 optlen += TCPOLEN_SACK_PERMITTED;
1254 }
1255 }
1256 #if MPTCP
1257 if (so->so_flags & SOF_MP_SUBFLOW) {
1258 /*
1259 * Its important to piggyback acks with data as ack only packets
1260 * may get lost and data packets that don't send Data ACKs
1261 * still advance the subflow level ACK and therefore make it
1262 * hard for the remote end to recover in low cwnd situations.
1263 */
1264 if (len != 0)
1265 tp->t_mpflags |= (TMPF_SEND_DSN |
1266 TMPF_MPTCP_ACKNOW);
1267 else
1268 tp->t_mpflags |= TMPF_MPTCP_ACKNOW;
1269 optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags,
1270 len, &dlenp, &finp, &dss_val, &sseqp);
1271 tp->t_mpflags &= ~TMPF_SEND_DSN;
1272 }
1273 #endif /* MPTCP */
1274
1275 if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) {
1276 /*
1277 * Send SACKs if necessary. This should be the last
1278 * option processed. Only as many SACKs are sent as
1279 * are permitted by the maximum options size.
1280 *
1281 * In general, SACK blocks consume 8*n+2 bytes.
1282 * So a full size SACK blocks option is 34 bytes
1283 * (to generate 4 SACK blocks). At a minimum,
1284 * we need 10 bytes (to generate 1 SACK block).
1285 * If TCP Timestamps (12 bytes) and TCP Signatures
1286 * (18 bytes) are both present, we'll just have
1287 * 10 bytes for SACK options 40 - (12 + 18).
1288 */
1289 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1290 (tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0 &&
1291 MAX_TCPOPTLEN - optlen - 2 >= TCPOLEN_SACK) {
1292 int nsack, padlen;
1293 u_char *bp = (u_char *)opt + optlen;
1294 u_int32_t *lp;
1295
1296 nsack = (MAX_TCPOPTLEN - optlen - 2) / TCPOLEN_SACK;
1297 nsack = min(nsack, tp->rcv_numsacks);
1298 sackoptlen = (2 + nsack * TCPOLEN_SACK);
1299
1300 /*
1301 * First we need to pad options so that the
1302 * SACK blocks can start at a 4-byte boundary
1303 * (sack option and length are at a 2 byte offset).
1304 */
1305 padlen = (MAX_TCPOPTLEN - optlen - sackoptlen) % 4;
1306 optlen += padlen;
1307 while (padlen-- > 0)
1308 *bp++ = TCPOPT_NOP;
1309
1310 tcpstat.tcps_sack_send_blocks++;
1311 *bp++ = TCPOPT_SACK;
1312 *bp++ = sackoptlen;
1313 lp = (u_int32_t *)(void *)bp;
1314 for (i = 0; i < nsack; i++) {
1315 struct sackblk sack = tp->sackblks[i];
1316 *lp++ = htonl(sack.start);
1317 *lp++ = htonl(sack.end);
1318 }
1319 optlen += sackoptlen;
1320 }
1321 }
1322
1323 /* Pad TCP options to a 4 byte boundary */
1324 if (optlen < MAX_TCPOPTLEN && (optlen % sizeof(u_int32_t))) {
1325 int pad = sizeof(u_int32_t) - (optlen % sizeof(u_int32_t));
1326 u_char *bp = (u_char *)opt + optlen;
1327
1328 optlen += pad;
1329 while (pad) {
1330 *bp++ = TCPOPT_EOL;
1331 pad--;
1332 }
1333 }
1334
1335 hdrlen += optlen;
1336
1337 #if INET6
1338 if (isipv6)
1339 ipoptlen = ip6_optlen(inp);
1340 else
1341 #endif
1342 {
1343 if (tp_inp_options) {
1344 ipoptlen = tp_inp_options->m_len -
1345 offsetof(struct ipoption, ipopt_list);
1346 } else
1347 ipoptlen = 0;
1348 }
1349 #if IPSEC
1350 ipoptlen += ipsec_optlen;
1351 #endif
1352
1353 /*
1354 * Adjust data length if insertion of options will
1355 * bump the packet length beyond the t_maxopd length.
1356 * Clear the FIN bit because we cut off the tail of
1357 * the segment.
1358 *
1359 * When doing TSO limit a burst to TCP_MAXWIN minus the
1360 * IP, TCP and Options length to keep ip->ip_len from
1361 * overflowing. Prevent the last segment from being
1362 * fractional thus making them all equal sized and set
1363 * the flag to continue sending. TSO is disabled when
1364 * IP options or IPSEC are present.
1365 */
1366 if (len + optlen + ipoptlen > tp->t_maxopd) {
1367 /*
1368 * If there is still more to send, don't close the connection.
1369 */
1370 flags &= ~TH_FIN;
1371 if (tso) {
1372 int32_t tso_maxlen;
1373
1374 tso_maxlen = tp->tso_max_segment_size ? tp->tso_max_segment_size : TCP_MAXWIN;
1375
1376 if (len > tso_maxlen - hdrlen - optlen) {
1377 len = tso_maxlen - hdrlen - optlen;
1378 len = len - (len % (tp->t_maxopd - optlen));
1379 sendalot = 1;
1380 } else if (tp->t_flags & TF_NEEDFIN)
1381 sendalot = 1;
1382 } else {
1383 len = tp->t_maxopd - optlen - ipoptlen;
1384 sendalot = 1;
1385 }
1386 }
1387 #if MPTCP
1388 /* Adjust the length in the DSS option, if it is lesser than len */
1389 if (dlenp) {
1390 /*
1391 * To test this path without SACK, artificially
1392 * decrement len with something like
1393 * if (len > 10)
1394 len -= 10;
1395 */
1396 if (ntohs(*dlenp) > len) {
1397 *dlenp = htons(len);
1398 /* Unset the FIN flag, if len was adjusted */
1399 if (finp) {
1400 *finp &= ~MDSS_F;
1401 }
1402 sendalot = 1;
1403 }
1404 }
1405 #endif /* MPTCP */
1406
1407 if (max_linkhdr + hdrlen > MCLBYTES)
1408 panic("tcphdr too big");
1409
1410 /* Check if there is enough data in the send socket
1411 * buffer to start measuring bw
1412 */
1413 if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 &&
1414 (tp->t_bwmeas != NULL) &&
1415 (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 &&
1416 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >=
1417 tp->t_bwmeas->bw_minsize) {
1418 tp->t_bwmeas->bw_size = min((so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)),
1419 tp->t_bwmeas->bw_maxsize);
1420 tp->t_flagsext |= TF_BWMEAS_INPROGRESS;
1421 tp->t_bwmeas->bw_start = tp->snd_max;
1422 tp->t_bwmeas->bw_ts = tcp_now;
1423 }
1424
1425 VERIFY(inp->inp_flowhash != 0);
1426 /*
1427 * Grab a header mbuf, attaching a copy of data to
1428 * be transmitted, and initialize the header from
1429 * the template for sends on this connection.
1430 */
1431 if (len) {
1432 if (tp->t_force && len == 1)
1433 tcpstat.tcps_sndprobe++;
1434 else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1435 tcpstat.tcps_sndrexmitpack++;
1436 tcpstat.tcps_sndrexmitbyte += len;
1437 if (nstat_collect) {
1438 nstat_route_tx(inp->inp_route.ro_rt, 1,
1439 len, NSTAT_TX_FLAG_RETRANSMIT);
1440 INP_ADD_STAT(inp, cell, wifi, txpackets, 1);
1441 INP_ADD_STAT(inp, cell, wifi, txbytes, len);
1442 tp->t_stat.txretransmitbytes += len;
1443 }
1444 } else {
1445 tcpstat.tcps_sndpack++;
1446 tcpstat.tcps_sndbyte += len;
1447
1448 if (nstat_collect) {
1449 INP_ADD_STAT(inp, cell, wifi, txpackets, 1);
1450 INP_ADD_STAT(inp, cell, wifi, txbytes, len);
1451 }
1452 }
1453 #if MPTCP
1454 if (tp->t_mpflags & TMPF_MPTCP_TRUE) {
1455 tcpstat.tcps_mp_sndpacks++;
1456 tcpstat.tcps_mp_sndbytes += len;
1457 }
1458 #endif /* MPTCP */
1459 /*
1460 * try to use the new interface that allocates all
1461 * the necessary mbuf hdrs under 1 mbuf lock and
1462 * avoids rescanning the socket mbuf list if
1463 * certain conditions are met. This routine can't
1464 * be used in the following cases...
1465 * 1) the protocol headers exceed the capacity of
1466 * of a single mbuf header's data area (no cluster attached)
1467 * 2) the length of the data being transmitted plus
1468 * the protocol headers fits into a single mbuf header's
1469 * data area (no cluster attached)
1470 */
1471 m = NULL;
1472
1473 /* minimum length we are going to allocate */
1474 allocated_len = MHLEN;
1475 if (MHLEN < hdrlen + max_linkhdr) {
1476 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1477 if (m == NULL) {
1478 error = ENOBUFS;
1479 goto out;
1480 }
1481 MCLGET(m, M_DONTWAIT);
1482 if ((m->m_flags & M_EXT) == 0) {
1483 m_freem(m);
1484 error = ENOBUFS;
1485 goto out;
1486 }
1487 m->m_data += max_linkhdr;
1488 m->m_len = hdrlen;
1489 allocated_len = MCLBYTES;
1490 }
1491 if (len <= allocated_len - hdrlen - max_linkhdr) {
1492 if (m == NULL) {
1493 VERIFY(allocated_len <= MHLEN);
1494 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1495 if (m == NULL) {
1496 error = ENOBUFS;
1497 goto out;
1498 }
1499 m->m_data += max_linkhdr;
1500 m->m_len = hdrlen;
1501 }
1502 /* makes sure we still have data left to be sent at this point */
1503 if (so->so_snd.sb_mb == NULL || off < 0) {
1504 if (m != NULL) m_freem(m);
1505 error = 0; /* should we return an error? */
1506 goto out;
1507 }
1508 m_copydata(so->so_snd.sb_mb, off, (int) len,
1509 mtod(m, caddr_t) + hdrlen);
1510 m->m_len += len;
1511 } else {
1512 uint32_t copymode;
1513 /*
1514 * Retain packet header metadata at the socket
1515 * buffer if this is is an MPTCP subflow,
1516 * otherwise move it.
1517 */
1518 copymode = M_COPYM_MOVE_HDR;
1519 #if MPTCP
1520 if ((tp->t_mpflags & TMPF_MPTCP_TRUE) ||
1521 (tp->t_mpflags & TMPF_TCP_FALLBACK)) {
1522 copymode = M_COPYM_NOOP_HDR;
1523 }
1524 #endif /* MPTCP */
1525 if (m != NULL) {
1526 m->m_next = m_copym_mode(so->so_snd.sb_mb, off,
1527 (int) len, M_DONTWAIT, copymode);
1528 if (m->m_next == NULL) {
1529 (void) m_free(m);
1530 error = ENOBUFS;
1531 goto out;
1532 }
1533 } else {
1534 /*
1535 * determine whether the mbuf pointer and
1536 * offset passed back by the 'last' call to
1537 * m_copym_with_hdrs are still valid... if the
1538 * head of the socket chain has changed (due
1539 * to an incoming ACK for instance), or the
1540 * offset into the chain we just computed is
1541 * different from the one last returned by
1542 * m_copym_with_hdrs (perhaps we're re-
1543 * transmitting a packet sent earlier), then
1544 * we can't pass the mbuf pointer and offset
1545 * into it as valid hints for m_copym_with_hdrs
1546 * to use (if valid, these hints allow
1547 * m_copym_with_hdrs to avoid rescanning from
1548 * the beginning of the socket buffer mbuf list.
1549 *
1550 * Setting the mbuf pointer to NULL is
1551 * sufficient to disable the hint mechanism.
1552 */
1553 if (m_head != so->so_snd.sb_mb || sack_rxmit ||
1554 last_off != off)
1555 m_lastm = NULL;
1556 last_off = off + len;
1557 m_head = so->so_snd.sb_mb;
1558
1559 /*
1560 * make sure we still have data left
1561 * to be sent at this point
1562 */
1563 if (m_head == NULL) {
1564 error = 0; /* should we return an error? */
1565 goto out;
1566 }
1567
1568 /*
1569 * m_copym_with_hdrs will always return the
1570 * last mbuf pointer and the offset into it that
1571 * it acted on to fullfill the current request,
1572 * whether a valid 'hint' was passed in or not.
1573 */
1574 if ((m = m_copym_with_hdrs(so->so_snd.sb_mb,
1575 off, len, M_DONTWAIT, &m_lastm, &m_off,
1576 copymode)) == NULL) {
1577 error = ENOBUFS;
1578 goto out;
1579 }
1580 m->m_data += max_linkhdr;
1581 m->m_len = hdrlen;
1582 }
1583 }
1584 /*
1585 * If we're sending everything we've got, set PUSH.
1586 * (This will keep happy those implementations which only
1587 * give data to the user when a buffer fills or
1588 * a PUSH comes in.)
1589 */
1590 if (off + len == so->so_snd.sb_cc)
1591 flags |= TH_PUSH;
1592 } else {
1593 if (tp->t_flags & TF_ACKNOW)
1594 tcpstat.tcps_sndacks++;
1595 else if (flags & (TH_SYN|TH_FIN|TH_RST))
1596 tcpstat.tcps_sndctrl++;
1597 else if (SEQ_GT(tp->snd_up, tp->snd_una))
1598 tcpstat.tcps_sndurg++;
1599 else
1600 tcpstat.tcps_sndwinup++;
1601
1602 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1603 if (m == NULL) {
1604 error = ENOBUFS;
1605 goto out;
1606 }
1607 if (MHLEN < (hdrlen + max_linkhdr)) {
1608 MCLGET(m, M_DONTWAIT);
1609 if ((m->m_flags & M_EXT) == 0) {
1610 m_freem(m);
1611 error = ENOBUFS;
1612 goto out;
1613 }
1614 }
1615 m->m_data += max_linkhdr;
1616 m->m_len = hdrlen;
1617 }
1618 m->m_pkthdr.rcvif = 0;
1619 #if MPTCP
1620 /* Before opt is copied to the mbuf, set the csum field */
1621 mptcp_output_csum(tp, m, len, hdrlen, dss_val, sseqp);
1622 #endif /* MPTCP */
1623 #if CONFIG_MACF_NET
1624 mac_mbuf_label_associate_inpcb(inp, m);
1625 #endif
1626 #if INET6
1627 if (isipv6) {
1628 ip6 = mtod(m, struct ip6_hdr *);
1629 th = (struct tcphdr *)(void *)(ip6 + 1);
1630 tcp_fillheaders(tp, ip6, th);
1631 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1632 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1633 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
1634 }
1635 svc_flags |= PKT_SCF_IPV6;
1636 } else
1637 #endif /* INET6 */
1638 {
1639 ip = mtod(m, struct ip *);
1640 ipov = (struct ipovly *)ip;
1641 th = (struct tcphdr *)(void *)(ip + 1);
1642 /* this picks up the pseudo header (w/o the length) */
1643 tcp_fillheaders(tp, ip, th);
1644 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1645 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1646 ip->ip_tos = IPTOS_ECN_ECT0;
1647 }
1648 }
1649
1650 /*
1651 * Fill in fields, remembering maximum advertised
1652 * window for use in delaying messages about window sizes.
1653 * If resending a FIN, be sure not to use a new sequence number.
1654 */
1655 if (flags & TH_FIN && (tp->t_flags & TF_SENTFIN) &&
1656 tp->snd_nxt == tp->snd_max)
1657 tp->snd_nxt--;
1658 /*
1659 * If we are doing retransmissions, then snd_nxt will
1660 * not reflect the first unsent octet. For ACK only
1661 * packets, we do not want the sequence number of the
1662 * retransmitted packet, we want the sequence number
1663 * of the next unsent octet. So, if there is no data
1664 * (and no SYN or FIN), use snd_max instead of snd_nxt
1665 * when filling in ti_seq. But if we are in persist
1666 * state, snd_max might reflect one byte beyond the
1667 * right edge of the window, so use snd_nxt in that
1668 * case, since we know we aren't doing a retransmission.
1669 * (retransmit and persist are mutually exclusive...)
1670 */
1671 if (sack_rxmit == 0) {
1672 if (len || (flags & (TH_SYN|TH_FIN)) || tp->t_timer[TCPT_PERSIST])
1673 th->th_seq = htonl(tp->snd_nxt);
1674 else
1675 th->th_seq = htonl(tp->snd_max);
1676 } else {
1677 th->th_seq = htonl(p->rxmit);
1678 p->rxmit += len;
1679 tp->sackhint.sack_bytes_rexmit += len;
1680 }
1681 th->th_ack = htonl(tp->rcv_nxt);
1682 tp->last_ack_sent = tp->rcv_nxt;
1683
1684 if (optlen) {
1685 bcopy(opt, th + 1, optlen);
1686 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1687 }
1688 th->th_flags = flags;
1689 /*
1690 * Calculate receive window. Don't shrink window,
1691 * but avoid silly window syndrome.
1692 */
1693 if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && recwin < (int)tp->t_maxseg)
1694 recwin = 0;
1695 if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt))
1696 recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt);
1697 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) {
1698 if (recwin > (int32_t)slowlink_wsize)
1699 recwin = slowlink_wsize;
1700 }
1701
1702 #if TRAFFIC_MGT
1703 if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) {
1704 if (tcp_recv_throttle(tp)) {
1705 uint32_t min_iaj_win =
1706 tcp_min_iaj_win * tp->t_maxseg;
1707 if (tp->iaj_rwintop == 0 ||
1708 SEQ_LT(tp->iaj_rwintop, tp->rcv_adv))
1709 tp->iaj_rwintop = tp->rcv_adv;
1710 if (SEQ_LT(tp->iaj_rwintop,
1711 tp->rcv_nxt + min_iaj_win))
1712 tp->iaj_rwintop = tp->rcv_nxt + min_iaj_win;
1713 recwin = min(tp->iaj_rwintop - tp->rcv_nxt, recwin);
1714 }
1715 }
1716 #endif /* TRAFFIC_MGT */
1717
1718 if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
1719 recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
1720 th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
1721
1722 /*
1723 * Adjust the RXWIN0SENT flag - indicate that we have advertised
1724 * a 0 window. This may cause the remote transmitter to stall. This
1725 * flag tells soreceive() to disable delayed acknowledgements when
1726 * draining the buffer. This can occur if the receiver is attempting
1727 * to read more data then can be buffered prior to transmitting on
1728 * the connection.
1729 */
1730 if (th->th_win == 0)
1731 tp->t_flags |= TF_RXWIN0SENT;
1732 else
1733 tp->t_flags &= ~TF_RXWIN0SENT;
1734 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
1735 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
1736 th->th_flags |= TH_URG;
1737 } else {
1738 /*
1739 * If no urgent pointer to send, then we pull
1740 * the urgent pointer to the left edge of the send window
1741 * so that it doesn't drift into the send window on sequence
1742 * number wraparound.
1743 */
1744 tp->snd_up = tp->snd_una; /* drag it along */
1745 }
1746
1747 /*
1748 * Put TCP length in extended header, and then
1749 * checksum extended header and data.
1750 */
1751 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
1752 #if INET6
1753 if (isipv6) {
1754 /*
1755 * ip6_plen is not need to be filled now, and will be filled
1756 * in ip6_output.
1757 */
1758 m->m_pkthdr.csum_flags = CSUM_TCPIPV6;
1759 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1760 if (len + optlen)
1761 th->th_sum = in_addword(th->th_sum,
1762 htons((u_short)(optlen + len)));
1763 }
1764 else
1765 #endif /* INET6 */
1766 {
1767 m->m_pkthdr.csum_flags = CSUM_TCP;
1768 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1769 if (len + optlen)
1770 th->th_sum = in_addword(th->th_sum,
1771 htons((u_short)(optlen + len)));
1772 }
1773
1774 /*
1775 * Enable TSO and specify the size of the segments.
1776 * The TCP pseudo header checksum is always provided.
1777 */
1778 if (tso) {
1779 #if INET6
1780 if (isipv6)
1781 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV6;
1782 else
1783 #endif /* INET6 */
1784 m->m_pkthdr.csum_flags |= CSUM_TSO_IPV4;
1785
1786 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
1787 } else {
1788 m->m_pkthdr.tso_segsz = 0;
1789 }
1790
1791 /*
1792 * In transmit state, time the transmission and arrange for
1793 * the retransmit. In persist state, just set snd_max.
1794 */
1795 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
1796 tcp_seq startseq = tp->snd_nxt;
1797
1798 /*
1799 * Advance snd_nxt over sequence space of this segment.
1800 */
1801 if (flags & (TH_SYN|TH_FIN)) {
1802 if (flags & TH_SYN)
1803 tp->snd_nxt++;
1804 if ((flags & TH_FIN) &&
1805 !(tp->t_flags & TF_SENTFIN)) {
1806 tp->snd_nxt++;
1807 tp->t_flags |= TF_SENTFIN;
1808 }
1809 }
1810 if (sack_rxmit)
1811 goto timer;
1812 tp->snd_nxt += len;
1813 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
1814 tp->snd_max = tp->snd_nxt;
1815 /*
1816 * Time this transmission if not a retransmission and
1817 * not currently timing anything.
1818 */
1819 if (tp->t_rtttime == 0) {
1820 tp->t_rtttime = tcp_now;
1821 tp->t_rtseq = startseq;
1822 tcpstat.tcps_segstimed++;
1823 }
1824 }
1825
1826 /*
1827 * Set retransmit timer if not currently set,
1828 * and not doing an ack or a keep-alive probe.
1829 * Initial value for retransmit timer is smoothed
1830 * round-trip time + 2 * round-trip time variance.
1831 * Initialize shift counter which is used for backoff
1832 * of retransmit time.
1833 */
1834 timer:
1835 if (tp->t_timer[TCPT_REXMT] == 0 &&
1836 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
1837 tp->snd_nxt != tp->snd_una ||
1838 (flags & TH_FIN))) {
1839 if (tp->t_timer[TCPT_PERSIST]) {
1840 tp->t_timer[TCPT_PERSIST] = 0;
1841 tp->t_rxtshift = 0;
1842 tp->t_rxtstart = 0;
1843 tp->t_persist_stop = 0;
1844 }
1845 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1846 }
1847 } else {
1848 /*
1849 * Persist case, update snd_max but since we are in
1850 * persist mode (no window) we do not update snd_nxt.
1851 */
1852 int xlen = len;
1853 if (flags & TH_SYN)
1854 ++xlen;
1855 if ((flags & TH_FIN) &&
1856 !(tp->t_flags & TF_SENTFIN)) {
1857 ++xlen;
1858 tp->t_flags |= TF_SENTFIN;
1859 }
1860 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
1861 tp->snd_max = tp->snd_nxt + len;
1862 }
1863
1864 #if TCPDEBUG
1865 /*
1866 * Trace.
1867 */
1868 if (so_options & SO_DEBUG)
1869 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
1870 #endif
1871
1872 /*
1873 * Fill in IP length and desired time to live and
1874 * send to IP level. There should be a better way
1875 * to handle ttl and tos; we could keep them in
1876 * the template, but need a way to checksum without them.
1877 */
1878 #if INET6
1879 /*
1880 * m->m_pkthdr.len should have been set before cksum calcuration,
1881 * because in6_cksum() need it.
1882 */
1883 if (isipv6) {
1884 /*
1885 * we separately set hoplimit for every segment, since the
1886 * user might want to change the value via setsockopt.
1887 * Also, desired default hop limit might be changed via
1888 * Neighbor Discovery.
1889 */
1890 ip6->ip6_hlim = in6_selecthlim(inp, inp->in6p_route.ro_rt ?
1891 inp->in6p_route.ro_rt->rt_ifp : NULL);
1892
1893 /* TODO: IPv6 IP6TOS_ECT bit on */
1894 KERNEL_DEBUG(DBG_LAYER_BEG,
1895 ((inp->inp_fport << 16) | inp->inp_lport),
1896 (((inp->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
1897 (inp->in6p_faddr.s6_addr16[0] & 0xffff)),
1898 sendalot,0,0);
1899 } else
1900 #endif /* INET6 */
1901 {
1902 ip->ip_len = m->m_pkthdr.len;
1903 ip->ip_ttl = inp->inp_ip_ttl; /* XXX */
1904 ip->ip_tos |= (inp->inp_ip_tos & ~IPTOS_ECN_MASK);/* XXX */
1905 KERNEL_DEBUG(DBG_LAYER_BEG,
1906 ((inp->inp_fport << 16) | inp->inp_lport),
1907 (((inp->inp_laddr.s_addr & 0xffff) << 16) |
1908 (inp->inp_faddr.s_addr & 0xffff)), 0,0,0);
1909 }
1910
1911 /*
1912 * See if we should do MTU discovery.
1913 * Look at the flag updated on the following criterias:
1914 * 1) Path MTU discovery is authorized by the sysctl
1915 * 2) The route isn't set yet (unlikely but could happen)
1916 * 3) The route is up
1917 * 4) the MTU is not locked (if it is, then discovery has been
1918 * disabled for that route)
1919 */
1920 #if INET6
1921 if (!isipv6)
1922 #endif /* INET6 */
1923 if (path_mtu_discovery && (tp->t_flags & TF_PMTUD))
1924 ip->ip_off |= IP_DF;
1925
1926 #if IPSEC
1927 if (ipsec_bypass == 0)
1928 ipsec_setsocket(m, so);
1929 #endif /*IPSEC*/
1930
1931 /*
1932 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
1933 */
1934 lost = 0;
1935
1936 /*
1937 * Embed the flow hash in pkt hdr and mark the packet as
1938 * capable of flow controlling
1939 */
1940 m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB;
1941 m->m_pkthdr.pkt_flowid = inp->inp_flowhash;
1942 m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC;
1943 #if MPTCP
1944 /* Disable flow advisory when using MPTCP. */
1945 if (!(tp->t_mpflags & TMPF_MPTCP_TRUE))
1946 #endif /* MPTCP */
1947 m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV;
1948 m->m_pkthdr.pkt_proto = IPPROTO_TCP;
1949
1950 m->m_nextpkt = NULL;
1951
1952 if (inp->inp_last_outifp != NULL &&
1953 !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) {
1954 /* Hint to prioritize this packet if
1955 * 1. if the packet has no data
1956 * 2. the interface supports transmit-start model and did
1957 * not disable ACK prioritization.
1958 * 3. Only ACK flag is set.
1959 * 4. there is no outstanding data on this connection.
1960 */
1961 if (tcp_prioritize_acks != 0 && len == 0 &&
1962 (inp->inp_last_outifp->if_eflags &
1963 (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART &&
1964 th->th_flags == TH_ACK && tp->snd_una == tp->snd_max &&
1965 tp->t_timer[TCPT_REXMT] == 0) {
1966 svc_flags |= PKT_SCF_TCP_ACK;
1967 }
1968 set_packet_service_class(m, so, MBUF_SC_UNSPEC, svc_flags);
1969 }
1970
1971 tp->t_pktlist_sentlen += len;
1972 tp->t_lastchain++;
1973
1974 #if INET6
1975 if (isipv6) {
1976 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
1977 struct ip6 *, ip6, struct tcpcb *, tp, struct tcphdr *,
1978 th);
1979 } else
1980 #endif /* INET6 */
1981 {
1982 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, inp,
1983 struct ip *, ip, struct tcpcb *, tp, struct tcphdr *, th);
1984 }
1985
1986 if (tp->t_pktlist_head != NULL) {
1987 tp->t_pktlist_tail->m_nextpkt = m;
1988 tp->t_pktlist_tail = m;
1989 } else {
1990 packchain_newlist++;
1991 tp->t_pktlist_head = tp->t_pktlist_tail = m;
1992 }
1993
1994 if ((lro_ackmore) && (!sackoptlen) && (!tp->t_timer[TCPT_PERSIST]) &&
1995 ((th->th_flags & TH_ACK) == TH_ACK) && (!len) &&
1996 (tp->t_state == TCPS_ESTABLISHED)) {
1997 /* For a pure ACK, see if you need to send more of them */
1998 mnext = tcp_send_lroacks(tp, m, th);
1999 if (mnext) {
2000 tp->t_pktlist_tail->m_nextpkt = mnext;
2001 if (mnext->m_nextpkt == NULL) {
2002 tp->t_pktlist_tail = mnext;
2003 tp->t_lastchain++;
2004 } else {
2005 struct mbuf *tail, *next;
2006 next = mnext->m_nextpkt;
2007 tail = next->m_nextpkt;
2008 while (tail) {
2009 next = tail;
2010 tail = tail->m_nextpkt;
2011 tp->t_lastchain++;
2012 }
2013 tp->t_pktlist_tail = next;
2014 }
2015 }
2016 }
2017
2018 if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
2019 (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
2020 (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || tp->t_force != 0 ||
2021 tp->t_lastchain >= tcp_packet_chaining) {
2022 error = 0;
2023
2024 /*
2025 * Reset the stack memory of offset as the socket
2026 * may get unlocked
2027 */
2028 m_lastm = NULL;
2029 while (inp->inp_sndinprog_cnt == 0 &&
2030 tp->t_pktlist_head != NULL) {
2031 packetlist = tp->t_pktlist_head;
2032 packchain_listadd = tp->t_lastchain;
2033 packchain_sent++;
2034 lost = tp->t_pktlist_sentlen;
2035 TCP_PKTLIST_CLEAR(tp);
2036
2037 error = tcp_ip_output(so, tp, packetlist,
2038 packchain_listadd, tp_inp_options,
2039 (so_options & SO_DONTROUTE),
2040 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
2041 #if INET6
2042 isipv6);
2043 #else /* INET6 */
2044 0);
2045 #endif /* !INET6 */
2046
2047 if (error) {
2048 /*
2049 * Take into account the rest of unsent
2050 * packets in the packet list for this tcp
2051 * into "lost", since we're about to free
2052 * the whole list below.
2053 */
2054 lost += tp->t_pktlist_sentlen;
2055 break;
2056 } else {
2057 lost = 0;
2058 }
2059 }
2060 /* tcp was closed while we were in ip; resume close */
2061 if (inp->inp_sndinprog_cnt == 0 &&
2062 (tp->t_flags & TF_CLOSING)) {
2063 tp->t_flags &= ~TF_CLOSING;
2064 (void) tcp_close(tp);
2065 return (0);
2066 }
2067 } else {
2068 error = 0;
2069 packchain_looped++;
2070 tcpstat.tcps_sndtotal++;
2071
2072 goto again;
2073 }
2074 if (error) {
2075 /*
2076 * Assume that the packets were lost, so back out the
2077 * sequence number advance, if any. Note that the "lost"
2078 * variable represents the amount of user data sent during
2079 * the recent call to ip_output_list() plus the amount of
2080 * user data in the packet list for this tcp at the moment.
2081 */
2082 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
2083 /*
2084 * No need to check for TH_FIN here because
2085 * the TF_SENTFIN flag handles that case.
2086 */
2087 if ((flags & TH_SYN) == 0) {
2088 if (sack_rxmit) {
2089 if (SEQ_GT((p->rxmit - lost),
2090 tp->snd_una)) {
2091 p->rxmit -= lost;
2092 } else {
2093 lost = p->rxmit - tp->snd_una;
2094 p->rxmit = tp->snd_una;
2095 }
2096 tp->sackhint.sack_bytes_rexmit -= lost;
2097 } else {
2098 if (SEQ_GT((tp->snd_nxt - lost),
2099 tp->snd_una))
2100 tp->snd_nxt -= lost;
2101 else
2102 tp->snd_nxt = tp->snd_una;
2103 }
2104 }
2105 }
2106 out:
2107 if (tp->t_pktlist_head != NULL)
2108 m_freem_list(tp->t_pktlist_head);
2109 TCP_PKTLIST_CLEAR(tp);
2110
2111 if (error == ENOBUFS) {
2112 if (!tp->t_timer[TCPT_REXMT] &&
2113 !tp->t_timer[TCPT_PERSIST])
2114 tp->t_timer[TCPT_REXMT] =
2115 OFFSET_FROM_START(tp, tp->t_rxtcur);
2116
2117 tp->snd_cwnd = tp->t_maxseg;
2118 tp->t_bytes_acked = 0;
2119
2120 tcp_check_timer_state(tp);
2121 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2122
2123 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
2124 struct tcpcb *, tp, struct tcphdr *, NULL,
2125 int32_t, TCP_CC_OUTPUT_ERROR);
2126 return (0);
2127 }
2128 if (error == EMSGSIZE) {
2129 /*
2130 * ip_output() will have already fixed the route
2131 * for us. tcp_mtudisc() will, as its last action,
2132 * initiate retransmission, so it is important to
2133 * not do so here.
2134 *
2135 * If TSO was active we either got an interface
2136 * without TSO capabilits or TSO was turned off.
2137 * Disable it for this connection as too and
2138 * immediatly retry with MSS sized segments generated
2139 * by this function.
2140 */
2141 if (tso)
2142 tp->t_flags &= ~TF_TSO;
2143
2144 tcp_mtudisc(inp, 0);
2145 tcp_check_timer_state(tp);
2146
2147 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2148 return 0;
2149 }
2150 /*
2151 * Unless this is due to interface restriction policy,
2152 * treat EHOSTUNREACH/ENETDOWN as a soft error.
2153 */
2154 if ((error == EHOSTUNREACH || error == ENETDOWN) &&
2155 TCPS_HAVERCVDSYN(tp->t_state) &&
2156 !((inp->inp_flags & INP_NO_IFT_CELLULAR) &&
2157 inp->inp_last_outifp != NULL &&
2158 IFNET_IS_CELLULAR(inp->inp_last_outifp))) {
2159 tp->t_softerror = error;
2160 tcp_check_timer_state(tp);
2161 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
2162 0, 0, 0, 0, 0);
2163 return (0);
2164 }
2165 tcp_check_timer_state(tp);
2166 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
2167 return (error);
2168 }
2169
2170 tcpstat.tcps_sndtotal++;
2171
2172 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
2173 if (sendalot)
2174 goto again;
2175
2176 tcp_check_timer_state(tp);
2177 return (0);
2178 }
2179
2180 static int
2181 tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
2182 int cnt, struct mbuf *opt, int flags, int sack_in_progress, int recwin,
2183 boolean_t isipv6)
2184 {
2185 int error = 0;
2186 boolean_t chain;
2187 boolean_t unlocked = FALSE;
2188 boolean_t ifdenied = FALSE;
2189 struct inpcb *inp = tp->t_inpcb;
2190 struct ip_out_args ipoa =
2191 { IFSCOPE_NONE, { 0 }, IPOAF_SELECT_SRCIF|IPOAF_BOUND_SRCADDR, 0 };
2192 struct route ro;
2193 struct ifnet *outif = NULL;
2194 #if INET6
2195 struct ip6_out_args ip6oa =
2196 { IFSCOPE_NONE, { 0 }, IP6OAF_SELECT_SRCIF|IP6OAF_BOUND_SRCADDR, 0 };
2197 struct route_in6 ro6;
2198 struct flowadv *adv =
2199 (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv);
2200 #else /* INET6 */
2201 struct flowadv *adv = &ipoa.ipoa_flowadv;
2202 #endif /* !INET6 */
2203
2204 /* If socket was bound to an ifindex, tell ip_output about it */
2205 if (inp->inp_flags & INP_BOUND_IF) {
2206 #if INET6
2207 if (isipv6) {
2208 ip6oa.ip6oa_boundif = inp->inp_boundifp->if_index;
2209 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
2210 } else
2211 #endif /* INET6 */
2212 {
2213 ipoa.ipoa_boundif = inp->inp_boundifp->if_index;
2214 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
2215 }
2216 }
2217
2218 if (inp->inp_flags & INP_NO_IFT_CELLULAR) {
2219 #if INET6
2220 if (isipv6)
2221 ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR;
2222 else
2223 #endif /* INET6 */
2224 ipoa.ipoa_flags |= IPOAF_NO_CELLULAR;
2225 }
2226 #if INET6
2227 if (isipv6)
2228 flags |= IPV6_OUTARGS;
2229 else
2230 #endif /* INET6 */
2231 flags |= IP_OUTARGS;
2232
2233 /* Copy the cached route and take an extra reference */
2234 #if INET6
2235 if (isipv6)
2236 in6p_route_copyout(inp, &ro6);
2237 else
2238 #endif /* INET6 */
2239 inp_route_copyout(inp, &ro);
2240
2241 /*
2242 * Data sent (as far as we can tell).
2243 * If this advertises a larger window than any other segment,
2244 * then remember the size of the advertised window.
2245 * Make sure ACK/DELACK conditions are cleared before
2246 * we unlock the socket.
2247 */
2248 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
2249 tp->rcv_adv = tp->rcv_nxt + recwin;
2250 tp->last_ack_sent = tp->rcv_nxt;
2251 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
2252 tp->t_timer[TCPT_DELACK] = 0;
2253 tp->t_unacksegs = 0;
2254
2255 /* Increment the count of outstanding send operations */
2256 inp->inp_sndinprog_cnt++;
2257
2258 /*
2259 * If allowed, unlock TCP socket while in IP
2260 * but only if the connection is established and
2261 * in a normal mode where reentrancy on the tcpcb won't be
2262 * an issue:
2263 * - there is no SACK episode
2264 * - we're not in Fast Recovery mode
2265 * - if we're not sending from an upcall.
2266 */
2267 if (tcp_output_unlocked && !so->so_upcallusecount &&
2268 (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) &&
2269 ((tp->t_flags & TF_FASTRECOVERY) == 0)) {
2270
2271 unlocked = TRUE;
2272 socket_unlock(so, 0);
2273 }
2274
2275 /*
2276 * Don't send down a chain of packets when:
2277 * - TCP chaining is disabled
2278 * - there is an IPsec rule set
2279 * - there is a non default rule set for the firewall
2280 */
2281
2282 chain = tcp_packet_chaining > 1
2283 #if IPSEC
2284 && ipsec_bypass
2285 #endif
2286 #if IPFIREWALL
2287 && (fw_enable == 0 || fw_bypass)
2288 #endif
2289 ; // I'm important, not extraneous
2290
2291
2292 while (pkt != NULL) {
2293 struct mbuf *npkt = pkt->m_nextpkt;
2294
2295 if (!chain) {
2296 pkt->m_nextpkt = NULL;
2297 /*
2298 * If we are not chaining, make sure to set the packet
2299 * list count to 0 so that IP takes the right path;
2300 * this is important for cases such as IPSec where a
2301 * single mbuf might result in multiple mbufs as part
2302 * of the encapsulation. If a non-zero count is passed
2303 * down to IP, the head of the chain might change and
2304 * we could end up skipping it (thus generating bogus
2305 * packets). Fixing it in IP would be desirable, but
2306 * for now this would do it.
2307 */
2308 cnt = 0;
2309 }
2310 #if INET6
2311 if (isipv6) {
2312 error = ip6_output_list(pkt, cnt,
2313 inp->in6p_outputopts, &ro6, flags, NULL, NULL,
2314 &ip6oa);
2315 ifdenied = (ip6oa.ip6oa_retflags & IP6OARF_IFDENIED);
2316 } else {
2317 #endif /* INET6 */
2318 error = ip_output_list(pkt, cnt, opt, &ro, flags, NULL,
2319 &ipoa);
2320 ifdenied = (ipoa.ipoa_retflags & IPOARF_IFDENIED);
2321 }
2322
2323 if (chain || error) {
2324 /*
2325 * If we sent down a chain then we are done since
2326 * the callee had taken care of everything; else
2327 * we need to free the rest of the chain ourselves.
2328 */
2329 if (!chain)
2330 m_freem_list(npkt);
2331 break;
2332 }
2333 pkt = npkt;
2334 }
2335
2336 if (unlocked)
2337 socket_lock(so, 0);
2338
2339 /*
2340 * Enter flow controlled state if the connection is established
2341 * and is not in recovery.
2342 *
2343 * A connection will enter suspended state even if it is in
2344 * recovery.
2345 */
2346 if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) ||
2347 adv->code == FADV_SUSPENDED) &&
2348 !(tp->t_flags & TF_CLOSING) &&
2349 tp->t_state == TCPS_ESTABLISHED) {
2350 int rc;
2351 rc = inp_set_fc_state(inp, adv->code);
2352
2353 if (rc == 1)
2354 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
2355 struct tcpcb *, tp, struct tcphdr *, NULL,
2356 int32_t, ((adv->code == FADV_FLOW_CONTROLLED) ?
2357 TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND));
2358 }
2359
2360 /*
2361 * When an interface queue gets suspended, some of the
2362 * packets are dropped. Return ENOBUFS, to update the
2363 * pcb state.
2364 */
2365 if (adv->code == FADV_SUSPENDED)
2366 error = ENOBUFS;
2367
2368 VERIFY(inp->inp_sndinprog_cnt > 0);
2369 if ( --inp->inp_sndinprog_cnt == 0)
2370 inp->inp_flags &= ~(INP_FC_FEEDBACK);
2371
2372 #if INET6
2373 if (isipv6) {
2374 if (ro6.ro_rt != NULL && (outif = ro6.ro_rt->rt_ifp) !=
2375 inp->in6p_last_outifp)
2376 inp->in6p_last_outifp = outif;
2377 } else
2378 #endif /* INET6 */
2379 if (ro.ro_rt != NULL && (outif = ro.ro_rt->rt_ifp) !=
2380 inp->inp_last_outifp)
2381 inp->inp_last_outifp = outif;
2382
2383 if (error != 0 && ifdenied && (inp->inp_flags & INP_NO_IFT_CELLULAR))
2384 soevent(inp->inp_socket,
2385 (SO_FILT_HINT_LOCKED|SO_FILT_HINT_IFDENIED));
2386
2387 /* Synchronize cached PCB route & options */
2388 #if INET6
2389 if (isipv6)
2390 in6p_route_copyin(inp, &ro6);
2391 else
2392 #endif /* INET6 */
2393 inp_route_copyin(inp, &ro);
2394
2395 if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 &&
2396 tp->t_inpcb->inp_route.ro_rt != NULL) {
2397 /* If we found the route and there is an rtt on it
2398 * reset the retransmit timer
2399 */
2400 tcp_getrt_rtt(tp, tp->t_inpcb->in6p_route.ro_rt);
2401 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
2402 }
2403 return (error);
2404 }
2405
2406 void
2407 tcp_setpersist(tp)
2408 register struct tcpcb *tp;
2409 {
2410 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
2411
2412 /* If a PERSIST_TIMER option was set we will limit the
2413 * time the persist timer will be active for that connection
2414 * in order to avoid DOS by using zero window probes.
2415 * see rdar://5805356
2416 */
2417
2418 if ((tp->t_persist_timeout != 0) &&
2419 (tp->t_timer[TCPT_PERSIST] == 0) &&
2420 (tp->t_persist_stop == 0)) {
2421 tp->t_persist_stop = tcp_now + tp->t_persist_timeout;
2422 }
2423
2424 /*
2425 * Start/restart persistance timer.
2426 */
2427 TCPT_RANGESET(tp->t_timer[TCPT_PERSIST],
2428 t * tcp_backoff[tp->t_rxtshift],
2429 TCPTV_PERSMIN, TCPTV_PERSMAX, 0);
2430 tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]);
2431
2432 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
2433 tp->t_rxtshift++;
2434 }
2435
2436 /*
2437 * Send as many acks as data coalesced. Every other packet when stretch
2438 * ACK is not enabled. Every 8 packets, if stretch ACK is enabled.
2439 */
2440 static struct mbuf*
2441 tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th)
2442 {
2443 struct mbuf *mnext = NULL, *ack_chain = NULL, *tail = NULL;
2444 int count = 0;
2445 tcp_seq org_ack = ntohl(th->th_ack);
2446 tcp_seq prev_ack = 0;
2447 int tack_offset = 28; /* XXX IPv6 and IP options not supported */
2448 int twin_offset = 34; /* XXX IPv6 and IP options not supported */
2449 int ack_size = (tp->t_flags & TF_STRETCHACK) ?
2450 (maxseg_unacked * tp->t_maxseg) : (tp->t_maxseg << 1);
2451 int segs_acked = (tp->t_flags & TF_STRETCHACK) ? maxseg_unacked : 2;
2452 struct mbuf *prev_ack_pkt = NULL;
2453 struct socket *so = tp->t_inpcb->inp_socket;
2454 unsigned short winsz = ntohs(th->th_win);
2455 unsigned int scaled_win = winsz<<tp->rcv_scale;
2456 tcp_seq win_rtedge = org_ack + scaled_win;
2457
2458 count = tp->t_lropktlen/tp->t_maxseg;
2459
2460 prev_ack = (org_ack - tp->t_lropktlen) + ack_size;
2461 if (prev_ack < org_ack) {
2462 ack_chain = m_dup(m, M_DONTWAIT);
2463 if (ack_chain) {
2464 th->th_ack = htonl(prev_ack);
2465 /* Keep adv window constant for duplicated ACK packets */
2466 scaled_win = win_rtedge - prev_ack;
2467 if (scaled_win > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
2468 scaled_win = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
2469 th->th_win = htons(scaled_win>>tp->rcv_scale);
2470 if (lrodebug == 5) {
2471 printf("%s: win = %d winsz = %d sc = %d"
2472 " lro_len %d %d\n",
2473 __func__, scaled_win>>tp->rcv_scale, winsz,
2474 tp->rcv_scale, tp->t_lropktlen, count);
2475 }
2476 tail = ack_chain;
2477 count -= segs_acked; /* accounts for prev_ack packet */
2478 count = (count <= segs_acked) ? 0 : count - segs_acked;
2479 tcpstat.tcps_sndacks++;
2480 so_tc_update_stats(m, so, m_get_service_class(m));
2481 } else {
2482 return NULL;
2483 }
2484 }
2485 else {
2486 tp->t_lropktlen = 0;
2487 return NULL;
2488 }
2489
2490 prev_ack_pkt = ack_chain;
2491
2492 while (count > 0) {
2493 if ((prev_ack + ack_size) < org_ack) {
2494 prev_ack += ack_size;
2495 } else {
2496 /*
2497 * The last ACK sent must have the ACK number that TCP
2498 * thinks is the last sent ACK number.
2499 */
2500 prev_ack = org_ack;
2501 }
2502 mnext = m_dup(prev_ack_pkt, M_DONTWAIT);
2503 if (mnext) {
2504 /* Keep adv window constant for duplicated ACK packets */
2505 scaled_win = win_rtedge - prev_ack;
2506 if (scaled_win > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
2507 scaled_win = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
2508 winsz = htons(scaled_win>>tp->rcv_scale);
2509 if (lrodebug == 5) {
2510 printf("%s: winsz = %d ack %x count %d\n",
2511 __func__, scaled_win>>tp->rcv_scale,
2512 prev_ack, count);
2513 }
2514 bcopy(&winsz, mtod(prev_ack_pkt, caddr_t) + twin_offset, 2);
2515 HTONL(prev_ack);
2516 bcopy(&prev_ack, mtod(prev_ack_pkt, caddr_t) + tack_offset, 4);
2517 NTOHL(prev_ack);
2518 tail->m_nextpkt = mnext;
2519 tail = mnext;
2520 count -= segs_acked;
2521 tcpstat.tcps_sndacks++;
2522 so_tc_update_stats(m, so, m_get_service_class(m));
2523 } else {
2524 if (lrodebug == 5) {
2525 printf("%s: failed to alloc mbuf.\n", __func__);
2526 }
2527 break;
2528 }
2529 prev_ack_pkt = mnext;
2530 }
2531 tp->t_lropktlen = 0;
2532 return ack_chain;
2533 }
2534
2535 static int
2536 tcp_recv_throttle (struct tcpcb *tp)
2537 {
2538 uint32_t base_rtt, newsize;
2539 int32_t qdelay;
2540 struct sockbuf *sbrcv = &tp->t_inpcb->inp_socket->so_rcv;
2541
2542 if (tcp_use_rtt_recvbg == 1 &&
2543 TSTMP_SUPPORTED(tp)) {
2544 /*
2545 * Timestamps are supported on this connection. Use
2546 * RTT to look for an increase in latency.
2547 */
2548
2549 /*
2550 * If the connection is already being throttled, leave it
2551 * in that state until rtt comes closer to base rtt
2552 */
2553 if (tp->t_flagsext & TF_RECV_THROTTLE)
2554 return (1);
2555
2556 base_rtt = get_base_rtt(tp);
2557
2558 if (base_rtt != 0 && tp->t_rttcur != 0) {
2559 qdelay = tp->t_rttcur - base_rtt;
2560 /*
2561 * if latency increased on a background flow,
2562 * return 1 to start throttling.
2563 */
2564 if (qdelay > target_qdelay) {
2565 tp->t_flagsext |= TF_RECV_THROTTLE;
2566
2567 /*
2568 * Reduce the recv socket buffer size to
2569 * minimize latecy.
2570 */
2571 if (sbrcv->sb_idealsize >
2572 tcp_recv_throttle_minwin) {
2573 newsize = sbrcv->sb_idealsize >> 1;
2574 /* Set a minimum of 16 K */
2575 newsize =
2576 max(newsize,
2577 tcp_recv_throttle_minwin);
2578 sbrcv->sb_idealsize = newsize;
2579 }
2580 return (1);
2581 } else {
2582 return (0);
2583 }
2584 }
2585 }
2586
2587 /*
2588 * Timestamps are not supported or there is no good RTT
2589 * measurement. Use IPDV in this case.
2590 */
2591 if (tp->acc_iaj > tcp_acc_iaj_react_limit)
2592 return (1);
2593
2594 return (0);
2595 }
mirror of XNU