]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/tcp_output.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2050.48.11.tar.gz
[apple/xnu.git] / bsd / netinet / tcp_output.c
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #define _IP_VHL
71
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/protosw.h>
80 #include <sys/socket.h>
81 #include <sys/socketvar.h>
82
83 #include <net/route.h>
84 #include <net/ntstat.h>
85 #include <net/if_var.h>
86 #include <net/if.h>
87 #include <net/if_types.h>
88
89 #include <netinet/in.h>
90 #include <netinet/in_systm.h>
91 #include <netinet/in_var.h>
92 #include <netinet/ip.h>
93 #include <netinet/in_pcb.h>
94 #include <netinet/ip_var.h>
95 #include <mach/sdt.h>
96 #if INET6
97 #include <netinet6/in6_pcb.h>
98 #include <netinet/ip6.h>
99 #include <netinet6/ip6_var.h>
100 #endif
101 #include <netinet/tcp.h>
102 #define TCPOUTFLAGS
103 #include <netinet/tcp_fsm.h>
104 #include <netinet/tcp_seq.h>
105 #include <netinet/tcp_timer.h>
106 #include <netinet/tcp_var.h>
107 #include <netinet/tcpip.h>
108 #include <netinet/tcp_cc.h>
109 #if TCPDEBUG
110 #include <netinet/tcp_debug.h>
111 #endif
112 #include <sys/kdebug.h>
113 #include <mach/sdt.h>
114
115 #if IPSEC
116 #include <netinet6/ipsec.h>
117 #endif /*IPSEC*/
118
119 #if CONFIG_MACF_NET
120 #include <security/mac_framework.h>
121 #endif /* MAC_SOCKET */
122
123 #include <netinet/lro_ext.h>
124
125 #define DBG_LAYER_BEG NETDBG_CODE(DBG_NETTCP, 1)
126 #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3)
127 #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
128
129 #ifdef notyet
130 extern struct mbuf *m_copypack();
131 #endif
132
133 int path_mtu_discovery = 1;
134 SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW | CTLFLAG_LOCKED,
135 &path_mtu_discovery, 1, "Enable Path MTU Discovery");
136
137 int ss_fltsz = 1;
138 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
139 &ss_fltsz, 1, "Slow start flight size");
140
141 int ss_fltsz_local = 8; /* starts with eight segments max */
142 SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
143 &ss_fltsz_local, 1, "Slow start flight size for local networks");
144
145 int tcp_do_tso = 1;
146 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED,
147 &tcp_do_tso, 0, "Enable TCP Segmentation Offload");
148
149
150 int tcp_ecn_outbound = 0;
151 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_initiate_out, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_outbound,
152 0, "Initiate ECN for outbound connections");
153
154 int tcp_ecn_inbound = 0;
155 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_inbound,
156 0, "Allow ECN negotiation for inbound connections");
157
158 int tcp_packet_chaining = 50;
159 SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining,
160 0, "Enable TCP output packet chaining");
161
162 int tcp_output_unlocked = 1;
163 SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked,
164 0, "Unlock TCP when sending packets down to IP");
165
166 int tcp_do_rfc3390 = 1;
167 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, CTLFLAG_RW | CTLFLAG_LOCKED,
168 &tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS");
169
170 int tcp_min_iaj_win = MIN_IAJ_WIN;
171 SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, CTLFLAG_RW | CTLFLAG_LOCKED,
172 &tcp_min_iaj_win, 1, "Minimum recv win based on inter-packet arrival jitter");
173
174 int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT;
175 SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, CTLFLAG_RW | CTLFLAG_LOCKED,
176 &tcp_acc_iaj_react_limit, 1, "Accumulated IAJ when receiver starts to react");
177
178 uint32_t tcp_do_autosendbuf = 1;
179 SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, CTLFLAG_RW | CTLFLAG_LOCKED,
180 &tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning");
181
182 uint32_t tcp_autosndbuf_inc = 8 * 1024;
183 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, CTLFLAG_RW | CTLFLAG_LOCKED,
184 &tcp_autosndbuf_inc, 1, "Increment in send socket bufffer size");
185
186 uint32_t tcp_autosndbuf_max = 512 * 1024;
187 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, CTLFLAG_RW | CTLFLAG_LOCKED,
188 &tcp_autosndbuf_max, 1, "Maximum send socket buffer size");
189
190 uint32_t tcp_prioritize_acks = 1;
191 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, CTLFLAG_RW | CTLFLAG_LOCKED,
192 &tcp_prioritize_acks, 1, "Prioritize pure acks");
193
194 static int32_t packchain_newlist = 0;
195 static int32_t packchain_looped = 0;
196 static int32_t packchain_sent = 0;
197
198 /* temporary: for testing */
199 #if IPSEC
200 extern int ipsec_bypass;
201 #endif
202
203 extern int slowlink_wsize; /* window correction for slow links */
204 #if IPFIREWALL
205 extern int fw_enable; /* firewall check for packet chaining */
206 extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */
207 #endif /* IPFIREWALL */
208
209 extern vm_size_t so_cache_zone_element_size;
210 #if RANDOM_IP_ID
211 extern int ip_use_randomid;
212 #endif /* RANDOM_IP_ID */
213 extern u_int32_t dlil_filter_count;
214 extern u_int32_t kipf_count;
215 extern int tcp_recv_bg;
216 extern int maxseg_unacked;
217
218 static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int,
219 struct mbuf *, int, int, int32_t, boolean_t);
220
221 extern uint32_t get_base_rtt(struct tcpcb *tp);
222 static struct mbuf* tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th);
223
224 static __inline__ u_int16_t
225 get_socket_id(struct socket * s)
226 {
227 u_int16_t val;
228
229 if (so_cache_zone_element_size == 0) {
230 return (0);
231 }
232 val = (u_int16_t)(((uintptr_t)s) / so_cache_zone_element_size);
233 if (val == 0) {
234 val = 0xffff;
235 }
236 return (val);
237 }
238
239 /*
240 * Tcp output routine: figure out what should be sent and send it.
241 *
242 * Returns: 0 Success
243 * EADDRNOTAVAIL
244 * ENOBUFS
245 * EMSGSIZE
246 * EHOSTUNREACH
247 * ENETDOWN
248 * ip_output_list:ENOMEM
249 * ip_output_list:EADDRNOTAVAIL
250 * ip_output_list:ENETUNREACH
251 * ip_output_list:EHOSTUNREACH
252 * ip_output_list:EACCES
253 * ip_output_list:EMSGSIZE
254 * ip_output_list:ENOBUFS
255 * ip_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
256 * ip6_output_list:EINVAL
257 * ip6_output_list:EOPNOTSUPP
258 * ip6_output_list:EHOSTUNREACH
259 * ip6_output_list:EADDRNOTAVAIL
260 * ip6_output_list:ENETUNREACH
261 * ip6_output_list:EMSGSIZE
262 * ip6_output_list:ENOBUFS
263 * ip6_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
264 */
265 int
266 tcp_output(struct tcpcb *tp)
267 {
268 struct socket *so = tp->t_inpcb->inp_socket;
269 int32_t len, recwin, sendwin, off;
270 int flags, error;
271 register struct mbuf *m;
272 struct ip *ip = NULL;
273 register struct ipovly *ipov = NULL;
274 #if INET6
275 struct ip6_hdr *ip6 = NULL;
276 #endif /* INET6 */
277 register struct tcphdr *th;
278 u_char opt[TCP_MAXOLEN];
279 unsigned ipoptlen, optlen, hdrlen;
280 int idle, sendalot, lost = 0;
281 int i, sack_rxmit;
282 int tso = 0;
283 int sack_bytes_rxmt;
284 struct sackhole *p;
285 #ifdef IPSEC
286 unsigned ipsec_optlen = 0;
287 #endif
288 int last_off = 0;
289 int m_off;
290 int idle_time = 0;
291 struct mbuf *m_lastm = NULL;
292 struct mbuf *m_head = NULL;
293 struct mbuf *packetlist = NULL;
294 struct mbuf *tp_inp_options = tp->t_inpcb->inp_depend4.inp4_options;
295 #if INET6
296 int isipv6 = tp->t_inpcb->inp_vflag & INP_IPV6 ;
297 #endif
298 short packchain_listadd = 0;
299 u_int16_t socket_id = get_socket_id(so);
300 int so_options = so->so_options;
301 struct rtentry *rt;
302 u_int32_t basertt, svc_flags = 0, allocated_len;
303 u_int32_t lro_ackmore = (tp->t_lropktlen != 0) ? 1 : 0;
304 struct mbuf *mnext = NULL;
305 int sackoptlen = 0;
306
307 /*
308 * Determine length of data that should be transmitted,
309 * and flags that will be used.
310 * If there is some data or critical controls (SYN, RST)
311 * to send, then transmit; otherwise, investigate further.
312 */
313 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
314
315 /* Since idle_time is signed integer, the following integer subtraction
316 * will take care of wrap around of tcp_now
317 */
318 idle_time = tcp_now - tp->t_rcvtime;
319 if (idle && idle_time >= TCP_IDLETIMEOUT(tp)) {
320 if (CC_ALGO(tp)->after_idle != NULL)
321 CC_ALGO(tp)->after_idle(tp);
322 DTRACE_TCP5(cc, void, NULL, struct inpcb *, tp->t_inpcb,
323 struct tcpcb *, tp, struct tcphdr *, NULL,
324 int32_t, TCP_CC_IDLE_TIMEOUT);
325 }
326 tp->t_flags &= ~TF_LASTIDLE;
327 if (idle) {
328 if (tp->t_flags & TF_MORETOCOME) {
329 tp->t_flags |= TF_LASTIDLE;
330 idle = 0;
331 }
332 }
333 again:
334 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
335
336 #if INET6
337 if (isipv6) {
338 KERNEL_DEBUG(DBG_LAYER_BEG,
339 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
340 (((tp->t_inpcb->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
341 (tp->t_inpcb->in6p_faddr.s6_addr16[0] & 0xffff)),
342 sendalot,0,0);
343 } else
344 #endif
345
346 {
347 KERNEL_DEBUG(DBG_LAYER_BEG,
348 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
349 (((tp->t_inpcb->inp_laddr.s_addr & 0xffff) << 16) |
350 (tp->t_inpcb->inp_faddr.s_addr & 0xffff)),
351 sendalot,0,0);
352 }
353 /*
354 * If the route generation id changed, we need to check that our
355 * local (source) IP address is still valid. If it isn't either
356 * return error or silently do nothing (assuming the address will
357 * come back before the TCP connection times out).
358 */
359 rt = tp->t_inpcb->inp_route.ro_rt;
360 if (rt != NULL && (!(rt->rt_flags & RTF_UP) ||
361 rt->generation_id != route_generation)) {
362 struct ifnet *ifp;
363 struct in_ifaddr *ia = NULL;
364 struct in6_ifaddr *ia6 = NULL;
365 int found_srcaddr = 0;
366
367 /* disable multipages at the socket */
368 somultipages(so, FALSE);
369
370 /* Disable TSO for the socket until we know more */
371 tp->t_flags &= ~TF_TSO;
372
373 if (isipv6) {
374 ia6 = ifa_foraddr6(&tp->t_inpcb->in6p_laddr);
375 if (ia6 != NULL)
376 found_srcaddr = 1;
377 } else {
378 ia = ifa_foraddr(tp->t_inpcb->inp_laddr.s_addr);
379 if (ia != NULL)
380 found_srcaddr = 1;
381 }
382
383 /* check that the source address is still valid */
384 if (found_srcaddr == 0) {
385
386 soevent(so,
387 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR));
388
389 if (tp->t_state >= TCPS_CLOSE_WAIT) {
390 tcp_drop(tp, EADDRNOTAVAIL);
391 return(EADDRNOTAVAIL);
392 }
393
394 /* set Retransmit timer if it wasn't set
395 * reset Persist timer and shift register as the
396 * advertised peer window may not be valid anymore
397 */
398
399 if (!tp->t_timer[TCPT_REXMT]) {
400 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
401 if (tp->t_timer[TCPT_PERSIST]) {
402 tp->t_timer[TCPT_PERSIST] = 0;
403 tp->t_rxtshift = 0;
404 tp->t_persist_stop = 0;
405 tp->rxt_start = 0;
406 }
407 }
408
409 if (tp->t_pktlist_head != NULL)
410 m_freem_list(tp->t_pktlist_head);
411 TCP_PKTLIST_CLEAR(tp);
412
413 /* drop connection if source address isn't available */
414 if (so->so_flags & SOF_NOADDRAVAIL) {
415 tcp_drop(tp, EADDRNOTAVAIL);
416 return(EADDRNOTAVAIL);
417 }
418 else {
419 tcp_check_timer_state(tp);
420 return(0); /* silently ignore, keep data in socket: address may be back */
421 }
422 }
423 if (ia != NULL)
424 IFA_REMREF(&ia->ia_ifa);
425
426 if (ia6 != NULL)
427 IFA_REMREF(&ia6->ia_ifa);
428
429 /*
430 * Address is still valid; check for multipages capability
431 * again in case the outgoing interface has changed.
432 */
433 RT_LOCK(rt);
434 if ((ifp = rt->rt_ifp) != NULL) {
435 somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
436 tcp_set_tso(tp, ifp);
437 }
438 if (rt->rt_flags & RTF_UP)
439 rt->generation_id = route_generation;
440 /*
441 * See if we should do MTU discovery. Don't do it if:
442 * 1) it is disabled via the sysctl
443 * 2) the route isn't up
444 * 3) the MTU is locked (if it is, then discovery has been
445 * disabled)
446 */
447
448 if (!path_mtu_discovery || ((rt != NULL) &&
449 (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU))))
450 tp->t_flags &= ~TF_PMTUD;
451 else
452 tp->t_flags |= TF_PMTUD;
453
454 RT_UNLOCK(rt);
455 }
456
457 /*
458 * If we've recently taken a timeout, snd_max will be greater than
459 * snd_nxt. There may be SACK information that allows us to avoid
460 * resending already delivered data. Adjust snd_nxt accordingly.
461 */
462 if (tp->sack_enable && SEQ_LT(tp->snd_nxt, tp->snd_max))
463 tcp_sack_adjust(tp);
464 sendalot = 0;
465 off = tp->snd_nxt - tp->snd_una;
466 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
467
468 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0)
469 sendwin = min(sendwin, slowlink_wsize);
470
471 flags = tcp_outflags[tp->t_state];
472 /*
473 * Send any SACK-generated retransmissions. If we're explicitly trying
474 * to send out new data (when sendalot is 1), bypass this function.
475 * If we retransmit in fast recovery mode, decrement snd_cwnd, since
476 * we're replacing a (future) new transmission with a retransmission
477 * now, and we previously incremented snd_cwnd in tcp_input().
478 */
479 /*
480 * Still in sack recovery , reset rxmit flag to zero.
481 */
482 sack_rxmit = 0;
483 sack_bytes_rxmt = 0;
484 len = 0;
485 p = NULL;
486 if (tp->sack_enable && IN_FASTRECOVERY(tp) &&
487 (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
488 int32_t cwin;
489
490 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
491 if (cwin < 0)
492 cwin = 0;
493 /* Do not retransmit SACK segments beyond snd_recover */
494 if (SEQ_GT(p->end, tp->snd_recover)) {
495 /*
496 * (At least) part of sack hole extends beyond
497 * snd_recover. Check to see if we can rexmit data
498 * for this hole.
499 */
500 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
501 /*
502 * Can't rexmit any more data for this hole.
503 * That data will be rexmitted in the next
504 * sack recovery episode, when snd_recover
505 * moves past p->rxmit.
506 */
507 p = NULL;
508 goto after_sack_rexmit;
509 } else
510 /* Can rexmit part of the current hole */
511 len = ((int32_t)min(cwin,
512 tp->snd_recover - p->rxmit));
513 } else {
514 len = ((int32_t)min(cwin, p->end - p->rxmit));
515 }
516 if (len > 0) {
517 off = p->rxmit - tp->snd_una; /* update off only if we really transmit SACK data */
518 sack_rxmit = 1;
519 sendalot = 1;
520 tcpstat.tcps_sack_rexmits++;
521 tcpstat.tcps_sack_rexmit_bytes +=
522 min(len, tp->t_maxseg);
523 if (nstat_collect) {
524 nstat_route_tx(tp->t_inpcb->inp_route.ro_rt, 1,
525 min(len, tp->t_maxseg), NSTAT_TX_FLAG_RETRANSMIT);
526 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
527 locked_add_64(&tp->t_inpcb->inp_stat->txbytes,
528 min(len, tp->t_maxseg));
529 tp->t_stat.txretransmitbytes += min(len, tp->t_maxseg);
530 }
531 } else {
532 len = 0;
533 }
534 }
535 after_sack_rexmit:
536 /*
537 * Get standard flags, and add SYN or FIN if requested by 'hidden'
538 * state flags.
539 */
540 if (tp->t_flags & TF_NEEDFIN)
541 flags |= TH_FIN;
542 if (tp->t_flags & TF_NEEDSYN)
543 flags |= TH_SYN;
544
545 /*
546 * If in persist timeout with window of 0, send 1 byte.
547 * Otherwise, if window is small but nonzero
548 * and timer expired, we will send what we can
549 * and go to transmit state.
550 */
551 if (tp->t_force) {
552 if (sendwin == 0) {
553 /*
554 * If we still have some data to send, then
555 * clear the FIN bit. Usually this would
556 * happen below when it realizes that we
557 * aren't sending all the data. However,
558 * if we have exactly 1 byte of unsent data,
559 * then it won't clear the FIN bit below,
560 * and if we are in persist state, we wind
561 * up sending the packet without recording
562 * that we sent the FIN bit.
563 *
564 * We can't just blindly clear the FIN bit,
565 * because if we don't have any more data
566 * to send then the probe will be the FIN
567 * itself.
568 */
569 if (off < so->so_snd.sb_cc)
570 flags &= ~TH_FIN;
571 sendwin = 1;
572 } else {
573 tp->t_timer[TCPT_PERSIST] = 0;
574 tp->t_rxtshift = 0;
575 tp->rxt_start = 0;
576 tp->t_persist_stop = 0;
577 }
578 }
579
580 /*
581 * If snd_nxt == snd_max and we have transmitted a FIN, the
582 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
583 * a negative length. This can also occur when TCP opens up
584 * its congestion window while receiving additional duplicate
585 * acks after fast-retransmit because TCP will reset snd_nxt
586 * to snd_max after the fast-retransmit.
587 *
588 * In the normal retransmit-FIN-only case, however, snd_nxt will
589 * be set to snd_una, the offset will be 0, and the length may
590 * wind up 0.
591 *
592 * If sack_rxmit is true we are retransmitting from the scoreboard
593 * in which case len is already set.
594 */
595 if (sack_rxmit == 0) {
596 if (sack_bytes_rxmt == 0)
597 len = min(so->so_snd.sb_cc, sendwin) - off;
598 else {
599 int32_t cwin;
600
601 /*
602 * We are inside of a SACK recovery episode and are
603 * sending new data, having retransmitted all the
604 * data possible in the scoreboard.
605 */
606 len = min(so->so_snd.sb_cc, tp->snd_wnd)
607 - off;
608 /*
609 * Don't remove this (len > 0) check !
610 * We explicitly check for len > 0 here (although it
611 * isn't really necessary), to work around a gcc
612 * optimization issue - to force gcc to compute
613 * len above. Without this check, the computation
614 * of len is bungled by the optimizer.
615 */
616 if (len > 0) {
617 cwin = tp->snd_cwnd -
618 (tp->snd_nxt - tp->sack_newdata) -
619 sack_bytes_rxmt;
620 if (cwin < 0)
621 cwin = 0;
622 len = imin(len, cwin);
623 }
624 else
625 len = 0;
626 }
627 }
628
629 /*
630 * Lop off SYN bit if it has already been sent. However, if this
631 * is SYN-SENT state and if segment contains data and if we don't
632 * know that foreign host supports TAO, suppress sending segment.
633 */
634 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
635 flags &= ~TH_SYN;
636 off--, len++;
637 if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
638 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
639 tp->t_pktlist_head != NULL) {
640 packetlist = tp->t_pktlist_head;
641 packchain_listadd = tp->t_lastchain;
642 packchain_sent++;
643 TCP_PKTLIST_CLEAR(tp);
644
645 error = tcp_ip_output(so, tp, packetlist,
646 packchain_listadd, tp_inp_options,
647 (so_options & SO_DONTROUTE),
648 (sack_rxmit | (sack_bytes_rxmt != 0)), 0,
649 #ifdef INET6
650 isipv6);
651 #else
652 0);
653 #endif
654
655
656 }
657 /*
658 * tcp was closed while we were in ip,
659 * resume close
660 */
661 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
662 (tp->t_flags & TF_CLOSING)) {
663 tp->t_flags &= ~TF_CLOSING;
664 (void) tcp_close(tp);
665 } else {
666 tcp_check_timer_state(tp);
667 }
668 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
669 0,0,0,0,0);
670 return(0);
671 }
672 }
673
674 /*
675 * Be careful not to send data and/or FIN on SYN segments.
676 * This measure is needed to prevent interoperability problems
677 * with not fully conformant TCP implementations.
678 */
679 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
680 len = 0;
681 flags &= ~TH_FIN;
682 }
683
684 /* The check here used to be (len < 0). Some times len is zero when
685 * the congestion window is closed and we need to check if persist timer
686 * has to be set in that case. But don't set persist until connection
687 * is established.
688 */
689 if (len <= 0 && !(flags & TH_SYN)) {
690 /*
691 * If FIN has been sent but not acked,
692 * but we haven't been called to retransmit,
693 * len will be < 0. Otherwise, window shrank
694 * after we sent into it. If window shrank to 0,
695 * cancel pending retransmit, pull snd_nxt back
696 * to (closed) window, and set the persist timer
697 * if it isn't already going. If the window didn't
698 * close completely, just wait for an ACK.
699 */
700 len = 0;
701 if (sendwin == 0) {
702 tp->t_timer[TCPT_REXMT] = 0;
703 tp->t_rxtshift = 0;
704 tp->rxt_start = 0;
705 tp->snd_nxt = tp->snd_una;
706 if (tp->t_timer[TCPT_PERSIST] == 0)
707 tcp_setpersist(tp);
708 }
709 }
710
711 /* Automatic sizing of send socket buffer. Increase the send socket buffer
712 * size if all of the following criteria are met
713 * 1. the receiver has enough buffer space for this data
714 * 2. send buffer is filled to 7/8th with data (so we actually
715 * have data to make use of it);
716 * 3. our send window (slow start and congestion controlled) is
717 * larger than sent but unacknowledged data in send buffer.
718 */
719 basertt = get_base_rtt(tp);
720 if (tcp_do_autosendbuf == 1 &&
721 !INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb) && !IN_FASTRECOVERY(tp) &&
722 (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
723 tcp_cansbgrow(&so->so_snd)) {
724 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
725 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
726 sendwin >= (so->so_snd.sb_cc -
727 (tp->snd_nxt - tp->snd_una))) {
728 /* Also increase the send buffer only if the
729 * round-trip time is not increasing because we do
730 * not want to contribute to latency by filling buffers.
731 * We also do not want to hold onto application's
732 * old data for too long. Interactive applications would
733 * rather discard old data.
734 */
735 if (tp->t_rttcur <=
736 (basertt + 25)) {
737 if (sbreserve(&so->so_snd,
738 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
739 tcp_autosndbuf_max)) == 1) {
740 so->so_snd.sb_idealsize = so->so_snd.sb_hiwat;
741 }
742 } else {
743 so->so_snd.sb_idealsize =
744 max(tcp_sendspace, so->so_snd.sb_hiwat -
745 (2 * tcp_autosndbuf_inc));
746 so->so_snd.sb_flags |= SB_TRIM;
747 }
748 }
749 }
750
751 /*
752 * Truncate to the maximum segment length or enable TCP Segmentation
753 * Offloading (if supported by hardware) and ensure that FIN is removed
754 * if the length no longer contains the last data byte.
755 *
756 * TSO may only be used if we are in a pure bulk sending state. The
757 * presence of TCP-MD5, SACK retransmits, SACK advertizements, ipfw rules
758 * and IP options prevent using TSO. With TSO the TCP header is the same
759 * (except for the sequence number) for all generated packets. This
760 * makes it impossible to transmit any options which vary per generated
761 * segment or packet.
762 *
763 * The length of TSO bursts is limited to TCP_MAXWIN. That limit and
764 * removal of FIN (if not already catched here) are handled later after
765 * the exact length of the TCP options are known.
766 */
767 #if IPSEC
768 /*
769 * Pre-calculate here as we save another lookup into the darknesses
770 * of IPsec that way and can actually decide if TSO is ok.
771 */
772 if (ipsec_bypass == 0)
773 ipsec_optlen = ipsec_hdrsiz_tcp(tp);
774 #endif
775
776 if (len > tp->t_maxseg) {
777 if ((tp->t_flags & TF_TSO) && tcp_do_tso &&
778 #if RANDOM_IP_ID
779 ip_use_randomid &&
780 #endif /* RANDOM_IP_ID */
781 kipf_count == 0 && dlil_filter_count == 0 &&
782 tp->rcv_numsacks == 0 && sack_rxmit == 0 && sack_bytes_rxmt == 0 &&
783 tp->t_inpcb->inp_options == NULL &&
784 tp->t_inpcb->in6p_options == NULL
785 #if IPSEC
786 && ipsec_optlen == 0
787 #endif
788 #if IPFIREWALL
789 && (fw_enable == 0 || fw_bypass)
790 #endif
791 ) {
792 tso = 1;
793 sendalot = 0;
794 } else {
795 len = tp->t_maxseg;
796 sendalot = 1;
797 tso = 0;
798 }
799 }
800 if (sack_rxmit) {
801 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
802 flags &= ~TH_FIN;
803 } else {
804 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
805 flags &= ~TH_FIN;
806 }
807
808 recwin = tcp_sbspace(tp);
809
810 /*
811 * Sender silly window avoidance. We transmit under the following
812 * conditions when len is non-zero:
813 *
814 * - we've timed out (e.g. persist timer)
815 * - we need to retransmit
816 * - We have a full segment (or more with TSO)
817 * - This is the last buffer in a write()/send() and we are
818 * either idle or running NODELAY
819 * - we have more then 1/2 the maximum send window's worth of
820 * data (receiver may be limited the window size)
821 */
822 if (len) {
823 if (tp->t_force) {
824 tp->t_flags &= ~TF_MAXSEGSNT;
825 goto send;
826 }
827 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
828 if (len >= tp->t_maxseg)
829 tp->t_flags |= TF_MAXSEGSNT;
830 else
831 tp->t_flags &= ~TF_MAXSEGSNT;
832 goto send;
833 }
834 if (sack_rxmit)
835 goto send;
836
837 /*
838 * Send new data on the connection only if it is
839 * not flow controlled
840 */
841 if (!INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb) ||
842 tp->t_state != TCPS_ESTABLISHED) {
843 if (len >= tp->t_maxseg) {
844 tp->t_flags |= TF_MAXSEGSNT;
845 goto send;
846 }
847 if (!(tp->t_flags & TF_MORETOCOME) &&
848 (idle || tp->t_flags & TF_NODELAY || tp->t_flags & TF_MAXSEGSNT) &&
849 (tp->t_flags & TF_NOPUSH) == 0 &&
850 len + off >= so->so_snd.sb_cc) {
851 tp->t_flags &= ~TF_MAXSEGSNT;
852 goto send;
853 }
854 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
855 tp->t_flags &= ~TF_MAXSEGSNT;
856 goto send;
857 }
858 } else {
859 tcpstat.tcps_fcholdpacket++;
860 }
861 }
862
863 /*
864 * Compare available window to amount of window
865 * known to peer (as advertised window less
866 * next expected input). If the difference is at least two
867 * max size segments, or at least 25% of the maximum possible
868 * window, then want to send a window update to peer.
869 * Skip this if the connection is in T/TCP half-open state.
870 */
871 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) {
872 /*
873 * "adv" is the amount we can increase the window,
874 * taking into account that we are limited by
875 * TCP_MAXWIN << tp->rcv_scale.
876 */
877 int32_t adv = imin(recwin, (int)TCP_MAXWIN << tp->rcv_scale) -
878 (tp->rcv_adv - tp->rcv_nxt);
879
880 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
881 /* Update only if the resulting scaled value of the window changed, or
882 * if there is a change in the sequence since the last ack.
883 * This avoids what appears as dupe ACKS (see rdar://5640997)
884 *
885 * If streaming is detected avoid sending too many window updates.
886 * We will depend on the delack timer to send a window update
887 * when needed.
888 */
889 if ((tp->t_flags & TF_STRETCHACK) == 0 &&
890 (tp->last_ack_sent != tp->rcv_nxt ||
891 ((recwin + adv) >> tp->rcv_scale) > recwin)) {
892 goto send;
893 }
894
895 /* Make sure that the delayed ack timer is set if we
896 * delayed sending a window update because of streaming
897 * detection.
898 */
899 if ((tp->t_flags & TF_STRETCHACK) != 0 &&
900 (tp->t_flags & TF_DELACK) == 0) {
901 tp->t_flags |= TF_DELACK;
902 tp->t_timer[TCPT_DELACK] = OFFSET_FROM_START(tp, tcp_delack);
903 }
904 }
905 if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat)
906 goto send;
907 }
908
909 /*
910 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
911 * is also a catch-all for the retransmit timer timeout case.
912 */
913 if (tp->t_flags & TF_ACKNOW)
914 goto send;
915 if ((flags & TH_RST) ||
916 ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
917 goto send;
918 if (SEQ_GT(tp->snd_up, tp->snd_una))
919 goto send;
920 /*
921 * If our state indicates that FIN should be sent
922 * and we have not yet done so, then we need to send.
923 */
924 if (flags & TH_FIN &&
925 ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
926 goto send;
927 /*
928 * In SACK, it is possible for tcp_output to fail to send a segment
929 * after the retransmission timer has been turned off. Make sure
930 * that the retransmission timer is set.
931 */
932 if (tp->sack_enable && (tp->t_state >= TCPS_ESTABLISHED) &&
933 SEQ_GT(tp->snd_max, tp->snd_una) &&
934 tp->t_timer[TCPT_REXMT] == 0 &&
935 tp->t_timer[TCPT_PERSIST] == 0) {
936 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
937 goto just_return;
938 }
939 /*
940 * TCP window updates are not reliable, rather a polling protocol
941 * using ``persist'' packets is used to insure receipt of window
942 * updates. The three ``states'' for the output side are:
943 * idle not doing retransmits or persists
944 * persisting to move a small or zero window
945 * (re)transmitting and thereby not persisting
946 *
947 * tp->t_timer[TCPT_PERSIST]
948 * is set when we are in persist state.
949 * tp->t_force
950 * is set when we are called to send a persist packet.
951 * tp->t_timer[TCPT_REXMT]
952 * is set when we are retransmitting
953 * The output side is idle when both timers are zero.
954 *
955 * If send window is too small, there is data to transmit, and no
956 * retransmit or persist is pending, then go to persist state.
957 * If nothing happens soon, send when timer expires:
958 * if window is nonzero, transmit what we can,
959 * otherwise force out a byte.
960 */
961 if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 &&
962 tp->t_timer[TCPT_PERSIST] == 0) {
963 tp->t_rxtshift = 0;
964 tp->rxt_start = 0;
965 tcp_setpersist(tp);
966 }
967 just_return:
968 /*
969 * If there is no reason to send a segment, just return.
970 * but if there is some packets left in the packet list, send them now.
971 */
972 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
973 tp->t_pktlist_head != NULL) {
974 packetlist = tp->t_pktlist_head;
975 packchain_listadd = tp->t_lastchain;
976 packchain_sent++;
977 TCP_PKTLIST_CLEAR(tp);
978
979 error = tcp_ip_output(so, tp, packetlist, packchain_listadd,
980 tp_inp_options, (so_options & SO_DONTROUTE),
981 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
982 #ifdef INET6
983 isipv6);
984 #else
985 0);
986 #endif
987 }
988 /* tcp was closed while we were in ip; resume close */
989 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
990 (tp->t_flags & TF_CLOSING)) {
991 tp->t_flags &= ~TF_CLOSING;
992 (void) tcp_close(tp);
993 } else {
994 tcp_check_timer_state(tp);
995 }
996 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
997 return (0);
998
999 send:
1000 /*
1001 * Before ESTABLISHED, force sending of initial options
1002 * unless TCP set not to do any options.
1003 * NOTE: we assume that the IP/TCP header plus TCP options
1004 * always fit in a single mbuf, leaving room for a maximum
1005 * link header, i.e.
1006 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1007 */
1008 optlen = 0;
1009 #if INET6
1010 if (isipv6)
1011 hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1012 else
1013 #endif
1014 hdrlen = sizeof (struct tcpiphdr);
1015 if (flags & TH_SYN) {
1016 tp->snd_nxt = tp->iss;
1017 if ((tp->t_flags & TF_NOOPT) == 0) {
1018 u_short mss;
1019
1020 opt[0] = TCPOPT_MAXSEG;
1021 opt[1] = TCPOLEN_MAXSEG;
1022 mss = htons((u_short) tcp_mssopt(tp));
1023 (void)memcpy(opt + 2, &mss, sizeof(mss));
1024 optlen = TCPOLEN_MAXSEG;
1025
1026 if ((tp->t_flags & TF_REQ_SCALE) &&
1027 ((flags & TH_ACK) == 0 ||
1028 (tp->t_flags & TF_RCVD_SCALE))) {
1029 *((u_int32_t *)(void *)(opt + optlen)) = htonl(
1030 TCPOPT_NOP << 24 |
1031 TCPOPT_WINDOW << 16 |
1032 TCPOLEN_WINDOW << 8 |
1033 tp->request_r_scale);
1034 optlen += 4;
1035 }
1036 }
1037
1038 }
1039
1040 /*
1041 RFC 3168 states that:
1042 - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
1043 to handle the TCP ECE flag, even if you also later send a
1044 non-ECN-setup SYN/SYN-ACK.
1045 - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
1046 the ip ECT flag.
1047
1048 It is not clear how the ECE flag would ever be set if you never
1049 set the IP ECT flag on outbound packets. All the same, we use
1050 the TE_SETUPSENT to indicate that we have committed to handling
1051 the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
1052 whether or not we should set the IP ECT flag on outbound packets.
1053 */
1054 /*
1055 * For a SYN-ACK, send an ECN setup SYN-ACK
1056 */
1057 if (tcp_ecn_inbound && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
1058 if ((tp->ecn_flags & TE_SETUPRECEIVED) != 0) {
1059 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1060 /* Setting TH_ECE makes this an ECN-setup SYN-ACK */
1061 flags |= TH_ECE;
1062
1063 /*
1064 * Record that we sent the ECN-setup and default to
1065 * setting IP ECT.
1066 */
1067 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1068 }
1069 else {
1070 /*
1071 * We sent an ECN-setup SYN-ACK but it was dropped.
1072 * Fallback to non-ECN-setup SYN-ACK and clear flag
1073 * that to indicate we should not send data with IP ECT set.
1074 *
1075 * Pretend we didn't receive an ECN-setup SYN.
1076 */
1077 tp->ecn_flags &= ~TE_SETUPRECEIVED;
1078 }
1079 }
1080 }
1081 else if (tcp_ecn_outbound && (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
1082 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1083 /* Setting TH_ECE and TH_CWR makes this an ECN-setup SYN */
1084 flags |= (TH_ECE | TH_CWR);
1085
1086 /*
1087 * Record that we sent the ECN-setup and default to
1088 * setting IP ECT.
1089 */
1090 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1091 }
1092 else {
1093 /*
1094 * We sent an ECN-setup SYN but it was dropped.
1095 * Fall back to no ECN and clear flag indicating
1096 * we should send data with IP ECT set.
1097 */
1098 tp->ecn_flags &= ~TE_SENDIPECT;
1099 }
1100 }
1101
1102 /*
1103 * Check if we should set the TCP CWR flag.
1104 * CWR flag is sent when we reduced the congestion window because
1105 * we received a TCP ECE or we performed a fast retransmit. We
1106 * never set the CWR flag on retransmitted packets. We only set
1107 * the CWR flag on data packets. Pure acks don't have this set.
1108 */
1109 if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
1110 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1111 flags |= TH_CWR;
1112 tp->ecn_flags &= ~TE_SENDCWR;
1113 }
1114
1115 /*
1116 * Check if we should set the TCP ECE flag.
1117 */
1118 if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
1119 flags |= TH_ECE;
1120 }
1121
1122 /*
1123 * Send a timestamp and echo-reply if this is a SYN and our side
1124 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1125 * and our peer have sent timestamps in our SYN's.
1126 */
1127 if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
1128 (flags & TH_RST) == 0 &&
1129 ((flags & TH_ACK) == 0 ||
1130 (tp->t_flags & TF_RCVD_TSTMP))) {
1131 u_int32_t *lp = (u_int32_t *)(void *)(opt + optlen);
1132
1133 /* Form timestamp option as shown in appendix A of RFC 1323. */
1134 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1135 *lp++ = htonl(tcp_now);
1136 *lp = htonl(tp->ts_recent);
1137 optlen += TCPOLEN_TSTAMP_APPA;
1138 }
1139
1140 /* Note the timestamp for receive buffer autosizing */
1141 if (tp->rfbuf_ts == 0 && (so->so_rcv.sb_flags & SB_AUTOSIZE))
1142 tp->rfbuf_ts = tcp_now;
1143
1144 if (tp->sack_enable && ((tp->t_flags & TF_NOOPT) == 0)) {
1145 /*
1146 * Tack on the SACK permitted option *last*.
1147 * And do padding of options after tacking this on.
1148 * This is because of MSS, TS, WinScale and Signatures are
1149 * all present, we have just 2 bytes left for the SACK
1150 * permitted option, which is just enough.
1151 */
1152 /*
1153 * If this is the first SYN of connection (not a SYN
1154 * ACK), include SACK permitted option. If this is a
1155 * SYN ACK, include SACK permitted option if peer has
1156 * already done so. This is only for active connect,
1157 * since the syncache takes care of the passive connect.
1158 */
1159 if ((flags & TH_SYN) &&
1160 (!(flags & TH_ACK) || (tp->t_flags & TF_SACK_PERMIT))) {
1161 u_char *bp;
1162 bp = (u_char *)opt + optlen;
1163
1164 *bp++ = TCPOPT_SACK_PERMITTED;
1165 *bp++ = TCPOLEN_SACK_PERMITTED;
1166 optlen += TCPOLEN_SACK_PERMITTED;
1167 }
1168
1169 /*
1170 * Send SACKs if necessary. This should be the last
1171 * option processed. Only as many SACKs are sent as
1172 * are permitted by the maximum options size.
1173 *
1174 * In general, SACK blocks consume 8*n+2 bytes.
1175 * So a full size SACK blocks option is 34 bytes
1176 * (to generate 4 SACK blocks). At a minimum,
1177 * we need 10 bytes (to generate 1 SACK block).
1178 * If TCP Timestamps (12 bytes) and TCP Signatures
1179 * (18 bytes) are both present, we'll just have
1180 * 10 bytes for SACK options 40 - (12 + 18).
1181 */
1182 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1183 (tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0 &&
1184 MAX_TCPOPTLEN - optlen - 2 >= TCPOLEN_SACK) {
1185 int nsack, padlen;
1186 u_char *bp = (u_char *)opt + optlen;
1187 u_int32_t *lp;
1188
1189 nsack = (MAX_TCPOPTLEN - optlen - 2) / TCPOLEN_SACK;
1190 nsack = min(nsack, tp->rcv_numsacks);
1191 sackoptlen = (2 + nsack * TCPOLEN_SACK);
1192
1193 /*
1194 * First we need to pad options so that the
1195 * SACK blocks can start at a 4-byte boundary
1196 * (sack option and length are at a 2 byte offset).
1197 */
1198 padlen = (MAX_TCPOPTLEN - optlen - sackoptlen) % 4;
1199 optlen += padlen;
1200 while (padlen-- > 0)
1201 *bp++ = TCPOPT_NOP;
1202
1203 tcpstat.tcps_sack_send_blocks++;
1204 *bp++ = TCPOPT_SACK;
1205 *bp++ = sackoptlen;
1206 lp = (u_int32_t *)(void *)bp;
1207 for (i = 0; i < nsack; i++) {
1208 struct sackblk sack = tp->sackblks[i];
1209 *lp++ = htonl(sack.start);
1210 *lp++ = htonl(sack.end);
1211 }
1212 optlen += sackoptlen;
1213 }
1214 }
1215
1216 /* Pad TCP options to a 4 byte boundary */
1217 if (optlen < MAX_TCPOPTLEN && (optlen % sizeof(u_int32_t))) {
1218 int pad = sizeof(u_int32_t) - (optlen % sizeof(u_int32_t));
1219 u_char *bp = (u_char *)opt + optlen;
1220
1221 optlen += pad;
1222 while (pad) {
1223 *bp++ = TCPOPT_EOL;
1224 pad--;
1225 }
1226 }
1227
1228 hdrlen += optlen;
1229
1230 #if INET6
1231 if (isipv6)
1232 ipoptlen = ip6_optlen(tp->t_inpcb);
1233 else
1234 #endif
1235 {
1236 if (tp_inp_options) {
1237 ipoptlen = tp_inp_options->m_len -
1238 offsetof(struct ipoption, ipopt_list);
1239 } else
1240 ipoptlen = 0;
1241 }
1242 #if IPSEC
1243 ipoptlen += ipsec_optlen;
1244 #endif
1245
1246 /*
1247 * Adjust data length if insertion of options will
1248 * bump the packet length beyond the t_maxopd length.
1249 * Clear the FIN bit because we cut off the tail of
1250 * the segment.
1251 *
1252 * When doing TSO limit a burst to TCP_MAXWIN minus the
1253 * IP, TCP and Options length to keep ip->ip_len from
1254 * overflowing. Prevent the last segment from being
1255 * fractional thus making them all equal sized and set
1256 * the flag to continue sending. TSO is disabled when
1257 * IP options or IPSEC are present.
1258 */
1259 if (len + optlen + ipoptlen > tp->t_maxopd) {
1260 /*
1261 * If there is still more to send, don't close the connection.
1262 */
1263 flags &= ~TH_FIN;
1264 if (tso) {
1265 int32_t tso_maxlen;
1266
1267 tso_maxlen = tp->tso_max_segment_size ? tp->tso_max_segment_size : TCP_MAXWIN;
1268
1269 if (len > tso_maxlen - hdrlen - optlen) {
1270 len = tso_maxlen - hdrlen - optlen;
1271 len = len - (len % (tp->t_maxopd - optlen));
1272 sendalot = 1;
1273 } else if (tp->t_flags & TF_NEEDFIN)
1274 sendalot = 1;
1275 } else {
1276 len = tp->t_maxopd - optlen - ipoptlen;
1277 sendalot = 1;
1278 }
1279 }
1280
1281 if (max_linkhdr + hdrlen > MCLBYTES)
1282 panic("tcphdr too big");
1283
1284 /* Check if there is enough data in the send socket
1285 * buffer to start measuring bw
1286 */
1287 if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 &&
1288 (tp->t_bwmeas != NULL) &&
1289 (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 &&
1290 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >=
1291 tp->t_bwmeas->bw_minsize) {
1292 tp->t_bwmeas->bw_size = min((so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)),
1293 tp->t_bwmeas->bw_maxsize);
1294 tp->t_flagsext |= TF_BWMEAS_INPROGRESS;
1295 tp->t_bwmeas->bw_start = tp->snd_max;
1296 tp->t_bwmeas->bw_ts = tcp_now;
1297 }
1298
1299 VERIFY(tp->t_inpcb->inp_flowhash != 0);
1300
1301 /*
1302 * Grab a header mbuf, attaching a copy of data to
1303 * be transmitted, and initialize the header from
1304 * the template for sends on this connection.
1305 */
1306 if (len) {
1307 if (tp->t_force && len == 1)
1308 tcpstat.tcps_sndprobe++;
1309 else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1310 tcpstat.tcps_sndrexmitpack++;
1311 tcpstat.tcps_sndrexmitbyte += len;
1312 if (nstat_collect) {
1313 nstat_route_tx(tp->t_inpcb->inp_route.ro_rt, 1,
1314 len, NSTAT_TX_FLAG_RETRANSMIT);
1315 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
1316 locked_add_64(&tp->t_inpcb->inp_stat->txbytes, len);
1317 tp->t_stat.txretransmitbytes += len;
1318 }
1319 } else {
1320 tcpstat.tcps_sndpack++;
1321 tcpstat.tcps_sndbyte += len;
1322 if (nstat_collect) {
1323 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
1324 locked_add_64(&tp->t_inpcb->inp_stat->txbytes, len);
1325 }
1326 }
1327 /*
1328 * try to use the new interface that allocates all
1329 * the necessary mbuf hdrs under 1 mbuf lock and
1330 * avoids rescanning the socket mbuf list if
1331 * certain conditions are met. This routine can't
1332 * be used in the following cases...
1333 * 1) the protocol headers exceed the capacity of
1334 * of a single mbuf header's data area (no cluster attached)
1335 * 2) the length of the data being transmitted plus
1336 * the protocol headers fits into a single mbuf header's
1337 * data area (no cluster attached)
1338 */
1339 m = NULL;
1340
1341 /* minimum length we are going to allocate */
1342 allocated_len = MHLEN;
1343 if (MHLEN < hdrlen + max_linkhdr) {
1344 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1345 if (m == NULL) {
1346 error = ENOBUFS;
1347 goto out;
1348 }
1349 MCLGET(m, M_DONTWAIT);
1350 if ((m->m_flags & M_EXT) == 0) {
1351 m_freem(m);
1352 error = ENOBUFS;
1353 goto out;
1354 }
1355 m->m_data += max_linkhdr;
1356 m->m_len = hdrlen;
1357 allocated_len = MCLBYTES;
1358 }
1359 if (len <= allocated_len - hdrlen - max_linkhdr) {
1360 if (m == NULL) {
1361 VERIFY(allocated_len <= MHLEN);
1362 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1363 if (m == NULL) {
1364 error = ENOBUFS;
1365 goto out;
1366 }
1367 m->m_data += max_linkhdr;
1368 m->m_len = hdrlen;
1369 }
1370 /* makes sure we still have data left to be sent at this point */
1371 if (so->so_snd.sb_mb == NULL || off < 0) {
1372 if (m != NULL) m_freem(m);
1373 error = 0; /* should we return an error? */
1374 goto out;
1375 }
1376 m_copydata(so->so_snd.sb_mb, off, (int) len,
1377 mtod(m, caddr_t) + hdrlen);
1378 m->m_len += len;
1379 } else {
1380 if (m != NULL) {
1381 m->m_next = m_copy(so->so_snd.sb_mb, off, (int) len);
1382 if (m->m_next == 0) {
1383 (void) m_free(m);
1384 error = ENOBUFS;
1385 goto out;
1386 }
1387 } else {
1388 /*
1389 * determine whether the mbuf pointer and offset passed back by the 'last' call
1390 * to m_copym_with_hdrs are still valid... if the head of the socket chain has
1391 * changed (due to an incoming ACK for instance), or the offset into the chain we
1392 * just computed is different from the one last returned by m_copym_with_hdrs (perhaps
1393 * we're re-transmitting a packet sent earlier), than we can't pass the mbuf pointer and
1394 * offset into it as valid hints for m_copym_with_hdrs to use (if valid, these hints allow
1395 * m_copym_with_hdrs to avoid rescanning from the beginning of the socket buffer mbuf list.
1396 * setting the mbuf pointer to NULL is sufficient to disable the hint mechanism.
1397 */
1398 if (m_head != so->so_snd.sb_mb || sack_rxmit || last_off != off)
1399 m_lastm = NULL;
1400 last_off = off + len;
1401 m_head = so->so_snd.sb_mb;
1402
1403 /* makes sure we still have data left to be sent at this point */
1404 if (m_head == NULL) {
1405 error = 0; /* should we return an error? */
1406 goto out;
1407 }
1408
1409 /*
1410 * m_copym_with_hdrs will always return the last mbuf pointer and the offset into it that
1411 * it acted on to fullfill the current request, whether a valid 'hint' was passed in or not
1412 */
1413 if ((m = m_copym_with_hdrs(so->so_snd.sb_mb, off, len, M_DONTWAIT, &m_lastm, &m_off)) == NULL) {
1414 error = ENOBUFS;
1415 goto out;
1416 }
1417 m->m_data += max_linkhdr;
1418 m->m_len = hdrlen;
1419 }
1420 }
1421 /*
1422 * If we're sending everything we've got, set PUSH.
1423 * (This will keep happy those implementations which only
1424 * give data to the user when a buffer fills or
1425 * a PUSH comes in.)
1426 */
1427 if (off + len == so->so_snd.sb_cc)
1428 flags |= TH_PUSH;
1429 } else {
1430 if (tp->t_flags & TF_ACKNOW)
1431 tcpstat.tcps_sndacks++;
1432 else if (flags & (TH_SYN|TH_FIN|TH_RST))
1433 tcpstat.tcps_sndctrl++;
1434 else if (SEQ_GT(tp->snd_up, tp->snd_una))
1435 tcpstat.tcps_sndurg++;
1436 else
1437 tcpstat.tcps_sndwinup++;
1438
1439 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1440 if (m == NULL) {
1441 error = ENOBUFS;
1442 goto out;
1443 }
1444 if (MHLEN < (hdrlen + max_linkhdr)) {
1445 MCLGET(m, M_DONTWAIT);
1446 if ((m->m_flags & M_EXT) == 0) {
1447 m_freem(m);
1448 error = ENOBUFS;
1449 goto out;
1450 }
1451 }
1452 m->m_data += max_linkhdr;
1453 m->m_len = hdrlen;
1454 }
1455 m->m_pkthdr.rcvif = 0;
1456 #if CONFIG_MACF_NET
1457 mac_mbuf_label_associate_inpcb(tp->t_inpcb, m);
1458 #endif
1459 #if INET6
1460 if (isipv6) {
1461 ip6 = mtod(m, struct ip6_hdr *);
1462 th = (struct tcphdr *)(void *)(ip6 + 1);
1463 tcp_fillheaders(tp, ip6, th);
1464 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1465 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1466 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
1467 }
1468 svc_flags |= PKT_SCF_IPV6;
1469 } else
1470 #endif /* INET6 */
1471 {
1472 ip = mtod(m, struct ip *);
1473 ipov = (struct ipovly *)ip;
1474 th = (struct tcphdr *)(void *)(ip + 1);
1475 /* this picks up the pseudo header (w/o the length) */
1476 tcp_fillheaders(tp, ip, th);
1477 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1478 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1479 ip->ip_tos = IPTOS_ECN_ECT0;
1480 }
1481 }
1482
1483 /*
1484 * Fill in fields, remembering maximum advertised
1485 * window for use in delaying messages about window sizes.
1486 * If resending a FIN, be sure not to use a new sequence number.
1487 */
1488 if (flags & TH_FIN && (tp->t_flags & TF_SENTFIN) &&
1489 tp->snd_nxt == tp->snd_max)
1490 tp->snd_nxt--;
1491 /*
1492 * If we are doing retransmissions, then snd_nxt will
1493 * not reflect the first unsent octet. For ACK only
1494 * packets, we do not want the sequence number of the
1495 * retransmitted packet, we want the sequence number
1496 * of the next unsent octet. So, if there is no data
1497 * (and no SYN or FIN), use snd_max instead of snd_nxt
1498 * when filling in ti_seq. But if we are in persist
1499 * state, snd_max might reflect one byte beyond the
1500 * right edge of the window, so use snd_nxt in that
1501 * case, since we know we aren't doing a retransmission.
1502 * (retransmit and persist are mutually exclusive...)
1503 */
1504 if (sack_rxmit == 0) {
1505 if (len || (flags & (TH_SYN|TH_FIN)) || tp->t_timer[TCPT_PERSIST])
1506 th->th_seq = htonl(tp->snd_nxt);
1507 else
1508 th->th_seq = htonl(tp->snd_max);
1509 } else {
1510 th->th_seq = htonl(p->rxmit);
1511 p->rxmit += len;
1512 tp->sackhint.sack_bytes_rexmit += len;
1513 }
1514 th->th_ack = htonl(tp->rcv_nxt);
1515 tp->last_ack_sent = tp->rcv_nxt;
1516
1517 if (optlen) {
1518 bcopy(opt, th + 1, optlen);
1519 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1520 }
1521 th->th_flags = flags;
1522 /*
1523 * Calculate receive window. Don't shrink window,
1524 * but avoid silly window syndrome.
1525 */
1526 if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && recwin < (int)tp->t_maxseg)
1527 recwin = 0;
1528 if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt))
1529 recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt);
1530 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) {
1531 if (recwin > (int32_t)slowlink_wsize)
1532 recwin = slowlink_wsize;
1533 }
1534
1535 #if TRAFFIC_MGT
1536 if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) {
1537 if (tp->acc_iaj > tcp_acc_iaj_react_limit) {
1538 uint32_t min_iaj_win = tcp_min_iaj_win * tp->t_maxseg;
1539 if (tp->iaj_rwintop == 0 ||
1540 SEQ_LT(tp->iaj_rwintop, tp->rcv_adv))
1541 tp->iaj_rwintop = tp->rcv_adv;
1542 if (SEQ_LT(tp->iaj_rwintop, tp->rcv_nxt + min_iaj_win))
1543 tp->iaj_rwintop = tp->rcv_nxt + min_iaj_win;
1544 recwin = min(tp->iaj_rwintop - tp->rcv_nxt, recwin);
1545 }
1546 }
1547 #endif /* TRAFFIC_MGT */
1548
1549 if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
1550 recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
1551 th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
1552
1553 /*
1554 * Adjust the RXWIN0SENT flag - indicate that we have advertised
1555 * a 0 window. This may cause the remote transmitter to stall. This
1556 * flag tells soreceive() to disable delayed acknowledgements when
1557 * draining the buffer. This can occur if the receiver is attempting
1558 * to read more data then can be buffered prior to transmitting on
1559 * the connection.
1560 */
1561 if (th->th_win == 0)
1562 tp->t_flags |= TF_RXWIN0SENT;
1563 else
1564 tp->t_flags &= ~TF_RXWIN0SENT;
1565 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
1566 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
1567 th->th_flags |= TH_URG;
1568 } else
1569 /*
1570 * If no urgent pointer to send, then we pull
1571 * the urgent pointer to the left edge of the send window
1572 * so that it doesn't drift into the send window on sequence
1573 * number wraparound.
1574 */
1575 tp->snd_up = tp->snd_una; /* drag it along */
1576
1577 /*
1578 * Put TCP length in extended header, and then
1579 * checksum extended header and data.
1580 */
1581 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
1582 #if INET6
1583 if (isipv6) {
1584 /*
1585 * ip6_plen is not need to be filled now, and will be filled
1586 * in ip6_output.
1587 */
1588 m->m_pkthdr.csum_flags = CSUM_TCPIPV6;
1589 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1590 if (len + optlen)
1591 th->th_sum = in_addword(th->th_sum,
1592 htons((u_short)(optlen + len)));
1593 }
1594 else
1595 #endif /* INET6 */
1596 {
1597 m->m_pkthdr.csum_flags = CSUM_TCP;
1598 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1599 if (len + optlen)
1600 th->th_sum = in_addword(th->th_sum,
1601 htons((u_short)(optlen + len)));
1602 }
1603
1604 /*
1605 * Enable TSO and specify the size of the segments.
1606 * The TCP pseudo header checksum is always provided.
1607 */
1608 if (tso) {
1609 #if INET6
1610 if (isipv6)
1611 m->m_pkthdr.csum_flags = CSUM_TSO_IPV6;
1612 else
1613 #endif /* INET6 */
1614 m->m_pkthdr.csum_flags = CSUM_TSO_IPV4;
1615
1616 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
1617 }
1618 else
1619 m->m_pkthdr.tso_segsz = 0;
1620
1621 /*
1622 * In transmit state, time the transmission and arrange for
1623 * the retransmit. In persist state, just set snd_max.
1624 */
1625 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
1626 tcp_seq startseq = tp->snd_nxt;
1627
1628 /*
1629 * Advance snd_nxt over sequence space of this segment.
1630 */
1631 if (flags & (TH_SYN|TH_FIN)) {
1632 if (flags & TH_SYN)
1633 tp->snd_nxt++;
1634 if (flags & TH_FIN) {
1635 tp->snd_nxt++;
1636 tp->t_flags |= TF_SENTFIN;
1637 }
1638 }
1639 if (sack_rxmit)
1640 goto timer;
1641 tp->snd_nxt += len;
1642 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
1643 tp->snd_max = tp->snd_nxt;
1644 /*
1645 * Time this transmission if not a retransmission and
1646 * not currently timing anything.
1647 */
1648 if (tp->t_rtttime == 0) {
1649 tp->t_rtttime = tcp_now;
1650 tp->t_rtseq = startseq;
1651 tcpstat.tcps_segstimed++;
1652 }
1653 }
1654
1655 /*
1656 * Set retransmit timer if not currently set,
1657 * and not doing an ack or a keep-alive probe.
1658 * Initial value for retransmit timer is smoothed
1659 * round-trip time + 2 * round-trip time variance.
1660 * Initialize shift counter which is used for backoff
1661 * of retransmit time.
1662 */
1663 timer:
1664 if (tp->t_timer[TCPT_REXMT] == 0 &&
1665 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
1666 tp->snd_nxt != tp->snd_una)) {
1667 if (tp->t_timer[TCPT_PERSIST]) {
1668 tp->t_timer[TCPT_PERSIST] = 0;
1669 tp->t_rxtshift = 0;
1670 tp->rxt_start = 0;
1671 tp->t_persist_stop = 0;
1672 }
1673 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1674 }
1675 } else {
1676 /*
1677 * Persist case, update snd_max but since we are in
1678 * persist mode (no window) we do not update snd_nxt.
1679 */
1680 int xlen = len;
1681 if (flags & TH_SYN)
1682 ++xlen;
1683 if (flags & TH_FIN) {
1684 ++xlen;
1685 tp->t_flags |= TF_SENTFIN;
1686 }
1687 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
1688 tp->snd_max = tp->snd_nxt + len;
1689 }
1690
1691 #if TCPDEBUG
1692 /*
1693 * Trace.
1694 */
1695 if (so_options & SO_DEBUG)
1696 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
1697 #endif
1698
1699 /*
1700 * Fill in IP length and desired time to live and
1701 * send to IP level. There should be a better way
1702 * to handle ttl and tos; we could keep them in
1703 * the template, but need a way to checksum without them.
1704 */
1705 #ifdef INET6
1706 /*
1707 * m->m_pkthdr.len should have been set before cksum calcuration,
1708 * because in6_cksum() need it.
1709 */
1710 if (isipv6) {
1711 /*
1712 * we separately set hoplimit for every segment, since the
1713 * user might want to change the value via setsockopt.
1714 * Also, desired default hop limit might be changed via
1715 * Neighbor Discovery.
1716 */
1717 ip6->ip6_hlim = in6_selecthlim(tp->t_inpcb,
1718 tp->t_inpcb->in6p_route.ro_rt ?
1719 tp->t_inpcb->in6p_route.ro_rt->rt_ifp
1720 : NULL);
1721
1722 /* TODO: IPv6 IP6TOS_ECT bit on */
1723 KERNEL_DEBUG(DBG_LAYER_BEG,
1724 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
1725 (((tp->t_inpcb->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
1726 (tp->t_inpcb->in6p_faddr.s6_addr16[0] & 0xffff)),
1727 sendalot,0,0);
1728 } else
1729 #endif /* INET6 */
1730 {
1731 ip->ip_len = m->m_pkthdr.len;
1732 ip->ip_ttl = tp->t_inpcb->inp_ip_ttl; /* XXX */
1733 ip->ip_tos |= (tp->t_inpcb->inp_ip_tos & ~IPTOS_ECN_MASK);/* XXX */
1734 KERNEL_DEBUG(DBG_LAYER_BEG,
1735 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
1736 (((tp->t_inpcb->inp_laddr.s_addr & 0xffff) << 16) |
1737 (tp->t_inpcb->inp_faddr.s_addr & 0xffff)),
1738 0,0,0);
1739 }
1740
1741 /*
1742 * See if we should do MTU discovery.
1743 * Look at the flag updated on the following criterias:
1744 * 1) Path MTU discovery is authorized by the sysctl
1745 * 2) The route isn't set yet (unlikely but could happen)
1746 * 3) The route is up
1747 * 4) the MTU is not locked (if it is, then discovery has been
1748 * disabled for that route)
1749 */
1750 #ifdef INET6
1751 if (!isipv6)
1752 #endif
1753 if (path_mtu_discovery && (tp->t_flags & TF_PMTUD))
1754 ip->ip_off |= IP_DF;
1755
1756 #if IPSEC
1757 if (ipsec_bypass == 0)
1758 ipsec_setsocket(m, so);
1759 #endif /*IPSEC*/
1760
1761 /*
1762 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
1763 */
1764 lost = 0;
1765 m->m_pkthdr.socket_id = socket_id;
1766
1767 /*
1768 * Embed the flow hash in pkt hdr and mark the packet as
1769 * capable of flow controlling
1770 */
1771 m->m_pkthdr.m_flowhash = tp->t_inpcb->inp_flowhash;
1772 m->m_pkthdr.m_fhflags |=
1773 (PF_TAG_TCP | PF_TAG_FLOWHASH | PF_TAG_FLOWADV);
1774
1775 m->m_nextpkt = NULL;
1776
1777 if (tp->t_inpcb->inp_last_outifp != NULL &&
1778 tp->t_inpcb->inp_last_outifp != lo_ifp) {
1779 /* Hint to prioritize this packet if
1780 * 1. if the packet has no data
1781 * 2. the interface supports transmit-start model and did
1782 * not disable ACK prioritization.
1783 * 3. Only ACK flag is set.
1784 * 4. there is no outstanding data on this connection.
1785 */
1786 if (tcp_prioritize_acks != 0 && len == 0 &&
1787 (tp->t_inpcb->inp_last_outifp->if_eflags &
1788 (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART &&
1789 th->th_flags == TH_ACK && tp->snd_una == tp->snd_max &&
1790 tp->t_timer[TCPT_REXMT] == 0) {
1791 svc_flags |= PKT_SCF_TCP_ACK;
1792 }
1793 set_packet_service_class(m, so, MBUF_SC_UNSPEC, svc_flags);
1794 }
1795
1796 tp->t_pktlist_sentlen += len;
1797 tp->t_lastchain++;
1798
1799 #ifdef INET6
1800 if (isipv6) {
1801 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, tp->t_inpcb,
1802 struct ip6 *, ip6, struct tcpcb *, tp, struct tcphdr *,
1803 th);
1804 } else
1805 #endif
1806 {
1807 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, tp->t_inpcb,
1808 struct ip *, ip, struct tcpcb *, tp, struct tcphdr *, th);
1809 }
1810
1811 if (tp->t_pktlist_head != NULL) {
1812 tp->t_pktlist_tail->m_nextpkt = m;
1813 tp->t_pktlist_tail = m;
1814 } else {
1815 packchain_newlist++;
1816 tp->t_pktlist_head = tp->t_pktlist_tail = m;
1817 }
1818
1819 if ((lro_ackmore) && (!sackoptlen) && (!tp->t_timer[TCPT_PERSIST]) &&
1820 ((th->th_flags & TH_ACK) == TH_ACK) && (!len) &&
1821 (tp->t_state == TCPS_ESTABLISHED)) {
1822 /* For a pure ACK, see if you need to send more of them */
1823 mnext = tcp_send_lroacks(tp, m, th);
1824 if (mnext) {
1825 tp->t_pktlist_tail->m_nextpkt = mnext;
1826 if (mnext->m_nextpkt == NULL) {
1827 tp->t_pktlist_tail = mnext;
1828 tp->t_lastchain++;
1829 } else {
1830 struct mbuf *tail, *next;
1831 next = mnext->m_nextpkt;
1832 tail = next->m_nextpkt;
1833 while (tail) {
1834 next = tail;
1835 tail = tail->m_nextpkt;
1836 tp->t_lastchain++;
1837 }
1838 tp->t_pktlist_tail = next;
1839 }
1840 }
1841 }
1842
1843 if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
1844 (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
1845 (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || tp->t_force != 0 ||
1846 tp->t_lastchain >= tcp_packet_chaining) {
1847 error = 0;
1848 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
1849 tp->t_pktlist_head != NULL) {
1850 packetlist = tp->t_pktlist_head;
1851 packchain_listadd = tp->t_lastchain;
1852 packchain_sent++;
1853 lost = tp->t_pktlist_sentlen;
1854 TCP_PKTLIST_CLEAR(tp);
1855
1856 error = tcp_ip_output(so, tp, packetlist,
1857 packchain_listadd, tp_inp_options,
1858 (so_options & SO_DONTROUTE),
1859 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
1860 #ifdef INET6
1861 isipv6);
1862 #else
1863 0);
1864 #endif
1865
1866 if (error) {
1867 /*
1868 * Take into account the rest of unsent
1869 * packets in the packet list for this tcp
1870 * into "lost", since we're about to free
1871 * the whole list below.
1872 */
1873 lost += tp->t_pktlist_sentlen;
1874 break;
1875 } else {
1876 lost = 0;
1877 }
1878 }
1879 /* tcp was closed while we were in ip; resume close */
1880 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
1881 (tp->t_flags & TF_CLOSING)) {
1882 tp->t_flags &= ~TF_CLOSING;
1883 (void) tcp_close(tp);
1884 return (0);
1885 }
1886 } else {
1887 error = 0;
1888 packchain_looped++;
1889 tcpstat.tcps_sndtotal++;
1890
1891 goto again;
1892 }
1893 if (error) {
1894 /*
1895 * Assume that the packets were lost, so back out the
1896 * sequence number advance, if any. Note that the "lost"
1897 * variable represents the amount of user data sent during
1898 * the recent call to ip_output_list() plus the amount of
1899 * user data in the packet list for this tcp at the moment.
1900 */
1901 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
1902 /*
1903 * No need to check for TH_FIN here because
1904 * the TF_SENTFIN flag handles that case.
1905 */
1906 if ((flags & TH_SYN) == 0) {
1907 if (sack_rxmit) {
1908 p->rxmit -= lost;
1909 tp->sackhint.sack_bytes_rexmit -= lost;
1910 } else
1911 tp->snd_nxt -= lost;
1912 }
1913 }
1914 out:
1915 if (tp->t_pktlist_head != NULL)
1916 m_freem_list(tp->t_pktlist_head);
1917 TCP_PKTLIST_CLEAR(tp);
1918
1919 if (error == ENOBUFS) {
1920 if (!tp->t_timer[TCPT_REXMT] &&
1921 !tp->t_timer[TCPT_PERSIST])
1922 tp->t_timer[TCPT_REXMT] =
1923 OFFSET_FROM_START(tp, tp->t_rxtcur);
1924
1925 tp->snd_cwnd = tp->t_maxseg;
1926 tp->t_bytes_acked = 0;
1927
1928 tcp_check_timer_state(tp);
1929 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1930
1931 DTRACE_TCP5(cc, void, NULL, struct inpcb *, tp->t_inpcb,
1932 struct tcpcb *, tp, struct tcphdr *, NULL,
1933 int32_t, TCP_CC_OUTPUT_ERROR);
1934 return (0);
1935 }
1936 if (error == EMSGSIZE) {
1937 /*
1938 * ip_output() will have already fixed the route
1939 * for us. tcp_mtudisc() will, as its last action,
1940 * initiate retransmission, so it is important to
1941 * not do so here.
1942 *
1943 * If TSO was active we either got an interface
1944 * without TSO capabilits or TSO was turned off.
1945 * Disable it for this connection as too and
1946 * immediatly retry with MSS sized segments generated
1947 * by this function.
1948 */
1949 if (tso)
1950 tp->t_flags &= ~TF_TSO;
1951
1952 tcp_mtudisc(tp->t_inpcb, 0);
1953 tcp_check_timer_state(tp);
1954
1955 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1956 return 0;
1957 }
1958 if ((error == EHOSTUNREACH || error == ENETDOWN)
1959 && TCPS_HAVERCVDSYN(tp->t_state)) {
1960 tp->t_softerror = error;
1961 tcp_check_timer_state(tp);
1962 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1963 return (0);
1964 }
1965 tcp_check_timer_state(tp);
1966 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1967 return (error);
1968 }
1969
1970 tcpstat.tcps_sndtotal++;
1971
1972 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
1973 if (sendalot)
1974 goto again;
1975
1976 tcp_check_timer_state(tp);
1977 return (0);
1978 }
1979
1980 static int
1981 tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
1982 int cnt, struct mbuf *opt, int flags, int sack_in_progress, int recwin,
1983 boolean_t isipv6)
1984 {
1985 int error = 0;
1986 boolean_t chain;
1987 boolean_t unlocked = FALSE;
1988 struct inpcb *inp = tp->t_inpcb;
1989 struct ip_out_args ipoa =
1990 { IFSCOPE_NONE, { 0 }, IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR };
1991 struct route ro;
1992 struct ifnet *outif = NULL;
1993 #ifdef INET6
1994 struct ip6_out_args ip6oa =
1995 { IFSCOPE_NONE, { 0 }, IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR };
1996 struct route_in6 ro6;
1997 struct flowadv *adv =
1998 (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv);
1999 #else
2000 struct flowadv *adv = &ipoa.ipoa_flowadv;
2001 #endif /* !INET6 */
2002
2003 /* If socket was bound to an ifindex, tell ip_output about it */
2004 if (inp->inp_flags & INP_BOUND_IF) {
2005 #ifdef INET6
2006 if (isipv6) {
2007 ip6oa.ip6oa_boundif = inp->inp_boundifp->if_index;
2008 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
2009 } else
2010 #endif
2011 {
2012 ipoa.ipoa_boundif = inp->inp_boundifp->if_index;
2013 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
2014 }
2015 }
2016
2017 if (inp->inp_flags & INP_NO_IFT_CELLULAR) {
2018 #ifdef INET6
2019 if (isipv6)
2020 ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR;
2021 else
2022 #endif
2023 ipoa.ipoa_flags |= IPOAF_NO_CELLULAR;
2024 }
2025 #ifdef INET6
2026 if (isipv6)
2027 flags |= IPV6_OUTARGS;
2028 else
2029 #endif
2030 flags |= IP_OUTARGS;
2031
2032 /* Copy the cached route and take an extra reference */
2033 #ifdef INET6
2034 if (isipv6)
2035 in6p_route_copyout(inp, &ro6);
2036 else
2037 #endif
2038 inp_route_copyout(inp, &ro);
2039
2040 /*
2041 * Data sent (as far as we can tell).
2042 * If this advertises a larger window than any other segment,
2043 * then remember the size of the advertised window.
2044 * Make sure ACK/DELACK conditions are cleared before
2045 * we unlock the socket.
2046 */
2047 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
2048 tp->rcv_adv = tp->rcv_nxt + recwin;
2049 tp->last_ack_sent = tp->rcv_nxt;
2050 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
2051 tp->t_timer[TCPT_DELACK] = 0;
2052 tp->t_unacksegs = 0;
2053
2054 /* Increment the count of outstanding send operations */
2055 inp->inp_sndinprog_cnt++;
2056
2057 /*
2058 * If allowed, unlock TCP socket while in IP
2059 * but only if the connection is established and
2060 * in a normal mode where reentrancy on the tcpcb won't be
2061 * an issue:
2062 * - there is no SACK episode
2063 * - we're not in Fast Recovery mode
2064 * - if we're not sending from an upcall.
2065 */
2066 if (tcp_output_unlocked && !so->so_upcallusecount &&
2067 (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) &&
2068 ((tp->t_flags & TF_FASTRECOVERY) == 0)) {
2069
2070 unlocked = TRUE;
2071 socket_unlock(so, 0);
2072 }
2073
2074 /*
2075 * Don't send down a chain of packets when:
2076 * - TCP chaining is disabled
2077 * - there is an IPsec rule set
2078 * - there is a non default rule set for the firewall
2079 */
2080
2081 chain = tcp_packet_chaining > 1
2082 #if IPSEC
2083 && ipsec_bypass
2084 #endif
2085 #if IPFIREWALL
2086 && (fw_enable == 0 || fw_bypass)
2087 #endif
2088 ; // I'm important, not extraneous
2089
2090
2091 while (pkt != NULL) {
2092 struct mbuf *npkt = pkt->m_nextpkt;
2093
2094 if (!chain) {
2095 pkt->m_nextpkt = NULL;
2096 /*
2097 * If we are not chaining, make sure to set the packet
2098 * list count to 0 so that IP takes the right path;
2099 * this is important for cases such as IPSec where a
2100 * single mbuf might result in multiple mbufs as part
2101 * of the encapsulation. If a non-zero count is passed
2102 * down to IP, the head of the chain might change and
2103 * we could end up skipping it (thus generating bogus
2104 * packets). Fixing it in IP would be desirable, but
2105 * for now this would do it.
2106 */
2107 cnt = 0;
2108 }
2109 #ifdef INET6
2110 if (isipv6)
2111 error = ip6_output_list(pkt, cnt,
2112 inp->in6p_outputopts, &ro6, flags, NULL, NULL,
2113 &ip6oa);
2114 else
2115 #endif
2116 error = ip_output_list(pkt, cnt, opt, &ro, flags, NULL,
2117 &ipoa);
2118
2119 if (chain || error) {
2120 /*
2121 * If we sent down a chain then we are done since
2122 * the callee had taken care of everything; else
2123 * we need to free the rest of the chain ourselves.
2124 */
2125 if (!chain)
2126 m_freem_list(npkt);
2127 break;
2128 }
2129 pkt = npkt;
2130 }
2131
2132 if (unlocked)
2133 socket_lock(so, 0);
2134
2135 /*
2136 * Enter flow controlled state if the connection is established
2137 * and is not in recovery.
2138 *
2139 * A connection will enter suspended state even if it is in
2140 * recovery.
2141 */
2142 if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) ||
2143 adv->code == FADV_SUSPENDED) &&
2144 !(tp->t_flags & TF_CLOSING) &&
2145 tp->t_state == TCPS_ESTABLISHED) {
2146 int rc;
2147 rc = inp_set_fc_state(inp, adv->code);
2148
2149 if (rc == 1)
2150 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
2151 struct tcpcb *, tp, struct tcphdr *, NULL,
2152 int32_t, ((adv->code == FADV_FLOW_CONTROLLED) ?
2153 TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND));
2154 }
2155
2156 /*
2157 * When an interface queue gets suspended, some of the
2158 * packets are dropped. Return ENOBUFS, to update the
2159 * pcb state.
2160 */
2161 if (adv->code == FADV_SUSPENDED)
2162 error = ENOBUFS;
2163
2164 VERIFY(inp->inp_sndinprog_cnt > 0);
2165 if ( --inp->inp_sndinprog_cnt == 0)
2166 inp->inp_flags &= ~(INP_FC_FEEDBACK);
2167
2168 #ifdef INET6
2169 if (isipv6) {
2170 if (ro6.ro_rt != NULL && (outif = ro6.ro_rt->rt_ifp) !=
2171 inp->in6p_last_outifp)
2172 inp->in6p_last_outifp = outif;
2173 } else
2174 #endif
2175 if (ro.ro_rt != NULL && (outif = ro.ro_rt->rt_ifp) !=
2176 inp->inp_last_outifp)
2177 inp->inp_last_outifp = outif;
2178
2179 if ((inp->inp_flags & INP_NO_IFT_CELLULAR) && outif != NULL &&
2180 outif->if_type == IFT_CELLULAR)
2181 soevent(inp->inp_socket,
2182 (SO_FILT_HINT_LOCKED|SO_FILT_HINT_IFDENIED));
2183
2184 /* Synchronize cached PCB route & options */
2185 #ifdef INET6
2186 if (isipv6)
2187 in6p_route_copyin(inp, &ro6);
2188 else
2189 #endif
2190 inp_route_copyin(inp, &ro);
2191
2192 if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 &&
2193 tp->t_inpcb->inp_route.ro_rt != NULL) {
2194 /* If we found the route and there is an rtt on it
2195 * reset the retransmit timer
2196 */
2197 tcp_getrt_rtt(tp, tp->t_inpcb->in6p_route.ro_rt);
2198 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
2199 }
2200 return (error);
2201 }
2202
2203 void
2204 tcp_setpersist(tp)
2205 register struct tcpcb *tp;
2206 {
2207 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
2208
2209 /* If a PERSIST_TIMER option was set we will limit the
2210 * time the persist timer will be active for that connection
2211 * in order to avoid DOS by using zero window probes.
2212 * see rdar://5805356
2213 */
2214
2215 if ((tp->t_persist_timeout != 0) &&
2216 (tp->t_timer[TCPT_PERSIST] == 0) &&
2217 (tp->t_persist_stop == 0)) {
2218 tp->t_persist_stop = tcp_now + tp->t_persist_timeout;
2219 }
2220
2221 /*
2222 * Start/restart persistance timer.
2223 */
2224 TCPT_RANGESET(tp->t_timer[TCPT_PERSIST],
2225 t * tcp_backoff[tp->t_rxtshift],
2226 TCPTV_PERSMIN, TCPTV_PERSMAX, 0);
2227 tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]);
2228
2229 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
2230 tp->t_rxtshift++;
2231 }
2232
2233 /*
2234 * Send as many acks as data coalesced. Every other packet when stretch
2235 * ACK is not enabled. Every 8 packets, if stretch ACK is enabled.
2236 */
2237 static struct mbuf*
2238 tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th)
2239 {
2240 struct mbuf *mnext = NULL, *ack_chain = NULL, *tail = NULL;
2241 int count = 0;
2242 tcp_seq org_ack = ntohl(th->th_ack);
2243 tcp_seq prev_ack = 0;
2244 int tack_offset = 28; /* XXX IPv6 not supported */
2245 int ack_size = (tp->t_flags & TF_STRETCHACK) ?
2246 (maxseg_unacked * tp->t_maxseg) : (tp->t_maxseg << 1);
2247 int segs_acked = (tp->t_flags & TF_STRETCHACK) ? maxseg_unacked : 2;
2248 struct mbuf *prev_ack_pkt = NULL;
2249 struct socket *so = tp->t_inpcb->inp_socket;
2250
2251 count = tp->t_lropktlen/tp->t_maxseg;
2252
2253 prev_ack = (org_ack - tp->t_lropktlen) + ack_size;
2254 if (prev_ack < org_ack) {
2255 ack_chain = m_dup(m, M_DONTWAIT);
2256 if (ack_chain) {
2257 th->th_ack = htonl(prev_ack);
2258 tail = ack_chain;
2259 count -= segs_acked; /* accounts for prev_ack packet */
2260 count = (count <= segs_acked) ? 0 : count - segs_acked;
2261 tcpstat.tcps_sndacks++;
2262 so_tc_update_stats(m, so, m_get_service_class(m));
2263 } else {
2264 return NULL;
2265 }
2266 }
2267 else {
2268 tp->t_lropktlen = 0;
2269 return NULL;
2270 }
2271
2272 prev_ack_pkt = ack_chain;
2273
2274 while (count > 0) {
2275 if ((prev_ack + ack_size) < org_ack) {
2276 prev_ack += ack_size;
2277 } else {
2278 /*
2279 * The last ACK sent must have the ACK number that TCP
2280 * thinks is the last sent ACK number.
2281 */
2282 prev_ack = org_ack;
2283 }
2284 mnext = m_dup(prev_ack_pkt, M_DONTWAIT);
2285 if (mnext) {
2286 HTONL(prev_ack);
2287 bcopy(&prev_ack, mtod(prev_ack_pkt, caddr_t) + tack_offset, 4);
2288 NTOHL(prev_ack);
2289 tail->m_nextpkt = mnext;
2290 tail = mnext;
2291 count -= segs_acked;
2292 tcpstat.tcps_sndacks++;
2293 so_tc_update_stats(m, so, m_get_service_class(m));
2294 if (lrodebug == 5) {
2295 printf("%s: lropktlen = %d count = %d, th_ack = %x \n",
2296 __func__, tp->t_lropktlen, count,
2297 th->th_ack);
2298 }
2299 } else {
2300 if (lrodebug == 5) {
2301 printf("%s: failed to alloc mbuf.\n", __func__);
2302 }
2303 break;
2304 }
2305 prev_ack_pkt = mnext;
2306 }
2307 tp->t_lropktlen = 0;
2308 return ack_chain;
2309 }
mirror of XNU