]> git.saurik.com Git - apple/xnu.git/blob - bsd/netinet/tcp_output.c
projects / apple / xnu.git / blob
? search:


8a9eeb9cf07bf4a5ca08dc405f66ce1dcde8a745
[apple/xnu.git] / bsd / netinet / tcp_output.c
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #define _IP_VHL
71
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/mbuf.h>
78 #include <sys/domain.h>
79 #include <sys/protosw.h>
80 #include <sys/socket.h>
81 #include <sys/socketvar.h>
82
83 #include <net/route.h>
84 #include <net/ntstat.h>
85 #include <net/if_var.h>
86 #include <net/if.h>
87 #include <net/if_types.h>
88
89 #include <netinet/in.h>
90 #include <netinet/in_systm.h>
91 #include <netinet/in_var.h>
92 #include <netinet/ip.h>
93 #include <netinet/in_pcb.h>
94 #include <netinet/ip_var.h>
95 #include <mach/sdt.h>
96 #if INET6
97 #include <netinet6/in6_pcb.h>
98 #include <netinet/ip6.h>
99 #include <netinet6/ip6_var.h>
100 #endif
101 #include <netinet/tcp.h>
102 #define TCPOUTFLAGS
103 #include <netinet/tcp_fsm.h>
104 #include <netinet/tcp_seq.h>
105 #include <netinet/tcp_timer.h>
106 #include <netinet/tcp_var.h>
107 #include <netinet/tcpip.h>
108 #include <netinet/tcp_cc.h>
109 #if TCPDEBUG
110 #include <netinet/tcp_debug.h>
111 #endif
112 #include <sys/kdebug.h>
113 #include <mach/sdt.h>
114
115 #if IPSEC
116 #include <netinet6/ipsec.h>
117 #endif /*IPSEC*/
118
119 #if CONFIG_MACF_NET
120 #include <security/mac_framework.h>
121 #endif /* MAC_SOCKET */
122
123 #include <netinet/lro_ext.h>
124
125 #define DBG_LAYER_BEG NETDBG_CODE(DBG_NETTCP, 1)
126 #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3)
127 #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
128
129 #ifdef notyet
130 extern struct mbuf *m_copypack();
131 #endif
132
133 int path_mtu_discovery = 1;
134 SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW | CTLFLAG_LOCKED,
135 &path_mtu_discovery, 1, "Enable Path MTU Discovery");
136
137 int ss_fltsz = 1;
138 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
139 &ss_fltsz, 1, "Slow start flight size");
140
141 int ss_fltsz_local = 8; /* starts with eight segments max */
142 SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, CTLFLAG_RW | CTLFLAG_LOCKED,
143 &ss_fltsz_local, 1, "Slow start flight size for local networks");
144
145 int tcp_do_tso = 1;
146 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED,
147 &tcp_do_tso, 0, "Enable TCP Segmentation Offload");
148
149
150 int tcp_ecn_outbound = 0;
151 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_initiate_out, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_outbound,
152 0, "Initiate ECN for outbound connections");
153
154 int tcp_ecn_inbound = 0;
155 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_ecn_inbound,
156 0, "Allow ECN negotiation for inbound connections");
157
158 int tcp_packet_chaining = 50;
159 SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining,
160 0, "Enable TCP output packet chaining");
161
162 int tcp_output_unlocked = 1;
163 SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked,
164 0, "Unlock TCP when sending packets down to IP");
165
166 int tcp_do_rfc3390 = 1;
167 SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, CTLFLAG_RW | CTLFLAG_LOCKED,
168 &tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS");
169
170 int tcp_min_iaj_win = MIN_IAJ_WIN;
171 SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, CTLFLAG_RW | CTLFLAG_LOCKED,
172 &tcp_min_iaj_win, 1, "Minimum recv win based on inter-packet arrival jitter");
173
174 int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT;
175 SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, CTLFLAG_RW | CTLFLAG_LOCKED,
176 &tcp_acc_iaj_react_limit, 1, "Accumulated IAJ when receiver starts to react");
177
178 uint32_t tcp_do_autosendbuf = 1;
179 SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, CTLFLAG_RW | CTLFLAG_LOCKED,
180 &tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning");
181
182 uint32_t tcp_autosndbuf_inc = 8 * 1024;
183 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, CTLFLAG_RW | CTLFLAG_LOCKED,
184 &tcp_autosndbuf_inc, 1, "Increment in send socket bufffer size");
185
186 uint32_t tcp_autosndbuf_max = 512 * 1024;
187 SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, CTLFLAG_RW | CTLFLAG_LOCKED,
188 &tcp_autosndbuf_max, 1, "Maximum send socket buffer size");
189
190 uint32_t tcp_prioritize_acks = 1;
191 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, CTLFLAG_RW | CTLFLAG_LOCKED,
192 &tcp_prioritize_acks, 1, "Prioritize pure acks");
193
194 static int32_t packchain_newlist = 0;
195 static int32_t packchain_looped = 0;
196 static int32_t packchain_sent = 0;
197
198 /* temporary: for testing */
199 #if IPSEC
200 extern int ipsec_bypass;
201 #endif
202
203 extern int slowlink_wsize; /* window correction for slow links */
204 #if IPFIREWALL
205 extern int fw_enable; /* firewall check for packet chaining */
206 extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */
207 #endif /* IPFIREWALL */
208
209 extern vm_size_t so_cache_zone_element_size;
210 #if RANDOM_IP_ID
211 extern int ip_use_randomid;
212 #endif /* RANDOM_IP_ID */
213 extern u_int32_t dlil_filter_count;
214 extern u_int32_t kipf_count;
215 extern int tcp_recv_bg;
216 extern int maxseg_unacked;
217
218 static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int,
219 struct mbuf *, int, int, int32_t, boolean_t);
220
221 extern uint32_t get_base_rtt(struct tcpcb *tp);
222 static struct mbuf* tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th);
223
224 static __inline__ u_int16_t
225 get_socket_id(struct socket * s)
226 {
227 u_int16_t val;
228
229 if (so_cache_zone_element_size == 0) {
230 return (0);
231 }
232 val = (u_int16_t)(((uintptr_t)s) / so_cache_zone_element_size);
233 if (val == 0) {
234 val = 0xffff;
235 }
236 return (val);
237 }
238
239 /*
240 * Tcp output routine: figure out what should be sent and send it.
241 *
242 * Returns: 0 Success
243 * EADDRNOTAVAIL
244 * ENOBUFS
245 * EMSGSIZE
246 * EHOSTUNREACH
247 * ENETDOWN
248 * ip_output_list:ENOMEM
249 * ip_output_list:EADDRNOTAVAIL
250 * ip_output_list:ENETUNREACH
251 * ip_output_list:EHOSTUNREACH
252 * ip_output_list:EACCES
253 * ip_output_list:EMSGSIZE
254 * ip_output_list:ENOBUFS
255 * ip_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
256 * ip6_output_list:EINVAL
257 * ip6_output_list:EOPNOTSUPP
258 * ip6_output_list:EHOSTUNREACH
259 * ip6_output_list:EADDRNOTAVAIL
260 * ip6_output_list:ENETUNREACH
261 * ip6_output_list:EMSGSIZE
262 * ip6_output_list:ENOBUFS
263 * ip6_output_list:??? [ignorable: mostly IPSEC/firewall/DLIL]
264 */
265 int
266 tcp_output(struct tcpcb *tp)
267 {
268 struct socket *so = tp->t_inpcb->inp_socket;
269 int32_t len, recwin, sendwin, off;
270 int flags, error;
271 register struct mbuf *m;
272 struct ip *ip = NULL;
273 register struct ipovly *ipov = NULL;
274 #if INET6
275 struct ip6_hdr *ip6 = NULL;
276 #endif /* INET6 */
277 register struct tcphdr *th;
278 u_char opt[TCP_MAXOLEN];
279 unsigned ipoptlen, optlen, hdrlen;
280 int idle, sendalot, lost = 0;
281 int i, sack_rxmit;
282 int tso = 0;
283 int sack_bytes_rxmt;
284 struct sackhole *p;
285 #ifdef IPSEC
286 unsigned ipsec_optlen = 0;
287 #endif
288 int last_off = 0;
289 int m_off;
290 int idle_time = 0;
291 struct mbuf *m_lastm = NULL;
292 struct mbuf *m_head = NULL;
293 struct mbuf *packetlist = NULL;
294 struct mbuf *tp_inp_options = tp->t_inpcb->inp_depend4.inp4_options;
295 #if INET6
296 int isipv6 = tp->t_inpcb->inp_vflag & INP_IPV6 ;
297 #endif
298 short packchain_listadd = 0;
299 u_int16_t socket_id = get_socket_id(so);
300 int so_options = so->so_options;
301 struct rtentry *rt;
302 u_int32_t basertt, svc_flags = 0;
303 u_int32_t lro_ackmore = (tp->t_lropktlen != 0) ? 1 : 0;
304 struct mbuf *mnext = NULL;
305 int sackoptlen = 0;
306
307 /*
308 * Determine length of data that should be transmitted,
309 * and flags that will be used.
310 * If there is some data or critical controls (SYN, RST)
311 * to send, then transmit; otherwise, investigate further.
312 */
313 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
314
315 /* Since idle_time is signed integer, the following integer subtraction
316 * will take care of wrap around of tcp_now
317 */
318 idle_time = tcp_now - tp->t_rcvtime;
319 if (idle && idle_time >= TCP_IDLETIMEOUT(tp)) {
320 if (CC_ALGO(tp)->after_idle != NULL)
321 CC_ALGO(tp)->after_idle(tp);
322 DTRACE_TCP5(cc, void, NULL, struct inpcb *, tp->t_inpcb,
323 struct tcpcb *, tp, struct tcphdr *, NULL,
324 int32_t, TCP_CC_IDLE_TIMEOUT);
325 }
326 tp->t_flags &= ~TF_LASTIDLE;
327 if (idle) {
328 if (tp->t_flags & TF_MORETOCOME) {
329 tp->t_flags |= TF_LASTIDLE;
330 idle = 0;
331 }
332 }
333 again:
334 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
335
336 #if INET6
337 if (isipv6) {
338 KERNEL_DEBUG(DBG_LAYER_BEG,
339 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
340 (((tp->t_inpcb->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
341 (tp->t_inpcb->in6p_faddr.s6_addr16[0] & 0xffff)),
342 sendalot,0,0);
343 } else
344 #endif
345
346 {
347 KERNEL_DEBUG(DBG_LAYER_BEG,
348 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
349 (((tp->t_inpcb->inp_laddr.s_addr & 0xffff) << 16) |
350 (tp->t_inpcb->inp_faddr.s_addr & 0xffff)),
351 sendalot,0,0);
352 }
353 /*
354 * If the route generation id changed, we need to check that our
355 * local (source) IP address is still valid. If it isn't either
356 * return error or silently do nothing (assuming the address will
357 * come back before the TCP connection times out).
358 */
359 rt = tp->t_inpcb->inp_route.ro_rt;
360 if (rt != NULL && (!(rt->rt_flags & RTF_UP) ||
361 rt->generation_id != route_generation)) {
362 struct ifnet *ifp;
363 struct in_ifaddr *ia = NULL;
364 struct in6_ifaddr *ia6 = NULL;
365 int found_srcaddr = 0;
366
367 /* disable multipages at the socket */
368 somultipages(so, FALSE);
369
370 /* Disable TSO for the socket until we know more */
371 tp->t_flags &= ~TF_TSO;
372
373 if (isipv6) {
374 ia6 = ifa_foraddr6(&tp->t_inpcb->in6p_laddr);
375 if (ia6 != NULL)
376 found_srcaddr = 1;
377 } else {
378 ia = ifa_foraddr(tp->t_inpcb->inp_laddr.s_addr);
379 if (ia != NULL)
380 found_srcaddr = 1;
381 }
382
383 /* check that the source address is still valid */
384 if (found_srcaddr == 0) {
385
386 soevent(so,
387 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_NOSRCADDR));
388
389 if (tp->t_state >= TCPS_CLOSE_WAIT) {
390 tcp_drop(tp, EADDRNOTAVAIL);
391 return(EADDRNOTAVAIL);
392 }
393
394 /* set Retransmit timer if it wasn't set
395 * reset Persist timer and shift register as the
396 * advertised peer window may not be valid anymore
397 */
398
399 if (!tp->t_timer[TCPT_REXMT]) {
400 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
401 if (tp->t_timer[TCPT_PERSIST]) {
402 tp->t_timer[TCPT_PERSIST] = 0;
403 tp->t_rxtshift = 0;
404 tp->t_persist_stop = 0;
405 tp->rxt_start = 0;
406 }
407 }
408
409 if (tp->t_pktlist_head != NULL)
410 m_freem_list(tp->t_pktlist_head);
411 TCP_PKTLIST_CLEAR(tp);
412
413 /* drop connection if source address isn't available */
414 if (so->so_flags & SOF_NOADDRAVAIL) {
415 tcp_drop(tp, EADDRNOTAVAIL);
416 return(EADDRNOTAVAIL);
417 }
418 else {
419 tcp_check_timer_state(tp);
420 return(0); /* silently ignore, keep data in socket: address may be back */
421 }
422 }
423 if (ia != NULL)
424 IFA_REMREF(&ia->ia_ifa);
425
426 if (ia6 != NULL)
427 IFA_REMREF(&ia6->ia_ifa);
428
429 /*
430 * Address is still valid; check for multipages capability
431 * again in case the outgoing interface has changed.
432 */
433 RT_LOCK(rt);
434 if ((ifp = rt->rt_ifp) != NULL) {
435 somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
436 tcp_set_tso(tp, ifp);
437 }
438 if (rt->rt_flags & RTF_UP)
439 rt->generation_id = route_generation;
440 /*
441 * See if we should do MTU discovery. Don't do it if:
442 * 1) it is disabled via the sysctl
443 * 2) the route isn't up
444 * 3) the MTU is locked (if it is, then discovery has been
445 * disabled)
446 */
447
448 if (!path_mtu_discovery || ((rt != NULL) &&
449 (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU))))
450 tp->t_flags &= ~TF_PMTUD;
451 else
452 tp->t_flags |= TF_PMTUD;
453
454 RT_UNLOCK(rt);
455 }
456
457 /*
458 * If we've recently taken a timeout, snd_max will be greater than
459 * snd_nxt. There may be SACK information that allows us to avoid
460 * resending already delivered data. Adjust snd_nxt accordingly.
461 */
462 if (tp->sack_enable && SEQ_LT(tp->snd_nxt, tp->snd_max))
463 tcp_sack_adjust(tp);
464 sendalot = 0;
465 off = tp->snd_nxt - tp->snd_una;
466 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
467
468 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0)
469 sendwin = min(sendwin, slowlink_wsize);
470
471 flags = tcp_outflags[tp->t_state];
472 /*
473 * Send any SACK-generated retransmissions. If we're explicitly trying
474 * to send out new data (when sendalot is 1), bypass this function.
475 * If we retransmit in fast recovery mode, decrement snd_cwnd, since
476 * we're replacing a (future) new transmission with a retransmission
477 * now, and we previously incremented snd_cwnd in tcp_input().
478 */
479 /*
480 * Still in sack recovery , reset rxmit flag to zero.
481 */
482 sack_rxmit = 0;
483 sack_bytes_rxmt = 0;
484 len = 0;
485 p = NULL;
486 if (tp->sack_enable && IN_FASTRECOVERY(tp) &&
487 (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
488 int32_t cwin;
489
490 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
491 if (cwin < 0)
492 cwin = 0;
493 /* Do not retransmit SACK segments beyond snd_recover */
494 if (SEQ_GT(p->end, tp->snd_recover)) {
495 /*
496 * (At least) part of sack hole extends beyond
497 * snd_recover. Check to see if we can rexmit data
498 * for this hole.
499 */
500 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
501 /*
502 * Can't rexmit any more data for this hole.
503 * That data will be rexmitted in the next
504 * sack recovery episode, when snd_recover
505 * moves past p->rxmit.
506 */
507 p = NULL;
508 goto after_sack_rexmit;
509 } else
510 /* Can rexmit part of the current hole */
511 len = ((int32_t)min(cwin,
512 tp->snd_recover - p->rxmit));
513 } else {
514 len = ((int32_t)min(cwin, p->end - p->rxmit));
515 }
516 if (len > 0) {
517 off = p->rxmit - tp->snd_una; /* update off only if we really transmit SACK data */
518 sack_rxmit = 1;
519 sendalot = 1;
520 tcpstat.tcps_sack_rexmits++;
521 tcpstat.tcps_sack_rexmit_bytes +=
522 min(len, tp->t_maxseg);
523 if (nstat_collect) {
524 nstat_route_tx(tp->t_inpcb->inp_route.ro_rt, 1,
525 min(len, tp->t_maxseg), NSTAT_TX_FLAG_RETRANSMIT);
526 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
527 locked_add_64(&tp->t_inpcb->inp_stat->txbytes,
528 min(len, tp->t_maxseg));
529 tp->t_stat.txretransmitbytes += min(len, tp->t_maxseg);
530 }
531 } else {
532 len = 0;
533 }
534 }
535 after_sack_rexmit:
536 /*
537 * Get standard flags, and add SYN or FIN if requested by 'hidden'
538 * state flags.
539 */
540 if (tp->t_flags & TF_NEEDFIN)
541 flags |= TH_FIN;
542 if (tp->t_flags & TF_NEEDSYN)
543 flags |= TH_SYN;
544
545 /*
546 * If in persist timeout with window of 0, send 1 byte.
547 * Otherwise, if window is small but nonzero
548 * and timer expired, we will send what we can
549 * and go to transmit state.
550 */
551 if (tp->t_force) {
552 if (sendwin == 0) {
553 /*
554 * If we still have some data to send, then
555 * clear the FIN bit. Usually this would
556 * happen below when it realizes that we
557 * aren't sending all the data. However,
558 * if we have exactly 1 byte of unsent data,
559 * then it won't clear the FIN bit below,
560 * and if we are in persist state, we wind
561 * up sending the packet without recording
562 * that we sent the FIN bit.
563 *
564 * We can't just blindly clear the FIN bit,
565 * because if we don't have any more data
566 * to send then the probe will be the FIN
567 * itself.
568 */
569 if (off < so->so_snd.sb_cc)
570 flags &= ~TH_FIN;
571 sendwin = 1;
572 } else {
573 tp->t_timer[TCPT_PERSIST] = 0;
574 tp->t_rxtshift = 0;
575 tp->rxt_start = 0;
576 tp->t_persist_stop = 0;
577 }
578 }
579
580 /*
581 * If snd_nxt == snd_max and we have transmitted a FIN, the
582 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
583 * a negative length. This can also occur when TCP opens up
584 * its congestion window while receiving additional duplicate
585 * acks after fast-retransmit because TCP will reset snd_nxt
586 * to snd_max after the fast-retransmit.
587 *
588 * In the normal retransmit-FIN-only case, however, snd_nxt will
589 * be set to snd_una, the offset will be 0, and the length may
590 * wind up 0.
591 *
592 * If sack_rxmit is true we are retransmitting from the scoreboard
593 * in which case len is already set.
594 */
595 if (sack_rxmit == 0) {
596 if (sack_bytes_rxmt == 0)
597 len = min(so->so_snd.sb_cc, sendwin) - off;
598 else {
599 int32_t cwin;
600
601 /*
602 * We are inside of a SACK recovery episode and are
603 * sending new data, having retransmitted all the
604 * data possible in the scoreboard.
605 */
606 len = min(so->so_snd.sb_cc, tp->snd_wnd)
607 - off;
608 /*
609 * Don't remove this (len > 0) check !
610 * We explicitly check for len > 0 here (although it
611 * isn't really necessary), to work around a gcc
612 * optimization issue - to force gcc to compute
613 * len above. Without this check, the computation
614 * of len is bungled by the optimizer.
615 */
616 if (len > 0) {
617 cwin = tp->snd_cwnd -
618 (tp->snd_nxt - tp->sack_newdata) -
619 sack_bytes_rxmt;
620 if (cwin < 0)
621 cwin = 0;
622 len = imin(len, cwin);
623 }
624 else
625 len = 0;
626 }
627 }
628
629 /*
630 * Lop off SYN bit if it has already been sent. However, if this
631 * is SYN-SENT state and if segment contains data and if we don't
632 * know that foreign host supports TAO, suppress sending segment.
633 */
634 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
635 flags &= ~TH_SYN;
636 off--, len++;
637 if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
638 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
639 tp->t_pktlist_head != NULL) {
640 packetlist = tp->t_pktlist_head;
641 packchain_listadd = tp->t_lastchain;
642 packchain_sent++;
643 TCP_PKTLIST_CLEAR(tp);
644
645 error = tcp_ip_output(so, tp, packetlist,
646 packchain_listadd, tp_inp_options,
647 (so_options & SO_DONTROUTE),
648 (sack_rxmit | (sack_bytes_rxmt != 0)), 0,
649 #ifdef INET6
650 isipv6);
651 #else
652 0);
653 #endif
654
655
656 }
657 /*
658 * tcp was closed while we were in ip,
659 * resume close
660 */
661 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
662 (tp->t_flags & TF_CLOSING)) {
663 tp->t_flags &= ~TF_CLOSING;
664 (void) tcp_close(tp);
665 } else {
666 tcp_check_timer_state(tp);
667 }
668 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
669 0,0,0,0,0);
670 return(0);
671 }
672 }
673
674 /*
675 * Be careful not to send data and/or FIN on SYN segments.
676 * This measure is needed to prevent interoperability problems
677 * with not fully conformant TCP implementations.
678 */
679 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
680 len = 0;
681 flags &= ~TH_FIN;
682 }
683
684 /* The check here used to be (len < 0). Some times len is zero when
685 * the congestion window is closed and we need to check if persist timer
686 * has to be set in that case. But don't set persist until connection
687 * is established.
688 */
689 if (len <= 0 && !(flags & TH_SYN)) {
690 /*
691 * If FIN has been sent but not acked,
692 * but we haven't been called to retransmit,
693 * len will be < 0. Otherwise, window shrank
694 * after we sent into it. If window shrank to 0,
695 * cancel pending retransmit, pull snd_nxt back
696 * to (closed) window, and set the persist timer
697 * if it isn't already going. If the window didn't
698 * close completely, just wait for an ACK.
699 */
700 len = 0;
701 if (sendwin == 0) {
702 tp->t_timer[TCPT_REXMT] = 0;
703 tp->t_rxtshift = 0;
704 tp->rxt_start = 0;
705 tp->snd_nxt = tp->snd_una;
706 if (tp->t_timer[TCPT_PERSIST] == 0)
707 tcp_setpersist(tp);
708 }
709 }
710
711 /* Automatic sizing of send socket buffer. Increase the send socket buffer
712 * size if all of the following criteria are met
713 * 1. the receiver has enough buffer space for this data
714 * 2. send buffer is filled to 7/8th with data (so we actually
715 * have data to make use of it);
716 * 3. our send window (slow start and congestion controlled) is
717 * larger than sent but unacknowledged data in send buffer.
718 */
719 basertt = get_base_rtt(tp);
720 if (tcp_do_autosendbuf == 1 &&
721 !INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb) && !IN_FASTRECOVERY(tp) &&
722 (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
723 tcp_cansbgrow(&so->so_snd)) {
724 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
725 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
726 sendwin >= (so->so_snd.sb_cc -
727 (tp->snd_nxt - tp->snd_una))) {
728 /* Also increase the send buffer only if the
729 * round-trip time is not increasing because we do
730 * not want to contribute to latency by filling buffers.
731 * We also do not want to hold onto application's
732 * old data for too long. Interactive applications would
733 * rather discard old data.
734 */
735 if (tp->t_rttcur <=
736 (basertt + 25)) {
737 if (sbreserve(&so->so_snd,
738 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
739 tcp_autosndbuf_max)) == 1) {
740 so->so_snd.sb_idealsize = so->so_snd.sb_hiwat;
741 }
742 } else {
743 so->so_snd.sb_idealsize =
744 max(tcp_sendspace, so->so_snd.sb_hiwat -
745 (2 * tcp_autosndbuf_inc));
746 so->so_snd.sb_flags |= SB_TRIM;
747 }
748 }
749 }
750
751 /*
752 * Truncate to the maximum segment length or enable TCP Segmentation
753 * Offloading (if supported by hardware) and ensure that FIN is removed
754 * if the length no longer contains the last data byte.
755 *
756 * TSO may only be used if we are in a pure bulk sending state. The
757 * presence of TCP-MD5, SACK retransmits, SACK advertizements, ipfw rules
758 * and IP options prevent using TSO. With TSO the TCP header is the same
759 * (except for the sequence number) for all generated packets. This
760 * makes it impossible to transmit any options which vary per generated
761 * segment or packet.
762 *
763 * The length of TSO bursts is limited to TCP_MAXWIN. That limit and
764 * removal of FIN (if not already catched here) are handled later after
765 * the exact length of the TCP options are known.
766 */
767 #if IPSEC
768 /*
769 * Pre-calculate here as we save another lookup into the darknesses
770 * of IPsec that way and can actually decide if TSO is ok.
771 */
772 if (ipsec_bypass == 0)
773 ipsec_optlen = ipsec_hdrsiz_tcp(tp);
774 #endif
775
776 if (len > tp->t_maxseg) {
777 if ((tp->t_flags & TF_TSO) && tcp_do_tso &&
778 #if RANDOM_IP_ID
779 ip_use_randomid &&
780 #endif /* RANDOM_IP_ID */
781 kipf_count == 0 && dlil_filter_count == 0 &&
782 tp->rcv_numsacks == 0 && sack_rxmit == 0 && sack_bytes_rxmt == 0 &&
783 tp->t_inpcb->inp_options == NULL &&
784 tp->t_inpcb->in6p_options == NULL
785 #if IPSEC
786 && ipsec_optlen == 0
787 #endif
788 #if IPFIREWALL
789 && (fw_enable == 0 || fw_bypass)
790 #endif
791 ) {
792 tso = 1;
793 sendalot = 0;
794 } else {
795 len = tp->t_maxseg;
796 sendalot = 1;
797 tso = 0;
798 }
799 }
800 if (sack_rxmit) {
801 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
802 flags &= ~TH_FIN;
803 } else {
804 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
805 flags &= ~TH_FIN;
806 }
807
808 recwin = tcp_sbspace(tp);
809
810 /*
811 * Sender silly window avoidance. We transmit under the following
812 * conditions when len is non-zero:
813 *
814 * - we've timed out (e.g. persist timer)
815 * - we need to retransmit
816 * - We have a full segment (or more with TSO)
817 * - This is the last buffer in a write()/send() and we are
818 * either idle or running NODELAY
819 * - we have more then 1/2 the maximum send window's worth of
820 * data (receiver may be limited the window size)
821 */
822 if (len) {
823 if (tp->t_force) {
824 tp->t_flags &= ~TF_MAXSEGSNT;
825 goto send;
826 }
827 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
828 tp->t_flags &= ~TF_MAXSEGSNT;
829 goto send;
830 }
831 if (sack_rxmit)
832 goto send;
833
834 /*
835 * Send new data on the connection only if it is
836 * not flow controlled
837 */
838 if (!INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb) ||
839 tp->t_state != TCPS_ESTABLISHED) {
840 if (len >= tp->t_maxseg) {
841 tp->t_flags |= TF_MAXSEGSNT;
842 goto send;
843 }
844 if (!(tp->t_flags & TF_MORETOCOME) &&
845 (idle || tp->t_flags & TF_NODELAY || tp->t_flags & TF_MAXSEGSNT) &&
846 (tp->t_flags & TF_NOPUSH) == 0 &&
847 len + off >= so->so_snd.sb_cc) {
848 tp->t_flags &= ~TF_MAXSEGSNT;
849 goto send;
850 }
851 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
852 tp->t_flags &= ~TF_MAXSEGSNT;
853 goto send;
854 }
855 } else {
856 tcpstat.tcps_fcholdpacket++;
857 }
858 }
859
860 /*
861 * Compare available window to amount of window
862 * known to peer (as advertised window less
863 * next expected input). If the difference is at least two
864 * max size segments, or at least 25% of the maximum possible
865 * window, then want to send a window update to peer.
866 * Skip this if the connection is in T/TCP half-open state.
867 */
868 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) {
869 /*
870 * "adv" is the amount we can increase the window,
871 * taking into account that we are limited by
872 * TCP_MAXWIN << tp->rcv_scale.
873 */
874 int32_t adv = imin(recwin, (int)TCP_MAXWIN << tp->rcv_scale) -
875 (tp->rcv_adv - tp->rcv_nxt);
876
877 if (adv >= (int32_t) (2 * tp->t_maxseg)) {
878 /* Update only if the resulting scaled value of the window changed, or
879 * if there is a change in the sequence since the last ack.
880 * This avoids what appears as dupe ACKS (see rdar://5640997)
881 *
882 * If streaming is detected avoid sending too many window updates.
883 * We will depend on the delack timer to send a window update
884 * when needed.
885 */
886 if ((tp->t_flags & TF_STRETCHACK) == 0 &&
887 (tp->last_ack_sent != tp->rcv_nxt ||
888 ((recwin + adv) >> tp->rcv_scale) > recwin)) {
889 goto send;
890 }
891
892 /* Make sure that the delayed ack timer is set if we
893 * delayed sending a window update because of streaming
894 * detection.
895 */
896 if ((tp->t_flags & TF_STRETCHACK) != 0 &&
897 (tp->t_flags & TF_DELACK) == 0) {
898 tp->t_flags |= TF_DELACK;
899 tp->t_timer[TCPT_DELACK] = OFFSET_FROM_START(tp, tcp_delack);
900 }
901 }
902 if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat)
903 goto send;
904 }
905
906 /*
907 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
908 * is also a catch-all for the retransmit timer timeout case.
909 */
910 if (tp->t_flags & TF_ACKNOW)
911 goto send;
912 if ((flags & TH_RST) ||
913 ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
914 goto send;
915 if (SEQ_GT(tp->snd_up, tp->snd_una))
916 goto send;
917 /*
918 * If our state indicates that FIN should be sent
919 * and we have not yet done so, then we need to send.
920 */
921 if (flags & TH_FIN &&
922 ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
923 goto send;
924 /*
925 * In SACK, it is possible for tcp_output to fail to send a segment
926 * after the retransmission timer has been turned off. Make sure
927 * that the retransmission timer is set.
928 */
929 if (tp->sack_enable && (tp->t_state >= TCPS_ESTABLISHED) &&
930 SEQ_GT(tp->snd_max, tp->snd_una) &&
931 tp->t_timer[TCPT_REXMT] == 0 &&
932 tp->t_timer[TCPT_PERSIST] == 0) {
933 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
934 goto just_return;
935 }
936 /*
937 * TCP window updates are not reliable, rather a polling protocol
938 * using ``persist'' packets is used to insure receipt of window
939 * updates. The three ``states'' for the output side are:
940 * idle not doing retransmits or persists
941 * persisting to move a small or zero window
942 * (re)transmitting and thereby not persisting
943 *
944 * tp->t_timer[TCPT_PERSIST]
945 * is set when we are in persist state.
946 * tp->t_force
947 * is set when we are called to send a persist packet.
948 * tp->t_timer[TCPT_REXMT]
949 * is set when we are retransmitting
950 * The output side is idle when both timers are zero.
951 *
952 * If send window is too small, there is data to transmit, and no
953 * retransmit or persist is pending, then go to persist state.
954 * If nothing happens soon, send when timer expires:
955 * if window is nonzero, transmit what we can,
956 * otherwise force out a byte.
957 */
958 if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 &&
959 tp->t_timer[TCPT_PERSIST] == 0) {
960 tp->t_rxtshift = 0;
961 tp->rxt_start = 0;
962 tcp_setpersist(tp);
963 }
964 just_return:
965 /*
966 * If there is no reason to send a segment, just return.
967 * but if there is some packets left in the packet list, send them now.
968 */
969 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
970 tp->t_pktlist_head != NULL) {
971 packetlist = tp->t_pktlist_head;
972 packchain_listadd = tp->t_lastchain;
973 packchain_sent++;
974 TCP_PKTLIST_CLEAR(tp);
975
976 error = tcp_ip_output(so, tp, packetlist, packchain_listadd,
977 tp_inp_options, (so_options & SO_DONTROUTE),
978 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
979 #ifdef INET6
980 isipv6);
981 #else
982 0);
983 #endif
984 }
985 /* tcp was closed while we were in ip; resume close */
986 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
987 (tp->t_flags & TF_CLOSING)) {
988 tp->t_flags &= ~TF_CLOSING;
989 (void) tcp_close(tp);
990 } else {
991 tcp_check_timer_state(tp);
992 }
993 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
994 return (0);
995
996 send:
997 /*
998 * Before ESTABLISHED, force sending of initial options
999 * unless TCP set not to do any options.
1000 * NOTE: we assume that the IP/TCP header plus TCP options
1001 * always fit in a single mbuf, leaving room for a maximum
1002 * link header, i.e.
1003 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1004 */
1005 optlen = 0;
1006 #if INET6
1007 if (isipv6)
1008 hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1009 else
1010 #endif
1011 hdrlen = sizeof (struct tcpiphdr);
1012 if (flags & TH_SYN) {
1013 tp->snd_nxt = tp->iss;
1014 if ((tp->t_flags & TF_NOOPT) == 0) {
1015 u_short mss;
1016
1017 opt[0] = TCPOPT_MAXSEG;
1018 opt[1] = TCPOLEN_MAXSEG;
1019 mss = htons((u_short) tcp_mssopt(tp));
1020 (void)memcpy(opt + 2, &mss, sizeof(mss));
1021 optlen = TCPOLEN_MAXSEG;
1022
1023 if ((tp->t_flags & TF_REQ_SCALE) &&
1024 ((flags & TH_ACK) == 0 ||
1025 (tp->t_flags & TF_RCVD_SCALE))) {
1026 *((u_int32_t *)(void *)(opt + optlen)) = htonl(
1027 TCPOPT_NOP << 24 |
1028 TCPOPT_WINDOW << 16 |
1029 TCPOLEN_WINDOW << 8 |
1030 tp->request_r_scale);
1031 optlen += 4;
1032 }
1033 }
1034
1035 }
1036
1037 /*
1038 RFC 3168 states that:
1039 - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
1040 to handle the TCP ECE flag, even if you also later send a
1041 non-ECN-setup SYN/SYN-ACK.
1042 - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
1043 the ip ECT flag.
1044
1045 It is not clear how the ECE flag would ever be set if you never
1046 set the IP ECT flag on outbound packets. All the same, we use
1047 the TE_SETUPSENT to indicate that we have committed to handling
1048 the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
1049 whether or not we should set the IP ECT flag on outbound packets.
1050 */
1051 /*
1052 * For a SYN-ACK, send an ECN setup SYN-ACK
1053 */
1054 if (tcp_ecn_inbound && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
1055 if ((tp->ecn_flags & TE_SETUPRECEIVED) != 0) {
1056 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1057 /* Setting TH_ECE makes this an ECN-setup SYN-ACK */
1058 flags |= TH_ECE;
1059
1060 /*
1061 * Record that we sent the ECN-setup and default to
1062 * setting IP ECT.
1063 */
1064 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1065 }
1066 else {
1067 /*
1068 * We sent an ECN-setup SYN-ACK but it was dropped.
1069 * Fallback to non-ECN-setup SYN-ACK and clear flag
1070 * that to indicate we should not send data with IP ECT set.
1071 *
1072 * Pretend we didn't receive an ECN-setup SYN.
1073 */
1074 tp->ecn_flags &= ~TE_SETUPRECEIVED;
1075 }
1076 }
1077 }
1078 else if (tcp_ecn_outbound && (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
1079 if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
1080 /* Setting TH_ECE and TH_CWR makes this an ECN-setup SYN */
1081 flags |= (TH_ECE | TH_CWR);
1082
1083 /*
1084 * Record that we sent the ECN-setup and default to
1085 * setting IP ECT.
1086 */
1087 tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
1088 }
1089 else {
1090 /*
1091 * We sent an ECN-setup SYN but it was dropped.
1092 * Fall back to no ECN and clear flag indicating
1093 * we should send data with IP ECT set.
1094 */
1095 tp->ecn_flags &= ~TE_SENDIPECT;
1096 }
1097 }
1098
1099 /*
1100 * Check if we should set the TCP CWR flag.
1101 * CWR flag is sent when we reduced the congestion window because
1102 * we received a TCP ECE or we performed a fast retransmit. We
1103 * never set the CWR flag on retransmitted packets. We only set
1104 * the CWR flag on data packets. Pure acks don't have this set.
1105 */
1106 if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
1107 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1108 flags |= TH_CWR;
1109 tp->ecn_flags &= ~TE_SENDCWR;
1110 }
1111
1112 /*
1113 * Check if we should set the TCP ECE flag.
1114 */
1115 if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
1116 flags |= TH_ECE;
1117 }
1118
1119 /*
1120 * Send a timestamp and echo-reply if this is a SYN and our side
1121 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1122 * and our peer have sent timestamps in our SYN's.
1123 */
1124 if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
1125 (flags & TH_RST) == 0 &&
1126 ((flags & TH_ACK) == 0 ||
1127 (tp->t_flags & TF_RCVD_TSTMP))) {
1128 u_int32_t *lp = (u_int32_t *)(void *)(opt + optlen);
1129
1130 /* Form timestamp option as shown in appendix A of RFC 1323. */
1131 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1132 *lp++ = htonl(tcp_now);
1133 *lp = htonl(tp->ts_recent);
1134 optlen += TCPOLEN_TSTAMP_APPA;
1135 }
1136
1137 /* Note the timestamp for receive buffer autosizing */
1138 if (tp->rfbuf_ts == 0 && (so->so_rcv.sb_flags & SB_AUTOSIZE))
1139 tp->rfbuf_ts = tcp_now;
1140
1141 if (tp->sack_enable && ((tp->t_flags & TF_NOOPT) == 0)) {
1142 /*
1143 * Tack on the SACK permitted option *last*.
1144 * And do padding of options after tacking this on.
1145 * This is because of MSS, TS, WinScale and Signatures are
1146 * all present, we have just 2 bytes left for the SACK
1147 * permitted option, which is just enough.
1148 */
1149 /*
1150 * If this is the first SYN of connection (not a SYN
1151 * ACK), include SACK permitted option. If this is a
1152 * SYN ACK, include SACK permitted option if peer has
1153 * already done so. This is only for active connect,
1154 * since the syncache takes care of the passive connect.
1155 */
1156 if ((flags & TH_SYN) &&
1157 (!(flags & TH_ACK) || (tp->t_flags & TF_SACK_PERMIT))) {
1158 u_char *bp;
1159 bp = (u_char *)opt + optlen;
1160
1161 *bp++ = TCPOPT_SACK_PERMITTED;
1162 *bp++ = TCPOLEN_SACK_PERMITTED;
1163 optlen += TCPOLEN_SACK_PERMITTED;
1164 }
1165
1166 /*
1167 * Send SACKs if necessary. This should be the last
1168 * option processed. Only as many SACKs are sent as
1169 * are permitted by the maximum options size.
1170 *
1171 * In general, SACK blocks consume 8*n+2 bytes.
1172 * So a full size SACK blocks option is 34 bytes
1173 * (to generate 4 SACK blocks). At a minimum,
1174 * we need 10 bytes (to generate 1 SACK block).
1175 * If TCP Timestamps (12 bytes) and TCP Signatures
1176 * (18 bytes) are both present, we'll just have
1177 * 10 bytes for SACK options 40 - (12 + 18).
1178 */
1179 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1180 (tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0 &&
1181 MAX_TCPOPTLEN - optlen - 2 >= TCPOLEN_SACK) {
1182 int nsack, padlen;
1183 u_char *bp = (u_char *)opt + optlen;
1184 u_int32_t *lp;
1185
1186 nsack = (MAX_TCPOPTLEN - optlen - 2) / TCPOLEN_SACK;
1187 nsack = min(nsack, tp->rcv_numsacks);
1188 sackoptlen = (2 + nsack * TCPOLEN_SACK);
1189
1190 /*
1191 * First we need to pad options so that the
1192 * SACK blocks can start at a 4-byte boundary
1193 * (sack option and length are at a 2 byte offset).
1194 */
1195 padlen = (MAX_TCPOPTLEN - optlen - sackoptlen) % 4;
1196 optlen += padlen;
1197 while (padlen-- > 0)
1198 *bp++ = TCPOPT_NOP;
1199
1200 tcpstat.tcps_sack_send_blocks++;
1201 *bp++ = TCPOPT_SACK;
1202 *bp++ = sackoptlen;
1203 lp = (u_int32_t *)(void *)bp;
1204 for (i = 0; i < nsack; i++) {
1205 struct sackblk sack = tp->sackblks[i];
1206 *lp++ = htonl(sack.start);
1207 *lp++ = htonl(sack.end);
1208 }
1209 optlen += sackoptlen;
1210 }
1211 }
1212
1213 /* Pad TCP options to a 4 byte boundary */
1214 if (optlen < MAX_TCPOPTLEN && (optlen % sizeof(u_int32_t))) {
1215 int pad = sizeof(u_int32_t) - (optlen % sizeof(u_int32_t));
1216 u_char *bp = (u_char *)opt + optlen;
1217
1218 optlen += pad;
1219 while (pad) {
1220 *bp++ = TCPOPT_EOL;
1221 pad--;
1222 }
1223 }
1224
1225 hdrlen += optlen;
1226
1227 #if INET6
1228 if (isipv6)
1229 ipoptlen = ip6_optlen(tp->t_inpcb);
1230 else
1231 #endif
1232 {
1233 if (tp_inp_options) {
1234 ipoptlen = tp_inp_options->m_len -
1235 offsetof(struct ipoption, ipopt_list);
1236 } else
1237 ipoptlen = 0;
1238 }
1239 #if IPSEC
1240 ipoptlen += ipsec_optlen;
1241 #endif
1242
1243 /*
1244 * Adjust data length if insertion of options will
1245 * bump the packet length beyond the t_maxopd length.
1246 * Clear the FIN bit because we cut off the tail of
1247 * the segment.
1248 *
1249 * When doing TSO limit a burst to TCP_MAXWIN minus the
1250 * IP, TCP and Options length to keep ip->ip_len from
1251 * overflowing. Prevent the last segment from being
1252 * fractional thus making them all equal sized and set
1253 * the flag to continue sending. TSO is disabled when
1254 * IP options or IPSEC are present.
1255 */
1256 if (len + optlen + ipoptlen > tp->t_maxopd) {
1257 /*
1258 * If there is still more to send, don't close the connection.
1259 */
1260 flags &= ~TH_FIN;
1261 if (tso) {
1262 int32_t tso_maxlen;
1263
1264 tso_maxlen = tp->tso_max_segment_size ? tp->tso_max_segment_size : TCP_MAXWIN;
1265
1266 if (len > tso_maxlen - hdrlen - optlen) {
1267 len = tso_maxlen - hdrlen - optlen;
1268 len = len - (len % (tp->t_maxopd - optlen));
1269 sendalot = 1;
1270 } else if (tp->t_flags & TF_NEEDFIN)
1271 sendalot = 1;
1272 } else {
1273 len = tp->t_maxopd - optlen - ipoptlen;
1274 sendalot = 1;
1275 }
1276 }
1277
1278 /*#ifdef DIAGNOSTIC*/
1279 #if INET6
1280 if (max_linkhdr + hdrlen > MCLBYTES)
1281 panic("tcphdr too big");
1282 #else
1283 if (max_linkhdr + hdrlen > MHLEN)
1284 panic("tcphdr too big");
1285 #endif
1286 /*#endif*/
1287
1288 /* Check if there is enough data in the send socket
1289 * buffer to start measuring bw
1290 */
1291 if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 &&
1292 (tp->t_bwmeas != NULL) &&
1293 (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 &&
1294 (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >=
1295 tp->t_bwmeas->bw_minsize) {
1296 tp->t_bwmeas->bw_size = min((so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)),
1297 tp->t_bwmeas->bw_maxsize);
1298 tp->t_flagsext |= TF_BWMEAS_INPROGRESS;
1299 tp->t_bwmeas->bw_start = tp->snd_max;
1300 tp->t_bwmeas->bw_ts = tcp_now;
1301 }
1302
1303 VERIFY(tp->t_inpcb->inp_flowhash != 0);
1304
1305 /*
1306 * Grab a header mbuf, attaching a copy of data to
1307 * be transmitted, and initialize the header from
1308 * the template for sends on this connection.
1309 */
1310 if (len) {
1311 if (tp->t_force && len == 1)
1312 tcpstat.tcps_sndprobe++;
1313 else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1314 tcpstat.tcps_sndrexmitpack++;
1315 tcpstat.tcps_sndrexmitbyte += len;
1316 if (nstat_collect) {
1317 nstat_route_tx(tp->t_inpcb->inp_route.ro_rt, 1, len, NSTAT_TX_FLAG_RETRANSMIT);
1318 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
1319 locked_add_64(&tp->t_inpcb->inp_stat->txbytes, len);
1320 tp->t_stat.txretransmitbytes += len;
1321 }
1322 } else {
1323 tcpstat.tcps_sndpack++;
1324 tcpstat.tcps_sndbyte += len;
1325 if (nstat_collect) {
1326 locked_add_64(&tp->t_inpcb->inp_stat->txpackets, 1);
1327 locked_add_64(&tp->t_inpcb->inp_stat->txbytes, len);
1328 }
1329 }
1330 #ifdef notyet
1331 if ((m = m_copypack(so->so_snd.sb_mb, off,
1332 (int)len, max_linkhdr + hdrlen)) == 0) {
1333 error = ENOBUFS;
1334 goto out;
1335 }
1336 /*
1337 * m_copypack left space for our hdr; use it.
1338 */
1339 m->m_len += hdrlen;
1340 m->m_data -= hdrlen;
1341 #else
1342 /*
1343 * try to use the new interface that allocates all
1344 * the necessary mbuf hdrs under 1 mbuf lock and
1345 * avoids rescanning the socket mbuf list if
1346 * certain conditions are met. This routine can't
1347 * be used in the following cases...
1348 * 1) the protocol headers exceed the capacity of
1349 * of a single mbuf header's data area (no cluster attached)
1350 * 2) the length of the data being transmitted plus
1351 * the protocol headers fits into a single mbuf header's
1352 * data area (no cluster attached)
1353 */
1354 m = NULL;
1355 #if INET6
1356 if (MHLEN < hdrlen + max_linkhdr) {
1357 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1358 if (m == NULL) {
1359 error = ENOBUFS;
1360 goto out;
1361 }
1362 MCLGET(m, M_DONTWAIT);
1363 if ((m->m_flags & M_EXT) == 0) {
1364 m_freem(m);
1365 error = ENOBUFS;
1366 goto out;
1367 }
1368 m->m_data += max_linkhdr;
1369 m->m_len = hdrlen;
1370 }
1371 #endif
1372 if (len <= MHLEN - hdrlen - max_linkhdr) {
1373 if (m == NULL) {
1374 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1375 if (m == NULL) {
1376 error = ENOBUFS;
1377 goto out;
1378 }
1379 m->m_data += max_linkhdr;
1380 m->m_len = hdrlen;
1381 }
1382 /* makes sure we still have data left to be sent at this point */
1383 if (so->so_snd.sb_mb == NULL || off < 0) {
1384 if (m != NULL) m_freem(m);
1385 error = 0; /* should we return an error? */
1386 goto out;
1387 }
1388 m_copydata(so->so_snd.sb_mb, off, (int) len,
1389 mtod(m, caddr_t) + hdrlen);
1390 m->m_len += len;
1391 } else {
1392 if (m != NULL) {
1393 m->m_next = m_copy(so->so_snd.sb_mb, off, (int) len);
1394 if (m->m_next == 0) {
1395 (void) m_free(m);
1396 error = ENOBUFS;
1397 goto out;
1398 }
1399 } else {
1400 /*
1401 * determine whether the mbuf pointer and offset passed back by the 'last' call
1402 * to m_copym_with_hdrs are still valid... if the head of the socket chain has
1403 * changed (due to an incoming ACK for instance), or the offset into the chain we
1404 * just computed is different from the one last returned by m_copym_with_hdrs (perhaps
1405 * we're re-transmitting a packet sent earlier), than we can't pass the mbuf pointer and
1406 * offset into it as valid hints for m_copym_with_hdrs to use (if valid, these hints allow
1407 * m_copym_with_hdrs to avoid rescanning from the beginning of the socket buffer mbuf list.
1408 * setting the mbuf pointer to NULL is sufficient to disable the hint mechanism.
1409 */
1410 if (m_head != so->so_snd.sb_mb || sack_rxmit || last_off != off)
1411 m_lastm = NULL;
1412 last_off = off + len;
1413 m_head = so->so_snd.sb_mb;
1414
1415 /* makes sure we still have data left to be sent at this point */
1416 if (m_head == NULL) {
1417 error = 0; /* should we return an error? */
1418 goto out;
1419 }
1420
1421 /*
1422 * m_copym_with_hdrs will always return the last mbuf pointer and the offset into it that
1423 * it acted on to fullfill the current request, whether a valid 'hint' was passed in or not
1424 */
1425 if ((m = m_copym_with_hdrs(so->so_snd.sb_mb, off, len, M_DONTWAIT, &m_lastm, &m_off)) == NULL) {
1426 error = ENOBUFS;
1427 goto out;
1428 }
1429 m->m_data += max_linkhdr;
1430 m->m_len = hdrlen;
1431 }
1432 }
1433 #endif
1434 /*
1435 * If we're sending everything we've got, set PUSH.
1436 * (This will keep happy those implementations which only
1437 * give data to the user when a buffer fills or
1438 * a PUSH comes in.)
1439 */
1440 if (off + len == so->so_snd.sb_cc)
1441 flags |= TH_PUSH;
1442 } else {
1443 if (tp->t_flags & TF_ACKNOW)
1444 tcpstat.tcps_sndacks++;
1445 else if (flags & (TH_SYN|TH_FIN|TH_RST))
1446 tcpstat.tcps_sndctrl++;
1447 else if (SEQ_GT(tp->snd_up, tp->snd_una))
1448 tcpstat.tcps_sndurg++;
1449 else
1450 tcpstat.tcps_sndwinup++;
1451
1452 MGETHDR(m, M_DONTWAIT, MT_HEADER); /* MAC-OK */
1453 if (m == NULL) {
1454 error = ENOBUFS;
1455 goto out;
1456 }
1457 #if INET6
1458 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
1459 MHLEN >= hdrlen) {
1460 MH_ALIGN(m, hdrlen);
1461 } else
1462 #endif
1463 m->m_data += max_linkhdr;
1464 m->m_len = hdrlen;
1465 }
1466 m->m_pkthdr.rcvif = 0;
1467 #if CONFIG_MACF_NET
1468 mac_mbuf_label_associate_inpcb(tp->t_inpcb, m);
1469 #endif
1470 #if INET6
1471 if (isipv6) {
1472 ip6 = mtod(m, struct ip6_hdr *);
1473 th = (struct tcphdr *)(void *)(ip6 + 1);
1474 tcp_fillheaders(tp, ip6, th);
1475 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1476 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1477 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
1478 }
1479 svc_flags |= PKT_SCF_IPV6;
1480 } else
1481 #endif /* INET6 */
1482 {
1483 ip = mtod(m, struct ip *);
1484 ipov = (struct ipovly *)ip;
1485 th = (struct tcphdr *)(void *)(ip + 1);
1486 /* this picks up the pseudo header (w/o the length) */
1487 tcp_fillheaders(tp, ip, th);
1488 if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
1489 !SEQ_LT(tp->snd_nxt, tp->snd_max) && !sack_rxmit) {
1490 ip->ip_tos = IPTOS_ECN_ECT0;
1491 }
1492 }
1493
1494 /*
1495 * Fill in fields, remembering maximum advertised
1496 * window for use in delaying messages about window sizes.
1497 * If resending a FIN, be sure not to use a new sequence number.
1498 */
1499 if (flags & TH_FIN && (tp->t_flags & TF_SENTFIN) &&
1500 tp->snd_nxt == tp->snd_max)
1501 tp->snd_nxt--;
1502 /*
1503 * If we are doing retransmissions, then snd_nxt will
1504 * not reflect the first unsent octet. For ACK only
1505 * packets, we do not want the sequence number of the
1506 * retransmitted packet, we want the sequence number
1507 * of the next unsent octet. So, if there is no data
1508 * (and no SYN or FIN), use snd_max instead of snd_nxt
1509 * when filling in ti_seq. But if we are in persist
1510 * state, snd_max might reflect one byte beyond the
1511 * right edge of the window, so use snd_nxt in that
1512 * case, since we know we aren't doing a retransmission.
1513 * (retransmit and persist are mutually exclusive...)
1514 */
1515 if (sack_rxmit == 0) {
1516 if (len || (flags & (TH_SYN|TH_FIN)) || tp->t_timer[TCPT_PERSIST])
1517 th->th_seq = htonl(tp->snd_nxt);
1518 else
1519 th->th_seq = htonl(tp->snd_max);
1520 } else {
1521 th->th_seq = htonl(p->rxmit);
1522 p->rxmit += len;
1523 tp->sackhint.sack_bytes_rexmit += len;
1524 }
1525 th->th_ack = htonl(tp->rcv_nxt);
1526 tp->last_ack_sent = tp->rcv_nxt;
1527
1528 if (optlen) {
1529 bcopy(opt, th + 1, optlen);
1530 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1531 }
1532 th->th_flags = flags;
1533 /*
1534 * Calculate receive window. Don't shrink window,
1535 * but avoid silly window syndrome.
1536 */
1537 if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && recwin < (int)tp->t_maxseg)
1538 recwin = 0;
1539 if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt))
1540 recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt);
1541 if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) {
1542 if (recwin > (int32_t)slowlink_wsize)
1543 recwin = slowlink_wsize;
1544 }
1545
1546 #if TRAFFIC_MGT
1547 if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) {
1548 if (tp->acc_iaj > tcp_acc_iaj_react_limit) {
1549 uint32_t min_iaj_win = tcp_min_iaj_win * tp->t_maxseg;
1550 if (tp->iaj_rwintop == 0 ||
1551 SEQ_LT(tp->iaj_rwintop, tp->rcv_adv))
1552 tp->iaj_rwintop = tp->rcv_adv;
1553 if (SEQ_LT(tp->iaj_rwintop, tp->rcv_nxt + min_iaj_win))
1554 tp->iaj_rwintop = tp->rcv_nxt + min_iaj_win;
1555 recwin = min(tp->iaj_rwintop - tp->rcv_nxt, recwin);
1556 }
1557 }
1558 #endif /* TRAFFIC_MGT */
1559
1560 if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale))
1561 recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale);
1562 th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
1563
1564 /*
1565 * Adjust the RXWIN0SENT flag - indicate that we have advertised
1566 * a 0 window. This may cause the remote transmitter to stall. This
1567 * flag tells soreceive() to disable delayed acknowledgements when
1568 * draining the buffer. This can occur if the receiver is attempting
1569 * to read more data then can be buffered prior to transmitting on
1570 * the connection.
1571 */
1572 if (th->th_win == 0)
1573 tp->t_flags |= TF_RXWIN0SENT;
1574 else
1575 tp->t_flags &= ~TF_RXWIN0SENT;
1576 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
1577 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
1578 th->th_flags |= TH_URG;
1579 } else
1580 /*
1581 * If no urgent pointer to send, then we pull
1582 * the urgent pointer to the left edge of the send window
1583 * so that it doesn't drift into the send window on sequence
1584 * number wraparound.
1585 */
1586 tp->snd_up = tp->snd_una; /* drag it along */
1587
1588 /*
1589 * Put TCP length in extended header, and then
1590 * checksum extended header and data.
1591 */
1592 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
1593 #if INET6
1594 if (isipv6) {
1595 /*
1596 * ip6_plen is not need to be filled now, and will be filled
1597 * in ip6_output.
1598 */
1599 m->m_pkthdr.csum_flags = CSUM_TCPIPV6;
1600 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1601 if (len + optlen)
1602 th->th_sum = in_addword(th->th_sum,
1603 htons((u_short)(optlen + len)));
1604 }
1605 else
1606 #endif /* INET6 */
1607 {
1608 m->m_pkthdr.csum_flags = CSUM_TCP;
1609 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1610 if (len + optlen)
1611 th->th_sum = in_addword(th->th_sum,
1612 htons((u_short)(optlen + len)));
1613 }
1614
1615 /*
1616 * Enable TSO and specify the size of the segments.
1617 * The TCP pseudo header checksum is always provided.
1618 */
1619 if (tso) {
1620 #if INET6
1621 if (isipv6)
1622 m->m_pkthdr.csum_flags = CSUM_TSO_IPV6;
1623 else
1624 #endif /* INET6 */
1625 m->m_pkthdr.csum_flags = CSUM_TSO_IPV4;
1626
1627 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
1628 }
1629 else
1630 m->m_pkthdr.tso_segsz = 0;
1631
1632 /*
1633 * In transmit state, time the transmission and arrange for
1634 * the retransmit. In persist state, just set snd_max.
1635 */
1636 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
1637 tcp_seq startseq = tp->snd_nxt;
1638
1639 /*
1640 * Advance snd_nxt over sequence space of this segment.
1641 */
1642 if (flags & (TH_SYN|TH_FIN)) {
1643 if (flags & TH_SYN)
1644 tp->snd_nxt++;
1645 if (flags & TH_FIN) {
1646 tp->snd_nxt++;
1647 tp->t_flags |= TF_SENTFIN;
1648 }
1649 }
1650 if (sack_rxmit)
1651 goto timer;
1652 tp->snd_nxt += len;
1653 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
1654 tp->snd_max = tp->snd_nxt;
1655 /*
1656 * Time this transmission if not a retransmission and
1657 * not currently timing anything.
1658 */
1659 if (tp->t_rtttime == 0) {
1660 tp->t_rtttime = tcp_now;
1661 tp->t_rtseq = startseq;
1662 tcpstat.tcps_segstimed++;
1663 }
1664 }
1665
1666 /*
1667 * Set retransmit timer if not currently set,
1668 * and not doing an ack or a keep-alive probe.
1669 * Initial value for retransmit timer is smoothed
1670 * round-trip time + 2 * round-trip time variance.
1671 * Initialize shift counter which is used for backoff
1672 * of retransmit time.
1673 */
1674 timer:
1675 if (tp->t_timer[TCPT_REXMT] == 0 &&
1676 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
1677 tp->snd_nxt != tp->snd_una)) {
1678 if (tp->t_timer[TCPT_PERSIST]) {
1679 tp->t_timer[TCPT_PERSIST] = 0;
1680 tp->t_rxtshift = 0;
1681 tp->rxt_start = 0;
1682 tp->t_persist_stop = 0;
1683 }
1684 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1685 }
1686 } else {
1687 /*
1688 * Persist case, update snd_max but since we are in
1689 * persist mode (no window) we do not update snd_nxt.
1690 */
1691 int xlen = len;
1692 if (flags & TH_SYN)
1693 ++xlen;
1694 if (flags & TH_FIN) {
1695 ++xlen;
1696 tp->t_flags |= TF_SENTFIN;
1697 }
1698 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
1699 tp->snd_max = tp->snd_nxt + len;
1700 }
1701
1702 #if TCPDEBUG
1703 /*
1704 * Trace.
1705 */
1706 if (so_options & SO_DEBUG)
1707 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
1708 #endif
1709
1710 /*
1711 * Fill in IP length and desired time to live and
1712 * send to IP level. There should be a better way
1713 * to handle ttl and tos; we could keep them in
1714 * the template, but need a way to checksum without them.
1715 */
1716 #ifdef INET6
1717 /*
1718 * m->m_pkthdr.len should have been set before cksum calcuration,
1719 * because in6_cksum() need it.
1720 */
1721 if (isipv6) {
1722 /*
1723 * we separately set hoplimit for every segment, since the
1724 * user might want to change the value via setsockopt.
1725 * Also, desired default hop limit might be changed via
1726 * Neighbor Discovery.
1727 */
1728 ip6->ip6_hlim = in6_selecthlim(tp->t_inpcb,
1729 tp->t_inpcb->in6p_route.ro_rt ?
1730 tp->t_inpcb->in6p_route.ro_rt->rt_ifp
1731 : NULL);
1732
1733 /* TODO: IPv6 IP6TOS_ECT bit on */
1734 KERNEL_DEBUG(DBG_LAYER_BEG,
1735 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
1736 (((tp->t_inpcb->in6p_laddr.s6_addr16[0] & 0xffff) << 16) |
1737 (tp->t_inpcb->in6p_faddr.s6_addr16[0] & 0xffff)),
1738 sendalot,0,0);
1739 } else
1740 #endif /* INET6 */
1741 {
1742 ip->ip_len = m->m_pkthdr.len;
1743 ip->ip_ttl = tp->t_inpcb->inp_ip_ttl; /* XXX */
1744 ip->ip_tos |= (tp->t_inpcb->inp_ip_tos & ~IPTOS_ECN_MASK);/* XXX */
1745 KERNEL_DEBUG(DBG_LAYER_BEG,
1746 ((tp->t_inpcb->inp_fport << 16) | tp->t_inpcb->inp_lport),
1747 (((tp->t_inpcb->inp_laddr.s_addr & 0xffff) << 16) |
1748 (tp->t_inpcb->inp_faddr.s_addr & 0xffff)),
1749 0,0,0);
1750 }
1751
1752 /*
1753 * See if we should do MTU discovery.
1754 * Look at the flag updated on the following criterias:
1755 * 1) Path MTU discovery is authorized by the sysctl
1756 * 2) The route isn't set yet (unlikely but could happen)
1757 * 3) The route is up
1758 * 4) the MTU is not locked (if it is, then discovery has been
1759 * disabled for that route)
1760 */
1761 #ifdef INET6
1762 if (!isipv6)
1763 #endif
1764 if (path_mtu_discovery && (tp->t_flags & TF_PMTUD))
1765 ip->ip_off |= IP_DF;
1766
1767 #if IPSEC
1768 if (ipsec_bypass == 0)
1769 ipsec_setsocket(m, so);
1770 #endif /*IPSEC*/
1771
1772 /*
1773 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
1774 */
1775 lost = 0;
1776 m->m_pkthdr.socket_id = socket_id;
1777
1778 /*
1779 * Embed the flow hash in pkt hdr and mark the packet as
1780 * capable of flow controlling
1781 */
1782 m->m_pkthdr.m_flowhash = tp->t_inpcb->inp_flowhash;
1783 m->m_pkthdr.m_fhflags |=
1784 (PF_TAG_TCP | PF_TAG_FLOWHASH | PF_TAG_FLOWADV);
1785
1786 m->m_nextpkt = NULL;
1787
1788 if (tp->t_inpcb->inp_last_outifp != NULL &&
1789 tp->t_inpcb->inp_last_outifp != lo_ifp) {
1790 /* Hint to prioritize this packet if
1791 * 1. if the packet has no data
1792 * 2. the interface supports transmit-start model and did
1793 * not disable ACK prioritization.
1794 * 3. Only ACK flag is set.
1795 * 4. there is no outstanding data on this connection.
1796 */
1797 if (tcp_prioritize_acks != 0 && len == 0 &&
1798 (tp->t_inpcb->inp_last_outifp->if_eflags &
1799 (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART &&
1800 th->th_flags == TH_ACK && tp->snd_una == tp->snd_max &&
1801 tp->t_timer[TCPT_REXMT] == 0) {
1802 svc_flags |= PKT_SCF_TCP_ACK;
1803 }
1804 set_packet_service_class(m, so, MBUF_SC_UNSPEC, svc_flags);
1805 }
1806
1807 tp->t_pktlist_sentlen += len;
1808 tp->t_lastchain++;
1809
1810 #ifdef INET6
1811 if (isipv6) {
1812 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, tp->t_inpcb,
1813 struct ip6 *, ip6, struct tcpcb *, tp, struct tcphdr *,
1814 th);
1815 } else
1816 #endif
1817 {
1818 DTRACE_TCP5(send, struct mbuf *, m, struct inpcb *, tp->t_inpcb,
1819 struct ip *, ip, struct tcpcb *, tp, struct tcphdr *, th);
1820 }
1821
1822 if (tp->t_pktlist_head != NULL) {
1823 tp->t_pktlist_tail->m_nextpkt = m;
1824 tp->t_pktlist_tail = m;
1825 } else {
1826 packchain_newlist++;
1827 tp->t_pktlist_head = tp->t_pktlist_tail = m;
1828 }
1829
1830 if ((lro_ackmore) && (!sackoptlen) && (!tp->t_timer[TCPT_PERSIST]) &&
1831 ((th->th_flags & TH_ACK) == TH_ACK) && (!len) &&
1832 (tp->t_state == TCPS_ESTABLISHED)) {
1833 /* For a pure ACK, see if you need to send more of them */
1834 mnext = tcp_send_lroacks(tp, m, th);
1835 if (mnext) {
1836 tp->t_pktlist_tail->m_nextpkt = mnext;
1837 if (mnext->m_nextpkt == NULL) {
1838 tp->t_pktlist_tail = mnext;
1839 tp->t_lastchain++;
1840 } else {
1841 struct mbuf *tail, *next;
1842 next = mnext->m_nextpkt;
1843 tail = next->m_nextpkt;
1844 while (tail) {
1845 next = tail;
1846 tail = tail->m_nextpkt;
1847 tp->t_lastchain++;
1848 }
1849 tp->t_pktlist_tail = next;
1850 }
1851 }
1852 }
1853
1854 if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
1855 (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
1856 (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || tp->t_force != 0 ||
1857 tp->t_lastchain >= tcp_packet_chaining) {
1858 error = 0;
1859 while (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
1860 tp->t_pktlist_head != NULL) {
1861 packetlist = tp->t_pktlist_head;
1862 packchain_listadd = tp->t_lastchain;
1863 packchain_sent++;
1864 lost = tp->t_pktlist_sentlen;
1865 TCP_PKTLIST_CLEAR(tp);
1866
1867 error = tcp_ip_output(so, tp, packetlist,
1868 packchain_listadd, tp_inp_options,
1869 (so_options & SO_DONTROUTE),
1870 (sack_rxmit | (sack_bytes_rxmt != 0)), recwin,
1871 #ifdef INET6
1872 isipv6);
1873 #else
1874 0);
1875 #endif
1876
1877 if (error) {
1878 /*
1879 * Take into account the rest of unsent
1880 * packets in the packet list for this tcp
1881 * into "lost", since we're about to free
1882 * the whole list below.
1883 */
1884 lost += tp->t_pktlist_sentlen;
1885 break;
1886 } else {
1887 lost = 0;
1888 }
1889 }
1890 /* tcp was closed while we were in ip; resume close */
1891 if (tp->t_inpcb->inp_sndinprog_cnt == 0 &&
1892 (tp->t_flags & TF_CLOSING)) {
1893 tp->t_flags &= ~TF_CLOSING;
1894 (void) tcp_close(tp);
1895 return (0);
1896 }
1897 } else {
1898 error = 0;
1899 packchain_looped++;
1900 tcpstat.tcps_sndtotal++;
1901
1902 goto again;
1903 }
1904 if (error) {
1905 /*
1906 * Assume that the packets were lost, so back out the
1907 * sequence number advance, if any. Note that the "lost"
1908 * variable represents the amount of user data sent during
1909 * the recent call to ip_output_list() plus the amount of
1910 * user data in the packet list for this tcp at the moment.
1911 */
1912 if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
1913 /*
1914 * No need to check for TH_FIN here because
1915 * the TF_SENTFIN flag handles that case.
1916 */
1917 if ((flags & TH_SYN) == 0) {
1918 if (sack_rxmit) {
1919 p->rxmit -= lost;
1920 tp->sackhint.sack_bytes_rexmit -= lost;
1921 } else
1922 tp->snd_nxt -= lost;
1923 }
1924 }
1925 out:
1926 if (tp->t_pktlist_head != NULL)
1927 m_freem_list(tp->t_pktlist_head);
1928 TCP_PKTLIST_CLEAR(tp);
1929
1930 if (error == ENOBUFS) {
1931 if (!tp->t_timer[TCPT_REXMT] &&
1932 !tp->t_timer[TCPT_PERSIST])
1933 tp->t_timer[TCPT_REXMT] =
1934 OFFSET_FROM_START(tp, tp->t_rxtcur);
1935
1936 tp->snd_cwnd = tp->t_maxseg;
1937 tp->t_bytes_acked = 0;
1938
1939 tcp_check_timer_state(tp);
1940 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1941
1942 DTRACE_TCP5(cc, void, NULL, struct inpcb *, tp->t_inpcb,
1943 struct tcpcb *, tp, struct tcphdr *, NULL,
1944 int32_t, TCP_CC_OUTPUT_ERROR);
1945 return (0);
1946 }
1947 if (error == EMSGSIZE) {
1948 /*
1949 * ip_output() will have already fixed the route
1950 * for us. tcp_mtudisc() will, as its last action,
1951 * initiate retransmission, so it is important to
1952 * not do so here.
1953 *
1954 * If TSO was active we either got an interface
1955 * without TSO capabilits or TSO was turned off.
1956 * Disable it for this connection as too and
1957 * immediatly retry with MSS sized segments generated
1958 * by this function.
1959 */
1960 if (tso)
1961 tp->t_flags &= ~TF_TSO;
1962
1963 tcp_mtudisc(tp->t_inpcb, 0);
1964 tcp_check_timer_state(tp);
1965
1966 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1967 return 0;
1968 }
1969 if ((error == EHOSTUNREACH || error == ENETDOWN)
1970 && TCPS_HAVERCVDSYN(tp->t_state)) {
1971 tp->t_softerror = error;
1972 tcp_check_timer_state(tp);
1973 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1974 return (0);
1975 }
1976 tcp_check_timer_state(tp);
1977 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
1978 return (error);
1979 }
1980
1981 tcpstat.tcps_sndtotal++;
1982
1983 KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
1984 if (sendalot)
1985 goto again;
1986
1987 tcp_check_timer_state(tp);
1988 return (0);
1989 }
1990
1991 static int
1992 tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
1993 int cnt, struct mbuf *opt, int flags, int sack_in_progress, int recwin,
1994 boolean_t isipv6)
1995 {
1996 int error = 0;
1997 boolean_t chain;
1998 boolean_t unlocked = FALSE;
1999 struct inpcb *inp = tp->t_inpcb;
2000 struct ip_out_args ipoa =
2001 { IFSCOPE_NONE, { 0 }, IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR };
2002 struct route ro;
2003 struct ifnet *outif = NULL;
2004 #ifdef INET6
2005 struct ip6_out_args ip6oa =
2006 { IFSCOPE_NONE, { 0 }, IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR };
2007 struct route_in6 ro6;
2008 struct flowadv *adv =
2009 (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv);
2010 #else
2011 struct flowadv *adv = &ipoa.ipoa_flowadv;
2012 #endif /* !INET6 */
2013
2014 /* If socket was bound to an ifindex, tell ip_output about it */
2015 if (inp->inp_flags & INP_BOUND_IF) {
2016 #ifdef INET6
2017 if (isipv6) {
2018 ip6oa.ip6oa_boundif = inp->inp_boundifp->if_index;
2019 ip6oa.ip6oa_flags |= IP6OAF_BOUND_IF;
2020 } else
2021 #endif
2022 {
2023 ipoa.ipoa_boundif = inp->inp_boundifp->if_index;
2024 ipoa.ipoa_flags |= IPOAF_BOUND_IF;
2025 }
2026 }
2027
2028 if (inp->inp_flags & INP_NO_IFT_CELLULAR) {
2029 #ifdef INET6
2030 if (isipv6)
2031 ip6oa.ip6oa_flags |= IP6OAF_NO_CELLULAR;
2032 else
2033 #endif
2034 ipoa.ipoa_flags |= IPOAF_NO_CELLULAR;
2035 }
2036 #ifdef INET6
2037 if (isipv6)
2038 flags |= IPV6_OUTARGS;
2039 else
2040 #endif
2041 flags |= IP_OUTARGS;
2042
2043 /* Copy the cached route and take an extra reference */
2044 #ifdef INET6
2045 if (isipv6)
2046 in6p_route_copyout(inp, &ro6);
2047 else
2048 #endif
2049 inp_route_copyout(inp, &ro);
2050
2051 /*
2052 * Data sent (as far as we can tell).
2053 * If this advertises a larger window than any other segment,
2054 * then remember the size of the advertised window.
2055 * Make sure ACK/DELACK conditions are cleared before
2056 * we unlock the socket.
2057 */
2058 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
2059 tp->rcv_adv = tp->rcv_nxt + recwin;
2060 tp->last_ack_sent = tp->rcv_nxt;
2061 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
2062 tp->t_timer[TCPT_DELACK] = 0;
2063 tp->t_unacksegs = 0;
2064
2065 /* Increment the count of outstanding send operations */
2066 inp->inp_sndinprog_cnt++;
2067
2068 /*
2069 * If allowed, unlock TCP socket while in IP
2070 * but only if the connection is established and
2071 * in a normal mode where reentrancy on the tcpcb won't be
2072 * an issue:
2073 * - there is no SACK episode
2074 * - we're not in Fast Recovery mode
2075 * - if we're not sending from an upcall.
2076 */
2077 if (tcp_output_unlocked && !so->so_upcallusecount &&
2078 (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) &&
2079 ((tp->t_flags & TF_FASTRECOVERY) == 0)) {
2080
2081 unlocked = TRUE;
2082 socket_unlock(so, 0);
2083 }
2084
2085 /*
2086 * Don't send down a chain of packets when:
2087 * - TCP chaining is disabled
2088 * - there is an IPsec rule set
2089 * - there is a non default rule set for the firewall
2090 */
2091
2092 chain = tcp_packet_chaining > 1
2093 #if IPSEC
2094 && ipsec_bypass
2095 #endif
2096 #if IPFIREWALL
2097 && (fw_enable == 0 || fw_bypass)
2098 #endif
2099 ; // I'm important, not extraneous
2100
2101
2102 while (pkt != NULL) {
2103 struct mbuf *npkt = pkt->m_nextpkt;
2104
2105 if (!chain) {
2106 pkt->m_nextpkt = NULL;
2107 /*
2108 * If we are not chaining, make sure to set the packet
2109 * list count to 0 so that IP takes the right path;
2110 * this is important for cases such as IPSec where a
2111 * single mbuf might result in multiple mbufs as part
2112 * of the encapsulation. If a non-zero count is passed
2113 * down to IP, the head of the chain might change and
2114 * we could end up skipping it (thus generating bogus
2115 * packets). Fixing it in IP would be desirable, but
2116 * for now this would do it.
2117 */
2118 cnt = 0;
2119 }
2120 #ifdef INET6
2121 if (isipv6)
2122 error = ip6_output_list(pkt, cnt,
2123 inp->in6p_outputopts, &ro6, flags, NULL, NULL,
2124 &ip6oa);
2125 else
2126 #endif
2127 error = ip_output_list(pkt, cnt, opt, &ro, flags, NULL,
2128 &ipoa);
2129
2130 if (chain || error) {
2131 /*
2132 * If we sent down a chain then we are done since
2133 * the callee had taken care of everything; else
2134 * we need to free the rest of the chain ourselves.
2135 */
2136 if (!chain)
2137 m_freem_list(npkt);
2138 break;
2139 }
2140 pkt = npkt;
2141 }
2142
2143 if (unlocked)
2144 socket_lock(so, 0);
2145
2146 /*
2147 * Enter flow controlled state if the connection is established
2148 * and is not in recovery.
2149 *
2150 * A connection will enter suspended state even if it is in
2151 * recovery.
2152 */
2153 if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) ||
2154 adv->code == FADV_SUSPENDED) &&
2155 !(tp->t_flags & TF_CLOSING) &&
2156 tp->t_state == TCPS_ESTABLISHED) {
2157 int rc;
2158 rc = inp_set_fc_state(inp, adv->code);
2159
2160 if (rc == 1)
2161 DTRACE_TCP5(cc, void, NULL, struct inpcb *, inp,
2162 struct tcpcb *, tp, struct tcphdr *, NULL,
2163 int32_t, ((adv->code == FADV_FLOW_CONTROLLED) ?
2164 TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND));
2165 }
2166
2167 /*
2168 * When an interface queue gets suspended, some of the
2169 * packets are dropped. Return ENOBUFS, to update the
2170 * pcb state.
2171 */
2172 if (adv->code == FADV_SUSPENDED)
2173 error = ENOBUFS;
2174
2175 VERIFY(inp->inp_sndinprog_cnt > 0);
2176 if ( --inp->inp_sndinprog_cnt == 0)
2177 inp->inp_flags &= ~(INP_FC_FEEDBACK);
2178
2179 #ifdef INET6
2180 if (isipv6) {
2181 if (ro6.ro_rt != NULL && (outif = ro6.ro_rt->rt_ifp) !=
2182 inp->in6p_last_outifp)
2183 inp->in6p_last_outifp = outif;
2184 } else
2185 #endif
2186 if (ro.ro_rt != NULL && (outif = ro.ro_rt->rt_ifp) !=
2187 inp->inp_last_outifp)
2188 inp->inp_last_outifp = outif;
2189
2190 if ((inp->inp_flags & INP_NO_IFT_CELLULAR) && outif != NULL &&
2191 outif->if_type == IFT_CELLULAR)
2192 soevent(inp->inp_socket,
2193 (SO_FILT_HINT_LOCKED|SO_FILT_HINT_IFDENIED));
2194
2195 /* Synchronize cached PCB route & options */
2196 #ifdef INET6
2197 if (isipv6)
2198 in6p_route_copyin(inp, &ro6);
2199 else
2200 #endif
2201 inp_route_copyin(inp, &ro);
2202
2203 if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 &&
2204 tp->t_inpcb->inp_route.ro_rt != NULL) {
2205 /* If we found the route and there is an rtt on it
2206 * reset the retransmit timer
2207 */
2208 tcp_getrt_rtt(tp, tp->t_inpcb->in6p_route.ro_rt);
2209 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
2210 }
2211 return (error);
2212 }
2213
2214 void
2215 tcp_setpersist(tp)
2216 register struct tcpcb *tp;
2217 {
2218 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
2219
2220 /* If a PERSIST_TIMER option was set we will limit the
2221 * time the persist timer will be active for that connection
2222 * in order to avoid DOS by using zero window probes.
2223 * see rdar://5805356
2224 */
2225
2226 if ((tp->t_persist_timeout != 0) &&
2227 (tp->t_timer[TCPT_PERSIST] == 0) &&
2228 (tp->t_persist_stop == 0)) {
2229 tp->t_persist_stop = tcp_now + tp->t_persist_timeout;
2230 }
2231
2232 /*
2233 * Start/restart persistance timer.
2234 */
2235 TCPT_RANGESET(tp->t_timer[TCPT_PERSIST],
2236 t * tcp_backoff[tp->t_rxtshift],
2237 TCPTV_PERSMIN, TCPTV_PERSMAX, 0);
2238 tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]);
2239
2240 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
2241 tp->t_rxtshift++;
2242 }
2243
2244 /*
2245 * Send as many acks as data coalesced. Every other packet when stretch
2246 * ACK is not enabled. Every 8 packets, if stretch ACK is enabled.
2247 */
2248 static struct mbuf*
2249 tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th)
2250 {
2251 struct mbuf *mnext = NULL, *ack_chain = NULL, *tail = NULL;
2252 int count = 0;
2253 tcp_seq org_ack = ntohl(th->th_ack);
2254 tcp_seq prev_ack = 0;
2255 int tack_offset = 28; /* XXX IPv6 not supported */
2256 int ack_size = (tp->t_flags & TF_STRETCHACK) ?
2257 (maxseg_unacked * tp->t_maxseg) : (tp->t_maxseg << 1);
2258 int segs_acked = (tp->t_flags & TF_STRETCHACK) ? maxseg_unacked : 2;
2259 struct mbuf *prev_ack_pkt = NULL;
2260 struct socket *so = tp->t_inpcb->inp_socket;
2261
2262 count = tp->t_lropktlen/tp->t_maxseg;
2263
2264 prev_ack = (org_ack - tp->t_lropktlen) + ack_size;
2265 if (prev_ack < org_ack) {
2266 ack_chain = m_dup(m, M_DONTWAIT);
2267 if (ack_chain) {
2268 th->th_ack = htonl(prev_ack);
2269 tail = ack_chain;
2270 count -= segs_acked; /* accounts for prev_ack packet */
2271 count = (count <= segs_acked) ? 0 : count - segs_acked;
2272 tcpstat.tcps_sndacks++;
2273 so_tc_update_stats(m, so, m_get_service_class(m));
2274 } else {
2275 return NULL;
2276 }
2277 }
2278 else {
2279 tp->t_lropktlen = 0;
2280 return NULL;
2281 }
2282
2283 prev_ack_pkt = ack_chain;
2284
2285 while (count > 0) {
2286 if ((prev_ack + ack_size) < org_ack) {
2287 prev_ack += ack_size;
2288 } else {
2289 /*
2290 * The last ACK sent must have the ACK number that TCP
2291 * thinks is the last sent ACK number.
2292 */
2293 prev_ack = org_ack;
2294 }
2295 mnext = m_dup(prev_ack_pkt, M_DONTWAIT);
2296 if (mnext) {
2297 HTONL(prev_ack);
2298 bcopy(&prev_ack, mtod(prev_ack_pkt, caddr_t) + tack_offset, 4);
2299 NTOHL(prev_ack);
2300 tail->m_nextpkt = mnext;
2301 tail = mnext;
2302 count -= segs_acked;
2303 tcpstat.tcps_sndacks++;
2304 so_tc_update_stats(m, so, m_get_service_class(m));
2305 if (lrodebug == 5) {
2306 printf("%s: lropktlen = %d count = %d, th_ack = %x \n",
2307 __func__, tp->t_lropktlen, count,
2308 th->th_ack);
2309 }
2310 } else {
2311 if (lrodebug == 5) {
2312 printf("%s: failed to alloc mbuf.\n", __func__);
2313 }
2314 break;
2315 }
2316 prev_ack_pkt = mnext;
2317 }
2318 tp->t_lropktlen = 0;
2319 return ack_chain;
2320 }
mirror of XNU