X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/ecc0ceb4089d506a0b8d16686a95817b331af9cb..9d749ea394c01276fa19e397e70f46858e849c76:/bsd/netinet/tcp_output.c?ds=inline diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c index 2dda8cb0f..52884d357 100644 --- a/bsd/netinet/tcp_output.c +++ b/bsd/netinet/tcp_output.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2018 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -90,6 +90,7 @@ #include #include #include +#include #include #include #include @@ -135,24 +136,25 @@ #define DBG_LAYER_END NETDBG_CODE(DBG_NETTCP, 3) #define DBG_FNC_TCP_OUTPUT NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1) -int path_mtu_discovery = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, - CTLFLAG_RW | CTLFLAG_LOCKED, &path_mtu_discovery, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, path_mtu_discovery, + CTLFLAG_RW | CTLFLAG_LOCKED, int, path_mtu_discovery, 1, "Enable Path MTU Discovery"); -int ss_fltsz = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED,&ss_fltsz, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz, 1, "Slow start flight size"); -int ss_fltsz_local = 8; /* starts with eight segments max */ -SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, - CTLFLAG_RW | CTLFLAG_LOCKED, &ss_fltsz_local, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, local_slowstart_flightsize, + CTLFLAG_RW | CTLFLAG_LOCKED, int, ss_fltsz_local, 8, "Slow start flight size for local networks"); int tcp_do_tso = 1; SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); + &tcp_do_tso, 0, "Enable TCP Segmentation Offload"); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ecn_setup_percentage, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_ecn_setup_percentage, 100, + "Max ECN setup percentage"); static int sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS @@ -170,6 +172,7 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS if ((tcp_ecn_outbound == 0 || tcp_ecn_outbound == 1) && (i == 0 || i == 1)) { tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); return(err); } if (tcp_ecn_outbound == 2 && (i == 0 || i == 1)) { @@ -205,10 +208,13 @@ sysctl_change_ecn_setting SYSCTL_HANDLER_ARGS ifnet_head_done(); } tcp_ecn_outbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_initiate_out, tcp_ecn_outbound); } /* Change the other one too as the work is done */ - if (i == 2 || tcp_ecn_inbound == 2) + if (i == 2 || tcp_ecn_inbound == 2) { tcp_ecn_inbound = i; + SYSCTL_SKMEM_UPDATE_FIELD(tcp.ecn_negotiate_in, tcp_ecn_inbound); + } return (err); } @@ -224,65 +230,53 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, sysctl_change_ecn_setting, "IU", "Initiate ECN for inbound connections"); -int tcp_packet_chaining = 50; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_packet_chaining, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, packetchain, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_packet_chaining, 50, "Enable TCP output packet chaining"); -int tcp_output_unlocked = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_output_unlocked, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, socket_unlocked_on_output, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_output_unlocked, 1, "Unlock TCP when sending packets down to IP"); -int tcp_do_rfc3390 = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3390, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_rfc3390, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rfc3390, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_do_rfc3390, 1, "Calculate intial slowstart cwnd depending on MSS"); -int tcp_min_iaj_win = MIN_IAJ_WIN; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, min_iaj_win, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_min_iaj_win, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, min_iaj_win, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_min_iaj_win, MIN_IAJ_WIN, "Minimum recv win based on inter-packet arrival jitter"); -int tcp_acc_iaj_react_limit = ACC_IAJ_REACT_LIMIT; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, acc_iaj_react_limit, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_acc_iaj_react_limit, 1, - "Accumulated IAJ when receiver starts to react"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, acc_iaj_react_limit, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_acc_iaj_react_limit, + ACC_IAJ_REACT_LIMIT, "Accumulated IAJ when receiver starts to react"); -uint32_t tcp_do_autosendbuf = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, doautosndbuf, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_do_autosendbuf, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, doautosndbuf, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_do_autosendbuf, 1, "Enable send socket buffer auto-tuning"); -uint32_t tcp_autosndbuf_inc = 8 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufinc, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_inc, 1, - "Increment in send socket bufffer size"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufinc, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_inc, + 8 * 1024, "Increment in send socket bufffer size"); -uint32_t tcp_autosndbuf_max = 512 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, autosndbufmax, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_autosndbuf_max, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, autosndbufmax, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_autosndbuf_max, 512 * 1024, "Maximum send socket buffer size"); -uint32_t tcp_prioritize_acks = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, ack_prioritize, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_prioritize_acks, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, ack_prioritize, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_prioritize_acks, 1, "Prioritize pure acks"); -uint32_t tcp_use_rtt_recvbg = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_recvbg, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_use_rtt_recvbg, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, rtt_recvbg, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_use_rtt_recvbg, 1, "Use RTT for bg recv algorithm"); -uint32_t tcp_recv_throttle_minwin = 16 * 1024; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, recv_throttle_minwin, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_recv_throttle_minwin, 1, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, recv_throttle_minwin, + CTLFLAG_RW | CTLFLAG_LOCKED, uint32_t, tcp_recv_throttle_minwin, 16 * 1024, "Minimum recv win for throttling"); -int32_t tcp_enable_tlp = 1; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, enable_tlp, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, enable_tlp, CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_enable_tlp, 1, "Enable Tail loss probe"); + int32_t, tcp_enable_tlp, 1, "Enable Tail loss probe"); static int32_t packchain_newlist = 0; static int32_t packchain_looped = 0; @@ -295,16 +289,15 @@ extern int ipsec_bypass; extern int slowlink_wsize; /* window correction for slow links */ #if IPFIREWALL -extern int fw_enable; /* firewall check for packet chaining */ -extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */ +extern int fw_enable; /* firewall check for packet chaining */ +extern int fw_bypass; /* firewall check: disable packet chaining if there is rules */ #endif /* IPFIREWALL */ extern u_int32_t dlil_filter_disable_tso_count; extern u_int32_t kipf_count; -extern int tcp_recv_bg; -static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int, - struct mbuf *, int, int, int32_t, boolean_t); +static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, + int, struct mbuf *, int, int, boolean_t); static struct mbuf* tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th); static int tcp_recv_throttle(struct tcpcb *tp); @@ -317,8 +310,18 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) if (tp->t_flags & TF_NOOPT) goto fallback; - if (!tcp_heuristic_do_tfo(tp)) + if ((so->so_flags1 & SOF1_DATA_AUTHENTICATED) && + !(tp->t_flagsext & TF_FASTOPEN_HEUR)) + return (len); + + if (!tcp_heuristic_do_tfo(tp)) { + tp->t_tfo_stats |= TFO_S_HEURISTICS_DISABLE; + tcpstat.tcps_tfo_heuristics_disable++; goto fallback; + } + + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) + return (len); optlen += TCPOLEN_MAXSEG; @@ -348,6 +351,10 @@ static int32_t tcp_tfo_check(struct tcpcb *tp, int32_t len) /* No cookie, so we request one */ return (0); + /* There is not enough space for the cookie, so we cannot do TFO */ + if (MAX_TCPOPTLEN - optlen < cookie_len) + goto fallback; + /* Do not send SYN+data if there is more in the queue than MSS */ if (so->so_snd.sb_cc > (tp->t_maxopd - MAX_TCPOPTLEN)) goto fallback; @@ -388,14 +395,25 @@ tcp_tfo_write_cookie_rep(struct tcpcb *tp, unsigned optlen, u_char *opt) } static unsigned -tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t *len, +tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t len, u_char *opt) { u_int8_t tfo_len = MAX_TCPOPTLEN - optlen - TCPOLEN_FASTOPEN_REQ; + struct socket *so = tp->t_inpcb->inp_socket; unsigned ret = 0; int res; u_char *bp; + if (so->so_flags1 & SOF1_DATA_AUTHENTICATED) { + /* If there is some data, let's track it */ + if (len > 0) { + tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT; + tcpstat.tcps_tfo_syn_data_sent++; + } + + return (0); + } + bp = opt + optlen; /* @@ -422,7 +440,7 @@ tcp_tfo_write_cookie(struct tcpcb *tp, unsigned optlen, int32_t *len, tp->t_tfo_flags |= TFO_F_COOKIE_SENT; /* If there is some data, let's track it */ - if (*len) { + if (len > 0) { tp->t_tfo_stats |= TFO_S_SYN_DATA_SENT; tcpstat.tcps_tfo_syn_data_sent++; } @@ -439,7 +457,6 @@ tcp_send_ecn_flags_on_syn(struct tcpcb *tp, struct socket *so) (tp->t_flagsext & TF_FASTOPEN))); } -#define TCP_ECN_SETUP_PERCENTAGE_MAX 5 void tcp_set_ecn(struct tcpcb *tp, struct ifnet *ifp) { @@ -500,7 +517,7 @@ check_heuristic: * Use the random value in iss for randomizing * this selection */ - if ((tp->iss % 100) >= TCP_ECN_SETUP_PERCENTAGE_MAX) + if ((tp->iss % 100) >= tcp_ecn_setup_percentage) tp->ecn_flags &= ~TE_ENABLE_ECN; } } @@ -561,26 +578,24 @@ tcp_output(struct tcpcb *tp) struct mbuf *tp_inp_options = inp->inp_depend4.inp4_options; #if INET6 int isipv6 = inp->inp_vflag & INP_IPV6 ; +#else + int isipv6 = 0; #endif short packchain_listadd = 0; int so_options = so->so_options; struct rtentry *rt; - u_int32_t basertt, svc_flags = 0, allocated_len; + u_int32_t svc_flags = 0, allocated_len; u_int32_t lro_ackmore = (tp->t_lropktlen != 0) ? 1 : 0; struct mbuf *mnext = NULL; int sackoptlen = 0; #if MPTCP - unsigned int *dlenp = NULL; - u_int8_t *finp = NULL; - u_int32_t *sseqp = NULL; - u_int64_t dss_val = 0; - boolean_t mptcp_acknow = FALSE; - boolean_t early_data_sent = FALSE; + boolean_t mptcp_acknow; #endif /* MPTCP */ boolean_t cell = FALSE; boolean_t wifi = FALSE; boolean_t wired = FALSE; boolean_t sack_rescue_rxt = FALSE; + int sotc = so->so_traffic_class; /* * Determine length of data that should be transmitted, @@ -619,11 +634,11 @@ tcp_output(struct tcpcb *tp) idle = 0; } } -#if MPTCP +#if MPTCP if (tp->t_mpflags & TMPF_RESET) { tcp_check_timer_state(tp); - /* - * Once a RST has been sent for an MPTCP subflow, + /* + * Once a RST has been sent for an MPTCP subflow, * the subflow socket stays around until deleted. * No packets such as FINs must be sent after RST. */ @@ -632,6 +647,10 @@ tcp_output(struct tcpcb *tp) #endif /* MPTCP */ again: +#if MPTCP + mptcp_acknow = FALSE; +#endif + KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0); #if INET6 @@ -702,9 +721,8 @@ again: OFFSET_FROM_START(tp, tp->t_rxtcur); if (tp->t_timer[TCPT_PERSIST]) { tp->t_timer[TCPT_PERSIST] = 0; - tp->t_rxtshift = 0; tp->t_persist_stop = 0; - tp->t_rxtstart = 0; + TCP_RESET_REXMT_STATE(tp); } } @@ -713,7 +731,7 @@ again: TCP_PKTLIST_CLEAR(tp); /* drop connection if source address isn't available */ - if (so->so_flags & SOF_NOADDRAVAIL) { + if (so->so_flags & SOF_NOADDRAVAIL) { tcp_drop(tp, EADDRNOTAVAIL); return(EADDRNOTAVAIL); } else { @@ -748,9 +766,9 @@ again: * has been disabled) */ - if (!path_mtu_discovery || ((rt != NULL) && + if (!path_mtu_discovery || ((rt != NULL) && (!(rt->rt_flags & RTF_UP) || - (rt->rt_rmx.rmx_locks & RTV_MTU)))) + (rt->rt_rmx.rmx_locks & RTV_MTU)))) tp->t_flags &= ~TF_PMTUD; else tp->t_flags |= TF_PMTUD; @@ -797,7 +815,7 @@ again: if (SACK_ENABLED(tp) && IN_FASTRECOVERY(tp) && (p = tcp_sack_output(tp, &sack_bytes_rxmt))) { int32_t cwin; - + cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt; if (cwin < 0) cwin = 0; @@ -825,7 +843,7 @@ again: len = ((int32_t)min(cwin, p->end - p->rxmit)); } if (len > 0) { - off = p->rxmit - tp->snd_una; + off = p->rxmit - tp->snd_una; sack_rxmit = 1; sendalot = 1; tcpstat.tcps_sack_rexmits++; @@ -874,9 +892,8 @@ after_sack_rexmit: sendwin = 1; } else { tp->t_timer[TCPT_PERSIST] = 0; - tp->t_rxtshift = 0; - tp->t_rxtstart = 0; tp->t_persist_stop = 0; + TCP_RESET_REXMT_STATE(tp); } } @@ -911,12 +928,12 @@ after_sack_rexmit: * sending new data, having retransmitted all the * data possible in the scoreboard. */ - len = min(so->so_snd.sb_cc, tp->snd_wnd) + len = min(so->so_snd.sb_cc, tp->snd_wnd) - off; /* * Don't remove this (len > 0) check ! - * We explicitly check for len > 0 here (although it - * isn't really necessary), to work around a gcc + * We explicitly check for len > 0 here (although it + * isn't really necessary), to work around a gcc * optimization issue - to force gcc to compute * len above. Without this check, the computation * of len is bungled by the optimizer. @@ -956,22 +973,6 @@ after_sack_rexmit: } } -#if MPTCP - if ((tp->t_mpflags & TMPF_FASTJOIN_SEND) && - (tp->t_state == TCPS_SYN_SENT) && - (!(tp->t_flags & TF_CLOSING)) && - (so->so_snd.sb_cc != 0) && - (tp->t_rxtshift == 0)) { - flags &= ~TH_SYN; - flags |= TH_ACK; - off = 0; - len = min(so->so_snd.sb_cc, tp->t_maxseg); - early_data_sent = TRUE; - } else if (early_data_sent) { - /* for now, we allow only one data segment to be sent */ - return (0); - } -#endif /* MPTCP */ /* * Lop off SYN bit if it has already been sent. However, if this * is SYN-SENT state and if segment contains data and if we don't @@ -980,7 +981,8 @@ after_sack_rexmit: if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) { if (tp->t_state != TCPS_SYN_RECEIVED || tfo_enabled(tp)) flags &= ~TH_SYN; - off--, len++; + off--; + len++; if (len > 0 && tp->t_state == TCPS_SYN_SENT) { while (inp->inp_sndinprog_cnt == 0 && tp->t_pktlist_head != NULL) { @@ -992,19 +994,13 @@ after_sack_rexmit: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), 0, -#if INET6 + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); -#else /* INET6 */ - 0); -#endif /* !INET6 */ - - } /* * tcp was closed while we were in ip, - * resume close + * resume close */ if (inp->inp_sndinprog_cnt == 0 && (tp->t_flags & TF_CLOSING)) { @@ -1039,9 +1035,9 @@ after_sack_rexmit: /* * The check here used to be (len < 0). Some times len is zero * when the congestion window is closed and we need to check - * if persist timer has to be set in that case. But don't set + * if persist timer has to be set in that case. But don't set * persist until connection is established. - */ + */ if (len <= 0 && !(flags & TH_SYN)) { /* * If FIN has been sent but not acked, @@ -1057,8 +1053,7 @@ after_sack_rexmit: if (sendwin == 0) { tp->t_timer[TCPT_REXMT] = 0; tp->t_timer[TCPT_PTO] = 0; - tp->t_rxtshift = 0; - tp->t_rxtstart = 0; + TCP_RESET_REXMT_STATE(tp); tp->snd_nxt = tp->snd_una; off = 0; if (tp->t_timer[TCPT_PERSIST] == 0) @@ -1075,34 +1070,17 @@ after_sack_rexmit: * 3. our send window (slow start and congestion controlled) is * larger than sent but unacknowledged data in send buffer. */ - basertt = get_base_rtt(tp); if (tcp_do_autosendbuf == 1 && !INP_WAIT_FOR_IF_FEEDBACK(inp) && !IN_FASTRECOVERY(tp) && (so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE && tcp_cansbgrow(&so->so_snd)) { if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat && so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) && - sendwin >= (so->so_snd.sb_cc - - (tp->snd_nxt - tp->snd_una))) { - /* Also increase the send buffer only if the - * round-trip time is not increasing because we do - * not want to contribute to latency by filling - * buffers. - * We also do not want to hold onto application's - * old data for too long. Interactive applications - * would rather discard old data. - */ - if (tp->t_rttcur <= (basertt + 25)) { - if (sbreserve(&so->so_snd, - min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc, - tcp_autosndbuf_max)) == 1) { - so->so_snd.sb_idealsize = so->so_snd.sb_hiwat; - } - } else { - so->so_snd.sb_idealsize = - max(tcp_sendspace, so->so_snd.sb_hiwat - - (2 * tcp_autosndbuf_inc)); - so->so_snd.sb_flags |= SB_TRIM; + sendwin >= (so->so_snd.sb_cc - (tp->snd_nxt - tp->snd_una))) { + if (sbreserve(&so->so_snd, + min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc, + tcp_autosndbuf_max)) == 1) { + so->so_snd.sb_idealsize = so->so_snd.sb_hiwat; } } } @@ -1164,30 +1142,36 @@ after_sack_rexmit: } #if MPTCP - if ((so->so_flags & SOF_MP_SUBFLOW) && + if ((so->so_flags & SOF_MP_SUBFLOW) && !(tp->t_mpflags & TMPF_TCP_FALLBACK)) { int newlen = len; - if ((tp->t_state >= TCPS_ESTABLISHED) && - ((tp->t_mpflags & TMPF_SND_MPPRIO) || - (tp->t_mpflags & TMPF_SND_REM_ADDR) || - (tp->t_mpflags & TMPF_SND_MPFAIL) || - (tp->t_mpflags & TMPF_MPCAP_RETRANSMIT))) { + if (tp->t_state >= TCPS_ESTABLISHED && + (tp->t_mpflags & TMPF_SND_MPPRIO || + tp->t_mpflags & TMPF_SND_REM_ADDR || + tp->t_mpflags & TMPF_SND_MPFAIL || + tp->t_mpflags & TMPF_SND_KEYS || + tp->t_mpflags & TMPF_SND_JACK)) { if (len > 0) { len = 0; } - sendalot = 1; + /* + * On a new subflow, don't try to send again, because + * we are still waiting for the fourth ack. + */ + if (!(tp->t_mpflags & TMPF_PREESTABLISHED)) + sendalot = 1; mptcp_acknow = TRUE; } else { mptcp_acknow = FALSE; } /* * The contiguous bytes in the subflow socket buffer can be - * discontiguous at the MPTCP level. Since only one DSS + * discontiguous at the MPTCP level. Since only one DSS * option can be sent in one packet, reduce length to match * the contiguous MPTCP level. Set sendalot to send remainder. */ if (len > 0) - newlen = mptcp_adj_sendlen(so, off, len); + newlen = mptcp_adj_sendlen(so, off); if (newlen < len) { len = newlen; sendalot = 1; @@ -1200,8 +1184,8 @@ after_sack_rexmit: * pull the amount of data that can be sent from the * unordered priority queues to the serial queue in * the socket buffer. If bytes are not yet available - * in the highest priority message, we may not be able - * to send any new data. + * in the highest priority message, we may not be able + * to send any new data. */ if (so->so_flags & SOF_ENABLE_MSGS) { if ((off + len) > @@ -1209,7 +1193,7 @@ after_sack_rexmit: sbpull_unordered_data(so, off, len); /* check if len needs to be modified */ - if ((off + len) > + if ((off + len) > so->so_msg_state->msg_serial_bytes) { len = so->so_msg_state->msg_serial_bytes - off; if (len <= 0) { @@ -1227,8 +1211,57 @@ after_sack_rexmit: if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc)) flags &= ~TH_FIN; } - + /* + * Compare available window to amount of window + * known to peer (as advertised window less + * next expected input). If the difference is at least two + * max size segments, or at least 25% of the maximum possible + * window, then want to send a window update to peer. + * Skip this if the connection is in T/TCP half-open state. + */ recwin = tcp_sbspace(tp); +#if MPTCP + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptcb *mp_tp = tptomptp(tp); + + if (mp_tp != NULL) { + mpte_lock_assert_held(mp_tp->mpt_mpte); + recwin = imin(recwin, mptcp_sbspace(mp_tp)); + } + } +#endif + + if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && + recwin < (int)tp->t_maxseg) + recwin = 0; + +#if TRAFFIC_MGT + if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) { + if (recwin > 0 && tcp_recv_throttle(tp)) { + uint32_t min_iaj_win = tcp_min_iaj_win * tp->t_maxseg; + uint32_t bg_rwintop = tp->rcv_adv; + if (SEQ_LT(bg_rwintop, tp->rcv_nxt + min_iaj_win)) + bg_rwintop = tp->rcv_nxt + min_iaj_win; + recwin = imin((int32_t)(bg_rwintop - tp->rcv_nxt), + recwin); + if (recwin < 0) + recwin = 0; + } + } +#endif /* TRAFFIC_MGT */ + + if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale)) + recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale); + + /* + * MPTCP needs to be able to announce a smaller window than previously, + * because the other subflow may have filled up the available window- + * space. So we have to be able to go backwards and announce a smaller + * window. + */ + if (!(so->so_flags & SOF_MP_SUBFLOW) && + recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) + recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt); /* * Sender silly window avoidance. We transmit under the following @@ -1258,12 +1291,20 @@ after_sack_rexmit: tp->t_state != TCPS_ESTABLISHED) { if (len >= tp->t_maxseg) goto send; + if (!(tp->t_flags & TF_MORETOCOME) && - (idle || tp->t_flags & TF_NODELAY || - tp->t_flags & TF_MAXSEGSNT || + (idle || tp->t_flags & TF_NODELAY || + (tp->t_flags & TF_MAXSEGSNT) || ALLOW_LIMITED_TRANSMIT(tp)) && (tp->t_flags & TF_NOPUSH) == 0 && - len + off >= so->so_snd.sb_cc) + (len + off >= so->so_snd.sb_cc || + /* + * MPTCP needs to respect the DSS-mappings. So, it + * may be sending data that *could* have been + * coalesced, but cannot because of + * mptcp_adj_sendlen(). + */ + so->so_flags & SOF_MP_SUBFLOW)) goto send; if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) goto send; @@ -1272,14 +1313,6 @@ after_sack_rexmit: } } - /* - * Compare available window to amount of window - * known to peer (as advertised window less - * next expected input). If the difference is at least two - * max size segments, or at least 25% of the maximum possible - * window, then want to send a window update to peer. - * Skip this if the connection is in T/TCP half-open state. - */ if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) { /* * "adv" is the amount we can increase the window, @@ -1297,34 +1330,35 @@ after_sack_rexmit: /* * Update only if the resulting scaled value of * the window changed, or if there is a change in - * the sequence since the last ack. This avoids + * the sequence since the last ack. This avoids * what appears as dupe ACKS (see rdar://5640997) * * If streaming is detected avoid sending too many - * window updates. We will depend on the delack + * window updates. We will depend on the delack * timer to send a window update when needed. */ if (!(tp->t_flags & TF_STRETCHACK) && - (tp->last_ack_sent != tp->rcv_nxt || + (tp->last_ack_sent != tp->rcv_nxt || ((oldwin + adv) >> tp->rcv_scale) > (oldwin >> tp->rcv_scale))) { goto send; } - /* - * Make sure that the delayed ack timer is set if - * we delayed sending a window update because of - * streaming detection. - */ - if ((tp->t_flags & TF_STRETCHACK) && - !(tp->t_flags & TF_DELACK)) { - tp->t_flags |= TF_DELACK; - tp->t_timer[TCPT_DELACK] = - OFFSET_FROM_START(tp, tcp_delack); - } } - if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat) - goto send; + if (4 * adv >= (int32_t) so->so_rcv.sb_hiwat) + goto send; + + /* + * Make sure that the delayed ack timer is set if + * we delayed sending a window update because of + * streaming detection. + */ + if ((tp->t_flags & TF_STRETCHACK) && + !(tp->t_flags & TF_DELACK)) { + tp->t_flags |= TF_DELACK; + tp->t_timer[TCPT_DELACK] = + OFFSET_FROM_START(tp, tcp_delack); + } } /* @@ -1354,14 +1388,14 @@ after_sack_rexmit: * after the retransmission timer has been turned off. Make sure * that the retransmission timer is set. */ - if (SACK_ENABLED(tp) && (tp->t_state >= TCPS_ESTABLISHED) && + if (SACK_ENABLED(tp) && (tp->t_state >= TCPS_ESTABLISHED) && SEQ_GT(tp->snd_max, tp->snd_una) && tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0) { tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur); goto just_return; - } + } /* * TCP window updates are not reliable, rather a polling protocol * using ``persist'' packets is used to insure receipt of window @@ -1386,8 +1420,7 @@ after_sack_rexmit: */ if (so->so_snd.sb_cc && tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0) { - tp->t_rxtshift = 0; - tp->t_rxtstart = 0; + TCP_RESET_REXMT_STATE(tp); tcp_setpersist(tp); } just_return: @@ -1405,12 +1438,7 @@ just_return: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), recwin, -#if INET6 - isipv6); -#else /* INET6 */ - 0); -#endif /* !INET6 */ + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); } /* tcp was closed while we were in ip; resume close */ if (inp->inp_sndinprog_cnt == 0 && @@ -1471,9 +1499,8 @@ send: optlen += 4; } #if MPTCP - if (mptcp_enable) { - optlen = mptcp_setup_syn_opts(so, flags, opt, - optlen); + if (mptcp_enable && (so->so_flags & SOF_MP_SUBFLOW)) { + optlen = mptcp_setup_syn_opts(so, opt, optlen); } #endif /* MPTCP */ } @@ -1502,7 +1529,7 @@ send: tp->rfbuf_ts = tcp_now; if (SACK_ENABLED(tp) && ((tp->t_flags & TF_NOOPT) == 0)) { - /* + /* * Tack on the SACK permitted option *last*. * And do padding of options after tacking this on. * This is because of MSS, TS, WinScale and Signatures are @@ -1541,14 +1568,14 @@ send: tp->t_mpflags |= TMPF_MPTCP_ACKNOW; } optlen = mptcp_setup_opts(tp, off, &opt[0], optlen, flags, - len, &dlenp, &finp, &dss_val, &sseqp, &mptcp_acknow); + len, &mptcp_acknow); tp->t_mpflags &= ~TMPF_SEND_DSN; } #endif /* MPTCP */ if (tfo_enabled(tp) && !(tp->t_flags & TF_NOOPT) && (flags & (TH_SYN | TH_ACK)) == TH_SYN) - optlen += tcp_tfo_write_cookie(tp, optlen, &len, opt); + optlen += tcp_tfo_write_cookie(tp, optlen, len, opt); if (tfo_enabled(tp) && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK) && @@ -1804,43 +1831,30 @@ send: sendalot = 1; } } -#if MPTCP - /* Adjust the length in the DSS option, if it is lesser than len */ - if (dlenp) { - /* - * To test this path without SACK, artificially - * decrement len with something like - * if (len > 10) - len -= 10; - */ - if (ntohs(*dlenp) > len) { - *dlenp = htons(len); - /* Unset the FIN flag, if len was adjusted */ - if (finp) { - *finp &= ~MDSS_F; - } - sendalot = 1; - } - } -#endif /* MPTCP */ if (max_linkhdr + hdrlen > MCLBYTES) panic("tcphdr too big"); /* Check if there is enough data in the send socket - * buffer to start measuring bw + * buffer to start measuring bandwidth */ if ((tp->t_flagsext & TF_MEASURESNDBW) != 0 && (tp->t_bwmeas != NULL) && - (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0 && - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)) >= - tp->t_bwmeas->bw_minsize) { - tp->t_bwmeas->bw_size = min( - (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), - tp->t_bwmeas->bw_maxsize); - tp->t_flagsext |= TF_BWMEAS_INPROGRESS; - tp->t_bwmeas->bw_start = tp->snd_max; - tp->t_bwmeas->bw_ts = tcp_now; + (tp->t_flagsext & TF_BWMEAS_INPROGRESS) == 0) { + tp->t_bwmeas->bw_size = min(min( + (so->so_snd.sb_cc - (tp->snd_max - tp->snd_una)), + tp->snd_cwnd), tp->snd_wnd); + if (tp->t_bwmeas->bw_minsize > 0 && + tp->t_bwmeas->bw_size < tp->t_bwmeas->bw_minsize) + tp->t_bwmeas->bw_size = 0; + if (tp->t_bwmeas->bw_maxsize > 0) + tp->t_bwmeas->bw_size = min(tp->t_bwmeas->bw_size, + tp->t_bwmeas->bw_maxsize); + if (tp->t_bwmeas->bw_size > 0) { + tp->t_flagsext |= TF_BWMEAS_INPROGRESS; + tp->t_bwmeas->bw_start = tp->snd_max; + tp->t_bwmeas->bw_ts = tcp_now; + } } VERIFY(inp->inp_flowhash != 0); @@ -1869,14 +1883,16 @@ send: } else { tcpstat.tcps_sndpack++; tcpstat.tcps_sndbyte += len; - + if (nstat_collect) { INP_ADD_STAT(inp, cell, wifi, wired, txpackets, 1); INP_ADD_STAT(inp, cell, wifi, wired, txbytes, len); } + inp_decr_sndbytes_unsent(so, len); } + inp_set_activity_bitmap(inp); #if MPTCP if (tp->t_mpflags & TMPF_MPTCP_TRUE) { tcpstat.tcps_mp_sndpacks++; @@ -1884,9 +1900,9 @@ send: } #endif /* MPTCP */ /* - * try to use the new interface that allocates all - * the necessary mbuf hdrs under 1 mbuf lock and - * avoids rescanning the socket mbuf list if + * try to use the new interface that allocates all + * the necessary mbuf hdrs under 1 mbuf lock and + * avoids rescanning the socket mbuf list if * certain conditions are met. This routine can't * be used in the following cases... * 1) the protocol headers exceed the capacity of @@ -1965,7 +1981,7 @@ send: error = 0; /* should we return an error? */ goto out; } - + /* * m_copym_with_hdrs will always return the * last mbuf pointer and the offset into it that @@ -2019,10 +2035,6 @@ send: m->m_len = hdrlen; } m->m_pkthdr.rcvif = 0; -#if MPTCP - /* Before opt is copied to the mbuf, set the csum field */ - mptcp_output_csum(tp, m, len, hdrlen, dss_val, sseqp); -#endif /* MPTCP */ #if CONFIG_MACF_NET mac_mbuf_label_associate_inpcb(inp, m); #endif @@ -2037,8 +2049,8 @@ send: } svc_flags |= PKT_SCF_IPV6; #if PF_ECN - m->m_pkthdr.pf_mtag.pftag_hdr = (void *)ip6; - m->m_pkthdr.pf_mtag.pftag_flags |= PF_TAG_HDR_INET6; + m_pftag(m)->pftag_hdr = (void *)ip6; + m_pftag(m)->pftag_flags |= PF_TAG_HDR_INET6; #endif /* PF_ECN */ } else #endif /* INET6 */ @@ -2054,8 +2066,8 @@ send: ip->ip_tos |= IPTOS_ECN_ECT0; } #if PF_ECN - m->m_pkthdr.pf_mtag.pftag_hdr = (void *)ip; - m->m_pkthdr.pf_mtag.pftag_flags |= PF_TAG_HDR_INET; + m_pftag(m)->pftag_hdr = (void *)ip; + m_pftag(m)->pftag_flags |= PF_TAG_HDR_INET; #endif /* PF_ECN */ } @@ -2087,6 +2099,10 @@ send: if (len || (flags & (TH_SYN|TH_FIN)) || tp->t_timer[TCPT_PERSIST]) { th->th_seq = htonl(tp->snd_nxt); + if (len > 0) { + m->m_pkthdr.tx_start_seq = tp->snd_nxt; + m->m_pkthdr.pkt_flags |= PKTF_START_SEQ; + } if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { if (SACK_ENABLED(tp) && len > 1) { tcp_rxtseg_insert(tp, tp->snd_nxt, @@ -2101,57 +2117,25 @@ send: } } else { th->th_seq = htonl(p->rxmit); + if (len > 0) { + m->m_pkthdr.pkt_flags |= + (PKTF_TCP_REXMT | PKTF_START_SEQ); + m->m_pkthdr.tx_start_seq = p->rxmit; + } tcp_rxtseg_insert(tp, p->rxmit, (p->rxmit + len - 1)); p->rxmit += len; tp->sackhint.sack_bytes_rexmit += len; - if (len > 0) - m->m_pkthdr.pkt_flags |= PKTF_TCP_REXMT; } th->th_ack = htonl(tp->rcv_nxt); tp->last_ack_sent = tp->rcv_nxt; -#if MPTCP - /* Initialize the ACK field to a value as 0 ack fields are dropped */ - if (early_data_sent) { - th->th_ack = th->th_seq + 1; - } -#endif /* MPTCP */ if (optlen) { bcopy(opt, th + 1, optlen); th->th_off = (sizeof (struct tcphdr) + optlen) >> 2; } th->th_flags = flags; - /* - * Calculate receive window. Don't shrink window, - * but avoid silly window syndrome. - */ - if (recwin < (int32_t)(so->so_rcv.sb_hiwat / 4) && recwin < (int)tp->t_maxseg) - recwin = 0; - if (recwin < (int32_t)(tp->rcv_adv - tp->rcv_nxt)) - recwin = (int32_t)(tp->rcv_adv - tp->rcv_nxt); - if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0) { - if (recwin > (int32_t)slowlink_wsize) - recwin = slowlink_wsize; - } - -#if TRAFFIC_MGT - if (tcp_recv_bg == 1 || IS_TCP_RECV_BG(so)) { - if (tcp_recv_throttle(tp)) { - uint32_t min_iaj_win = - tcp_min_iaj_win * tp->t_maxseg; - if (tp->iaj_rwintop == 0 || - SEQ_LT(tp->iaj_rwintop, tp->rcv_adv)) - tp->iaj_rwintop = tp->rcv_adv; - if (SEQ_LT(tp->iaj_rwintop, - tp->rcv_nxt + min_iaj_win)) - tp->iaj_rwintop = tp->rcv_nxt + min_iaj_win; - recwin = min(tp->iaj_rwintop - tp->rcv_nxt, recwin); - } - } -#endif /* TRAFFIC_MGT */ - - if (recwin > (int32_t)(TCP_MAXWIN << tp->rcv_scale)) - recwin = (int32_t)(TCP_MAXWIN << tp->rcv_scale); th->th_win = htons((u_short) (recwin>>tp->rcv_scale)); + if (recwin > 0 && SEQ_LT(tp->rcv_adv, tp->rcv_nxt + recwin)) + tp->rcv_adv = tp->rcv_nxt + recwin; /* * Adjust the RXWIN0SENT flag - indicate that we have advertised @@ -2183,6 +2167,17 @@ send: * checksum extended header and data. */ m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ + + /* + * If this is potentially the last packet on the stream, then mark + * it in order to enable some optimizations in the underlying + * layers + */ + if (tp->t_state != TCPS_ESTABLISHED && + (tp->t_state == TCPS_CLOSING || tp->t_state == TCPS_TIME_WAIT + || tp->t_state == TCPS_LAST_ACK || (th->th_flags & TH_RST))) + m->m_pkthdr.pkt_flags |= PKTF_LAST_PKT; + #if INET6 if (isipv6) { /* @@ -2192,7 +2187,7 @@ send: m->m_pkthdr.csum_flags = CSUM_TCPIPV6; m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); if (len + optlen) - th->th_sum = in_addword(th->th_sum, + th->th_sum = in_addword(th->th_sum, htons((u_short)(optlen + len))); } else @@ -2201,7 +2196,7 @@ send: m->m_pkthdr.csum_flags = CSUM_TCP; m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); if (len + optlen) - th->th_sum = in_addword(th->th_sum, + th->th_sum = in_addword(th->th_sum, htons((u_short)(optlen + len))); } @@ -2236,7 +2231,7 @@ send: if (flags & (TH_SYN|TH_FIN)) { if (flags & TH_SYN) tp->snd_nxt++; - if ((flags & TH_FIN) && + if ((flags & TH_FIN) && !(tp->t_flags & TF_SENTFIN)) { tp->snd_nxt++; tp->t_flags |= TF_SENTFIN; @@ -2253,6 +2248,7 @@ send: } if (SEQ_GT(tp->snd_nxt, tp->snd_max)) { tp->snd_max = tp->snd_nxt; + tp->t_sndtime = tcp_now; /* * Time this transmission if not a retransmission and * not currently timing anything. @@ -2277,9 +2273,8 @@ timer: tp->snd_nxt != tp->snd_una || (flags & TH_FIN))) { if (tp->t_timer[TCPT_PERSIST]) { tp->t_timer[TCPT_PERSIST] = 0; - tp->t_rxtshift = 0; - tp->t_rxtstart = 0; tp->t_persist_stop = 0; + TCP_RESET_REXMT_STATE(tp); } tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur); @@ -2303,7 +2298,7 @@ timer: /* * Using SRTT alone to set PTO can cause spurious * retransmissions on wireless networks where there - * is a lot of variance in RTT. Taking variance + * is a lot of variance in RTT. Taking variance * into account will avoid this. */ srtt = tp->t_srtt >> TCP_RTT_SHIFT; @@ -2323,7 +2318,7 @@ timer: */ pto = tp->t_rxtcur; - /* Reset the next RTO to be after PTO. */ + /* Reset the next RTO to be after PTO. */ TCPT_RANGESET(new_rto, (pto + TCP_REXMTVAL(tp)), max(tp->t_rttmin, tp->t_rttcur + 2), @@ -2341,13 +2336,15 @@ timer: int xlen = len; if (flags & TH_SYN) ++xlen; - if ((flags & TH_FIN) && + if ((flags & TH_FIN) && !(tp->t_flags & TF_SENTFIN)) { ++xlen; tp->t_flags |= TF_SENTFIN; } - if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) + if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max)) { tp->snd_max = tp->snd_nxt + len; + tp->t_sndtime = tcp_now; + } } #if TCPDEBUG @@ -2421,8 +2418,12 @@ timer: error = EHOSTUNREACH; goto out; } - necp_mark_packet_from_socket(m, inp, policy_id, route_rule_id); + + if (net_qos_policy_restricted != 0) { + necp_socket_update_qos_marking(inp, inp->inp_route.ro_rt, + NULL, route_rule_id); + } } #endif /* NECP */ @@ -2442,12 +2443,7 @@ timer: */ m->m_pkthdr.pkt_flowsrc = FLOWSRC_INPCB; m->m_pkthdr.pkt_flowid = inp->inp_flowhash; - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC; -#if MPTCP - /* Disable flow advisory when using MPTCP. */ - if (!(tp->t_mpflags & TMPF_MPTCP_TRUE)) -#endif /* MPTCP */ - m->m_pkthdr.pkt_flags |= PKTF_FLOW_ADV; + m->m_pkthdr.pkt_flags |= (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC | PKTF_FLOW_ADV); m->m_pkthdr.pkt_proto = IPPROTO_TCP; m->m_nextpkt = NULL; @@ -2456,19 +2452,28 @@ timer: !(inp->inp_last_outifp->if_flags & IFF_LOOPBACK)) { /* Hint to prioritize this packet if * 1. if the packet has no data - * 2. the interface supports transmit-start model and did + * 2. the interface supports transmit-start model and did * not disable ACK prioritization. * 3. Only ACK flag is set. * 4. there is no outstanding data on this connection. */ if (tcp_prioritize_acks != 0 && len == 0 && - (inp->inp_last_outifp->if_eflags & - (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART && - th->th_flags == TH_ACK && tp->snd_una == tp->snd_max && - tp->t_timer[TCPT_REXMT] == 0) { - svc_flags |= PKT_SCF_TCP_ACK; - } - set_packet_service_class(m, so, MBUF_SC_UNSPEC, svc_flags); + (inp->inp_last_outifp->if_eflags & + (IFEF_TXSTART | IFEF_NOACKPRI)) == IFEF_TXSTART) { + if (th->th_flags == TH_ACK && + tp->snd_una == tp->snd_max && + tp->t_timer[TCPT_REXMT] == 0) + svc_flags |= PKT_SCF_TCP_ACK; + if (th->th_flags & TH_SYN) + svc_flags |= PKT_SCF_TCP_SYN; + } + set_packet_service_class(m, so, sotc, svc_flags); + } else { + /* + * Optimization for loopback just set the mbuf + * service class + */ + (void) m_set_service_class(m, so_tc2msc(sotc)); } tp->t_pktlist_sentlen += len; @@ -2497,7 +2502,7 @@ timer: if ((lro_ackmore) && (!sackoptlen) && (!tp->t_timer[TCPT_PERSIST]) && ((th->th_flags & TH_ACK) == TH_ACK) && (!len) && (tp->t_state == TCPS_ESTABLISHED)) { - /* For a pure ACK, see if you need to send more of them */ + /* For a pure ACK, see if you need to send more of them */ mnext = tcp_send_lroacks(tp, m, th); if (mnext) { tp->t_pktlist_tail->m_nextpkt = mnext; @@ -2520,7 +2525,7 @@ timer: if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) || (tp->snd_cwnd <= (tp->snd_wnd / 8)) || - (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || + (tp->t_flags & TF_ACKNOW) || (tp->t_flagsext & TF_FORCE) || tp->t_lastchain >= tcp_packet_chaining) { error = 0; @@ -2535,12 +2540,7 @@ timer: error = tcp_ip_output(so, tp, packetlist, packchain_listadd, tp_inp_options, (so_options & SO_DONTROUTE), - (sack_rxmit | (sack_bytes_rxmt != 0)), recwin, -#if INET6 - isipv6); -#else /* INET6 */ - 0); -#endif /* !INET6 */ + (sack_rxmit || (sack_bytes_rxmt != 0)), isipv6); if (error) { /* * Take into account the rest of unsent @@ -2607,9 +2607,16 @@ out: TCP_PKTLIST_CLEAR(tp); if (error == ENOBUFS) { + /* + * Set retransmit timer if not currently set + * when we failed to send a segment that can be + * retransmitted (i.e. not pure ack or rst) + */ if (!tp->t_timer[TCPT_REXMT] && - !tp->t_timer[TCPT_PERSIST]) - tp->t_timer[TCPT_REXMT] = + !tp->t_timer[TCPT_PERSIST] && + (len != 0 || (flags & (TH_SYN | TH_FIN)) != 0 || + so->so_snd.sb_cc > 0)) + tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur); tp->snd_cwnd = tp->t_maxseg; tp->t_bytes_acked = 0; @@ -2646,7 +2653,7 @@ out: * treat EHOSTUNREACH/ENETDOWN as a soft error. */ if ((error == EHOSTUNREACH || error == ENETDOWN) && - TCPS_HAVERCVDSYN(tp->t_state) && + TCPS_HAVERCVDSYN(tp->t_state) && !inp_restricted_send(inp, inp->inp_last_outifp)) { tp->t_softerror = error; error = 0; @@ -2668,22 +2675,32 @@ out: static int tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, - int cnt, struct mbuf *opt, int flags, int sack_in_progress, int recwin, - boolean_t isipv6) + int cnt, struct mbuf *opt, int flags, int sack_in_progress, boolean_t isipv6) { int error = 0; boolean_t chain; boolean_t unlocked = FALSE; boolean_t ifdenied = FALSE; struct inpcb *inp = tp->t_inpcb; - struct ip_out_args ipoa = - { IFSCOPE_NONE, { 0 }, IPOAF_SELECT_SRCIF|IPOAF_BOUND_SRCADDR, 0 }; + struct ip_out_args ipoa; struct route ro; struct ifnet *outif = NULL; + + bzero(&ipoa, sizeof(ipoa)); + ipoa.ipoa_boundif = IFSCOPE_NONE; + ipoa.ipoa_flags = IPOAF_SELECT_SRCIF | IPOAF_BOUND_SRCADDR; + ipoa.ipoa_sotc = SO_TC_UNSPEC; + ipoa.ipoa_netsvctype = _NET_SERVICE_TYPE_UNSPEC; #if INET6 - struct ip6_out_args ip6oa = - { IFSCOPE_NONE, { 0 }, IP6OAF_SELECT_SRCIF|IP6OAF_BOUND_SRCADDR, 0 }; + struct ip6_out_args ip6oa; struct route_in6 ro6; + + bzero(&ip6oa, sizeof(ip6oa)); + ip6oa.ip6oa_boundif = IFSCOPE_NONE; + ip6oa.ip6oa_flags = IP6OAF_SELECT_SRCIF | IP6OAF_BOUND_SRCADDR; + ip6oa.ip6oa_sotc = SO_TC_UNSPEC; + ip6oa.ip6oa_netsvctype = _NET_SERVICE_TYPE_UNSPEC; + struct flowadv *adv = (isipv6 ? &ip6oa.ip6oa_flowadv : &ipoa.ipoa_flowadv); #else /* INET6 */ @@ -2711,7 +2728,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, else #endif /* INET6 */ ipoa.ipoa_flags |= IPOAF_NO_CELLULAR; - } + } if (INP_NO_EXPENSIVE(inp)) { #if INET6 if (isipv6) @@ -2719,7 +2736,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, else #endif /* INET6 */ ipoa.ipoa_flags |= IPOAF_NO_EXPENSIVE; - + } if (INP_AWDL_UNRESTRICTED(inp)) { #if INET6 @@ -2728,7 +2745,28 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, else #endif /* INET6 */ ipoa.ipoa_flags |= IPOAF_AWDL_UNRESTRICTED; - + + } +#if INET6 + if (INP_INTCOPROC_ALLOWED(inp) && isipv6) { + ip6oa.ip6oa_flags |= IP6OAF_INTCOPROC_ALLOWED; + } + if (isipv6) { + ip6oa.ip6oa_sotc = so->so_traffic_class; + ip6oa.ip6oa_netsvctype = so->so_netsvctype; + } else +#endif /* INET6 */ + { + ipoa.ipoa_sotc = so->so_traffic_class; + ipoa.ipoa_netsvctype = so->so_netsvctype; + } + if ((so->so_flags1 & SOF1_QOSMARKING_ALLOWED)) { +#if INET6 + if (isipv6) + ip6oa.ip6oa_flags |= IP6OAF_QOSMARKING_ALLOWED; + else +#endif /* INET6 */ + ipoa.ipoa_flags |= IPOAF_QOSMARKING_ALLOWED; } #if INET6 if (isipv6) @@ -2746,14 +2784,9 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, inp_route_copyout(inp, &ro); /* - * Data sent (as far as we can tell). - * If this advertises a larger window than any other segment, - * then remember the size of the advertised window. * Make sure ACK/DELACK conditions are cleared before * we unlock the socket. */ - if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) - tp->rcv_adv = tp->rcv_nxt + recwin; tp->last_ack_sent = tp->rcv_nxt; tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); tp->t_timer[TCPT_DELACK] = 0; @@ -2773,12 +2806,12 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, */ if (tcp_output_unlocked && !so->so_upcallusecount && (tp->t_state == TCPS_ESTABLISHED) && (sack_in_progress == 0) && - !IN_FASTRECOVERY(tp)) { + !IN_FASTRECOVERY(tp) && !(so->so_flags & SOF_MP_SUBFLOW)) { unlocked = TRUE; socket_unlock(so, 0); } - + /* * Don't send down a chain of packets when: * - TCP chaining is disabled @@ -2843,27 +2876,29 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, if (unlocked) socket_lock(so, 0); - /* + /* * Enter flow controlled state if the connection is established - * and is not in recovery. + * and is not in recovery. Flow control is allowed only if there + * is outstanding data. * - * A connection will enter suspended state even if it is in + * A connection will enter suspended state even if it is in * recovery. */ if (((adv->code == FADV_FLOW_CONTROLLED && !IN_FASTRECOVERY(tp)) || - adv->code == FADV_SUSPENDED) && + adv->code == FADV_SUSPENDED) && !(tp->t_flags & TF_CLOSING) && - tp->t_state == TCPS_ESTABLISHED) { + tp->t_state == TCPS_ESTABLISHED && + SEQ_GT(tp->snd_max, tp->snd_una)) { int rc; rc = inp_set_fc_state(inp, adv->code); - if (rc == 1) - tcp_ccdbg_trace(tp, NULL, + if (rc == 1) + tcp_ccdbg_trace(tp, NULL, ((adv->code == FADV_FLOW_CONTROLLED) ? TCP_CC_FLOW_CONTROL : TCP_CC_SUSPEND)); } - /* + /* * When an interface queue gets suspended, some of the * packets are dropped. Return ENOBUFS, to update the * pcb state. @@ -2877,18 +2912,28 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, #if INET6 if (isipv6) { - if (ro6.ro_rt != NULL && (outif = ro6.ro_rt->rt_ifp) != - inp->in6p_last_outifp) - inp->in6p_last_outifp = outif; + if (ro6.ro_rt != NULL) + outif = ro6.ro_rt->rt_ifp; } else #endif /* INET6 */ - if (ro.ro_rt != NULL && (outif = ro.ro_rt->rt_ifp) != - inp->inp_last_outifp) - inp->inp_last_outifp = outif; + if (ro.ro_rt != NULL) + outif = ro.ro_rt->rt_ifp; + + if (outif != NULL && outif != inp->inp_last_outifp && + so->so_snd.sb_cc > 0) { + /* Update the send byte count */ + if (so->so_snd.sb_flags & SB_SNDBYTE_CNT) { + inp_decr_sndbytes_total(so, so->so_snd.sb_cc); + inp_decr_sndbytes_allunsent(so, tp->snd_una); + so->so_snd.sb_flags &= ~SB_SNDBYTE_CNT; + } + inp->inp_last_outifp = outif; + + } - if (error != 0 && ifdenied && + if (error != 0 && ifdenied && (INP_NO_CELLULAR(inp) || INP_NO_EXPENSIVE(inp))) - soevent(inp->inp_socket, + soevent(so, (SO_FILT_HINT_LOCKED|SO_FILT_HINT_IFDENIED)); /* Synchronize cached PCB route & options */ @@ -2899,7 +2944,7 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, #endif /* INET6 */ inp_route_copyin(inp, &ro); - if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 && + if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift == 0 && tp->t_inpcb->inp_route.ro_rt != NULL) { /* If we found the route and there is an rtt on it * reset the retransmit timer @@ -2910,9 +2955,10 @@ tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt, return (error); } +int tcptv_persmin_val = TCPTV_PERSMIN; + void -tcp_setpersist(tp) - register struct tcpcb *tp; +tcp_setpersist(struct tcpcb *tp) { int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1; @@ -2933,7 +2979,7 @@ tcp_setpersist(tp) */ TCPT_RANGESET(tp->t_timer[TCPT_PERSIST], t * tcp_backoff[tp->t_rxtshift], - TCPTV_PERSMIN, TCPTV_PERSMAX, 0); + tcptv_persmin_val, TCPTV_PERSMAX, 0); tp->t_timer[TCPT_PERSIST] = OFFSET_FROM_START(tp, tp->t_timer[TCPT_PERSIST]); if (tp->t_rxtshift < TCP_MAXRXTSHIFT) @@ -2951,8 +2997,8 @@ tcp_send_lroacks(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th) int count = 0; tcp_seq org_ack = ntohl(th->th_ack); tcp_seq prev_ack = 0; - int tack_offset = 28; /* XXX IPv6 and IP options not supported */ - int twin_offset = 34; /* XXX IPv6 and IP options not supported */ + int tack_offset = 28; /* IPv6 and IP options not supported */ + int twin_offset = 34; /* IPv6 and IP options not supported */ int ack_size = (tp->t_flags & TF_STRETCHACK) ? (maxseg_unacked * tp->t_maxseg) : (tp->t_maxseg << 1); int segs_acked = (tp->t_flags & TF_STRETCHACK) ? maxseg_unacked : 2; @@ -3047,12 +3093,12 @@ tcp_recv_throttle (struct tcpcb *tp) if (tcp_use_rtt_recvbg == 1 && TSTMP_SUPPORTED(tp)) { - /* + /* * Timestamps are supported on this connection. Use * RTT to look for an increase in latency. */ - /* + /* * If the connection is already being throttled, leave it * in that state until rtt comes closer to base rtt */ @@ -3060,7 +3106,7 @@ tcp_recv_throttle (struct tcpcb *tp) return (1); base_rtt = get_base_rtt(tp); - + if (base_rtt != 0 && tp->t_rttcur != 0) { /* * if latency increased on a background flow, @@ -3074,12 +3120,12 @@ tcp_recv_throttle (struct tcpcb *tp) * Reduce the recv socket buffer size to * minimize latecy. */ - if (sbrcv->sb_idealsize > + if (sbrcv->sb_idealsize > tcp_recv_throttle_minwin) { newsize = sbrcv->sb_idealsize >> 1; /* Set a minimum of 16 K */ - newsize = - max(newsize, + newsize = + max(newsize, tcp_recv_throttle_minwin); sbrcv->sb_idealsize = newsize; } @@ -3096,6 +3142,6 @@ tcp_recv_throttle (struct tcpcb *tp) */ if (tp->acc_iaj > tcp_acc_iaj_react_limit) return (1); - + return (0); }