X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..527f99514973766e9c0382a4d8550dfb00f54939:/bsd/netinet/tcp_timer.c diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index 046163f7b..2c861120e 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2000-2015 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -106,29 +106,6 @@ #include #include -#define TIMERENTRY_TO_TP(te) ((struct tcpcb *)((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next))) - -#define VERIFY_NEXT_LINK(elm,field) do { \ - if (LIST_NEXT((elm),field) != NULL && \ - LIST_NEXT((elm),field)->field.le_prev != \ - &((elm)->field.le_next)) \ - panic("Bad link elm %p next->prev != elm", (elm)); \ -} while(0) - -#define VERIFY_PREV_LINK(elm,field) do { \ - if (*(elm)->field.le_prev != (elm)) \ - panic("Bad link elm %p prev->next != elm", (elm)); \ -} while(0) - -#define TCP_SET_TIMER_MODE(mode, i) do { \ - if (IS_TIMER_HZ_10MS(i)) \ - (mode) |= TCP_TIMERLIST_10MS_MODE; \ - else if (IS_TIMER_HZ_100MS(i)) \ - (mode) |= TCP_TIMERLIST_100MS_MODE; \ - else \ - (mode) |= TCP_TIMERLIST_500MS_MODE; \ -} while(0) - /* Max number of times a stretch ack can be delayed on a connection */ #define TCP_STRETCHACK_DELAY_THRESHOLD 5 @@ -147,10 +124,10 @@ struct tcptailq tcp_tw_tailq; static int sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS { -#pragma unused(arg1, arg2) +#pragma unused(arg2) int error, s, tt; - tt = *(int *)oidp->oid_arg1; + tt = *(int *)arg1; s = tt * 1000 / TCP_RETRANSHZ;; error = sysctl_handle_int(oidp, &s, 0, req); @@ -161,36 +138,67 @@ sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS if (tt < 1) return (EINVAL); - *(int *)oidp->oid_arg1 = tt; - return (0); + *(int *)arg1 = tt; + SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1); + return (0); } -int tcp_keepinit; +#if SYSCTL_SKMEM +int tcp_keepinit = TCPTV_KEEP_INIT; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepidle = TCPTV_KEEP_IDLE; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle), + sysctl_msec_to_ticks, "I", ""); + +int tcp_keepintvl = TCPTV_KEEPINTVL; +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl), + sysctl_msec_to_ticks, "I", ""); + +SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive"); + +int tcp_msl = TCPTV_MSL; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_msl, offsetof(skmem_sysctl, tcp.msl), + sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#else /* SYSCTL_SKMEM */ +int tcp_keepinit; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepidle; +int tcp_keepidle; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepintvl; +int tcp_keepintvl; SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_keepcnt; +int tcp_keepcnt; SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepcnt, 0, "number of times to repeat keepalive"); -int tcp_msl; +int tcp_msl; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +#endif /* SYSCTL_SKMEM */ -/* +/* * Avoid DoS via TCP Robustness in Persist Condition * (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt) * by allowing a system wide maximum persistence timeout value when in @@ -199,26 +207,32 @@ SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, * Expressed in milliseconds to be consistent without timeout related * values, the TCP socket option is in seconds. */ +#if SYSCTL_SKMEM +u_int32_t tcp_max_persist_timeout = 0; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout), + sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#else /* SYSCTL_SKMEM */ u_int32_t tcp_max_persist_timeout = 0; SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, - &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I", + &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP"); +#endif /* SYSCTL_SKMEM */ -static int always_keepalive = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, - CTLFLAG_RW | CTLFLAG_LOCKED, - &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0, + "Assume SO_KEEPALIVE on all TCP connections"); /* * This parameter determines how long the timer list will stay in fast or - * quick mode even though all connections are idle. In this state, the + * quick mode even though all connections are idle. In this state, the * timer will run more frequently anticipating new data. */ -int timer_fastmode_idlemax = TCP_FASTMODE_IDLERUN_MAX; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, - CTLFLAG_RW | CTLFLAG_LOCKED, - &timer_fastmode_idlemax, 0, "Maximum idle generations in fast mode"); +SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax, + CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax, + TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode"); /* * See tcp_syn_backoff[] for interval values between SYN retransmits; @@ -227,10 +241,9 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, * SYN retransmits. Setting it to 0 disables the dropping off of those * two options. */ -static int tcp_broken_peer_syn_rxmit_thres = 10; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rexmit_thres, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_broken_peer_syn_rxmit_thres, 0, - "Number of retransmitted SYNs before disabling RFC 1323 " +SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres, + CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres, + 10, "Number of retransmitted SYNs before disabling RFC 1323 " "options on local connections"); static int tcp_timer_advanced = 0; @@ -240,32 +253,28 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced, static int tcp_resched_timerlist = 0; SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist, - CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0, + CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0, "Number of times timer list was rescheduled as part of processing a packet"); -int tcp_pmtud_black_hole_detect = 1 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_detection, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_detect, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1, "Path MTU Discovery Black Hole Detection"); -int tcp_pmtud_black_hole_mss = 1200 ; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_mss, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_pmtud_black_hole_mss, 0, +SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss, + CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200, "Path MTU Discovery Black Hole Detection lowered MSS"); +static u_int32_t tcp_mss_rec_medium = 1200; +static u_int32_t tcp_mss_rec_low = 512; + #define TCP_REPORT_STATS_INTERVAL 43200 /* 12 hours, in seconds */ int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL; -#if (DEVELOPMENT || DEBUG) -SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval, - CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0, - "Report stats interval"); -#endif /* (DEVELOPMENT || DEBUG) */ /* performed garbage collection of "used" sockets */ static boolean_t tcp_gc_done = FALSE; /* max idle probes */ -int tcp_maxpersistidle; +int tcp_maxpersistidle = TCPTV_KEEP_IDLE; /* * TCP delack timer is set to 100 ms. Since the processing of timer list @@ -290,19 +299,9 @@ static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode, static void tcp_sched_timers(struct tcpcb *tp); static inline void tcp_set_lotimer_index(struct tcpcb *); __private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp); +static inline void tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp); __private_extern__ void tcp_report_stats(void); -/* - * Macro to compare two timers. If there is a reset of the sign bit, - * it is safe to assume that the timer has wrapped around. By doing - * signed comparision, we take care of wrap around such that the value - * with the sign bit reset is actually ahead of the other. - */ -inline int32_t -timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) { - return (int32_t)((t1 + toff1) - (t2 + toff2)); -}; - static u_int64_t tcp_last_report_time; /* @@ -327,6 +326,9 @@ struct tcp_last_report_stats { u_int32_t tcps_ecn_conn_plnoce; u_int32_t tcps_ecn_conn_pl_ce; u_int32_t tcps_ecn_conn_nopl_ce; + u_int32_t tcps_ecn_fallback_synloss; + u_int32_t tcps_ecn_fallback_reorder; + u_int32_t tcps_ecn_fallback_ce; /* TFO-related statistics */ u_int32_t tcps_tfo_syn_data_rcv; @@ -339,6 +341,40 @@ struct tcp_last_report_stats { u_int32_t tcps_tfo_syn_data_acked; u_int32_t tcps_tfo_syn_loss; u_int32_t tcps_tfo_blackhole; + u_int32_t tcps_tfo_cookie_wrong; + u_int32_t tcps_tfo_no_cookie_rcv; + u_int32_t tcps_tfo_heuristics_disable; + u_int32_t tcps_tfo_sndblackhole; + + /* MPTCP-related statistics */ + u_int32_t tcps_mptcp_handover_attempt; + u_int32_t tcps_mptcp_interactive_attempt; + u_int32_t tcps_mptcp_aggregate_attempt; + u_int32_t tcps_mptcp_fp_handover_attempt; + u_int32_t tcps_mptcp_fp_interactive_attempt; + u_int32_t tcps_mptcp_fp_aggregate_attempt; + u_int32_t tcps_mptcp_heuristic_fallback; + u_int32_t tcps_mptcp_fp_heuristic_fallback; + u_int32_t tcps_mptcp_handover_success_wifi; + u_int32_t tcps_mptcp_handover_success_cell; + u_int32_t tcps_mptcp_interactive_success; + u_int32_t tcps_mptcp_aggregate_success; + u_int32_t tcps_mptcp_fp_handover_success_wifi; + u_int32_t tcps_mptcp_fp_handover_success_cell; + u_int32_t tcps_mptcp_fp_interactive_success; + u_int32_t tcps_mptcp_fp_aggregate_success; + u_int32_t tcps_mptcp_handover_cell_from_wifi; + u_int32_t tcps_mptcp_handover_wifi_from_cell; + u_int32_t tcps_mptcp_interactive_cell_from_wifi; + u_int64_t tcps_mptcp_handover_cell_bytes; + u_int64_t tcps_mptcp_interactive_cell_bytes; + u_int64_t tcps_mptcp_aggregate_cell_bytes; + u_int64_t tcps_mptcp_handover_all_bytes; + u_int64_t tcps_mptcp_interactive_all_bytes; + u_int64_t tcps_mptcp_aggregate_all_bytes; + u_int32_t tcps_mptcp_back_to_wifi; + u_int32_t tcps_mptcp_wifi_proxy; + u_int32_t tcps_mptcp_cell_proxy; }; @@ -352,6 +388,93 @@ struct tcp_last_report_stats { static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay); static boolean_t tcp_garbage_collect(struct inpcb *, int); +#define TIMERENTRY_TO_TP(te) ((struct tcpcb *)((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next))) + +#define VERIFY_NEXT_LINK(elm,field) do { \ + if (LIST_NEXT((elm),field) != NULL && \ + LIST_NEXT((elm),field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while(0) + +#define VERIFY_PREV_LINK(elm,field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while(0) + +#define TCP_SET_TIMER_MODE(mode, i) do { \ + if (IS_TIMER_HZ_10MS(i)) \ + (mode) |= TCP_TIMERLIST_10MS_MODE; \ + else if (IS_TIMER_HZ_100MS(i)) \ + (mode) |= TCP_TIMERLIST_100MS_MODE; \ + else \ + (mode) |= TCP_TIMERLIST_500MS_MODE; \ +} while(0) + +#if (DEVELOPMENT || DEBUG) +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_medium, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_medium, 0, + "Medium MSS based on recommendation in link status report"); +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_low, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_low, 0, + "Low MSS based on recommendation in link status report"); + +static int32_t tcp_change_mss_recommended = 0; +static int +sysctl_change_mss_recommended SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int i, err = 0, changed = 0; + struct ifnet *ifp; + struct if_link_status ifsr; + struct if_cellular_status_v1 *new_cell_sr; + err = sysctl_io_number(req, tcp_change_mss_recommended, + sizeof (int32_t), &i, &changed); + if (changed) { + ifnet_head_lock_shared(); + TAILQ_FOREACH(ifp, &ifnet_head, if_link) { + if (IFNET_IS_CELLULAR(ifp)) { + bzero(&ifsr, sizeof (ifsr)); + new_cell_sr = &ifsr.ifsr_u.ifsr_cell.if_cell_u.if_status_v1; + ifsr.ifsr_version = IF_CELLULAR_STATUS_REPORT_CURRENT_VERSION; + ifsr.ifsr_len = sizeof(*new_cell_sr); + + /* Set MSS recommended */ + new_cell_sr->valid_bitmask |= IF_CELL_UL_MSS_RECOMMENDED_VALID; + new_cell_sr->mss_recommended = i; + err = ifnet_link_status_report(ifp, new_cell_sr, sizeof (new_cell_sr)); + if (err == 0) { + tcp_change_mss_recommended = i; + } else { + break; + } + } + } + ifnet_head_done(); + } + return (err); +} + +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, change_mss_recommended, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_change_mss_recommended, + 0, sysctl_change_mss_recommended, "IU", "Change MSS recommended"); + +SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0, + "Report stats interval"); +#endif /* (DEVELOPMENT || DEBUG) */ + +/* + * Macro to compare two timers. If there is a reset of the sign bit, + * it is safe to assume that the timer has wrapped around. By doing + * signed comparision, we take care of wrap around such that the value + * with the sign bit reset is actually ahead of the other. + */ +inline int32_t +timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) { + return (int32_t)((t1 + toff1) - (t2 + toff2)); +}; + /* * Add to tcp timewait list, delay is given in milliseconds. */ @@ -363,13 +486,13 @@ add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay) uint32_t timer; /* pcb list should be locked when we get here */ - lck_rw_assert(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + LCK_RW_ASSERT(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); /* We may get here multiple times, so check */ if (!(inp->inp_flags2 & INP2_TIMEWAIT)) { pcbinfo->ipi_twcount++; inp->inp_flags2 |= INP2_TIMEWAIT; - + /* Remove from global inp list */ LIST_REMOVE(inp, inp_list); } else { @@ -378,7 +501,7 @@ add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay) /* Compute the time at which this socket can be closed */ timer = tcp_now + delay; - + /* We will use the TCPT_2MSL timer for tracking this delay */ if (TIMER_IS_ON_LIST(tp)) @@ -399,9 +522,9 @@ add_to_time_wait(struct tcpcb *tp, uint32_t delay) nstat_pcb_detach(tp->t_inpcb); if (!lck_rw_try_lock_exclusive(pcbinfo->ipi_lock)) { - tcp_unlock(tp->t_inpcb->inp_socket, 0, 0); + socket_unlock(tp->t_inpcb->inp_socket, 0); lck_rw_lock_exclusive(pcbinfo->ipi_lock); - tcp_lock(tp->t_inpcb->inp_socket, 0, 0); + socket_lock(tp->t_inpcb->inp_socket, 0); } add_to_time_wait_locked(tp, delay); lck_rw_done(pcbinfo->ipi_lock); @@ -422,12 +545,22 @@ static boolean_t tcp_garbage_collect(struct inpcb *inp, int istimewait) { boolean_t active = FALSE; - struct socket *so; + struct socket *so, *mp_so = NULL; struct tcpcb *tp; so = inp->inp_socket; tp = intotcpcb(inp); + if (so->so_flags & SOF_MP_SUBFLOW) { + mp_so = mptetoso(tptomptp(tp)->mpt_mpte); + if (!socket_try_lock(mp_so)) { + mp_so = NULL; + active = TRUE; + goto out; + } + mp_so->so_usecount++; + } + /* * Skip if still in use or busy; it would have been more efficient * if we were to test so_usecount against 0, but this isn't possible @@ -435,20 +568,21 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) * overflow sockets that are eligible for garbage collection have * their usecounts set to 1. */ - if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) - return (TRUE); + if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) { + active = TRUE; + goto out; + } /* Check again under the lock */ if (so->so_usecount > 1) { if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } - if (istimewait && - TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && - tp->t_state != TCPS_CLOSED) { + if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && + tp->t_state != TCPS_CLOSED) { /* Become a regular mutex */ lck_mtx_convert_spin(&inp->inpcb_mtx); tcp_close(tp); @@ -480,24 +614,26 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) #endif /* INET6 */ in_pcbdetach(inp); } + VERIFY(so->so_usecount > 0); so->so_usecount--; if (inp->inp_wantcnt == WNT_STOPUSING) active = TRUE; lck_mtx_unlock(&inp->inpcb_mtx); - return (active); + goto out; } else if (inp->inp_wantcnt != WNT_STOPUSING) { lck_mtx_unlock(&inp->inpcb_mtx); - return (FALSE); + active = FALSE; + goto out; } /* - * We get here because the PCB is no longer searchable - * (WNT_STOPUSING); detach (if needed) and dispose if it is dead - * (usecount is 0). This covers all cases, including overflow - * sockets and those that are considered as "embryonic", - * i.e. created by sonewconn() in TCP input path, and have + * We get here because the PCB is no longer searchable + * (WNT_STOPUSING); detach (if needed) and dispose if it is dead + * (usecount is 0). This covers all cases, including overflow + * sockets and those that are considered as "embryonic", + * i.e. created by sonewconn() in TCP input path, and have * not yet been committed. For the former, we reduce the usecount - * to 0 as done by the code above. For the latter, the usecount + * to 0 as done by the code above. For the latter, the usecount * would have reduced to 0 as part calling soabort() when the * socket is dropped at the end of tcp_input(). */ @@ -508,7 +644,7 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) lck_mtx_convert_spin(&inp->inpcb_mtx); /* - * If this tp still happens to be on the timer list, + * If this tp still happens to be on the timer list, * take it out */ if (TIMER_IS_ON_LIST(tp)) { @@ -523,12 +659,28 @@ tcp_garbage_collect(struct inpcb *inp, int istimewait) #endif /* INET6 */ in_pcbdetach(inp); } + + if (mp_so) { + mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub); + + /* so is now unlinked from mp_so - let's drop the lock */ + socket_unlock(mp_so, 1); + mp_so = NULL; + } + in_pcbdispose(inp); - return (FALSE); + active = FALSE; + goto out; } lck_mtx_unlock(&inp->inpcb_mtx); - return (TRUE); + active = TRUE; + +out: + if (mp_so) + socket_unlock(mp_so, 1); + + return (active); } /* @@ -585,16 +737,16 @@ tcp_gc(struct inpcbinfo *ipi) /* Now cleanup the time wait ones */ TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) { /* - * We check the timestamp here without holding the + * We check the timestamp here without holding the * socket lock for better performance. If there are * any pcbs in time-wait, the timer will get rescheduled. * Hence some error in this check can be tolerated. * * Sometimes a socket on time-wait queue can be closed if * 2MSL timer expired but the application still has a - * usecount on it. + * usecount on it. */ - if (tw_tp->t_state == TCPS_CLOSED || + if (tw_tp->t_state == TCPS_CLOSED || TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) { if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1); @@ -620,10 +772,9 @@ tcp_gc(struct inpcbinfo *ipi) * Cancel all timers for TCP tp. */ void -tcp_canceltimers(tp) - struct tcpcb *tp; +tcp_canceltimers(struct tcpcb *tp) { - register int i; + int i; tcp_remove_timer(tp); for (i = 0; i < TCPT_NTIMERS; i++) @@ -640,12 +791,13 @@ int tcp_backoff[TCP_MAXRXTSHIFT + 1] = static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */ -void tcp_rexmt_save_state(struct tcpcb *tp) +void +tcp_rexmt_save_state(struct tcpcb *tp) { u_int32_t fsize; if (TSTMP_SUPPORTED(tp)) { /* - * Since timestamps are supported on the connection, + * Since timestamps are supported on the connection, * we can do recovery as described in rfc 4015. */ fsize = tp->snd_max - tp->snd_una; @@ -656,9 +808,9 @@ void tcp_rexmt_save_state(struct tcpcb *tp) * Timestamp option is not supported on this connection. * Record ssthresh and cwnd so they can * be recovered if this turns out to be a "bad" retransmit. - * A retransmit is considered "bad" if an ACK for this + * A retransmit is considered "bad" if an ACK for this * segment is received within RTT/2 interval; the assumption - * here is that the ACK was already in flight. See + * here is that the ACK was already in flight. See * "On Estimating End-to-End Network Path Properties" by * Allman and Paxson for more details. */ @@ -679,33 +831,35 @@ void tcp_rexmt_save_state(struct tcpcb *tp) * Revert to the older segment size if there is an indication that PMTU * blackhole detection was not needed. */ -void tcp_pmtud_revert_segment_size(struct tcpcb *tp) +void +tcp_pmtud_revert_segment_size(struct tcpcb *tp) { int32_t optlen; VERIFY(tp->t_pmtud_saved_maxopd > 0); - tp->t_flags |= TF_PMTUD; - tp->t_flags &= ~TF_BLACKHOLE; + tp->t_flags |= TF_PMTUD; + tp->t_flags &= ~TF_BLACKHOLE; optlen = tp->t_maxopd - tp->t_maxseg; tp->t_maxopd = tp->t_pmtud_saved_maxopd; tp->t_maxseg = tp->t_maxopd - optlen; /* - * Reset the slow-start flight size as it + * Reset the slow-start flight size as it * may depend on the new MSS */ if (CC_ALGO(tp)->cwnd_init != NULL) CC_ALGO(tp)->cwnd_init(tp); tp->t_pmtud_start_ts = 0; tcpstat.tcps_pmtudbh_reverted++; + + /* change MSS according to recommendation, if there was one */ + tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL); } /* * TCP timer processing. */ struct tcpcb * -tcp_timers(tp, timer) - register struct tcpcb *tp; - int timer; +tcp_timers(struct tcpcb *tp, int timer) { int32_t rexmt, optlen = 0, idle_time = 0; struct socket *so; @@ -738,7 +892,7 @@ tcp_timers(tp, timer) if (tp->t_state != TCPS_TIME_WAIT && tp->t_state != TCPS_FIN_WAIT_2 && ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) { - tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, + tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, (u_int32_t)TCP_CONN_KEEPINTVL(tp)); } else { tp = tcp_close(tp); @@ -752,7 +906,9 @@ tcp_timers(tp, timer) * to a longer retransmit interval and retransmit one segment. */ case TCPT_REXMT: - accsleep_ms = mach_absolutetime_asleep / 1000000UL; + absolutetime_to_nanoseconds(mach_absolutetime_asleep, + &accsleep_ms); + accsleep_ms = accsleep_ms / 1000000UL; if (accsleep_ms > tp->t_accsleep_ms) last_sleep_ms = accsleep_ms - tp->t_accsleep_ms; /* @@ -772,6 +928,20 @@ tcp_timers(tp, timer) ((tp->t_flagsext & TF_RXTFINDROP) != 0 && (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) || (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) { + if (tp->t_state == TCPS_ESTABLISHED && + tp->t_rxt_minimum_timeout > 0) { + /* + * Avoid dropping a connection if minimum + * timeout is set and that time did not + * pass. We will retry sending + * retransmissions at the maximum interval + */ + if (TSTMP_LT(tcp_now, (tp->t_rxtstart + + tp->t_rxt_minimum_timeout))) { + tp->t_rxtshift = TCP_MAXRXTSHIFT - 1; + goto retransmit_packet; + } + } if ((tp->t_flagsext & TF_RXTFINDROP) != 0) { tcpstat.tcps_rxtfindrop++; } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) { @@ -779,29 +949,43 @@ tcp_timers(tp, timer) } else { tcpstat.tcps_timeoutdrop++; } + if (tp->t_rxtshift >= TCP_MAXRXTSHIFT) { + if (TCP_ECN_ENABLED(tp)) { + INP_INC_IFNET_STAT(tp->t_inpcb, + ecn_on.rxmit_drop); + } else { + INP_INC_IFNET_STAT(tp->t_inpcb, + ecn_off.rxmit_drop); + } + } tp->t_rxtshift = TCP_MAXRXTSHIFT; - postevent(so, 0, EV_TIMEOUT); - soevent(so, + postevent(so, 0, EV_TIMEOUT); + soevent(so, (SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT)); + + if (TCP_ECN_ENABLED(tp) && + tp->t_state == TCPS_ESTABLISHED) + tcp_heuristic_ecn_droprxmt(tp); + tp = tcp_drop(tp, tp->t_softerror ? tp->t_softerror : ETIMEDOUT); break; } - +retransmit_packet: tcpstat.tcps_rexmttimeo++; tp->t_accsleep_ms = accsleep_ms; - if (tp->t_rxtshift == 1 && + if (tp->t_rxtshift == 1 && tp->t_state == TCPS_ESTABLISHED) { /* Set the time at which retransmission started. */ tp->t_rxtstart = tcp_now; - /* + /* * if this is the first retransmit timeout, save * the state so that we can recover if the timeout * is spurious. - */ + */ tcp_rexmt_save_state(tp); } #if MPTCP @@ -811,6 +995,12 @@ tcp_timers(tp, timer) mptcp_act_on_txfail(so); } + + if (so->so_flags & SOF_MP_SUBFLOW) { + struct mptses *mpte = tptomptp(tp)->mpt_mpte; + + mptcp_check_subflows_and_add(mpte); + } #endif /* MPTCP */ if (tp->t_adaptive_wtimo > 0 && @@ -846,10 +1036,11 @@ tcp_timers(tp, timer) if (tp->t_state == TCPS_SYN_RECEIVED) tcp_disable_tfo(tp); - if ((tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) || - tp->t_rxtshift > 2)) { + tp->t_rxtshift > 4)) { /* * For regular retransmissions, a first one is being * done for tail-loss probe. @@ -864,6 +1055,21 @@ tcp_timers(tp, timer) so->so_error = ENODATA; sorwakeup(so); sowwakeup(so); + + tp->t_tfo_stats |= TFO_S_SEND_BLACKHOLE; + tcpstat.tcps_tfo_sndblackhole++; + } + + if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) && + tp->t_rxtshift > 3) { + if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) { + tcp_heuristic_tfo_middlebox(tp); + + so->so_error = ENODATA; + sorwakeup(so); + sowwakeup(so); + } } if (tp->t_state == TCPS_SYN_SENT) { @@ -871,19 +1077,17 @@ tcp_timers(tp, timer) tp->t_stat.synrxtshift = tp->t_rxtshift; /* When retransmitting, disable TFO */ - if (tfo_enabled(tp)) { + if (tfo_enabled(tp) && + (!(so->so_flags1 & SOF1_DATA_AUTHENTICATED) || + (tp->t_flagsext & TF_FASTOPEN_HEUR))) { tp->t_flagsext &= ~TF_FASTOPEN; tp->t_tfo_flags |= TFO_F_SYN_LOSS; - - tp->t_tfo_stats |= TFO_S_SYN_LOSS; - tcpstat.tcps_tfo_syn_loss++; } } else { rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; } - TCPT_RANGESET(tp->t_rxtcur, rexmt, - tp->t_rttmin, TCPTV_REXMTMAX, + TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX, TCP_ADD_REXMTSLOP(tp)); tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur); @@ -901,7 +1105,7 @@ tcp_timers(tp, timer) ((tp->t_flags & TF_MAXSEGSNT) || tp->t_pmtud_lastseg_size > tcp_pmtud_black_hole_mss) && tp->t_rxtshift == 2) { - /* + /* * Enter Path MTU Black-hole Detection mechanism: * - Disable Path MTU Discovery (IP "DF" bit). * - Reduce MTU to lower value than what we @@ -930,11 +1134,12 @@ tcp_timers(tp, timer) tp->t_maxseg = tp->t_maxopd - optlen; /* - * Reset the slow-start flight size + * Reset the slow-start flight size * as it may depend on the new MSS */ if (CC_ALGO(tp)->cwnd_init != NULL) CC_ALGO(tp)->cwnd_init(tp); + tp->snd_cwnd = tp->t_maxseg; } /* * If further retransmissions are still @@ -943,10 +1148,11 @@ tcp_timers(tp, timer) * MSS and blackhole detection flags. */ else { - + if ((tp->t_flags & TF_BLACKHOLE) && (tp->t_rxtshift > 4)) { tcp_pmtud_revert_segment_size(tp); + tp->snd_cwnd = tp->t_maxseg; } } } @@ -1007,7 +1213,7 @@ tcp_timers(tp, timer) * been retransmitted by way of the retransmission timer at * least once, the value of ssthresh is held constant */ - if (tp->t_rxtshift == 1 && + if (tp->t_rxtshift == 1 && CC_ALGO(tp)->after_timeout != NULL) { CC_ALGO(tp)->after_timeout(tp); /* @@ -1043,15 +1249,15 @@ fc_output: * backoff, drop the connection if the idle time * (no responses to probes) reaches the maximum * backoff that we would use if retransmitting. - * - * Drop the connection if we reached the maximum allowed time for - * Zero Window Probes without a non-zero update from the peer. + * + * Drop the connection if we reached the maximum allowed time for + * Zero Window Probes without a non-zero update from the peer. * See rdar://5805356 */ if ((tp->t_rxtshift == TCP_MAXRXTSHIFT && (idle_time >= tcp_maxpersistidle || - idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) || - ((tp->t_persist_stop != 0) && + idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) || + ((tp->t_persist_stop != 0) && TSTMP_LEQ(tp->t_persist_stop, tcp_now))) { tcpstat.tcps_persistdrop++; postevent(so, 0, EV_TIMEOUT); @@ -1077,7 +1283,7 @@ fc_output: * Regular TCP connections do not send keepalives after closing * MPTCP must not also, after sending Data FINs. */ - struct mptcb *mp_tp = tp->t_mptcb; + struct mptcb *mp_tp = tptomptp(tp); if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && (tp->t_state > TCPS_ESTABLISHED)) { goto dropit; @@ -1117,6 +1323,7 @@ fc_output: tra.nocell = INP_NO_CELLULAR(inp); tra.noexpensive = INP_NO_EXPENSIVE(inp); tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp); + tra.intcoproc_allowed = INP_INTCOPROC_ALLOWED(inp); if (tp->t_inpcb->inp_flags & INP_BOUND_IF) tra.ifscope = tp->t_inpcb->inp_boundifp->if_index; else @@ -1137,9 +1344,9 @@ fc_output: if (tp->t_flagsext & TF_DETECT_READSTALL) { struct ifnet *outifp = tp->t_inpcb->inp_last_outifp; bool reenable_probe = false; - /* + /* * The keep alive packets sent to detect a read - * stall did not get a response from the + * stall did not get a response from the * peer. Generate more keep-alives to confirm this. * If the number of probes sent reaches the limit, * generate an event. @@ -1184,12 +1391,14 @@ fc_output: tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START( tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)), tp->t_timer[TCPT_KEEP]); - } else if (tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { + } else if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && + tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) { /* Still no data! Let's assume a TFO-error and err out... */ tcp_heuristic_tfo_middlebox(tp); so->so_error = ENODATA; sorwakeup(so); + tp->t_tfo_stats |= TFO_S_RECV_BLACKHOLE; tcpstat.tcps_tfo_blackhole++; } break; @@ -1202,7 +1411,7 @@ fc_output: /* * If delayed ack timer fired while stretching * acks, count the number of times the streaming - * detection was not correct. If this exceeds a + * detection was not correct. If this exceeds a * threshold, disable strech ack on this * connection * @@ -1223,6 +1432,7 @@ fc_output: tp->rcv_nostrack_ts = tcp_now; tcpstat.tcps_nostretchack++; tp->t_stretchack_delayed = 0; + tp->rcv_nostrack_pkts = 0; } tcp_reset_stretch_ack(tp); } @@ -1248,18 +1458,19 @@ fc_output: if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) { tcpstat.tcps_timeoutdrop++; postevent(so, 0, EV_TIMEOUT); - soevent(so, - (SO_FILT_HINT_LOCKED| + soevent(so, + (SO_FILT_HINT_LOCKED| SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, tp->t_softerror ? - tp->t_softerror : ETIMEDOUT); + tp->t_softerror : ETIMEDOUT); break; } tcpstat.tcps_join_rxmts++; + tp->t_mpflags |= TMPF_SND_JACK; tp->t_flags |= TF_ACKNOW; /* - * No backoff is implemented for simplicity for this + * No backoff is implemented for simplicity for this * corner case. */ (void) tcp_output(tp); @@ -1341,6 +1552,7 @@ fc_output: tp->t_timer[TCPT_REXMT] = 0; tcpstat.tcps_sack_recovery_episode++; + tp->t_sack_recovery_episode++; tp->sack_newdata = tp->snd_nxt; tp->snd_cwnd = tp->t_maxseg; tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY); @@ -1368,18 +1580,18 @@ tcp_remove_timer(struct tcpcb *tp) { struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (!(TIMER_IS_ON_LIST(tp))) { return; } lck_mtx_lock(listp->mtx); - + /* Check if pcb is on timer list again after acquiring the lock */ if (!(TIMER_IS_ON_LIST(tp))) { lck_mtx_unlock(listp->mtx); return; } - + if (listp->next_te != NULL && listp->next_te == &tp->tentry) listp->next_te = LIST_NEXT(&tp->tentry, le); @@ -1436,15 +1648,16 @@ need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode) } void -tcp_sched_timerlist(uint32_t offset) +tcp_sched_timerlist(uint32_t offset) { uint64_t deadline = 0; struct tcptimerlist *listp = &tcp_timer_list; - lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED); + LCK_MTX_ASSERT(listp->mtx, LCK_MTX_ASSERT_OWNED); offset = min(offset, TCP_TIMERLIST_MAX_OFFSET); listp->runtime = tcp_now + offset; + listp->schedtime = tcp_now; if (listp->runtime == 0) { listp->runtime++; offset++; @@ -1459,7 +1672,7 @@ tcp_sched_timerlist(uint32_t offset) /* * Function to run the timers for a connection. * - * Returns the offset of next timer to be run for this connection which + * Returns the offset of next timer to be run for this connection which * can be used to reschedule the timerlist. * * te_mode is an out parameter that indicates the modes of active @@ -1480,17 +1693,17 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, bzero(needtorun, sizeof(needtorun)); *te_mode = 0; - tcp_lock(tp->t_inpcb->inp_socket, 1, 0); + socket_lock(tp->t_inpcb->inp_socket, 1); so = tp->t_inpcb->inp_socket; - /* Release the want count on inp */ + /* Release the want count on inp */ if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1) == WNT_STOPUSING) { if (TIMER_IS_ON_LIST(tp)) { tcp_remove_timer(tp); } - /* Looks like the TCP connection got closed while we + /* Looks like the TCP connection got closed while we * were waiting for the lock.. Done */ goto done; @@ -1505,7 +1718,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, tp->t_flagsext |= TF_PROBING; tcp_timers(tp, TCPT_PTO); tp->t_timer[TCPT_PTO] = 0; - tp->t_flagsext &= TF_PROBING; + tp->t_flagsext &= ~TF_PROBING; } /* @@ -1515,7 +1728,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, */ if ((index = tp->tentry.index) == TCPT_NONE) goto done; - + timer_val = tp->t_timer[index]; diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0); @@ -1533,7 +1746,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, if (tp == NULL) goto done; } - + /* * Check if there are any other timers that need to be run. * While doing it, adjust the timer values wrt tcp_now. @@ -1557,7 +1770,7 @@ tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode, } } } - + tp->tentry.timer_start = tcp_now; tp->tentry.index = lo_index; VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0); @@ -1596,12 +1809,13 @@ done: offset = 0; } - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); return(offset); } void -tcp_run_timerlist(void * arg1, void * arg2) { +tcp_run_timerlist(void * arg1, void * arg2) +{ #pragma unused(arg1, arg2) struct tcptimerentry *te, *next_te; struct tcptimerlist *listp = &tcp_timer_list; @@ -1615,8 +1829,29 @@ tcp_run_timerlist(void * arg1, void * arg2) { lck_mtx_lock(listp->mtx); + int32_t drift = tcp_now - listp->runtime; + if (drift <= 1) { + tcpstat.tcps_timer_drift_le_1_ms++; + } else if (drift <= 10) { + tcpstat.tcps_timer_drift_le_10_ms++; + } else if (drift <= 20) { + tcpstat.tcps_timer_drift_le_20_ms++; + } else if (drift <= 50) { + tcpstat.tcps_timer_drift_le_50_ms++; + } else if (drift <= 100) { + tcpstat.tcps_timer_drift_le_100_ms++; + } else if (drift <= 200) { + tcpstat.tcps_timer_drift_le_200_ms++; + } else if (drift <= 500) { + tcpstat.tcps_timer_drift_le_500_ms++; + } else if (drift <= 1000) { + tcpstat.tcps_timer_drift_le_1000_ms++; + } else { + tcpstat.tcps_timer_drift_gt_1000_ms++; + } + listp->running = TRUE; - + LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) { uint32_t offset = 0; uint32_t runtime = te->runtime; @@ -1641,7 +1876,7 @@ tcp_run_timerlist(void * arg1, void * arg2) { * Some how this pcb went into dead state while * on the timer list, just take it off the list. * Since the timer list entry pointers are - * protected by the timer list lock, we can + * protected by the timer list lock, we can * do it here without the socket lock. */ if (TIMER_IS_ON_LIST(tp)) { @@ -1662,7 +1897,7 @@ tcp_run_timerlist(void * arg1, void * arg2) { * release the lock, this pointer will be updated to the * element after that. */ - listp->next_te = next_te; + listp->next_te = next_te; VERIFY_NEXT_LINK(&tp->tentry, le); VERIFY_PREV_LINK(&tp->tentry, le); @@ -1671,7 +1906,7 @@ tcp_run_timerlist(void * arg1, void * arg2) { offset = tcp_run_conn_timer(tp, &te_mode, listp->probe_if_index); - + lck_mtx_lock(listp->mtx); next_te = listp->next_te; @@ -1745,8 +1980,8 @@ tcp_run_timerlist(void * arg1, void * arg2) { * Function to check if the timerlist needs to be rescheduled to run this * connection's timers correctly. */ -void -tcp_sched_timers(struct tcpcb *tp) +void +tcp_sched_timers(struct tcpcb *tp) { struct tcptimerentry *te = &tp->tentry; u_int16_t index = te->index; @@ -1785,26 +2020,27 @@ tcp_sched_timers(struct tcpcb *tp) list_locked = TRUE; } - LIST_INSERT_HEAD(&listp->lhead, te, le); - tp->t_flags |= TF_TIMER_ONLIST; + if (!TIMER_IS_ON_LIST(tp)) { + LIST_INSERT_HEAD(&listp->lhead, te, le); + tp->t_flags |= TF_TIMER_ONLIST; - listp->entries++; - if (listp->entries > listp->maxentries) - listp->maxentries = listp->entries; + listp->entries++; + if (listp->entries > listp->maxentries) + listp->maxentries = listp->entries; - /* if the list is not scheduled, just schedule it */ - if (!listp->scheduled) - goto schedule; + /* if the list is not scheduled, just schedule it */ + if (!listp->scheduled) + goto schedule; + } } - /* * Timer entry is currently on the list, check if the list needs * to be rescheduled. */ if (need_to_resched_timerlist(te->runtime, mode)) { tcp_resched_timerlist++; - + if (!list_locked) { lck_mtx_lock(listp->mtx); list_locked = TRUE; @@ -1863,7 +2099,7 @@ done: return; } - + static inline void tcp_set_lotimer_index(struct tcpcb *tp) { @@ -1883,7 +2119,7 @@ tcp_set_lotimer_index(struct tcpcb *tp) VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0); if (tp->tentry.index != TCPT_NONE) { - tp->tentry.runtime = tp->tentry.timer_start + tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[tp->tentry.index]; if (tp->tentry.runtime == 0) tp->tentry.runtime++; @@ -1893,7 +2129,7 @@ tcp_set_lotimer_index(struct tcpcb *tp) void tcp_check_timer_state(struct tcpcb *tp) { - lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + socket_lock_assert_owned(tp->t_inpcb->inp_socket); if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) return; @@ -1917,6 +2153,19 @@ tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest) return; } +static inline void +tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest) +{ + /* handle wrap around */ + int64_t diff = (int64_t) (cur - *prev); + if (diff > 0) + *dest = diff; + else + *dest = 0; + *prev = cur; + return; +} + __private_extern__ void tcp_report_stats(void) { @@ -1925,7 +2174,7 @@ tcp_report_stats(void) struct sockaddr_in6 dst6; struct rtentry *rt = NULL; static struct tcp_last_report_stats prev; - u_int64_t var, uptime; + u_int64_t var, uptime; #define stat data.u.tcp_stats if (((uptime = net_uptime()) - tcp_last_report_time) < @@ -1990,13 +2239,13 @@ tcp_report_stats(void) } /* RTO after tail loss, shift by 10 for precision */ - if (tcpstat.tcps_sndrexmitpack > 0 + if (tcpstat.tcps_sndrexmitpack > 0 && tcpstat.tcps_tailloss_rto > 0) { var = tcpstat.tcps_tailloss_rto << 10; stat.send_tlrto_rate = (var * 100) / tcpstat.tcps_sndrexmitpack; } - + /* packet reordering */ if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) { var = tcpstat.tcps_reordered_pkts << 10; @@ -2046,6 +2295,12 @@ tcp_report_stats(void) &prev.tcps_ecn_conn_pl_ce, &stat.ecn_conn_pl_ce); tcp_cumulative_stat(tcpstat.tcps_ecn_conn_nopl_ce, &prev.tcps_ecn_conn_nopl_ce, &stat.ecn_conn_nopl_ce); + tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_synloss, + &prev.tcps_ecn_fallback_synloss, &stat.ecn_fallback_synloss); + tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_reorder, + &prev.tcps_ecn_fallback_reorder, &stat.ecn_fallback_reorder); + tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_ce, + &prev.tcps_ecn_fallback_ce, &stat.ecn_fallback_ce); tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_rcv, &prev.tcps_tfo_syn_data_rcv, &stat.tfo_syn_data_rcv); tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req_rcv, @@ -2066,6 +2321,73 @@ tcp_report_stats(void) &prev.tcps_tfo_syn_loss, &stat.tfo_syn_loss); tcp_cumulative_stat(tcpstat.tcps_tfo_blackhole, &prev.tcps_tfo_blackhole, &stat.tfo_blackhole); + tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_wrong, + &prev.tcps_tfo_cookie_wrong, &stat.tfo_cookie_wrong); + tcp_cumulative_stat(tcpstat.tcps_tfo_no_cookie_rcv, + &prev.tcps_tfo_no_cookie_rcv, &stat.tfo_no_cookie_rcv); + tcp_cumulative_stat(tcpstat.tcps_tfo_heuristics_disable, + &prev.tcps_tfo_heuristics_disable, &stat.tfo_heuristics_disable); + tcp_cumulative_stat(tcpstat.tcps_tfo_sndblackhole, + &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole); + + + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt, + &prev.tcps_mptcp_handover_attempt , &stat.mptcp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt, + &prev.tcps_mptcp_interactive_attempt , &stat.mptcp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt, + &prev.tcps_mptcp_aggregate_attempt , &stat.mptcp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt, + &prev.tcps_mptcp_fp_handover_attempt , &stat.mptcp_fp_handover_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt, + &prev.tcps_mptcp_fp_interactive_attempt , &stat.mptcp_fp_interactive_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt, + &prev.tcps_mptcp_fp_aggregate_attempt , &stat.mptcp_fp_aggregate_attempt); + tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback, + &prev.tcps_mptcp_heuristic_fallback , &stat.mptcp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback, + &prev.tcps_mptcp_fp_heuristic_fallback , &stat.mptcp_fp_heuristic_fallback); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi, + &prev.tcps_mptcp_handover_success_wifi , &stat.mptcp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell, + &prev.tcps_mptcp_handover_success_cell , &stat.mptcp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success, + &prev.tcps_mptcp_interactive_success , &stat.mptcp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success, + &prev.tcps_mptcp_aggregate_success , &stat.mptcp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi, + &prev.tcps_mptcp_fp_handover_success_wifi , &stat.mptcp_fp_handover_success_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell, + &prev.tcps_mptcp_fp_handover_success_cell , &stat.mptcp_fp_handover_success_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success, + &prev.tcps_mptcp_fp_interactive_success , &stat.mptcp_fp_interactive_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success, + &prev.tcps_mptcp_fp_aggregate_success , &stat.mptcp_fp_aggregate_success); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi, + &prev.tcps_mptcp_handover_cell_from_wifi , &stat.mptcp_handover_cell_from_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell, + &prev.tcps_mptcp_handover_wifi_from_cell , &stat.mptcp_handover_wifi_from_cell); + tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi, + &prev.tcps_mptcp_interactive_cell_from_wifi , &stat.mptcp_interactive_cell_from_wifi); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes, + &prev.tcps_mptcp_handover_cell_bytes , &stat.mptcp_handover_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes, + &prev.tcps_mptcp_interactive_cell_bytes , &stat.mptcp_interactive_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes, + &prev.tcps_mptcp_aggregate_cell_bytes , &stat.mptcp_aggregate_cell_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes, + &prev.tcps_mptcp_handover_all_bytes , &stat.mptcp_handover_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes, + &prev.tcps_mptcp_interactive_all_bytes , &stat.mptcp_interactive_all_bytes); + tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes, + &prev.tcps_mptcp_aggregate_all_bytes , &stat.mptcp_aggregate_all_bytes); + tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi, + &prev.tcps_mptcp_back_to_wifi , &stat.mptcp_back_to_wifi); + tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy, + &prev.tcps_mptcp_wifi_proxy , &stat.mptcp_wifi_proxy); + tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy, + &prev.tcps_mptcp_cell_proxy , &stat.mptcp_cell_proxy); + nstat_sysinfo_send_data(&data); @@ -2164,6 +2486,9 @@ tcp_disable_read_probe(struct tcpcb *tp) ((tp->t_flagsext & TF_DETECT_READSTALL) || tp->t_rtimo_probes > 0)) { tcp_keepalive_reset(tp); + + if (tp->t_mpsub) + mptcp_reset_keepalive(tp); } } @@ -2198,21 +2523,21 @@ tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable) continue; /* Acquire lock to look at the state of the connection */ - tcp_lock(inp->inp_socket, 1, 0); + socket_lock(inp->inp_socket, 1); /* Release the want count */ - if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(inp->inp_socket, 1, 0); + if (inp->inp_ppcb == NULL || + (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) { + socket_unlock(inp->inp_socket, 1); continue; } - tp = intotcpcb(inp); if (enable) tcp_enable_read_probe(tp, ifp); else tcp_disable_read_probe(tp); - tcp_unlock(inp->inp_socket, 1, 0); + socket_unlock(inp->inp_socket, 1); } lck_rw_done(pcbinfo->ipi_lock); @@ -2241,6 +2566,73 @@ done: return; } +inline void +tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp) +{ + struct if_cellular_status_v1 *ifsr; + u_int32_t optlen; + ifsr = &ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1; + if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) { + optlen = tp->t_maxopd - tp->t_maxseg; + + if (ifsr->mss_recommended == + IF_CELL_UL_MSS_RECOMMENDED_NONE && + tp->t_cached_maxopd > 0 && + tp->t_maxopd < tp->t_cached_maxopd) { + tp->t_maxopd = tp->t_cached_maxopd; + tcpstat.tcps_mss_to_default++; + } else if (ifsr->mss_recommended == + IF_CELL_UL_MSS_RECOMMENDED_MEDIUM && + tp->t_maxopd > tcp_mss_rec_medium) { + tp->t_cached_maxopd = tp->t_maxopd; + tp->t_maxopd = tcp_mss_rec_medium; + tcpstat.tcps_mss_to_medium++; + } else if (ifsr->mss_recommended == + IF_CELL_UL_MSS_RECOMMENDED_LOW && + tp->t_maxopd > tcp_mss_rec_low) { + tp->t_cached_maxopd = tp->t_maxopd; + tp->t_maxopd = tcp_mss_rec_low; + tcpstat.tcps_mss_to_low++; + } + tp->t_maxseg = tp->t_maxopd - optlen; + + /* + * clear the cached value if it is same as the current + */ + if (tp->t_maxopd == tp->t_cached_maxopd) + tp->t_cached_maxopd = 0; + } +} + +void +tcp_update_mss_locked(struct socket *so, struct ifnet *ifp) +{ + struct inpcb *inp = sotoinpcb(so); + struct tcpcb *tp = intotcpcb(inp); + + if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) + return; + + if (!IFNET_IS_CELLULAR(ifp)) { + /* + * This optimization is implemented for cellular + * networks only + */ + return; + } + if ( tp->t_state <= TCPS_CLOSE_WAIT) { + /* + * If the connection is currently doing or has done PMTU + * blackhole detection, do not change the MSS + */ + if (tp->t_flags & TF_BLACKHOLE) + return; + if (ifp->if_link_status == NULL) + return; + tcp_update_mss_core(tp, ifp); + } +} + void tcp_itimer(struct inpcbinfo *ipi) { @@ -2259,19 +2651,41 @@ tcp_itimer(struct inpcbinfo *ipi) LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) { struct socket *so; + struct ifnet *ifp; - if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) + if (inp->inp_ppcb == NULL || + in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) continue; so = inp->inp_socket; - tcp_lock(so, 1, 0); + ifp = inp->inp_last_outifp; + socket_lock(so, 1); if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) { - tcp_unlock(so, 1, 0); + socket_unlock(so, 1); continue; } so_check_extended_bk_idle_time(so); - tcp_unlock(so, 1, 0); + if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) { + tcp_update_mss_locked(so, NULL); + } + socket_unlock(so, 1); + + /* + * Defunct all system-initiated background sockets if the + * socket is using the cellular interface and the interface + * has its LQM set to abort. + */ + if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) && + IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) && + ifp != NULL && IFNET_IS_CELLULAR(ifp) && + (ifp->if_interface_state.valid_bitmask & + IF_INTERFACE_STATE_LQM_STATE_VALID) && + ifp->if_interface_state.lqm_state == + IFNET_LQM_THRESH_ABORT) { + socket_defunct(current_proc(), so, + SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL); + } } + ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT); lck_rw_done(ipi->ipi_lock); } -