X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d7e50217d7adf6e52786a38bcaa4cd698cb9a79e..143464d58d2bd6378e74eec636961ceb0d32fb91:/bsd/netinet/tcp_timer.c diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index 3d92a0d82..b1ac3138b 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -1,16 +1,19 @@ /* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2014 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ - * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -20,7 +23,7 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995 @@ -67,10 +70,15 @@ #include #include #include - +#include +#include +#include +#include #include /* before tcp_seq.h, for tcp_random18() */ +#include #include +#include #include #include @@ -84,28 +92,61 @@ #include #include #include +#include +#if INET6 +#include +#endif #include #if TCPDEBUG #include #endif #include +#include +#include +extern void postevent(struct socket *, struct sockbuf *, + int); #define DBG_FNC_TCP_FAST NETDBG_CODE(DBG_NETTCP, (5 << 8)) #define DBG_FNC_TCP_SLOW NETDBG_CODE(DBG_NETTCP, (5 << 8) | 1) +#define TIMERENTRY_TO_TP(te) ((struct tcpcb *)((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next))) + +#define VERIFY_NEXT_LINK(elm,field) do { \ + if (LIST_NEXT((elm),field) != NULL && \ + LIST_NEXT((elm),field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while(0) + +#define VERIFY_PREV_LINK(elm,field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while(0) + +/* tcp timer list */ +struct tcptimerlist tcp_timer_list; + +/* List of pcbs in timewait state, protected by tcbinfo's ipi_lock */ +struct tcptailq tcp_tw_tailq; + +static int background_io_trigger = 5; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, background_io_trigger, CTLFLAG_RW | CTLFLAG_LOCKED, + &background_io_trigger, 0, "Background IO Trigger Setting"); + static int sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS { +#pragma unused(arg1, arg2) int error, s, tt; tt = *(int *)oidp->oid_arg1; - s = tt * 1000 / hz; + s = tt * 1000 / TCP_RETRANSHZ;; error = sysctl_handle_int(oidp, &s, 0, req); if (error || !req->newptr) return (error); - tt = s * hz / 1000; + tt = s * TCP_RETRANSHZ / 1000; if (tt < 1) return (EINVAL); @@ -114,217 +155,398 @@ sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS } int tcp_keepinit; -SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT|CTLFLAG_RW, +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", ""); int tcp_keepidle; -SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT|CTLFLAG_RW, +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", ""); int tcp_keepintvl; -SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT|CTLFLAG_RW, +SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", ""); -int tcp_delacktime; -SYSCTL_PROC(_net_inet_tcp, TCPCTL_DELACKTIME, delacktime, - CTLTYPE_INT|CTLFLAG_RW, &tcp_delacktime, 0, sysctl_msec_to_ticks, "I", - "Time before a delayed ACK is sent"); - +int tcp_keepcnt; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_keepcnt, 0, "number of times to repeat keepalive"); + int tcp_msl; -SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT|CTLFLAG_RW, +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); +/* + * Avoid DoS via TCP Robustness in Persist Condition (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt) + * by allowing a system wide maximum persistence timeout value when in Zero Window Probe mode. + * Expressed in milliseconds to be consistent without timeout related values, the TCP socket option is in seconds. + */ +u_int32_t tcp_max_persist_timeout = 0; +SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I", "Maximum persistence timout for ZWP"); + static int always_keepalive = 0; -SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, CTLFLAG_RW, +SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, CTLFLAG_RW | CTLFLAG_LOCKED, &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections"); -static int tcp_keepcnt = TCPTV_KEEPCNT; +/* This parameter determines how long the timer list will stay in fast mode even + * though all connections are idle. In fast mode, the timer will fire more frequently + * anticipating new data. + */ +int timer_fastmode_idlemax = TCP_FASTMODE_IDLEGEN_MAX; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, timer_fastmode_idlemax, CTLFLAG_RW | CTLFLAG_LOCKED, + &timer_fastmode_idlemax, 0, "Maximum idle generations in fast mode"); + +/* + * See tcp_syn_backoff[] for interval values between SYN retransmits; + * the value set below defines the number of retransmits, before we + * disable the timestamp and window scaling options during subsequent + * SYN retransmits. Setting it to 0 disables the dropping off of those + * two options. + */ +static int tcp_broken_peer_syn_rxmit_thres = 7; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rxmit_thres, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_broken_peer_syn_rxmit_thres, 0, "Number of retransmitted SYNs before " + "TCP disables rfc1323 and rfc1644 during the rest of attempts"); + +/* A higher threshold on local connections for disabling RFC 1323 options */ +static int tcp_broken_peer_syn_rxmit_thres_local = 10; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, broken_peer_syn_rexmit_thres_local, + CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_broken_peer_syn_rxmit_thres_local, 0, + "Number of retransmitted SYNs before disabling RFC 1323 options on local connections"); + +static int tcp_timer_advanced = 0; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced, CTLFLAG_RD | CTLFLAG_LOCKED, + &tcp_timer_advanced, 0, "Number of times one of the timers was advanced"); + +static int tcp_resched_timerlist = 0; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist, CTLFLAG_RD | CTLFLAG_LOCKED, + &tcp_resched_timerlist, 0, + "Number of times timer list was rescheduled as part of processing a packet"); + +int tcp_pmtud_black_hole_detect = 1 ; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_detection, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_pmtud_black_hole_detect, 0, "Path MTU Discovery Black Hole Detection"); + +int tcp_pmtud_black_hole_mss = 1200 ; +SYSCTL_INT(_net_inet_tcp, OID_AUTO, pmtud_blackhole_mss, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_pmtud_black_hole_mss, 0, "Path MTU Discovery Black Hole Detection lowered MSS"); + +/* performed garbage collection of "used" sockets */ +static boolean_t tcp_gc_done = FALSE; + /* max idle probes */ int tcp_maxpersistidle; - /* max idle time in persist */ -int tcp_maxidle; -struct inpcbhead time_wait_slots[N_TIME_WAIT_SLOTS]; -int cur_tw_slot = 0; - -u_long *delack_bitmask; +/* TCP delack timer is set to 100 ms. Since the processing of timer list in fast + * mode will happen no faster than 100 ms, the delayed ack timer will fire some where + * between 100 and 200 ms. + */ +int tcp_delack = TCP_RETRANSHZ / 10; +#if MPTCP +/* + * MP_JOIN retransmission of 3rd ACK will be every 500 msecs without backoff + */ +int tcp_jack_rxmt = TCP_RETRANSHZ / 2; +#endif /* MPTCP */ -void add_to_time_wait(tp) - struct tcpcb *tp; -{ - int tw_slot; +/* The frequency of running through the TCP timer list in + * fast and slow mode can be configured. + */ +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, timer_fastquantum, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_timer_list.fast_quantum, TCP_FASTTIMER_QUANTUM, + "Frequency of running timer list in fast mode"); - LIST_REMOVE(tp->t_inpcb, inp_list); +SYSCTL_UINT(_net_inet_tcp, OID_AUTO, timer_slowquantum, CTLFLAG_RW | CTLFLAG_LOCKED, + &tcp_timer_list.slow_quantum, TCP_SLOWTIMER_QUANTUM, + "Frequency of running timer list in slow mode"); - if (tp->t_timer[TCPT_2MSL] == 0) - tp->t_timer[TCPT_2MSL] = 1; +static void tcp_remove_timer(struct tcpcb *tp); +static void tcp_sched_timerlist(uint32_t offset); +static uint32_t tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index); +static void tcp_sched_timers(struct tcpcb *tp); +static inline void tcp_set_lotimer_index(struct tcpcb *); +static void tcp_rexmt_save_state(struct tcpcb *tp); +void tcp_remove_from_time_wait(struct inpcb *inp); - tp->t_rcvtime += tp->t_timer[TCPT_2MSL] & (N_TIME_WAIT_SLOTS - 1); - tw_slot = (tp->t_timer[TCPT_2MSL] & (N_TIME_WAIT_SLOTS - 1)) + cur_tw_slot; - if (tw_slot >= N_TIME_WAIT_SLOTS) - tw_slot -= N_TIME_WAIT_SLOTS; +/* Macro to compare two timers. If there is a reset of the sign bit, it is + * safe to assume that the timer has wrapped around. By doing signed comparision, + * we take care of wrap around such that the value with the sign bit reset is + * actually ahead of the other. + */ - LIST_INSERT_HEAD(&time_wait_slots[tw_slot], tp->t_inpcb, inp_list); -} +static inline int32_t +timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2) { + return (int32_t)((t1 + toff1) - (t2 + toff2)); +}; +/* Returns true if the timer is on the timer list */ +#define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST) +/* Run the TCP timerlist atleast once every hour */ +#define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ) +static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay); +void add_to_time_wait(struct tcpcb *tp, uint32_t delay) ; +static boolean_t tcp_garbage_collect(struct inpcb *, int); /* - * Fast timeout routine for processing delayed acks + * Add to tcp timewait list, delay is given in milliseconds. */ +static void +add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay) +{ + struct inpcbinfo *pcbinfo = &tcbinfo; + struct inpcb *inp = tp->t_inpcb; + uint32_t timer; + + /* pcb list should be locked when we get here */ + lck_rw_assert(pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE); + + /* We may get here multiple times, so check */ + if (!(inp->inp_flags2 & INP2_TIMEWAIT)) { + pcbinfo->ipi_twcount++; + inp->inp_flags2 |= INP2_TIMEWAIT; + + /* Remove from global inp list */ + LIST_REMOVE(inp, inp_list); + } else { + TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry); + } + + /* Compute the time at which this socket can be closed */ + timer = tcp_now + delay; + + /* We will use the TCPT_2MSL timer for tracking this delay */ + + if (TIMER_IS_ON_LIST(tp)) + tcp_remove_timer(tp); + tp->t_timer[TCPT_2MSL] = timer; + + TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry); +} + void -tcp_fasttimo() +add_to_time_wait(struct tcpcb *tp, uint32_t delay) { - register struct inpcb *inp; - register struct tcpcb *tp; + struct inpcbinfo *pcbinfo = &tcbinfo; + if (!lck_rw_try_lock_exclusive(pcbinfo->ipi_lock)) { + tcp_unlock(tp->t_inpcb->inp_socket, 0, 0); + lck_rw_lock_exclusive(pcbinfo->ipi_lock); + tcp_lock(tp->t_inpcb->inp_socket, 0, 0); + } + add_to_time_wait_locked(tp, delay); + lck_rw_done(pcbinfo->ipi_lock); - register u_long i,j; - register u_long temp_mask; - register u_long elem_base = 0; - struct inpcbhead *head; - int s = splnet(); + inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY); +} - static - int delack_checked = 0; +/* If this is on time wait queue, remove it. */ +void +tcp_remove_from_time_wait(struct inpcb *inp) +{ + struct tcpcb *tp = intotcpcb(inp); + if (inp->inp_flags2 & INP2_TIMEWAIT) + TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry); +} - KERNEL_DEBUG(DBG_FNC_TCP_FAST | DBG_FUNC_START, 0,0,0,0,0); +static boolean_t +tcp_garbage_collect(struct inpcb *inp, int istimewait) +{ + boolean_t active = FALSE; + struct socket *so; + struct tcpcb *tp; - if (!tcp_delack_enabled) - return; + so = inp->inp_socket; + tp = intotcpcb(inp); - for (i=0; i < (tcbinfo.hashsize / 32); i++) { - if (delack_bitmask[i]) { - temp_mask = 1; - for (j=0; j < 32; j++) { - if (temp_mask & delack_bitmask[i]) { - head = &tcbinfo.hashbase[elem_base + j]; - for (inp=head->lh_first; inp != 0; inp = inp->inp_hash.le_next) { - delack_checked++; - if ((tp = (struct tcpcb *)inp->inp_ppcb) && (tp->t_flags & TF_DELACK)) { - tp->t_flags &= ~TF_DELACK; - tp->t_flags |= TF_ACKNOW; - tcpstat.tcps_delack++; - (void) tcp_output(tp); - } - } - } - temp_mask <<= 1; + /* + * Skip if still in use or busy; it would have been more efficient + * if we were to test so_usecount against 0, but this isn't possible + * due to the current implementation of tcp_dropdropablreq() where + * overflow sockets that are eligible for garbage collection have + * their usecounts set to 1. + */ + if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) + return (TRUE); + + /* Check again under the lock */ + if (so->so_usecount > 1) { + if (inp->inp_wantcnt == WNT_STOPUSING) + active = TRUE; + lck_mtx_unlock(&inp->inpcb_mtx); + return (active); + } + + if (istimewait && + TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) && + tp->t_state != TCPS_CLOSED) { + /* Become a regular mutex */ + lck_mtx_convert_spin(&inp->inpcb_mtx); + tcp_close(tp); + } + + /* + * Overflowed socket dropped from the listening queue? Do this + * only if we are called to clean up the time wait slots, since + * tcp_dropdropablreq() considers a socket to have been fully + * dropped after add_to_time_wait() is finished. + * Also handle the case of connections getting closed by the peer + * while in the queue as seen with rdar://6422317 + * + */ + if (so->so_usecount == 1 && + ((istimewait && (so->so_flags & SOF_OVERFLOW)) || + ((tp != NULL) && (tp->t_state == TCPS_CLOSED) && + (so->so_head != NULL) && + ((so->so_state & (SS_INCOMP|SS_CANTSENDMORE|SS_CANTRCVMORE)) == + (SS_INCOMP|SS_CANTSENDMORE|SS_CANTRCVMORE))))) { + + if (inp->inp_state != INPCB_STATE_DEAD) { + /* Become a regular mutex */ + lck_mtx_convert_spin(&inp->inpcb_mtx); +#if INET6 + if (SOCK_CHECK_DOM(so, PF_INET6)) + in6_pcbdetach(inp); + else +#endif /* INET6 */ + in_pcbdetach(inp); } - delack_bitmask[i] = 0; - } - elem_base += 32; - } - KERNEL_DEBUG(DBG_FNC_TCP_FAST | DBG_FUNC_END, delack_checked,tcpstat.tcps_delack,0,0,0); - splx(s); + so->so_usecount--; + if (inp->inp_wantcnt == WNT_STOPUSING) + active = TRUE; + lck_mtx_unlock(&inp->inpcb_mtx); + return (active); + } else if (inp->inp_wantcnt != WNT_STOPUSING) { + lck_mtx_unlock(&inp->inpcb_mtx); + return (FALSE); + } + /* + * We get here because the PCB is no longer searchable + * (WNT_STOPUSING); detach (if needed) and dispose if it is dead + * (usecount is 0). This covers all cases, including overflow + * sockets and those that are considered as "embryonic", + * i.e. created by sonewconn() in TCP input path, and have + * not yet been committed. For the former, we reduce the usecount + * to 0 as done by the code above. For the latter, the usecount + * would have reduced to 0 as part calling soabort() when the + * socket is dropped at the end of tcp_input(). + */ + if (so->so_usecount == 0) { + DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp, + struct tcpcb *, tp, int32_t, TCPS_CLOSED); + /* Become a regular mutex */ + lck_mtx_convert_spin(&inp->inpcb_mtx); + + /* + * If this tp still happens to be on the timer list, + * take it out + */ + if (TIMER_IS_ON_LIST(tp)) { + tcp_remove_timer(tp); + } + + if (inp->inp_state != INPCB_STATE_DEAD) { +#if INET6 + if (SOCK_CHECK_DOM(so, PF_INET6)) + in6_pcbdetach(inp); + else +#endif /* INET6 */ + in_pcbdetach(inp); + } + in_pcbdispose(inp); + return (FALSE); + } + + lck_mtx_unlock(&inp->inpcb_mtx); + return (TRUE); } /* - * Tcp protocol timeout routine called every 500 ms. - * Updates the timers in all active tcb's and - * causes finite state machine actions if timers expire. + * TCP garbage collector callback (inpcb_timer_func_t). + * + * Returns the number of pcbs that will need to be gc-ed soon, + * returnining > 0 will keep timer active. */ void -tcp_slowtimo() +tcp_gc(struct inpcbinfo *ipi) { - register struct inpcb *ip, *ipnxt; - register struct tcpcb *tp; - register int i; - int s; + struct inpcb *inp, *nxt; + struct tcpcb *tw_tp, *tw_ntp; #if TCPDEBUG int ostate; #endif -#if KDEBUG - static int tws_checked; +#if KDEBUG + static int tws_checked = 0; #endif - KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0,0,0,0,0); - s = splnet(); + KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0); - tcp_maxidle = tcp_keepcnt * tcp_keepintvl; + /* + * Update tcp_now here as it may get used while + * processing the slow timer. + */ + calculate_tcp_clock(); - ip = tcb.lh_first; - if (ip == NULL) { - splx(s); - return; - } /* - * Search through tcb's and update active timers. + * Garbage collect socket/tcpcb: We need to acquire the list lock + * exclusively to do this */ - for (; ip != NULL; ip = ipnxt) { - ipnxt = ip->inp_list.le_next; - tp = intotcpcb(ip); - if (tp == 0 || tp->t_state == TCPS_LISTEN) - continue; + + if (lck_rw_try_lock_exclusive(ipi->ipi_lock) == FALSE) { + /* don't sweat it this time; cleanup was done last time */ + if (tcp_gc_done == TRUE) { + tcp_gc_done = FALSE; + KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, + tws_checked, cur_tw_slot, 0, 0, 0); + /* Lock upgrade failed, give up this round */ + atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); + return; + } + /* Upgrade failed, lost lock now take it again exclusive */ + lck_rw_lock_exclusive(ipi->ipi_lock); + } + tcp_gc_done = TRUE; + + LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) { + if (tcp_garbage_collect(inp, 0)) + atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1); + } + + /* Now cleanup the time wait ones */ + TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) { /* - * Bogus state when port owned by SharedIP with loopback as the - * only configured interface: BlueBox does not filters loopback + * We check the timestamp here without holding the + * socket lock for better performance. If there are + * any pcbs in time-wait, the timer will get rescheduled. + * Hence some error in this check can be tolerated. + * + * Sometimes a socket on time-wait queue can be closed if + * 2MSL timer expired but the application still has a + * usecount on it. */ - if (tp->t_state == TCP_NSTATES) - continue; - - for (i = 0; i < TCPT_NTIMERS; i++) { - if (tp->t_timer[i] && --tp->t_timer[i] == 0) { -#if TCPDEBUG - ostate = tp->t_state; -#endif - tp = tcp_timers(tp, i); - if (tp == NULL) - goto tpgone; -#if TCPDEBUG - if (tp->t_inpcb->inp_socket->so_options - & SO_DEBUG) - tcp_trace(TA_USER, ostate, tp, - (void *)0, - (struct tcphdr *)0, - PRU_SLOWTIMO); -#endif - } + if (tw_tp->t_state == TCPS_CLOSED || + TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) { + if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) + atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1); } - tp->t_rcvtime++; - tp->t_starttime++; - if (tp->t_rtttime) - tp->t_rtttime++; -tpgone: - ; } -#if KDEBUG - tws_checked = 0; -#endif - KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_NONE, tws_checked,0,0,0,0); + /* take into account pcbs that are still in time_wait_slots */ + atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount); - /* - * Process the items in the current time-wait slot - */ + lck_rw_done(ipi->ipi_lock); - for (ip = time_wait_slots[cur_tw_slot].lh_first; ip; ip = ipnxt) - { -#if KDEBUG - tws_checked++; -#endif - ipnxt = ip->inp_list.le_next; - tp = intotcpcb(ip); - if (tp->t_timer[TCPT_2MSL] >= N_TIME_WAIT_SLOTS) { - tp->t_timer[TCPT_2MSL] -= N_TIME_WAIT_SLOTS; - tp->t_rcvtime += N_TIME_WAIT_SLOTS; - } - else - tp->t_timer[TCPT_2MSL] = 0; + /* Clean up the socache while we are here */ + if (so_cache_timer()) + atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1); - if (tp->t_timer[TCPT_2MSL] == 0) - tp = tcp_timers(tp, TCPT_2MSL); - } + KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked, + cur_tw_slot, 0, 0, 0); - if (++cur_tw_slot >= N_TIME_WAIT_SLOTS) - cur_tw_slot = 0; - tcp_now++; /* for timestamps */ - splx(s); - KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked, cur_tw_slot,0,0,0); + return; } /* @@ -336,8 +558,11 @@ tcp_canceltimers(tp) { register int i; + tcp_remove_timer(tp); for (i = 0; i < TCPT_NTIMERS; i++) tp->t_timer[i] = 0; + tp->tentry.timer_start = tcp_now; + tp->tentry.index = TCPT_NONE; } int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] = @@ -348,6 +573,41 @@ int tcp_backoff[TCP_MAXRXTSHIFT + 1] = static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */ +static void tcp_rexmt_save_state(struct tcpcb *tp) +{ + u_int32_t fsize; + if (TSTMP_SUPPORTED(tp)) { + /* + * Since timestamps are supported on the connection, + * we can do recovery as described in rfc 4015. + */ + fsize = tp->snd_max - tp->snd_una; + tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh); + tp->snd_recover_prev = tp->snd_recover; + } else { + /* + * Timestamp option is not supported on this connection. + * Record ssthresh and cwnd so they can + * be recovered if this turns out to be a "bad" retransmit. + * A retransmit is considered "bad" if an ACK for this + * segment is received within RTT/2 interval; the assumption + * here is that the ACK was already in flight. See + * "On Estimating End-to-End Network Path Properties" by + * Allman and Paxson for more details. + */ + tp->snd_cwnd_prev = tp->snd_cwnd; + tp->snd_ssthresh_prev = tp->snd_ssthresh; + tp->snd_recover_prev = tp->snd_recover; + if (IN_FASTRECOVERY(tp)) + tp->t_flags |= TF_WASFRECOVERY; + else + tp->t_flags &= ~TF_WASFRECOVERY; + } + tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2; + tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT); + tp->t_flagsext &= ~(TF_RECOMPUTE_RTT); +} + /* * TCP timer processing. */ @@ -357,30 +617,43 @@ tcp_timers(tp, timer) int timer; { register int rexmt; - struct socket *so_tmp; + struct socket *so; struct tcptemp *t_template; + int optlen = 0; + int idle_time = 0; + +#if TCPDEBUG + int ostate; +#endif #if INET6 int isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV4) == 0; #endif /* INET6 */ + so = tp->t_inpcb->inp_socket; + idle_time = tcp_now - tp->t_rcvtime; switch (timer) { /* * 2 MSL timeout in shutdown went off. If we're closed but * still waiting for peer to close and connection has been idle - * too long, or if 2MSL time is up from TIME_WAIT, delete connection - * control block. Otherwise, check again in a bit. + * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2, + * delete connection control block. + * Otherwise, (this case shouldn't happen) check again in a bit + * we keep the socket in the main list in that case. */ case TCPT_2MSL: + tcp_free_sackholes(tp); if (tp->t_state != TCPS_TIME_WAIT && - tp->t_rcvtime <= tcp_maxidle) { - tp->t_timer[TCPT_2MSL] = tcp_keepintvl; - add_to_time_wait(tp); - } - else + tp->t_state != TCPS_FIN_WAIT_2 && + ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) { + tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp, + (u_int32_t)TCP_CONN_KEEPINTVL(tp)); + } else { tp = tcp_close(tp); + return(tp); + } break; /* @@ -389,48 +662,159 @@ tcp_timers(tp, timer) * to a longer retransmit interval and retransmit one segment. */ case TCPT_REXMT: - if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) { + /* Drop a connection in the retransmit timer + * 1. If we have retransmitted more than TCP_MAXRXTSHIFT times + * 2. If the time spent in this retransmission episode is more than + * the time limit set with TCP_RXT_CONNDROPTIME socket option + * 3. If TCP_RXT_FINDROP socket option was set and we have already + * retransmitted the FIN 3 times without receiving an ack + */ + if (++tp->t_rxtshift > TCP_MAXRXTSHIFT || + (tp->t_rxt_conndroptime > 0 && tp->t_rxtstart > 0 && + (tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime) || + ((tp->t_flagsext & TF_RXTFINDROP) != 0 && + (tp->t_flags & TF_SENTFIN) != 0 && + tp->t_rxtshift >= 4)) { + + if ((tp->t_flagsext & TF_RXTFINDROP) != 0) { + tcpstat.tcps_rxtfindrop++; + } else { + tcpstat.tcps_timeoutdrop++; + } tp->t_rxtshift = TCP_MAXRXTSHIFT; - tcpstat.tcps_timeoutdrop++; - so_tmp = tp->t_inpcb->inp_socket; + postevent(so, 0, EV_TIMEOUT); + soevent(so, + (SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, tp->t_softerror ? tp->t_softerror : ETIMEDOUT); - postevent(so_tmp, 0, EV_TIMEOUT); + break; } - if (tp->t_rxtshift == 1) { - /* - * first retransmit; record ssthresh and cwnd so they can - * be recovered if this turns out to be a "bad" retransmit. - * A retransmit is considered "bad" if an ACK for this - * segment is received within RTT/2 interval; the assumption - * here is that the ACK was already in flight. See - * "On Estimating End-to-End Network Path Properties" by - * Allman and Paxson for more details. - */ - tp->snd_cwnd_prev = tp->snd_cwnd; - tp->snd_ssthresh_prev = tp->snd_ssthresh; - tp->t_badrxtwin = tcp_now + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); - } tcpstat.tcps_rexmttimeo++; - if (tp->t_state == TCPS_SYN_SENT) + + if (tp->t_rxtshift == 1 && + tp->t_state == TCPS_ESTABLISHED) { + /* Set the time at which retransmission started. */ + tp->t_rxtstart = tcp_now; + + /* + * if this is the first retransmit timeout, save + * the state so that we can recover if the timeout + * is spurious. + */ + tcp_rexmt_save_state(tp); + } +#if MPTCP + if ((tp->t_rxtshift == mptcp_fail_thresh) && + (tp->t_state == TCPS_ESTABLISHED) && + (tp->t_mpflags & TMPF_MPTCP_TRUE)) { + mptcp_act_on_txfail(so); + + } +#endif /* MPTCP */ + + if (tp->t_adaptive_wtimo > 0 && + tp->t_rxtshift > tp->t_adaptive_wtimo && + TCPS_HAVEESTABLISHED(tp->t_state)) { + /* Send an event to the application */ + soevent(so, + (SO_FILT_HINT_LOCKED| + SO_FILT_HINT_ADAPTIVE_WTIMO)); + } + + if (tp->t_state == TCPS_SYN_SENT) { rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift]; + tp->t_stat.synrxtshift = tp->t_rxtshift; + } else rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; TCPT_RANGESET(tp->t_rxtcur, rexmt, - tp->t_rttmin, TCPTV_REXMTMAX); - tp->t_timer[TCPT_REXMT] = tp->t_rxtcur; + tp->t_rttmin, TCPTV_REXMTMAX, + TCP_ADD_REXMTSLOP(tp)); + tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur); + + if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb)) + goto fc_output; + tcp_free_sackholes(tp); /* - * Disable rfc1323 and rfc1644 if we havn't got any response to - * our third SYN to work-around some broken terminal servers - * (most of which have hopefully been retired) that have bad VJ - * header compression code which trashes TCP segments containing - * unknown-to-them TCP options. + * Check for potential Path MTU Discovery Black Hole */ - if ((tp->t_state == TCPS_SYN_SENT) && (tp->t_rxtshift == 3)) - tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_REQ_CC); + + if (tcp_pmtud_black_hole_detect && (tp->t_state == TCPS_ESTABLISHED)) { + if (((tp->t_flags & (TF_PMTUD|TF_MAXSEGSNT)) == (TF_PMTUD|TF_MAXSEGSNT)) && + (tp->t_rxtshift == 2)) { + /* + * Enter Path MTU Black-hole Detection mechanism: + * - Disable Path MTU Discovery (IP "DF" bit). + * - Reduce MTU to lower value than what we negociated with peer. + */ + /* Disable Path MTU Discovery for now */ + tp->t_flags &= ~TF_PMTUD; + /* Record that we may have found a black hole */ + tp->t_flags |= TF_BLACKHOLE; + optlen = tp->t_maxopd - tp->t_maxseg; + /* Keep track of previous MSS */ + tp->t_pmtud_saved_maxopd = tp->t_maxopd; + /* Reduce the MSS to intermediary value */ + if (tp->t_maxopd > tcp_pmtud_black_hole_mss) { + tp->t_maxopd = tcp_pmtud_black_hole_mss; + } else { + tp->t_maxopd = /* use the default MSS */ +#if INET6 + isipv6 ? tcp_v6mssdflt : +#endif /* INET6 */ + tcp_mssdflt; + } + tp->t_maxseg = tp->t_maxopd - optlen; + + /* + * Reset the slow-start flight size + * as it may depend on the new MSS + */ + if (CC_ALGO(tp)->cwnd_init != NULL) + CC_ALGO(tp)->cwnd_init(tp); + } + /* + * If further retransmissions are still unsuccessful with a lowered MTU, + * maybe this isn't a Black Hole and we restore the previous MSS and + * blackhole detection flags. + */ + else { + + if ((tp->t_flags & TF_BLACKHOLE) && (tp->t_rxtshift > 4)) { + tp->t_flags |= TF_PMTUD; + tp->t_flags &= ~TF_BLACKHOLE; + optlen = tp->t_maxopd - tp->t_maxseg; + tp->t_maxopd = tp->t_pmtud_saved_maxopd; + tp->t_maxseg = tp->t_maxopd - optlen; + /* + * Reset the slow-start flight size as it + * may depend on the new MSS + */ + if (CC_ALGO(tp)->cwnd_init != NULL) + CC_ALGO(tp)->cwnd_init(tp); + } + } + } + + + /* + * Disable rfc1323 and rfc1644 if we haven't got any response to + * our SYN (after we reach the threshold) to work-around some + * broken terminal servers (most of which have hopefully been + * retired) that have bad VJ header compression code which + * trashes TCP segments containing unknown-to-them TCP options. + * Do this only on non-local connections. + */ + if (tp->t_state == TCPS_SYN_SENT && + ((!(tp->t_flags & TF_LOCAL) && + tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) || + ((tp->t_flags & TF_LOCAL) && + tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres_local))) + tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_REQ_CC); + /* * If losing, let the lower level know and try for * a better route. Also, if we backed off this far, @@ -463,38 +847,30 @@ tcp_timers(tp, timer) * If timing a segment in this window, stop the timer. */ tp->t_rtttime = 0; - /* - * Close the congestion window down to one segment - * (we'll open it by one segment for each ack we get). - * Since we probably have a window's worth of unacked - * data accumulated, this "slow start" keeps us from - * dumping all that data as back-to-back packets (which - * might overwhelm an intermediate gateway). - * - * There are two phases to the opening: Initially we - * open by one mss on each ack. This makes the window - * size increase exponentially with time. If the - * window is larger than the path can handle, this - * exponential growth results in dropped packet(s) - * almost immediately. To get more time between - * drops but still "push" the network to take advantage - * of improving conditions, we switch from exponential - * to linear window opening at some threshhold size. - * For a threshhold, we use half the current window - * size, truncated to a multiple of the mss. - * - * (the minimum cwnd that will give us exponential - * growth is 2 mss. We don't allow the threshhold - * to go below this.) + + EXIT_FASTRECOVERY(tp); + + /* RFC 5681 says: when a TCP sender detects segment loss + * using retransmit timer and the given segment has already + * been retransmitted by way of the retransmission timer at + * least once, the value of ssthresh is held constant + */ + if (tp->t_rxtshift == 1 && + CC_ALGO(tp)->after_timeout != NULL) + CC_ALGO(tp)->after_timeout(tp); + + + /* CWR notifications are to be sent on new data right after + * RTOs, Fast Retransmits and ECE notification receipts. */ - { - u_int win = min(tp->snd_wnd, tp->snd_cwnd) / 2 / tp->t_maxseg; - if (win < 2) - win = 2; - tp->snd_cwnd = tp->t_maxseg; - tp->snd_ssthresh = win * tp->t_maxseg; - tp->t_dupacks = 0; + if ((tp->ecn_flags & TE_ECN_ON) == TE_ECN_ON) { + tp->ecn_flags |= TE_SENDCWR; } +fc_output: + DTRACE_TCP5(cc, void, NULL, struct inpcb *, tp->t_inpcb, + struct tcpcb *, tp, struct tcphdr *, NULL, + int32_t, TCP_CC_REXMT_TIMEOUT); + (void) tcp_output(tp); break; @@ -510,14 +886,21 @@ tcp_timers(tp, timer) * backoff, drop the connection if the idle time * (no responses to probes) reaches the maximum * backoff that we would use if retransmitting. + * + * Drop the connection if we reached the maximum allowed time for + * Zero Window Probes without a non-zero update from the peer. + * See rdar://5805356 */ - if (tp->t_rxtshift == TCP_MAXRXTSHIFT && - (tp->t_rcvtime >= tcp_maxpersistidle || - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { + if ((tp->t_rxtshift == TCP_MAXRXTSHIFT && + (idle_time >= tcp_maxpersistidle || + idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) || + ((tp->t_persist_stop != 0) && + TSTMP_LEQ(tp->t_persist_stop, tcp_now))) { tcpstat.tcps_persistdrop++; - so_tmp = tp->t_inpcb->inp_socket; + postevent(so, 0, EV_TIMEOUT); + soevent(so, + (SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, ETIMEDOUT); - postevent(so_tmp, 0, EV_TIMEOUT); break; } tcp_setpersist(tp); @@ -532,12 +915,27 @@ tcp_timers(tp, timer) */ case TCPT_KEEP: tcpstat.tcps_keeptimeo++; +#if MPTCP + /* + * Regular TCP connections do not send keepalives after closing + * MPTCP must not also, after sending Data FINs. + */ + struct mptcb *mp_tp = tp->t_mptcb; + if ((tp->t_mpflags & TMPF_MPTCP_TRUE) && + (mp_tp == NULL)) { + goto dropit; + } else if (mp_tp != NULL) { + if ((mptcp_ok_to_keepalive(mp_tp) == 0)) + goto dropit; + } +#endif /* MPTCP */ if (tp->t_state < TCPS_ESTABLISHED) goto dropit; if ((always_keepalive || - tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) && - tp->t_state <= TCPS_CLOSING) { - if (tp->t_rcvtime >= tcp_keepidle + tcp_maxidle) + (tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) || + (tp->t_flagsext & TF_DETECT_READSTALL)) && + (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) { + if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp)) goto dropit; /* * Send a packet designed to force a response @@ -554,15 +952,104 @@ tcp_timers(tp, timer) tcpstat.tcps_keepprobe++; t_template = tcp_maketemplate(tp); if (t_template) { + unsigned int ifscope, nocell = 0; + + if (tp->t_inpcb->inp_flags & INP_BOUND_IF) + ifscope = tp->t_inpcb->inp_boundifp->if_index; + else + ifscope = IFSCOPE_NONE; + + /* + * If the socket isn't allowed to use the + * cellular interface, indicate it as such. + */ + if (tp->t_inpcb->inp_flags & INP_NO_IFT_CELLULAR) + nocell = 1; + tcp_respond(tp, t_template->tt_ipgen, &t_template->tt_t, (struct mbuf *)NULL, - tp->rcv_nxt, tp->snd_una - 1, 0); + tp->rcv_nxt, tp->snd_una - 1, 0, ifscope, + nocell); (void) m_free(dtom(t_template)); + if (tp->t_flagsext & TF_DETECT_READSTALL) + tp->t_rtimo_probes++; + } + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + TCP_CONN_KEEPINTVL(tp)); + } else { + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp, + TCP_CONN_KEEPIDLE(tp)); + } + if (tp->t_flagsext & TF_DETECT_READSTALL) { + /* + * The keep alive packets sent to detect a read + * stall did not get a response from the + * peer. Generate more keep-alives to confirm this. + * If the number of probes sent reaches the limit, + * generate an event. + */ + if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) { + /* Generate an event */ + soevent(so, + (SO_FILT_HINT_LOCKED| + SO_FILT_HINT_ADAPTIVE_RTIMO)); + tcp_keepalive_reset(tp); + } else { + tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START( + tp, TCP_REXMTVAL(tp)); + } + } + break; + case TCPT_DELACK: + if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) { + tp->t_flags &= ~TF_DELACK; + tp->t_timer[TCPT_DELACK] = 0; + tp->t_flags |= TF_ACKNOW; + + /* If delayed ack timer fired while stretching acks + * go back to acking every other packet + */ + if ((tp->t_flags & TF_STRETCHACK) != 0) + tcp_reset_stretch_ack(tp); + + /* If we are measuring inter packet arrival jitter for + * throttling a connection, this delayed ack might be + * the reason for accumulating some jitter. So let's + * restart the measurement. + */ + CLEAR_IAJ_STATE(tp); + + tcpstat.tcps_delack++; + (void) tcp_output(tp); + } + break; + +#if MPTCP + case TCPT_JACK_RXMT: + if ((tp->t_state == TCPS_ESTABLISHED) && + (tp->t_mpflags & TMPF_PREESTABLISHED) && + (tp->t_mpflags & TMPF_JOINED_FLOW)) { + if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) { + tcpstat.tcps_timeoutdrop++; + postevent(so, 0, EV_TIMEOUT); + soevent(so, + (SO_FILT_HINT_LOCKED| + SO_FILT_HINT_TIMEOUT)); + tp = tcp_drop(tp, tp->t_softerror ? + tp->t_softerror : ETIMEDOUT); + break; } - tp->t_timer[TCPT_KEEP] = tcp_keepintvl; - } else - tp->t_timer[TCPT_KEEP] = tcp_keepidle; + tcpstat.tcps_join_rxmts++; + tp->t_flags |= TF_ACKNOW; + + /* + * No backoff is implemented for simplicity for this + * corner case. + */ + (void) tcp_output(tp); + } break; +#endif /* MPTCP */ #if TCPDEBUG if (tp->t_inpcb->inp_socket->so_options & SO_DEBUG) @@ -571,10 +1058,491 @@ tcp_timers(tp, timer) #endif dropit: tcpstat.tcps_keepdrops++; - so_tmp = tp->t_inpcb->inp_socket; + postevent(so, 0, EV_TIMEOUT); + soevent(so, + (SO_FILT_HINT_LOCKED|SO_FILT_HINT_TIMEOUT)); tp = tcp_drop(tp, ETIMEDOUT); - postevent(so_tmp, 0, EV_TIMEOUT); break; } return (tp); } + +/* Remove a timer entry from timer list */ +void +tcp_remove_timer(struct tcpcb *tp) +{ + struct tcptimerlist *listp = &tcp_timer_list; + + lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + if (!(TIMER_IS_ON_LIST(tp))) { + return; + } + lck_mtx_lock(listp->mtx); + + /* Check if pcb is on timer list again after acquiring the lock */ + if (!(TIMER_IS_ON_LIST(tp))) { + lck_mtx_unlock(listp->mtx); + return; + } + + if (listp->next_te != NULL && listp->next_te == &tp->tentry) + listp->next_te = LIST_NEXT(&tp->tentry, le); + + LIST_REMOVE(&tp->tentry, le); + tp->t_flags &= ~(TF_TIMER_ONLIST); + + listp->entries--; + + tp->tentry.le.le_next = NULL; + tp->tentry.le.le_prev = NULL; + lck_mtx_unlock(listp->mtx); +} + +/* Function to check if the timerlist needs to be rescheduled to run + * the timer entry correctly. Basically, this is to check if we can avoid + * taking the list lock. + */ + +static boolean_t +need_to_resched_timerlist(uint32_t runtime, uint16_t index) { + struct tcptimerlist *listp = &tcp_timer_list; + int32_t diff; + boolean_t is_fast; + + if (index == TCPT_NONE) + return FALSE; + is_fast = !(IS_TIMER_SLOW(index)); + + /* If the list is being processed then the state of the list is in flux. + * In this case always acquire the lock and set the state correctly. + */ + if (listp->running) + return TRUE; + + if (!listp->scheduled) + return (TRUE); + + diff = timer_diff(listp->runtime, 0, runtime, 0); + if (diff <= 0) { + /* The list is going to run before this timer */ + return FALSE; + } else { + if (is_fast) { + if (diff <= listp->fast_quantum) + return FALSE; + } else { + if (diff <= listp->slow_quantum) + return FALSE; + } + } + return TRUE; +} + +void +tcp_sched_timerlist(uint32_t offset) +{ + + uint64_t deadline = 0; + struct tcptimerlist *listp = &tcp_timer_list; + + lck_mtx_assert(listp->mtx, LCK_MTX_ASSERT_OWNED); + + offset = min(offset, TCP_TIMERLIST_MAX_OFFSET); + listp->runtime = tcp_now + offset; + if (listp->runtime == 0) + listp->runtime++; + + clock_interval_to_deadline(offset, NSEC_PER_SEC / TCP_RETRANSHZ, + &deadline); + + thread_call_enter_delayed(listp->call, deadline); + listp->scheduled = TRUE; +} + +/* Function to run the timers for a connection. + * + * Returns the offset of next timer to be run for this connection which + * can be used to reschedule the timerlist. + */ +uint32_t +tcp_run_conn_timer(struct tcpcb *tp, uint16_t *next_index) { + + struct socket *so; + uint16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE; + uint32_t timer_val, offset = 0, lo_timer = 0; + int32_t diff; + boolean_t needtorun[TCPT_NTIMERS]; + int count = 0; + + VERIFY(tp != NULL); + bzero(needtorun, sizeof(needtorun)); + + tcp_lock(tp->t_inpcb->inp_socket, 1, 0); + + so = tp->t_inpcb->inp_socket; + /* Release the want count on inp */ + if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1) == WNT_STOPUSING) { + if (TIMER_IS_ON_LIST(tp)) { + tcp_remove_timer(tp); + } + + /* Looks like the TCP connection got closed while we + * were waiting for the lock.. Done + */ + goto done; + } + + /* Since the timer thread needs to wait for tcp lock, it may race + * with another thread that can cancel or reschedule the timer that is + * about to run. Check if we need to run anything. + */ + if ((index = tp->tentry.index) == TCPT_NONE) + goto done; + timer_val = tp->t_timer[index]; + + diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0); + if (diff > 0) { + if (tp->tentry.index != TCPT_NONE) { + offset = diff; + *(next_index) = tp->tentry.index; + } + goto done; + } + + tp->t_timer[index] = 0; + if (timer_val > 0) { + tp = tcp_timers(tp, index); + if (tp == NULL) + goto done; + } + + /* Check if there are any other timers that need to be run. While doing it, + * adjust the timer values wrt tcp_now. + */ + for (i = 0; i < TCPT_NTIMERS; ++i) { + if (tp->t_timer[i] != 0) { + diff = timer_diff(tp->tentry.timer_start, tp->t_timer[i], tcp_now, 0); + if (diff <= 0) { + tp->t_timer[i] = 0; + needtorun[i] = TRUE; + count++; + } else { + tp->t_timer[i] = diff; + needtorun[i] = FALSE; + if (lo_timer == 0 || diff < lo_timer) { + lo_timer = diff; + lo_index = i; + } + } + } + } + + tp->tentry.timer_start = tcp_now; + tp->tentry.index = lo_index; + if (lo_index != TCPT_NONE) { + tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index]; + if (tp->tentry.runtime == 0) + tp->tentry.runtime++; + } + + if (count > 0) { + /* run any other timers that are also outstanding at this time. */ + for (i = 0; i < TCPT_NTIMERS; ++i) { + if (needtorun[i]) { + tp->t_timer[i] = 0; + tp = tcp_timers(tp, i); + if (tp == NULL) { + offset = 0; + *(next_index) = TCPT_NONE; + goto done; + } + } + } + tcp_set_lotimer_index(tp); + } + + if (tp->tentry.index < TCPT_NONE) { + offset = tp->t_timer[tp->tentry.index]; + *(next_index) = tp->tentry.index; + } + +done: + if (tp != NULL && tp->tentry.index == TCPT_NONE) { + tcp_remove_timer(tp); + offset = 0; + } + tcp_unlock(so, 1, 0); + return offset; +} + +void +tcp_run_timerlist(void * arg1, void * arg2) { + +#pragma unused(arg1, arg2) + + struct tcptimerentry *te, *next_te; + struct tcptimerlist *listp = &tcp_timer_list; + struct tcpcb *tp; + uint32_t next_timer = 0; + uint16_t index = TCPT_NONE; + boolean_t need_fast = FALSE; + uint32_t active_count = 0; + uint32_t mode = TCP_TIMERLIST_FASTMODE; + + calculate_tcp_clock(); + + lck_mtx_lock(listp->mtx); + + listp->running = TRUE; + + LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) { + uint32_t offset = 0; + uint32_t runtime = te->runtime; + if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now)) { + offset = timer_diff(runtime, 0, tcp_now, 0); + if (next_timer == 0 || offset < next_timer) { + next_timer = offset; + } + continue; + } + active_count++; + + tp = TIMERENTRY_TO_TP(te); + + /* Acquire an inp wantcnt on the inpcb so that the socket won't get + * detached even if tcp_close is called + */ + if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0) == WNT_STOPUSING) { + /* Some how this pcb went into dead state while on the timer list, + * just take it off the list. Since the timer list entry pointers + * are protected by the timer list lock, we can do it here + */ + if (TIMER_IS_ON_LIST(tp)) { + tp->t_flags &= ~(TF_TIMER_ONLIST); + LIST_REMOVE(&tp->tentry, le); + listp->entries--; + + tp->tentry.le.le_next = NULL; + tp->tentry.le.le_prev = NULL; + } + continue; + } + + /* Store the next timerentry pointer before releasing the list lock. + * If that entry has to be removed when we release the lock, this + * pointer will be updated to the element after that. + */ + listp->next_te = next_te; + + VERIFY_NEXT_LINK(&tp->tentry, le); + VERIFY_PREV_LINK(&tp->tentry, le); + + lck_mtx_unlock(listp->mtx); + + index = TCPT_NONE; + offset = tcp_run_conn_timer(tp, &index); + + lck_mtx_lock(listp->mtx); + + next_te = listp->next_te; + listp->next_te = NULL; + + if (offset > 0) { + if (index < TCPT_NONE) { + /* Check if this is a fast_timer. */ + if (!need_fast && !(IS_TIMER_SLOW(index))) { + need_fast = TRUE; + } + + if (next_timer == 0 || offset < next_timer) { + next_timer = offset; + } + } + } + } + + if (!LIST_EMPTY(&listp->lhead)) { + if (listp->mode == TCP_TIMERLIST_FASTMODE) { + if (need_fast || active_count > 0 || + listp->pref_mode == TCP_TIMERLIST_FASTMODE) { + listp->idlegen = 0; + } else { + listp->idlegen++; + if (listp->idlegen > timer_fastmode_idlemax) { + mode = TCP_TIMERLIST_SLOWMODE; + listp->idlegen = 0; + } + } + } else { + if (!need_fast) { + mode = TCP_TIMERLIST_SLOWMODE; + } + } + + if (mode == TCP_TIMERLIST_FASTMODE || + listp->pref_mode == TCP_TIMERLIST_FASTMODE) { + next_timer = listp->fast_quantum; + } else { + if (listp->pref_offset != 0 && + listp->pref_offset < next_timer) + next_timer = listp->pref_offset; + if (next_timer < listp->slow_quantum) + next_timer = listp->slow_quantum; + } + + listp->mode = mode; + + tcp_sched_timerlist(next_timer); + } else { + /* + * No need to reschedule this timer, but always run + * periodically at a much higher granularity. + */ + tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET); + } + + listp->running = FALSE; + listp->pref_mode = 0; + listp->pref_offset = 0; + + lck_mtx_unlock(listp->mtx); +} + +/* Function to verify if a change in timer state is required for a connection */ +void +tcp_sched_timers(struct tcpcb *tp) +{ + struct tcptimerentry *te = &tp->tentry; + uint16_t index = te->index; + struct tcptimerlist *listp = &tcp_timer_list; + int32_t offset = 0; + boolean_t is_fast; + int list_locked = 0; + + if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) { + /* Just return without adding the dead pcb to the list */ + if (TIMER_IS_ON_LIST(tp)) { + tcp_remove_timer(tp); + } + return; + } + + if (index == TCPT_NONE) { + tcp_remove_timer(tp); + return; + } + + is_fast = !(IS_TIMER_SLOW(index)); + offset = timer_diff(te->runtime, 0, tcp_now, 0); + if (offset <= 0) { + offset = 1; + tcp_timer_advanced++; + } + if (is_fast) + offset = listp->fast_quantum; + + if (!TIMER_IS_ON_LIST(tp)) { + if (!list_locked) { + lck_mtx_lock(listp->mtx); + list_locked = 1; + } + + LIST_INSERT_HEAD(&listp->lhead, te, le); + tp->t_flags |= TF_TIMER_ONLIST; + + listp->entries++; + if (listp->entries > listp->maxentries) + listp->maxentries = listp->entries; + + /* if the list is not scheduled, just schedule it */ + if (!listp->scheduled) + goto schedule; + + } + + + /* timer entry is currently on the list */ + if (need_to_resched_timerlist(te->runtime, index)) { + tcp_resched_timerlist++; + + if (!list_locked) { + lck_mtx_lock(listp->mtx); + list_locked = 1; + } + + VERIFY_NEXT_LINK(te, le); + VERIFY_PREV_LINK(te, le); + + if (listp->running) { + if (is_fast) { + listp->pref_mode = TCP_TIMERLIST_FASTMODE; + } else if (listp->pref_offset == 0 || + offset < listp->pref_offset) { + listp->pref_offset = offset; + } + } else { + /* + * The list could have got scheduled while this + * thread was waiting for the lock + */ + if (listp->scheduled) { + int32_t diff; + diff = timer_diff(listp->runtime, 0, + tcp_now, offset); + if (diff <= 0) + goto done; + else + goto schedule; + } else { + goto schedule; + } + } + } + goto done; + +schedule: + if (is_fast) { + listp->mode = TCP_TIMERLIST_FASTMODE; + listp->idlegen = 0; + } + tcp_sched_timerlist(offset); + +done: + if (list_locked) + lck_mtx_unlock(listp->mtx); + + return; +} + +void +tcp_set_lotimer_index(struct tcpcb *tp) { + uint16_t i, lo_index = TCPT_NONE; + uint32_t lo_timer = 0; + for (i = 0; i < TCPT_NTIMERS; ++i) { + if (tp->t_timer[i] != 0 && + (lo_timer == 0 || tp->t_timer[i] < lo_timer)) { + lo_timer = tp->t_timer[i]; + lo_index = i; + } + } + tp->tentry.index = lo_index; + if (lo_index != TCPT_NONE) { + tp->tentry.runtime = tp->tentry.timer_start + tp->t_timer[lo_index]; + if (tp->tentry.runtime == 0) + tp->tentry.runtime++; + } +} + +void +tcp_check_timer_state(struct tcpcb *tp) { + + lck_mtx_assert(&tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED); + + if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) + return; + + tcp_set_lotimer_index(tp); + + tcp_sched_timers(tp); + return; +}