bsd/netinet/tcp_lro.c

   1 /*
   2  * Copyright (c) 2011-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/param.h>
  30 #include <sys/systm.h>
  31 #include <sys/sysctl.h>
  32 #include <sys/mbuf.h>
  33 #include <sys/mcache.h>
  34 #include <sys/socket.h>
  35 #include <sys/socketvar.h>
  36 #include <net/if_types.h>
  37 #include <net/route.h>
  38 #include <netinet/in.h>
  39 #include <netinet/in_systm.h>
  40 #include <net/if.h>
  41 #include <net/dlil.h>
  42 #include <netinet/ip.h>
  43 #include <netinet/ip_var.h>
  44 #include <netinet/in_var.h>
  45 #include <netinet/tcp.h>
  46 #include <netinet/tcp_seq.h>
  47 #include <netinet/tcpip.h>
  48 #include <netinet/tcp_var.h>
  49 #include <netinet/tcp_lro.h>
  50 #include <netinet/lro_ext.h>
  51 #include <kern/locks.h>
  52
  53 unsigned int lrocount = 0; /* A counter used for debugging only */
  54 unsigned int lro_seq_outoforder = 0; /* Counter for debugging */
  55 unsigned int lro_seq_mismatch = 0; /* Counter for debugging */
  56 unsigned int lro_flushes = 0; /* Counter for tracking number of flushes */
  57 unsigned int lro_single_flushes = 0;
  58 unsigned int lro_double_flushes = 0;
  59 unsigned int lro_good_flushes = 0;
  60
  61 unsigned int coalesc_sz = LRO_MX_COALESCE_PKTS;
  62 SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro_sz, CTLFLAG_RW | CTLFLAG_LOCKED,
  63     &coalesc_sz, 0, "Max coalescing size");
  64
  65 unsigned int coalesc_time = LRO_MX_TIME_TO_BUFFER;
  66 SYSCTL_INT(_net_inet_tcp, OID_AUTO, lro_time, CTLFLAG_RW | CTLFLAG_LOCKED,
  67     &coalesc_time, 0, "Max coalescing time");
  68
  69 struct lro_flow lro_flow_list[TCP_LRO_NUM_FLOWS];
  70
  71 char lro_flow_map[TCP_LRO_FLOW_MAP];
  72
  73 static lck_attr_t *tcp_lro_mtx_attr = NULL;             /* mutex attributes */
  74 static lck_grp_t *tcp_lro_mtx_grp = NULL;               /* mutex group */
  75 static lck_grp_attr_t *tcp_lro_mtx_grp_attr = NULL;     /* mutex group attrs */
  76 decl_lck_mtx_data(, tcp_lro_lock);      /* Used to synchronize updates */
  77
  78 unsigned int lro_byte_count = 0;
  79
  80 uint64_t lro_deadline = 0; /* LRO's sense of time - protected by tcp_lro_lock */
  81 uint32_t lro_timer_set = 0;
  82
  83 /* Some LRO stats */
  84 u_int32_t lro_pkt_count = 0; /* Number of packets encountered in an LRO period */
  85 thread_call_t tcp_lro_timer;
  86
  87 extern u_int32_t kipf_count;
  88
  89 static void     tcp_lro_timer_proc(void*, void*);
  90 static void     lro_update_stats(struct mbuf*);
  91 static void     lro_update_flush_stats(struct mbuf *);
  92 static void     tcp_lro_flush_flows(void);
  93 static void     tcp_lro_sched_timer(uint64_t);
  94 static void     lro_proto_input(struct mbuf *);
  95
  96 static struct mbuf *lro_tcp_xsum_validate(struct mbuf*, struct ip *,
  97     struct tcphdr*);
  98 static struct mbuf *tcp_lro_process_pkt(struct mbuf*, int);
  99
 100 void
 101 tcp_lro_init(void)
 102 {
 103         int i;
 104
 105         bzero(lro_flow_list, sizeof(struct lro_flow) * TCP_LRO_NUM_FLOWS);
 106         for (i = 0; i < TCP_LRO_FLOW_MAP; i++) {
 107                 lro_flow_map[i] = TCP_LRO_FLOW_UNINIT;
 108         }
 109
 110         /*
 111          * allocate lock group attribute, group and attribute for tcp_lro_lock
 112          */
 113         tcp_lro_mtx_grp_attr = lck_grp_attr_alloc_init();
 114         tcp_lro_mtx_grp = lck_grp_alloc_init("tcplro", tcp_lro_mtx_grp_attr);
 115         tcp_lro_mtx_attr = lck_attr_alloc_init();
 116         lck_mtx_init(&tcp_lro_lock, tcp_lro_mtx_grp, tcp_lro_mtx_attr);
 117
 118         tcp_lro_timer = thread_call_allocate(tcp_lro_timer_proc, NULL);
 119         if (tcp_lro_timer == NULL) {
 120                 panic_plain("%s: unable to allocate lro timer", __func__);
 121         }
 122
 123         return;
 124 }
 125
 126 static int
 127 tcp_lro_matching_tuple(struct ip* ip_hdr, struct tcphdr *tcp_hdr, int *hash,
 128     int *flow_id )
 129 {
 130         struct lro_flow *flow;
 131         tcp_seq seqnum;
 132         unsigned int off = 0;
 133         int payload_len = 0;
 134
 135         *hash = LRO_HASH(ip_hdr->ip_src.s_addr, ip_hdr->ip_dst.s_addr,
 136             tcp_hdr->th_sport, tcp_hdr->th_dport, (TCP_LRO_FLOW_MAP - 1));
 137
 138         *flow_id = lro_flow_map[*hash];
 139         if (*flow_id == TCP_LRO_FLOW_NOTFOUND) {
 140                 return TCP_LRO_NAN;
 141         }
 142
 143         seqnum = tcp_hdr->th_seq;
 144         off = tcp_hdr->th_off << 2;
 145         payload_len = ip_hdr->ip_len - off;
 146
 147         flow = &lro_flow_list[*flow_id];
 148
 149         if ((flow->lr_faddr.s_addr == ip_hdr->ip_src.s_addr) &&
 150             (flow->lr_laddr.s_addr == ip_hdr->ip_dst.s_addr) &&
 151             (flow->lr_fport == tcp_hdr->th_sport) &&
 152             (flow->lr_lport == tcp_hdr->th_dport)) {
 153                 if (flow->lr_tcphdr == NULL) {
 154                         if (ntohl(seqnum) == flow->lr_seq) {
 155                                 return TCP_LRO_COALESCE;
 156                         }
 157                         if (lrodebug >= 4) {
 158                                 printf("%s: seqnum = %x, lr_seq = %x\n",
 159                                     __func__, ntohl(seqnum), flow->lr_seq);
 160                         }
 161                         lro_seq_mismatch++;
 162                         if (SEQ_GT(ntohl(seqnum), flow->lr_seq)) {
 163                                 lro_seq_outoforder++;
 164                                 /*
 165                                  * Whenever we receive out of order packets it
 166                                  * signals loss and recovery and LRO doesn't
 167                                  * let flows recover quickly. So eject.
 168                                  */
 169                                 flow->lr_flags |= LRO_EJECT_REQ;
 170                         }
 171                         return TCP_LRO_NAN;
 172                 }
 173
 174                 if (flow->lr_flags & LRO_EJECT_REQ) {
 175                         if (lrodebug) {
 176                                 printf("%s: eject. \n", __func__);
 177                         }
 178                         return TCP_LRO_EJECT_FLOW;
 179                 }
 180                 if (SEQ_GT(tcp_hdr->th_ack, flow->lr_tcphdr->th_ack)) {
 181                         if (lrodebug) {
 182                                 printf("%s: th_ack = %x flow_ack = %x \n",
 183                                     __func__, tcp_hdr->th_ack,
 184                                     flow->lr_tcphdr->th_ack);
 185                         }
 186                         return TCP_LRO_EJECT_FLOW;
 187                 }
 188
 189                 if (ntohl(seqnum) == (ntohl(lro_flow_list[*flow_id].lr_tcphdr->th_seq) + lro_flow_list[*flow_id].lr_len)) {
 190                         return TCP_LRO_COALESCE;
 191                 } else {
 192                         /* LRO does not handle loss recovery well, eject */
 193                         flow->lr_flags |= LRO_EJECT_REQ;
 194                         return TCP_LRO_EJECT_FLOW;
 195                 }
 196         }
 197         if (lrodebug) {
 198                 printf("tcp_lro_matching_tuple: collision \n");
 199         }
 200         return TCP_LRO_COLLISION;
 201 }
 202
 203 static void
 204 tcp_lro_init_flow(int flow_id, struct ip* ip_hdr, struct tcphdr *tcp_hdr,
 205     int hash, u_int32_t timestamp, int payload_len)
 206 {
 207         struct lro_flow *flow = NULL;
 208
 209         flow = &lro_flow_list[flow_id];
 210
 211         flow->lr_hash_map = hash;
 212         flow->lr_faddr.s_addr = ip_hdr->ip_src.s_addr;
 213         flow->lr_laddr.s_addr = ip_hdr->ip_dst.s_addr;
 214         flow->lr_fport = tcp_hdr->th_sport;
 215         flow->lr_lport = tcp_hdr->th_dport;
 216         lro_flow_map[hash] = flow_id;
 217         flow->lr_timestamp = timestamp;
 218         flow->lr_seq = ntohl(tcp_hdr->th_seq) + payload_len;
 219         flow->lr_flags = 0;
 220         return;
 221 }
 222
 223 static void
 224 tcp_lro_coalesce(int flow_id, struct mbuf *lro_mb, struct tcphdr *tcphdr,
 225     int payload_len, int drop_hdrlen, struct tcpopt *topt,
 226     u_int32_t* tsval, u_int32_t* tsecr, int thflags)
 227 {
 228         struct lro_flow *flow = NULL;
 229         struct mbuf *last;
 230         struct ip *ip = NULL;
 231
 232         flow =  &lro_flow_list[flow_id];
 233         if (flow->lr_mhead) {
 234                 if (lrodebug) {
 235                         printf("%s: lr_mhead %x %d \n", __func__, flow->lr_seq,
 236                             payload_len);
 237                 }
 238                 m_adj(lro_mb, drop_hdrlen);
 239
 240                 last = flow->lr_mtail;
 241                 while (last->m_next != NULL) {
 242                         last = last->m_next;
 243                 }
 244                 last->m_next = lro_mb;
 245
 246                 flow->lr_mtail = lro_mb;
 247
 248                 ip = mtod(flow->lr_mhead, struct ip *);
 249                 ip->ip_len += lro_mb->m_pkthdr.len;
 250                 flow->lr_mhead->m_pkthdr.len += lro_mb->m_pkthdr.len;
 251
 252                 if (flow->lr_len == 0) {
 253                         panic_plain("%s: Inconsistent LRO flow state", __func__);
 254                 }
 255                 flow->lr_len += payload_len;
 256                 flow->lr_seq += payload_len;
 257                 /*
 258                  * This bit is re-OR'd each time a packet is added to the
 259                  * large coalesced packet.
 260                  */
 261                 flow->lr_mhead->m_pkthdr.pkt_flags |= PKTF_SW_LRO_PKT;
 262                 flow->lr_mhead->m_pkthdr.lro_npkts++; /* for tcpstat.tcps_rcvpack */
 263                 if (flow->lr_mhead->m_pkthdr.lro_pktlen <
 264                     lro_mb->m_pkthdr.lro_pktlen) {
 265                         /*
 266                          * For TCP Inter Arrival Jitter calculation, return max
 267                          * size encountered while coalescing a stream of pkts.
 268                          */
 269                         flow->lr_mhead->m_pkthdr.lro_pktlen =
 270                             lro_mb->m_pkthdr.lro_pktlen;
 271                 }
 272                 /* Update the timestamp value */
 273                 if (topt->to_flags & TOF_TS) {
 274                         if ((flow->lr_tsval) &&
 275                             (TSTMP_GT(topt->to_tsval, ntohl(*(flow->lr_tsval))))) {
 276                                 *(flow->lr_tsval) = htonl(topt->to_tsval);
 277                         }
 278                         if ((flow->lr_tsecr) &&
 279                             (topt->to_tsecr != 0) &&
 280                             (TSTMP_GT(topt->to_tsecr, ntohl(*(flow->lr_tsecr))))) {
 281                                 if (lrodebug >= 2) {
 282                                         printf("%s: instantaneous RTT = %d \n", __func__,
 283                                             topt->to_tsecr - ntohl(*(flow->lr_tsecr)));
 284                                 }
 285                                 *(flow->lr_tsecr) = htonl(topt->to_tsecr);
 286                         }
 287                 }
 288                 /* Coalesce the flags */
 289                 if (thflags) {
 290                         flow->lr_tcphdr->th_flags |= thflags;
 291                 }
 292                 /* Update receive window */
 293                 flow->lr_tcphdr->th_win = tcphdr->th_win;
 294         } else {
 295                 if (lro_mb) {
 296                         flow->lr_mhead = flow->lr_mtail = lro_mb;
 297                         flow->lr_mhead->m_pkthdr.pkt_flags |= PKTF_SW_LRO_PKT;
 298                         flow->lr_tcphdr = tcphdr;
 299                         if ((topt) && (topt->to_flags & TOF_TS)) {
 300                                 ASSERT(tsval != NULL);
 301                                 ASSERT(tsecr != NULL);
 302                                 flow->lr_tsval = tsval;
 303                                 flow->lr_tsecr = tsecr;
 304                         }
 305                         flow->lr_len = payload_len;
 306                         calculate_tcp_clock();
 307                         flow->lr_timestamp = tcp_now;
 308                         tcp_lro_sched_timer(0);
 309                 }
 310                 flow->lr_seq = ntohl(tcphdr->th_seq) + payload_len;
 311         }
 312         if (lro_mb) {
 313                 tcpstat.tcps_coalesced_pack++;
 314         }
 315         return;
 316 }
 317
 318 static struct mbuf *
 319 tcp_lro_eject_flow(int flow_id)
 320 {
 321         struct mbuf *mb = NULL;
 322
 323         mb = lro_flow_list[flow_id].lr_mhead;
 324         ASSERT(lro_flow_map[lro_flow_list[flow_id].lr_hash_map] == flow_id);
 325         lro_flow_map[lro_flow_list[flow_id].lr_hash_map] = TCP_LRO_FLOW_UNINIT;
 326         bzero(&lro_flow_list[flow_id], sizeof(struct lro_flow));
 327
 328         return mb;
 329 }
 330
 331 static struct mbuf*
 332 tcp_lro_eject_coalesced_pkt(int flow_id)
 333 {
 334         struct mbuf *mb = NULL;
 335         mb = lro_flow_list[flow_id].lr_mhead;
 336         lro_flow_list[flow_id].lr_mhead =
 337             lro_flow_list[flow_id].lr_mtail = NULL;
 338         lro_flow_list[flow_id].lr_tcphdr = NULL;
 339         return mb;
 340 }
 341
 342 static struct mbuf*
 343 tcp_lro_insert_flow(struct mbuf *lro_mb, struct ip *ip_hdr,
 344     struct tcphdr *tcp_hdr, int payload_len,
 345     int drop_hdrlen, int hash, struct tcpopt *topt,
 346     u_int32_t *tsval, u_int32_t *tsecr)
 347 {
 348         int i;
 349         int slot_available = 0;
 350         int candidate_flow = 0;
 351         u_int32_t oldest_timestamp;
 352         struct mbuf *mb = NULL;
 353         int collision = 0;
 354
 355         oldest_timestamp = tcp_now;
 356
 357         /* handle collision */
 358         if (lro_flow_map[hash] != TCP_LRO_FLOW_UNINIT) {
 359                 if (lrodebug) {
 360                         collision = 1;
 361                 }
 362                 candidate_flow = lro_flow_map[hash];
 363                 tcpstat.tcps_flowtbl_collision++;
 364                 goto kick_flow;
 365         }
 366
 367         for (i = 0; i < TCP_LRO_NUM_FLOWS; i++) {
 368                 if (lro_flow_list[i].lr_mhead == NULL) {
 369                         candidate_flow = i;
 370                         slot_available = 1;
 371                         break;
 372                 }
 373                 if (oldest_timestamp >= lro_flow_list[i].lr_timestamp) {
 374                         candidate_flow = i;
 375                         oldest_timestamp = lro_flow_list[i].lr_timestamp;
 376                 }
 377         }
 378
 379         if (!slot_available) {
 380                 tcpstat.tcps_flowtbl_full++;
 381 kick_flow:
 382                 /* kick the oldest flow */
 383                 mb = tcp_lro_eject_flow(candidate_flow);
 384
 385                 if (lrodebug) {
 386                         if (!slot_available) {
 387                                 printf("%s: slot unavailable.\n", __func__);
 388                         }
 389                         if (collision) {
 390                                 printf("%s: collision.\n", __func__);
 391                         }
 392                 }
 393         } else {
 394                 candidate_flow = i; /* this is now the flow to be used */
 395         }
 396
 397         tcp_lro_init_flow(candidate_flow, ip_hdr, tcp_hdr, hash,
 398             tcp_now, payload_len);
 399         tcp_lro_coalesce(candidate_flow, lro_mb, tcp_hdr, payload_len,
 400             drop_hdrlen, topt, tsval, tsecr, 0);
 401         return mb;
 402 }
 403
 404 struct mbuf*
 405 tcp_lro_process_pkt(struct mbuf *lro_mb, int drop_hdrlen)
 406 {
 407         int flow_id = TCP_LRO_FLOW_UNINIT;
 408         int hash;
 409         unsigned int off = 0;
 410         int eject_flow = 0;
 411         int optlen;
 412         int retval = 0;
 413         struct mbuf *mb = NULL;
 414         int payload_len = 0;
 415         u_char *optp = NULL;
 416         int thflags = 0;
 417         struct tcpopt to;
 418         int ret_response = TCP_LRO_CONSUMED;
 419         int coalesced = 0, tcpflags = 0, unknown_tcpopts = 0;
 420         u_int8_t ecn;
 421         struct ip *ip_hdr;
 422         struct tcphdr *tcp_hdr;
 423
 424         if (lro_mb->m_len < drop_hdrlen) {
 425                 if ((lro_mb = m_pullup(lro_mb, drop_hdrlen)) == NULL) {
 426                         tcpstat.tcps_rcvshort++;
 427                         m_freem(lro_mb);
 428                         if (lrodebug) {
 429                                 printf("tcp_lro_process_pkt:mbuf too short.\n");
 430                         }
 431                         return NULL;
 432                 }
 433         }
 434
 435         ip_hdr = mtod(lro_mb, struct ip*);
 436         tcp_hdr = (struct tcphdr *)((caddr_t)ip_hdr + sizeof(struct ip));
 437
 438         /* Just in case */
 439         lro_mb->m_pkthdr.pkt_flags &= ~PKTF_SW_LRO_DID_CSUM;
 440
 441         if ((lro_mb = lro_tcp_xsum_validate(lro_mb, ip_hdr, tcp_hdr)) == NULL) {
 442                 if (lrodebug) {
 443                         printf("tcp_lro_process_pkt: TCP xsum failed.\n");
 444                 }
 445                 return NULL;
 446         }
 447
 448         /* Update stats */
 449         lro_pkt_count++;
 450
 451         /* Avoids checksumming in tcp_input */
 452         lro_mb->m_pkthdr.pkt_flags |= PKTF_SW_LRO_DID_CSUM;
 453
 454         off = tcp_hdr->th_off << 2;
 455         optlen = off - sizeof(struct tcphdr);
 456         payload_len = ip_hdr->ip_len - off;
 457         optp = (u_char *)(tcp_hdr + 1);
 458         /*
 459          * Do quick retrieval of timestamp options ("options
 460          * prediction?").  If timestamp is the only option and it's
 461          * formatted as recommended in RFC 1323 appendix A, we
 462          * quickly get the values now and not bother calling
 463          * tcp_dooptions(), etc.
 464          */
 465         bzero(&to, sizeof(to));
 466         if ((optlen == TCPOLEN_TSTAMP_APPA ||
 467             (optlen > TCPOLEN_TSTAMP_APPA &&
 468             optp[TCPOLEN_TSTAMP_APPA] == TCPOPT_EOL)) &&
 469             *(u_int32_t *)optp == htonl(TCPOPT_TSTAMP_HDR) &&
 470             (tcp_hdr->th_flags & TH_SYN) == 0) {
 471                 to.to_flags |= TOF_TS;
 472                 to.to_tsval = ntohl(*(u_int32_t *)(void *)(optp + 4));
 473                 to.to_tsecr = ntohl(*(u_int32_t *)(void *)(optp + 8));
 474         } else {
 475                 /*
 476                  * If TCP timestamps are not in use, or not the first option,
 477                  * skip LRO path since timestamps are used to avoid LRO
 478                  * from introducing additional latencies for retransmissions
 479                  * and other slow-paced transmissions.
 480                  */
 481                 to.to_flags = to.to_tsecr = 0;
 482                 eject_flow = 1;
 483         }
 484
 485         /* list all the conditions that can trigger a flow ejection here */
 486
 487         thflags = tcp_hdr->th_flags;
 488         if (thflags & (TH_SYN | TH_URG | TH_ECE | TH_CWR | TH_PUSH | TH_RST | TH_FIN)) {
 489                 eject_flow = tcpflags = 1;
 490         }
 491
 492         if (optlen && !((optlen == TCPOLEN_TSTAMP_APPA) &&
 493             (to.to_flags & TOF_TS))) {
 494                 eject_flow = unknown_tcpopts = 1;
 495         }
 496
 497         if (payload_len <= LRO_MIN_COALESC_SZ) { /* zero payload ACK */
 498                 eject_flow = 1;
 499         }
 500
 501         /* Can't coalesce ECN marked packets. */
 502         ecn = ip_hdr->ip_tos & IPTOS_ECN_MASK;
 503         if (ecn == IPTOS_ECN_CE) {
 504                 /*
 505                  * ECN needs quick notification
 506                  */
 507                 if (lrodebug) {
 508                         printf("%s: ECE bits set.\n", __func__);
 509                 }
 510                 eject_flow = 1;
 511         }
 512
 513         lck_mtx_lock_spin(&tcp_lro_lock);
 514
 515         retval = tcp_lro_matching_tuple(ip_hdr, tcp_hdr, &hash, &flow_id);
 516
 517         switch (retval) {
 518         case TCP_LRO_NAN:
 519                 lck_mtx_unlock(&tcp_lro_lock);
 520                 ret_response = TCP_LRO_FLOW_NOTFOUND;
 521                 break;
 522
 523         case TCP_LRO_COALESCE:
 524                 if ((payload_len != 0) && (unknown_tcpopts == 0) &&
 525                     (tcpflags == 0) && (ecn != IPTOS_ECN_CE) && (to.to_flags & TOF_TS)) {
 526                         tcp_lro_coalesce(flow_id, lro_mb, tcp_hdr, payload_len,
 527                             drop_hdrlen, &to,
 528                             (to.to_flags & TOF_TS) ? (u_int32_t *)(void *)(optp + 4) : NULL,
 529                             (to.to_flags & TOF_TS) ? (u_int32_t *)(void *)(optp + 8) : NULL,
 530                             thflags);
 531                         if (lrodebug >= 2) {
 532                                 printf("tcp_lro_process_pkt: coalesce len = %d. flow_id = %d payload_len = %d drop_hdrlen = %d optlen = %d lport = %d seqnum = %x.\n",
 533                                     lro_flow_list[flow_id].lr_len, flow_id,
 534                                     payload_len, drop_hdrlen, optlen,
 535                                     ntohs(lro_flow_list[flow_id].lr_lport),
 536                                     ntohl(tcp_hdr->th_seq));
 537                         }
 538                         if (lro_flow_list[flow_id].lr_mhead->m_pkthdr.lro_npkts >= coalesc_sz) {
 539                                 eject_flow = 1;
 540                         }
 541                         coalesced = 1;
 542                 }
 543                 if (eject_flow) {
 544                         mb = tcp_lro_eject_coalesced_pkt(flow_id);
 545                         lro_flow_list[flow_id].lr_seq = ntohl(tcp_hdr->th_seq) +
 546                             payload_len;
 547                         calculate_tcp_clock();
 548                         u_int8_t timestamp = tcp_now - lro_flow_list[flow_id].lr_timestamp;
 549                         lck_mtx_unlock(&tcp_lro_lock);
 550                         if (mb) {
 551                                 mb->m_pkthdr.lro_elapsed = timestamp;
 552                                 lro_proto_input(mb);
 553                         }
 554                         if (!coalesced) {
 555                                 if (lrodebug >= 2) {
 556                                         printf("%s: pkt payload_len = %d \n", __func__, payload_len);
 557                                 }
 558                                 lro_proto_input(lro_mb);
 559                         }
 560                 } else {
 561                         lck_mtx_unlock(&tcp_lro_lock);
 562                 }
 563                 break;
 564
 565         case TCP_LRO_EJECT_FLOW:
 566                 mb = tcp_lro_eject_coalesced_pkt(flow_id);
 567                 calculate_tcp_clock();
 568                 u_int8_t timestamp = tcp_now - lro_flow_list[flow_id].lr_timestamp;
 569                 lck_mtx_unlock(&tcp_lro_lock);
 570                 if (mb) {
 571                         if (lrodebug) {
 572                                 printf("tcp_lro_process_pkt eject_flow, len = %d\n", mb->m_pkthdr.len);
 573                         }
 574                         mb->m_pkthdr.lro_elapsed = timestamp;
 575                         lro_proto_input(mb);
 576                 }
 577
 578                 lro_proto_input(lro_mb);
 579                 break;
 580
 581         case TCP_LRO_COLLISION:
 582                 lck_mtx_unlock(&tcp_lro_lock);
 583                 ret_response = TCP_LRO_FLOW_NOTFOUND;
 584                 break;
 585
 586         default:
 587                 lck_mtx_unlock(&tcp_lro_lock);
 588                 panic_plain("%s: unrecognized type %d", __func__, retval);
 589         }
 590
 591         if (ret_response == TCP_LRO_FLOW_NOTFOUND) {
 592                 lro_proto_input(lro_mb);
 593         }
 594         return NULL;
 595 }
 596
 597 static void
 598 tcp_lro_timer_proc(void *arg1, void *arg2)
 599 {
 600 #pragma unused(arg1, arg2)
 601
 602         lck_mtx_lock_spin(&tcp_lro_lock);
 603         lro_timer_set = 0;
 604         lck_mtx_unlock(&tcp_lro_lock);
 605         tcp_lro_flush_flows();
 606 }
 607
 608 static void
 609 tcp_lro_flush_flows(void)
 610 {
 611         int i = 0;
 612         struct mbuf *mb;
 613         struct lro_flow *flow;
 614         int tcpclock_updated = 0;
 615
 616         lck_mtx_lock(&tcp_lro_lock);
 617
 618         while (i < TCP_LRO_NUM_FLOWS) {
 619                 flow = &lro_flow_list[i];
 620                 if (flow->lr_mhead != NULL) {
 621                         if (!tcpclock_updated) {
 622                                 calculate_tcp_clock();
 623                                 tcpclock_updated = 1;
 624                         }
 625
 626                         if (lrodebug >= 2) {
 627                                 printf("tcp_lro_flush_flows: len =%d n_pkts = %d %d %d \n",
 628                                     flow->lr_len,
 629                                     flow->lr_mhead->m_pkthdr.lro_npkts,
 630                                     flow->lr_timestamp, tcp_now);
 631                         }
 632
 633                         u_int8_t timestamp = tcp_now - flow->lr_timestamp;
 634
 635                         mb = tcp_lro_eject_flow(i);
 636
 637                         if (mb) {
 638                                 mb->m_pkthdr.lro_elapsed = timestamp;
 639                                 lck_mtx_unlock(&tcp_lro_lock);
 640                                 lro_update_flush_stats(mb);
 641                                 lro_proto_input(mb);
 642                                 lck_mtx_lock(&tcp_lro_lock);
 643                         }
 644                 }
 645                 i++;
 646         }
 647         lck_mtx_unlock(&tcp_lro_lock);
 648 }
 649
 650 /*
 651  * Must be called with tcp_lro_lock held.
 652  * The hint is non-zero for longer waits. The wait time dictated by coalesc_time
 653  * takes precedence, so lro_timer_set is not set for the hint case
 654  */
 655 static void
 656 tcp_lro_sched_timer(uint64_t hint)
 657 {
 658         if (lro_timer_set) {
 659                 return;
 660         }
 661
 662         lro_timer_set = 1;
 663         if (!hint) {
 664                 /* the intent is to wake up every coalesc_time msecs */
 665                 clock_interval_to_deadline(coalesc_time,
 666                     (NSEC_PER_SEC / TCP_RETRANSHZ), &lro_deadline);
 667         } else {
 668                 clock_interval_to_deadline(hint, NSEC_PER_SEC / TCP_RETRANSHZ,
 669                     &lro_deadline);
 670         }
 671         thread_call_enter_delayed(tcp_lro_timer, lro_deadline);
 672 }
 673
 674 struct mbuf*
 675 tcp_lro(struct mbuf *m, unsigned int hlen)
 676 {
 677         struct ip *ip_hdr;
 678         unsigned int tlen;
 679         struct tcphdr * tcp_hdr = NULL;
 680         unsigned int off = 0;
 681
 682         if (kipf_count != 0) {
 683                 return m;
 684         }
 685
 686         /*
 687          * Experiments on cellular show that the RTT is much higher
 688          * than the coalescing time of 5 msecs, causing lro to flush
 689          * 80% of the time on a single packet. Increasing
 690          * coalescing time for cellular does not show marked
 691          * improvement to throughput either. Loopback perf is hurt
 692          * by the 5 msec latency and it already sends large packets.
 693          */
 694         if (IFNET_IS_CELLULAR(m->m_pkthdr.rcvif) ||
 695             (m->m_pkthdr.rcvif->if_type == IFT_LOOP)) {
 696                 return m;
 697         }
 698
 699         ip_hdr = mtod(m, struct ip*);
 700
 701         /* don't deal with IP options */
 702         if (hlen != sizeof(struct ip)) {
 703                 return m;
 704         }
 705
 706         /* only TCP is coalesced */
 707         if (ip_hdr->ip_p != IPPROTO_TCP) {
 708                 return m;
 709         }
 710
 711         if (m->m_len < (int32_t) sizeof(struct tcpiphdr)) {
 712                 if (lrodebug) {
 713                         printf("tcp_lro m_pullup \n");
 714                 }
 715                 if ((m = m_pullup(m, sizeof(struct tcpiphdr))) == NULL) {
 716                         tcpstat.tcps_rcvshort++;
 717                         if (lrodebug) {
 718                                 printf("ip_lro: rcvshort.\n");
 719                         }
 720                         return NULL;
 721                 }
 722                 ip_hdr = mtod(m, struct ip*);
 723         }
 724
 725         tcp_hdr = (struct tcphdr *)((caddr_t)ip_hdr + hlen);
 726         tlen = ip_hdr->ip_len;  //ignore IP header bytes len
 727         m->m_pkthdr.lro_pktlen = tlen; /* Used to return max pkt encountered to tcp */
 728         m->m_pkthdr.lro_npkts = 1; /* Initialize a counter to hold num pkts coalesced */
 729         m->m_pkthdr.lro_elapsed = 0; /* Initialize the field to carry elapsed time */
 730         off = tcp_hdr->th_off << 2;
 731         if (off < sizeof(struct tcphdr) || off > tlen) {
 732                 tcpstat.tcps_rcvbadoff++;
 733                 if (lrodebug) {
 734                         printf("ip_lro: TCP off greater than TCP header.\n");
 735                 }
 736                 return m;
 737         }
 738
 739         return tcp_lro_process_pkt(m, hlen + off);
 740 }
 741
 742 static void
 743 lro_proto_input(struct mbuf *m)
 744 {
 745         struct ip* ip_hdr = mtod(m, struct ip*);
 746
 747         if (lrodebug >= 3) {
 748                 printf("lro_proto_input: ip_len = %d \n",
 749                     ip_hdr->ip_len);
 750         }
 751         lro_update_stats(m);
 752         ip_proto_dispatch_in_wrapper(m, ip_hdr->ip_hl << 2, ip_hdr->ip_p);
 753 }
 754
 755 static struct mbuf *
 756 lro_tcp_xsum_validate(struct mbuf *m, struct ip *ip, struct tcphdr * th)
 757 {
 758         /* Expect 32-bit aligned data pointer on strict-align platforms */
 759         MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(m);
 760
 761         /* we shouldn't get here for IP with options; hence sizeof (ip) */
 762         if (tcp_input_checksum(AF_INET, m, th, sizeof(*ip), ip->ip_len)) {
 763                 if (lrodebug) {
 764                         printf("%s: bad xsum and drop m = 0x%llx.\n", __func__,
 765                             (uint64_t)VM_KERNEL_ADDRPERM(m));
 766                 }
 767                 m_freem(m);
 768                 return NULL;
 769         }
 770
 771         return m;
 772 }
 773
 774 /*
 775  * When TCP detects a stable, steady flow without out of ordering,
 776  * with a sufficiently high cwnd, it invokes LRO.
 777  */
 778 int
 779 tcp_start_coalescing(struct ip *ip_hdr, struct tcphdr *tcp_hdr, int tlen)
 780 {
 781         int hash;
 782         int flow_id;
 783         struct mbuf *eject_mb;
 784         struct lro_flow *lf;
 785
 786         hash = LRO_HASH(ip_hdr->ip_src.s_addr, ip_hdr->ip_dst.s_addr,
 787             tcp_hdr->th_sport, tcp_hdr->th_dport,
 788             (TCP_LRO_FLOW_MAP - 1));
 789
 790
 791         lck_mtx_lock_spin(&tcp_lro_lock);
 792         flow_id = lro_flow_map[hash];
 793         if (flow_id != TCP_LRO_FLOW_NOTFOUND) {
 794                 lf = &lro_flow_list[flow_id];
 795                 if ((lf->lr_faddr.s_addr == ip_hdr->ip_src.s_addr) &&
 796                     (lf->lr_laddr.s_addr == ip_hdr->ip_dst.s_addr) &&
 797                     (lf->lr_fport == tcp_hdr->th_sport) &&
 798                     (lf->lr_lport == tcp_hdr->th_dport)) {
 799                         if ((lf->lr_tcphdr == NULL) &&
 800                             (lf->lr_seq != (tcp_hdr->th_seq + tlen))) {
 801                                 lf->lr_seq = tcp_hdr->th_seq + tlen;
 802                         }
 803                         lf->lr_flags &= ~LRO_EJECT_REQ;
 804                 }
 805                 lck_mtx_unlock(&tcp_lro_lock);
 806                 return 0;
 807         }
 808
 809         HTONL(tcp_hdr->th_seq);
 810         HTONL(tcp_hdr->th_ack);
 811         eject_mb =
 812             tcp_lro_insert_flow(NULL, ip_hdr, tcp_hdr, tlen, 0, hash,
 813             NULL, NULL, NULL);
 814
 815         lck_mtx_unlock(&tcp_lro_lock);
 816
 817         NTOHL(tcp_hdr->th_seq);
 818         NTOHL(tcp_hdr->th_ack);
 819         if (lrodebug >= 3) {
 820                 printf("%s: src = %x dst = %x sport = %d dport = %d seq %x \n",
 821                     __func__, ip_hdr->ip_src.s_addr, ip_hdr->ip_dst.s_addr,
 822                     tcp_hdr->th_sport, tcp_hdr->th_dport, tcp_hdr->th_seq);
 823         }
 824         ASSERT(eject_mb == NULL);
 825         return 0;
 826 }
 827
 828 /*
 829  * When TCP detects loss or idle condition, it stops offloading
 830  * to LRO.
 831  */
 832 int
 833 tcp_lro_remove_state(struct in_addr saddr, struct in_addr daddr,
 834     unsigned short sport, unsigned short dport)
 835 {
 836         int hash, flow_id;
 837         struct lro_flow *lf;
 838
 839         hash = LRO_HASH(daddr.s_addr, saddr.s_addr, dport, sport,
 840             (TCP_LRO_FLOW_MAP - 1));
 841         lck_mtx_lock_spin(&tcp_lro_lock);
 842         flow_id = lro_flow_map[hash];
 843         if (flow_id == TCP_LRO_FLOW_UNINIT) {
 844                 lck_mtx_unlock(&tcp_lro_lock);
 845                 return 0;
 846         }
 847         lf = &lro_flow_list[flow_id];
 848         if ((lf->lr_faddr.s_addr == daddr.s_addr) &&
 849             (lf->lr_laddr.s_addr == saddr.s_addr) &&
 850             (lf->lr_fport == dport) &&
 851             (lf->lr_lport == sport)) {
 852                 if (lrodebug) {
 853                         printf("%s: %x %x\n", __func__,
 854                             lf->lr_flags, lf->lr_seq);
 855                 }
 856                 lf->lr_flags |= LRO_EJECT_REQ;
 857         }
 858         lck_mtx_unlock(&tcp_lro_lock);
 859         return 0;
 860 }
 861
 862 void
 863 tcp_update_lro_seq(__uint32_t rcv_nxt, struct in_addr saddr, struct in_addr daddr,
 864     unsigned short sport, unsigned short dport)
 865 {
 866         int hash, flow_id;
 867         struct lro_flow *lf;
 868
 869         hash = LRO_HASH(daddr.s_addr, saddr.s_addr, dport, sport,
 870             (TCP_LRO_FLOW_MAP - 1));
 871         lck_mtx_lock_spin(&tcp_lro_lock);
 872         flow_id = lro_flow_map[hash];
 873         if (flow_id == TCP_LRO_FLOW_UNINIT) {
 874                 lck_mtx_unlock(&tcp_lro_lock);
 875                 return;
 876         }
 877         lf = &lro_flow_list[flow_id];
 878         if ((lf->lr_faddr.s_addr == daddr.s_addr) &&
 879             (lf->lr_laddr.s_addr == saddr.s_addr) &&
 880             (lf->lr_fport == dport) &&
 881             (lf->lr_lport == sport) &&
 882             (lf->lr_tcphdr == NULL)) {
 883                 lf->lr_seq = (tcp_seq)rcv_nxt;
 884         }
 885         lck_mtx_unlock(&tcp_lro_lock);
 886         return;
 887 }
 888
 889 static void
 890 lro_update_stats(struct mbuf *m)
 891 {
 892         switch (m->m_pkthdr.lro_npkts) {
 893         case 0: /* fall through */
 894         case 1:
 895                 break;
 896
 897         case 2:
 898                 tcpstat.tcps_lro_twopack++;
 899                 break;
 900
 901         case 3: /* fall through */
 902         case 4:
 903                 tcpstat.tcps_lro_multpack++;
 904                 break;
 905
 906         default:
 907                 tcpstat.tcps_lro_largepack++;
 908                 break;
 909         }
 910         return;
 911 }
 912
 913 static void
 914 lro_update_flush_stats(struct mbuf *m)
 915 {
 916         lro_flushes++;
 917         switch (m->m_pkthdr.lro_npkts) {
 918         case 0: ASSERT(0);
 919         case 1: lro_single_flushes++;
 920                 break;
 921         case 2: lro_double_flushes++;
 922                 break;
 923         default: lro_good_flushes++;
 924                 break;
 925         }
 926         return;
 927 }