X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..a39ff7e25e19b3a8c3020042a3872ca9ec9659f1:/bsd/net/pf_norm.c diff --git a/bsd/net/pf_norm.c b/bsd/net/pf_norm.c index 8f21362d5..3cb22e9ce 100644 --- a/bsd/net/pf_norm.c +++ b/bsd/net/pf_norm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Apple Inc. All rights reserved. + * Copyright (c) 2007-2016 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -89,9 +89,16 @@ #include struct pf_frent { - LIST_ENTRY(pf_frent) fr_next; - struct ip *fr_ip; - struct mbuf *fr_m; + LIST_ENTRY(pf_frent) fr_next; + struct mbuf *fr_m; +#define fr_ip fr_u.fru_ipv4 +#define fr_ip6 fr_u.fru_ipv6 + union { + struct ip *fru_ipv4; + struct ip6_hdr *fru_ipv6; + } fr_u; + struct ip6_frag fr_ip6f_opt; + int fr_ip6f_hlen; }; struct pf_frcache { @@ -108,12 +115,18 @@ struct pf_frcache { struct pf_fragment { RB_ENTRY(pf_fragment) fr_entry; TAILQ_ENTRY(pf_fragment) frag_next; - struct in_addr fr_src; - struct in_addr fr_dst; + struct pf_addr fr_srcx; + struct pf_addr fr_dstx; u_int8_t fr_p; /* protocol of this fragment */ u_int8_t fr_flags; /* status flags */ - u_int16_t fr_id; /* fragment id for reassemble */ u_int16_t fr_max; /* fragment data max */ +#define fr_id fr_uid.fru_id4 +#define fr_id6 fr_uid.fru_id6 + union { + u_int16_t fru_id4; + u_int32_t fru_id6; + } fr_uid; + int fr_af; u_int32_t fr_timeout; #define fr_queue fr_u.fru_queue #define fr_cache fr_u.fru_cache @@ -121,6 +134,8 @@ struct pf_fragment { LIST_HEAD(pf_fragq, pf_frent) fru_queue; /* buffering */ LIST_HEAD(pf_cacheq, pf_frcache) fru_cache; /* non-buf */ } fr_u; + uint32_t fr_csum_flags; /* checksum flags */ + uint32_t fr_csum; /* partial checksum value */ }; static TAILQ_HEAD(pf_fragqueue, pf_fragment) pf_fragqueue; @@ -134,22 +149,29 @@ RB_PROTOTYPE_SC(__private_extern__, pf_frag_tree, pf_fragment, fr_entry, RB_GENERATE(pf_frag_tree, pf_fragment, fr_entry, pf_frag_compare); /* Private prototypes */ +static void pf_ip6hdr2key(struct pf_fragment *, struct ip6_hdr *, + struct ip6_frag *); static void pf_ip2key(struct pf_fragment *, struct ip *); static void pf_remove_fragment(struct pf_fragment *); static void pf_flush_fragments(void); static void pf_free_fragment(struct pf_fragment *); -static struct pf_fragment *pf_find_fragment(struct ip *, struct pf_frag_tree *); -static struct mbuf *pf_reassemble(struct mbuf **, struct pf_fragment **, +static struct pf_fragment *pf_find_fragment_by_key(struct pf_fragment *, + struct pf_frag_tree *); +static __inline struct pf_fragment * + pf_find_fragment_by_ipv4_header(struct ip *, struct pf_frag_tree *); +static __inline struct pf_fragment * + pf_find_fragment_by_ipv6_header(struct ip6_hdr *, struct ip6_frag *, + struct pf_frag_tree *); +static struct mbuf *pf_reassemble(struct mbuf *, struct pf_fragment **, struct pf_frent *, int); static struct mbuf *pf_fragcache(struct mbuf **, struct ip *, struct pf_fragment **, int, int, int *); -#ifndef NO_APPLE_MODIFICATIONS +static struct mbuf *pf_reassemble6(struct mbuf **, struct pf_fragment **, + struct pf_frent *, int); +static struct mbuf *pf_frag6cache(struct mbuf **, struct ip6_hdr*, + struct ip6_frag *, struct pf_fragment **, int, int, int, int *); static int pf_normalize_tcpopt(struct pf_rule *, int, struct pfi_kif *, - struct pf_pdesc *, struct mbuf *, struct tcphdr *, int, int *); -#else -static int pf_normalize_tcpopt(struct pf_rule *, struct mbuf *, - struct tcphdr *, int, sa_family_t); -#endif + struct pf_pdesc *, pbuf_t *, struct tcphdr *, int, int *); #define DPFPRINTF(x) do { \ if (pf_status.debug >= PF_DEBUG_MISC) { \ @@ -211,18 +233,74 @@ pf_frag_compare(struct pf_fragment *a, struct pf_fragment *b) { int diff; - if ((diff = a->fr_id - b->fr_id)) + if ((diff = a->fr_af - b->fr_af)) return (diff); else if ((diff = a->fr_p - b->fr_p)) return (diff); - else if (a->fr_src.s_addr < b->fr_src.s_addr) - return (-1); - else if (a->fr_src.s_addr > b->fr_src.s_addr) - return (1); - else if (a->fr_dst.s_addr < b->fr_dst.s_addr) - return (-1); - else if (a->fr_dst.s_addr > b->fr_dst.s_addr) - return (1); + else { + struct pf_addr *sa = &a->fr_srcx; + struct pf_addr *sb = &b->fr_srcx; + struct pf_addr *da = &a->fr_dstx; + struct pf_addr *db = &b->fr_dstx; + + switch (a->fr_af) { +#ifdef INET + case AF_INET: + if ((diff = a->fr_id - b->fr_id)) + return (diff); + else if (sa->v4addr.s_addr < sb->v4addr.s_addr) + return (-1); + else if (sa->v4addr.s_addr > sb->v4addr.s_addr) + return (1); + else if (da->v4addr.s_addr < db->v4addr.s_addr) + return (-1); + else if (da->v4addr.s_addr > db->v4addr.s_addr) + return (1); + break; +#endif +#ifdef INET6 + case AF_INET6: + if ((diff = a->fr_id6 - b->fr_id6)) + return (diff); + else if (sa->addr32[3] < sb->addr32[3]) + return (-1); + else if (sa->addr32[3] > sb->addr32[3]) + return (1); + else if (sa->addr32[2] < sb->addr32[2]) + return (-1); + else if (sa->addr32[2] > sb->addr32[2]) + return (1); + else if (sa->addr32[1] < sb->addr32[1]) + return (-1); + else if (sa->addr32[1] > sb->addr32[1]) + return (1); + else if (sa->addr32[0] < sb->addr32[0]) + return (-1); + else if (sa->addr32[0] > sb->addr32[0]) + return (1); + else if (da->addr32[3] < db->addr32[3]) + return (-1); + else if (da->addr32[3] > db->addr32[3]) + return (1); + else if (da->addr32[2] < db->addr32[2]) + return (-1); + else if (da->addr32[2] > db->addr32[2]) + return (1); + else if (da->addr32[1] < db->addr32[1]) + return (-1); + else if (da->addr32[1] > db->addr32[1]) + return (1); + else if (da->addr32[0] < db->addr32[0]) + return (-1); + else if (da->addr32[0] > db->addr32[0]) + return (1); + break; +#endif + default: + VERIFY(!0 && "only IPv4 and IPv6 supported!"); + break; + } + } return (0); } @@ -238,7 +316,21 @@ pf_purge_expired_fragments(void) if (frag->fr_timeout > expire) break; - DPFPRINTF(("expiring %d(%p)\n", frag->fr_id, frag)); + switch (frag->fr_af) { + case AF_INET: + DPFPRINTF(("expiring IPv4 %d(0x%llx) from queue.\n", + ntohs(frag->fr_id), + (uint64_t)VM_KERNEL_ADDRPERM(frag))); + break; + case AF_INET6: + DPFPRINTF(("expiring IPv6 %d(0x%llx) from queue.\n", + ntohl(frag->fr_id6), + (uint64_t)VM_KERNEL_ADDRPERM(frag))); + break; + default: + VERIFY(0 && "only IPv4 and IPv6 supported"); + break; + } pf_free_fragment(frag); } @@ -247,7 +339,21 @@ pf_purge_expired_fragments(void) if (frag->fr_timeout > expire) break; - DPFPRINTF(("expiring %d(%p)\n", frag->fr_id, frag)); + switch (frag->fr_af) { + case AF_INET: + DPFPRINTF(("expiring IPv4 %d(0x%llx) from cache.\n", + ntohs(frag->fr_id), + (uint64_t)VM_KERNEL_ADDRPERM(frag))); + break; + case AF_INET6: + DPFPRINTF(("expiring IPv6 %d(0x%llx) from cache.\n", + ntohl(frag->fr_id6), + (uint64_t)VM_KERNEL_ADDRPERM(frag))); + break; + default: + VERIFY(0 && "only IPv4 and IPv6 supported"); + break; + } pf_free_fragment(frag); VERIFY(TAILQ_EMPTY(&pf_cachequeue) || TAILQ_LAST(&pf_cachequeue, pf_cachequeue) != frag); @@ -321,24 +427,33 @@ pf_free_fragment(struct pf_fragment *frag) pf_remove_fragment(frag); } +static void +pf_ip6hdr2key(struct pf_fragment *key, struct ip6_hdr *ip6, + struct ip6_frag *fh) +{ + key->fr_p = fh->ip6f_nxt; + key->fr_id6 = fh->ip6f_ident; + key->fr_af = AF_INET6; + key->fr_srcx.v6addr = ip6->ip6_src; + key->fr_dstx.v6addr = ip6->ip6_dst; +} + static void pf_ip2key(struct pf_fragment *key, struct ip *ip) { key->fr_p = ip->ip_p; key->fr_id = ip->ip_id; - key->fr_src.s_addr = ip->ip_src.s_addr; - key->fr_dst.s_addr = ip->ip_dst.s_addr; + key->fr_af = AF_INET; + key->fr_srcx.v4addr.s_addr = ip->ip_src.s_addr; + key->fr_dstx.v4addr.s_addr = ip->ip_dst.s_addr; } static struct pf_fragment * -pf_find_fragment(struct ip *ip, struct pf_frag_tree *tree) +pf_find_fragment_by_key(struct pf_fragment *key, struct pf_frag_tree *tree) { - struct pf_fragment key; - struct pf_fragment *frag; - - pf_ip2key(&key, ip); - - frag = RB_FIND(pf_frag_tree, tree, &key); + struct pf_fragment *frag; + + frag = RB_FIND(pf_frag_tree, tree, key); if (frag != NULL) { /* XXX Are we sure we want to update the timeout? */ frag->fr_timeout = pf_time_second(); @@ -350,9 +465,26 @@ pf_find_fragment(struct ip *ip, struct pf_frag_tree *tree) TAILQ_INSERT_HEAD(&pf_cachequeue, frag, frag_next); } } - + return (frag); } + +static __inline struct pf_fragment * +pf_find_fragment_by_ipv4_header(struct ip *ip, struct pf_frag_tree *tree) +{ + struct pf_fragment key; + pf_ip2key(&key, ip); + return pf_find_fragment_by_key(&key, tree); +} + +static __inline struct pf_fragment * +pf_find_fragment_by_ipv6_header(struct ip6_hdr *ip6, struct ip6_frag *fh, + struct pf_frag_tree *tree) +{ + struct pf_fragment key; + pf_ip6hdr2key(&key, ip6, fh); + return pf_find_fragment_by_key(&key, tree); +} /* Removes a fragment from the fragment queue and frees the fragment */ @@ -372,20 +504,79 @@ pf_remove_fragment(struct pf_fragment *frag) #define FR_IP_OFF(fr) ((ntohs((fr)->fr_ip->ip_off) & IP_OFFMASK) << 3) static struct mbuf * -pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, +pf_reassemble(struct mbuf *m0, struct pf_fragment **frag, struct pf_frent *frent, int mff) { - struct mbuf *m = *m0, *m2; + struct mbuf *m = m0, *m2; struct pf_frent *frea, *next; struct pf_frent *frep = NULL; struct ip *ip = frent->fr_ip; - int hlen = ip->ip_hl << 2; + uint32_t hlen = ip->ip_hl << 2; u_int16_t off = (ntohs(ip->ip_off) & IP_OFFMASK) << 3; u_int16_t ip_len = ntohs(ip->ip_len) - ip->ip_hl * 4; u_int16_t fr_max = ip_len + off; + uint32_t csum, csum_flags; VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); + /* + * Leverage partial checksum offload for IP fragments. Narrow down + * the scope to cover only UDP without IP options, as that is the + * most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Ignore cases + * where the value includes the entire IPv4 header span, as the sum + * for those octets would already be 0 by the time we get here; IP + * has already performed its header checksum validation. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip->ip_p == IPPROTO_UDP && hlen == sizeof (struct ip) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + int32_t trailer = (m_pktlen(m) - ntohs(ip->ip_len)); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if ((start != 0 && start != hlen) || trailer != 0) { +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + HTONS(ip->ip_len); + HTONS(ip->ip_off); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + /* callee folds in sum */ + csum = m_adj_sum16(m, start, hlen, + (ip->ip_len - hlen), csum); + if (hlen > start) + swbytes += (hlen - start); + else + swbytes += (start - hlen); +#if BYTE_ORDER != BIG_ENDIAN + if (start < hlen) { + NTOHS(ip->ip_off); + NTOHS(ip->ip_len); + } +#endif /* BYTE_ORDER != BIG_ENDIAN */ + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + /* Strip off ip header */ m->m_data += hlen; m->m_len -= hlen; @@ -402,11 +593,16 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, (*frag)->fr_flags = 0; (*frag)->fr_max = 0; - (*frag)->fr_src = frent->fr_ip->ip_src; - (*frag)->fr_dst = frent->fr_ip->ip_dst; + (*frag)->fr_af = AF_INET; + (*frag)->fr_srcx.v4addr = frent->fr_ip->ip_src; + (*frag)->fr_dstx.v4addr = frent->fr_ip->ip_dst; (*frag)->fr_p = frent->fr_ip->ip_p; (*frag)->fr_id = frent->fr_ip->ip_id; (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } LIST_INIT(&(*frag)->fr_queue); RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); @@ -417,6 +613,16 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment **frag, goto insert; } + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; + /* * Find a fragment after the current one: * - off contains the real shifted offset. @@ -534,8 +740,26 @@ insert: m_cat(m, m2); } - ip->ip_src = (*frag)->fr_src; - ip->ip_dst = (*frag)->fr_dst; + ip->ip_src = (*frag)->fr_srcx.v4addr; + ip->ip_dst = (*frag)->fr_dstx.v4addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = + CSUM_DATA_VALID | CSUM_PSEUDO_HDR | + CSUM_IP_CHECKED | CSUM_IP_VALID; + } /* Remove from fragment queue */ pf_remove_fragment(*frag); @@ -555,7 +779,8 @@ insert: m->m_pkthdr.len = plen; } - DPFPRINTF(("complete: %p(%d)\n", m, ntohs(ip->ip_len))); + DPFPRINTF(("complete: 0x%llx(%d)\n", + (uint64_t)VM_KERNEL_ADDRPERM(m), ntohs(ip->ip_len))); return (m); drop_fragment: @@ -600,8 +825,9 @@ pf_fragcache(struct mbuf **m0, struct ip *h, struct pf_fragment **frag, int mff, (*frag)->fr_flags = PFFRAG_NOBUFFER; (*frag)->fr_max = 0; - (*frag)->fr_src = h->ip_src; - (*frag)->fr_dst = h->ip_dst; + (*frag)->fr_af = AF_INET; + (*frag)->fr_srcx.v4addr = h->ip_src; + (*frag)->fr_dstx.v4addr = h->ip_dst; (*frag)->fr_p = h->ip_p; (*frag)->fr_id = h->ip_id; (*frag)->fr_timeout = pf_time_second(); @@ -865,21 +1091,650 @@ drop_fragment: return (NULL); } +#define FR_IP6_OFF(fr) \ + (ntohs((fr)->fr_ip6f_opt.ip6f_offlg & IP6F_OFF_MASK)) +#define FR_IP6_PLEN(fr) (ntohs((fr)->fr_ip6->ip6_plen)) +struct mbuf * +pf_reassemble6(struct mbuf **m0, struct pf_fragment **frag, + struct pf_frent *frent, int mff) +{ + struct mbuf *m, *m2; + struct pf_frent *frea, *frep, *next; + struct ip6_hdr *ip6; + struct ip6_frag *ip6f; + int plen, off, fr_max; + uint32_t uoff, csum, csum_flags; + + VERIFY(*frag == NULL || BUFFER_FRAGMENTS(*frag)); + m = *m0; + frep = NULL; + ip6 = frent->fr_ip6; + ip6f = &frent->fr_ip6f_opt; + off = FR_IP6_OFF(frent); + uoff = frent->fr_ip6f_hlen; + plen = FR_IP6_PLEN(frent); + fr_max = off + plen - (frent->fr_ip6f_hlen - sizeof *ip6); + + DPFPRINTF(("0x%llx IPv6 frag plen %u off %u fr_ip6f_hlen %u " + "fr_max %u m_len %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, off, + frent->fr_ip6f_hlen, fr_max, m->m_len)); + + /* + * Leverage partial checksum offload for simple UDP/IP fragments, + * as that is the most common case. + * + * Perform 1's complement adjustment of octets that got included/ + * excluded in the hardware-calculated checksum value. Also take + * care of any trailing bytes and subtract out their partial sum. + */ + if (ip6f->ip6f_nxt == IPPROTO_UDP && + uoff == (sizeof (*ip6) + sizeof (*ip6f)) && + (m->m_pkthdr.csum_flags & + (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PARTIAL)) { + uint32_t start = m->m_pkthdr.csum_rx_start; + uint32_t ip_len = (sizeof (*ip6) + ntohs(ip6->ip6_plen)); + int32_t trailer = (m_pktlen(m) - ip_len); + uint32_t swbytes = (uint32_t)trailer; + + csum = m->m_pkthdr.csum_rx_val; + + ASSERT(trailer >= 0); + if (start != uoff || trailer != 0) { + uint16_t s = 0, d = 0; + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) { + s = ip6->ip6_src.s6_addr16[1]; + ip6->ip6_src.s6_addr16[1] = 0 ; + } + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) { + d = ip6->ip6_dst.s6_addr16[1]; + ip6->ip6_dst.s6_addr16[1] = 0; + } + + /* callee folds in sum */ + csum = m_adj_sum16(m, start, uoff, + (ip_len - uoff), csum); + if (uoff > start) + swbytes += (uoff - start); + else + swbytes += (start - uoff); + + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_src)) + ip6->ip6_src.s6_addr16[1] = s; + if (IN6_IS_SCOPE_EMBED(&ip6->ip6_dst)) + ip6->ip6_dst.s6_addr16[1] = d; + + } + csum_flags = m->m_pkthdr.csum_flags; + + if (swbytes != 0) + udp_in6_cksum_stats(swbytes); + if (trailer != 0) + m_adj(m, -trailer); + } else { + csum = 0; + csum_flags = 0; + } + + /* Invalidate checksum */ + m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID; + + /* strip off headers up to the fragment payload */ + m->m_data += frent->fr_ip6f_hlen; + m->m_len -= frent->fr_ip6f_hlen; + + /* Create a new reassembly queue for this packet */ + if (*frag == NULL) { + *frag = pool_get(&pf_frag_pl, PR_NOWAIT); + if (*frag == NULL) { + pf_flush_fragments(); + *frag = pool_get(&pf_frag_pl, PR_NOWAIT); + if (*frag == NULL) + goto drop_fragment; + } + + (*frag)->fr_flags = 0; + (*frag)->fr_max = 0; + (*frag)->fr_af = AF_INET6; + (*frag)->fr_srcx.v6addr = frent->fr_ip6->ip6_src; + (*frag)->fr_dstx.v6addr = frent->fr_ip6->ip6_dst; + (*frag)->fr_p = frent->fr_ip6f_opt.ip6f_nxt; + (*frag)->fr_id6 = frent->fr_ip6f_opt.ip6f_ident; + (*frag)->fr_timeout = pf_time_second(); + if (csum_flags != 0) { + (*frag)->fr_csum_flags = csum_flags; + (*frag)->fr_csum = csum; + } + LIST_INIT(&(*frag)->fr_queue); + + RB_INSERT(pf_frag_tree, &pf_frag_tree, *frag); + TAILQ_INSERT_HEAD(&pf_fragqueue, *frag, frag_next); + + /* We do not have a previous fragment */ + frep = NULL; + goto insert; + } + + /* + * If this fragment contains similar checksum offload info + * as that of the existing ones, accumulate checksum. Otherwise, + * invalidate checksum offload info for the entire datagram. + */ + if (csum_flags != 0 && csum_flags == (*frag)->fr_csum_flags) + (*frag)->fr_csum += csum; + else if ((*frag)->fr_csum_flags != 0) + (*frag)->fr_csum_flags = 0; + + /* + * Find a fragment after the current one: + * - off contains the real shifted offset. + */ + LIST_FOREACH(frea, &(*frag)->fr_queue, fr_next) { + if (FR_IP6_OFF(frea) > off) + break; + frep = frea; + } + + VERIFY(frep != NULL || frea != NULL); + + if (frep != NULL && + FR_IP6_OFF(frep) + FR_IP6_PLEN(frep) - frep->fr_ip6f_hlen > off) + { + u_int16_t precut; + + precut = FR_IP6_OFF(frep) + FR_IP6_PLEN(frep) - + frep->fr_ip6f_hlen - off; + if (precut >= plen) + goto drop_fragment; + m_adj(frent->fr_m, precut); + DPFPRINTF(("overlap -%d\n", precut)); + /* Enforce 8 byte boundaries */ + frent->fr_ip6f_opt.ip6f_offlg = + htons(ntohs(frent->fr_ip6f_opt.ip6f_offlg) + + (precut >> 3)); + off = FR_IP6_OFF(frent); + plen -= precut; + ip6->ip6_plen = htons(plen); + } + + for (; frea != NULL && plen + off > FR_IP6_OFF(frea); frea = next) { + u_int16_t aftercut; + + aftercut = plen + off - FR_IP6_OFF(frea); + DPFPRINTF(("adjust overlap %d\n", aftercut)); + if (aftercut < FR_IP6_PLEN(frea) - frea->fr_ip6f_hlen) { + frea->fr_ip6->ip6_plen = htons(FR_IP6_PLEN(frea) - + aftercut); + frea->fr_ip6f_opt.ip6f_offlg = + htons(ntohs(frea->fr_ip6f_opt.ip6f_offlg) + + (aftercut >> 3)); + m_adj(frea->fr_m, aftercut); + break; + } + + /* This fragment is completely overlapped, lose it */ + next = LIST_NEXT(frea, fr_next); + m_freem(frea->fr_m); + LIST_REMOVE(frea, fr_next); + pool_put(&pf_frent_pl, frea); + pf_nfrents--; + } + + insert: + /* Update maximum data size */ + if ((*frag)->fr_max < fr_max) + (*frag)->fr_max = fr_max; + /* This is the last segment */ + if (!mff) + (*frag)->fr_flags |= PFFRAG_SEENLAST; + + if (frep == NULL) + LIST_INSERT_HEAD(&(*frag)->fr_queue, frent, fr_next); + else + LIST_INSERT_AFTER(frep, frent, fr_next); + + /* Check if we are completely reassembled */ + if (!((*frag)->fr_flags & PFFRAG_SEENLAST)) + return (NULL); + + /* Check if we have all the data */ + off = 0; + for (frep = LIST_FIRST(&(*frag)->fr_queue); frep; frep = next) { + next = LIST_NEXT(frep, fr_next); + off += FR_IP6_PLEN(frep) - (frent->fr_ip6f_hlen - sizeof *ip6); + DPFPRINTF(("frep at %d, next %d, max %d\n", + off, next == NULL ? -1 : FR_IP6_OFF(next), + (*frag)->fr_max)); + if (off < (*frag)->fr_max && + (next == NULL || FR_IP6_OFF(next) != off)) { + DPFPRINTF(("missing fragment at %d, next %d, max %d\n", + off, next == NULL ? -1 : FR_IP6_OFF(next), + (*frag)->fr_max)); + return (NULL); + } + } + DPFPRINTF(("%d < %d?\n", off, (*frag)->fr_max)); + if (off < (*frag)->fr_max) + return (NULL); + + /* We have all the data */ + frent = LIST_FIRST(&(*frag)->fr_queue); + VERIFY(frent != NULL); + if (frent->fr_ip6f_hlen + off > IP_MAXPACKET) { + DPFPRINTF(("drop: too big: %d\n", off)); + pf_free_fragment(*frag); + *frag = NULL; + return (NULL); + } + + ip6 = frent->fr_ip6; + ip6->ip6_nxt = (*frag)->fr_p; + ip6->ip6_plen = htons(off); + ip6->ip6_src = (*frag)->fr_srcx.v6addr; + ip6->ip6_dst = (*frag)->fr_dstx.v6addr; + + if ((*frag)->fr_csum_flags != 0) { + csum = (*frag)->fr_csum; + + ADDCARRY(csum); + + m->m_pkthdr.csum_rx_val = csum; + m->m_pkthdr.csum_rx_start = sizeof (struct ip6_hdr); + m->m_pkthdr.csum_flags = (*frag)->fr_csum_flags; + } else if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) || + (m->m_pkthdr.pkt_flags & PKTF_LOOP)) { + /* loopback checksums are always OK */ + m->m_pkthdr.csum_data = 0xffff; + m->m_pkthdr.csum_flags &= ~CSUM_PARTIAL; + m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + } + + /* Remove from fragment queue */ + pf_remove_fragment(*frag); + *frag = NULL; + + m = frent->fr_m; + m->m_len += sizeof(struct ip6_hdr); + m->m_data -= sizeof(struct ip6_hdr); + memmove(m->m_data, ip6, sizeof(struct ip6_hdr)); + + next = LIST_NEXT(frent, fr_next); + pool_put(&pf_frent_pl, frent); + pf_nfrents--; + for (frent = next; next != NULL; frent = next) { + m2 = frent->fr_m; + + m_cat(m, m2); + next = LIST_NEXT(frent, fr_next); + pool_put(&pf_frent_pl, frent); + pf_nfrents--; + } + + /* XXX this should be done elsewhere */ + if (m->m_flags & M_PKTHDR) { + int pktlen = 0; + for (m2 = m; m2; m2 = m2->m_next) + pktlen += m2->m_len; + m->m_pkthdr.len = pktlen; + } + + DPFPRINTF(("complete: 0x%llx ip6_plen %d m_pkthdr.len %d\n", + (uint64_t)VM_KERNEL_ADDRPERM(m), ntohs(ip6->ip6_plen), + m->m_pkthdr.len)); + + return m; + + drop_fragment: + /* Oops - fail safe - drop packet */ + pool_put(&pf_frent_pl, frent); + --pf_nfrents; + m_freem(m); + return NULL; +} + +static struct mbuf * +pf_frag6cache(struct mbuf **m0, struct ip6_hdr *h, struct ip6_frag *fh, + struct pf_fragment **frag, int hlen, int mff, int drop, int *nomem) +{ + struct mbuf *m = *m0; + u_int16_t plen, off, fr_max; + struct pf_frcache *frp, *fra, *cur = NULL; + int hosed = 0; + + VERIFY(*frag == NULL || !BUFFER_FRAGMENTS(*frag)); + m = *m0; + off = ntohs(fh->ip6f_offlg & IP6F_OFF_MASK); + plen = ntohs(h->ip6_plen) - (hlen - sizeof *h); + + /* + * Apple Modification: dimambro@apple.com. The hlen, being passed + * into this function Includes all the headers associated with + * the packet, and may include routing headers, so to get to + * the data payload as stored in the original IPv6 header we need + * to subtract al those headers and the IP header. + * + * The 'max' local variable should also contain the offset from the start + * of the reassembled packet to the octet just past the end of the octets + * in the current fragment where: + * - 'off' is the offset from the start of the reassembled packet to the + * first octet in the fragment, + * - 'plen' is the length of the "payload data length" Excluding all the + * IPv6 headers of the fragment. + * - 'hlen' is computed in pf_normalize_ip6() as the offset from the start + * of the IPv6 packet to the beginning of the data. + */ + fr_max = off + plen; + + DPFPRINTF(("0x%llx plen %u off %u fr_max %u\n", + (uint64_t)VM_KERNEL_ADDRPERM(m), plen, off, fr_max)); + + /* Create a new range queue for this packet */ + if (*frag == NULL) { + *frag = pool_get(&pf_cache_pl, PR_NOWAIT); + if (*frag == NULL) { + pf_flush_fragments(); + *frag = pool_get(&pf_cache_pl, PR_NOWAIT); + if (*frag == NULL) + goto no_mem; + } + + /* Get an entry for the queue */ + cur = pool_get(&pf_cent_pl, PR_NOWAIT); + if (cur == NULL) { + pool_put(&pf_cache_pl, *frag); + *frag = NULL; + goto no_mem; + } + pf_ncache++; + + (*frag)->fr_flags = PFFRAG_NOBUFFER; + (*frag)->fr_max = 0; + (*frag)->fr_af = AF_INET6; + (*frag)->fr_srcx.v6addr = h->ip6_src; + (*frag)->fr_dstx.v6addr = h->ip6_dst; + (*frag)->fr_p = fh->ip6f_nxt; + (*frag)->fr_id6 = fh->ip6f_ident; + (*frag)->fr_timeout = pf_time_second(); + + cur->fr_off = off; + cur->fr_end = fr_max; + LIST_INIT(&(*frag)->fr_cache); + LIST_INSERT_HEAD(&(*frag)->fr_cache, cur, fr_next); + + RB_INSERT(pf_frag_tree, &pf_cache_tree, *frag); + TAILQ_INSERT_HEAD(&pf_cachequeue, *frag, frag_next); + + DPFPRINTF(("frag6cache[%d]: new %d-%d\n", ntohl(fh->ip6f_ident), + off, fr_max)); + + goto pass; + } + + /* + * Find a fragment after the current one: + * - off contains the real shifted offset. + */ + frp = NULL; + LIST_FOREACH(fra, &(*frag)->fr_cache, fr_next) { + if (fra->fr_off > off) + break; + frp = fra; + } + + VERIFY(frp != NULL || fra != NULL); + + if (frp != NULL) { + int precut; + + precut = frp->fr_end - off; + if (precut >= plen) { + /* Fragment is entirely a duplicate */ + DPFPRINTF(("frag6cache[%u]: dead (%d-%d) %d-%d\n", + ntohl(fh->ip6f_ident), frp->fr_off, frp->fr_end, + off, fr_max)); + goto drop_fragment; + } + if (precut == 0) { + /* They are adjacent. Fixup cache entry */ + DPFPRINTF(("frag6cache[%u]: adjacent (%d-%d) %d-%d\n", + ntohl(fh->ip6f_ident), frp->fr_off, frp->fr_end, + off, fr_max)); + frp->fr_end = fr_max; + } else if (precut > 0) { + /* The first part of this payload overlaps with a + * fragment that has already been passed. + * Need to trim off the first part of the payload. + * But to do so easily, we need to create another + * mbuf to throw the original header into. + */ + + DPFPRINTF(("frag6cache[%u]: chop %d (%d-%d) %d-%d\n", + ntohl(fh->ip6f_ident), precut, frp->fr_off, + frp->fr_end, off, fr_max)); + + off += precut; + fr_max -= precut; + /* Update the previous frag to encompass this one */ + frp->fr_end = fr_max; + + if (!drop) { + /* XXX Optimization opportunity + * This is a very heavy way to trim the payload. + * we could do it much faster by diddling mbuf + * internals but that would be even less legible + * than this mbuf magic. For my next trick, + * I'll pull a rabbit out of my laptop. + */ + *m0 = m_copym(m, 0, hlen, M_NOWAIT); + if (*m0 == NULL) + goto no_mem; + VERIFY((*m0)->m_next == NULL); + m_adj(m, precut + hlen); + m_cat(*m0, m); + m = *m0; + if (m->m_flags & M_PKTHDR) { + int pktlen = 0; + struct mbuf *t; + for (t = m; t; t = t->m_next) + pktlen += t->m_len; + m->m_pkthdr.len = pktlen; + } + + h = mtod(m, struct ip6_hdr *); + + VERIFY((int)m->m_len == + ntohs(h->ip6_plen) - precut); + fh->ip6f_offlg &= ~IP6F_OFF_MASK; + fh->ip6f_offlg |= + htons(ntohs(fh->ip6f_offlg & IP6F_OFF_MASK) + + (precut >> 3)); + h->ip6_plen = htons(ntohs(h->ip6_plen) - + precut); + } else { + hosed++; + } + } else { + /* There is a gap between fragments */ + + DPFPRINTF(("frag6cache[%u]: gap %d (%d-%d) %d-%d\n", + ntohl(fh->ip6f_ident), -precut, frp->fr_off, + frp->fr_end, off, fr_max)); + + cur = pool_get(&pf_cent_pl, PR_NOWAIT); + if (cur == NULL) + goto no_mem; + pf_ncache++; + + cur->fr_off = off; + cur->fr_end = fr_max; + LIST_INSERT_AFTER(frp, cur, fr_next); + } + } + + if (fra != NULL) { + int aftercut; + int merge = 0; + + aftercut = fr_max - fra->fr_off; + if (aftercut == 0) { + /* Adjacent fragments */ + DPFPRINTF(("frag6cache[%u]: adjacent %d-%d (%d-%d)\n", + ntohl(fh->ip6f_ident), off, fr_max, fra->fr_off, + fra->fr_end)); + fra->fr_off = off; + merge = 1; + } else if (aftercut > 0) { + /* Need to chop off the tail of this fragment */ + DPFPRINTF(("frag6cache[%u]: chop %d %d-%d (%d-%d)\n", + ntohl(fh->ip6f_ident), aftercut, off, fr_max, + fra->fr_off, fra->fr_end)); + fra->fr_off = off; + fr_max -= aftercut; + + merge = 1; + + if (!drop) { + m_adj(m, -aftercut); + if (m->m_flags & M_PKTHDR) { + int pktlen = 0; + struct mbuf *t; + for (t = m; t; t = t->m_next) + pktlen += t->m_len; + m->m_pkthdr.len = pktlen; + } + h = mtod(m, struct ip6_hdr *); + VERIFY((int)m->m_len == + ntohs(h->ip6_plen) - aftercut); + h->ip6_plen = + htons(ntohs(h->ip6_plen) - aftercut); + } else { + hosed++; + } + } else if (frp == NULL) { + /* There is a gap between fragments */ + DPFPRINTF(("frag6cache[%u]: gap %d %d-%d (%d-%d)\n", + ntohl(fh->ip6f_ident), -aftercut, off, fr_max, + fra->fr_off, fra->fr_end)); + + cur = pool_get(&pf_cent_pl, PR_NOWAIT); + if (cur == NULL) + goto no_mem; + pf_ncache++; + + cur->fr_off = off; + cur->fr_end = fr_max; + LIST_INSERT_BEFORE(fra, cur, fr_next); + } + + /* Need to glue together two separate fragment descriptors */ + if (merge) { + if (cur && fra->fr_off <= cur->fr_end) { + /* Need to merge in a previous 'cur' */ + DPFPRINTF(("frag6cache[%u]: adjacent(merge " + "%d-%d) %d-%d (%d-%d)\n", + ntohl(fh->ip6f_ident), cur->fr_off, + cur->fr_end, off, fr_max, fra->fr_off, + fra->fr_end)); + fra->fr_off = cur->fr_off; + LIST_REMOVE(cur, fr_next); + pool_put(&pf_cent_pl, cur); + pf_ncache--; + cur = NULL; + } else if (frp && fra->fr_off <= frp->fr_end) { + /* Need to merge in a modified 'frp' */ + VERIFY(cur == NULL); + DPFPRINTF(("frag6cache[%u]: adjacent(merge " + "%d-%d) %d-%d (%d-%d)\n", + ntohl(fh->ip6f_ident), frp->fr_off, + frp->fr_end, off, fr_max, fra->fr_off, + fra->fr_end)); + fra->fr_off = frp->fr_off; + LIST_REMOVE(frp, fr_next); + pool_put(&pf_cent_pl, frp); + pf_ncache--; + frp = NULL; + } + } + } + + if (hosed) { + /* + * We must keep tracking the overall fragment even when + * we're going to drop it anyway so that we know when to + * free the overall descriptor. Thus we drop the frag late. + */ + goto drop_fragment; + } + + pass: + /* Update maximum data size */ + if ((*frag)->fr_max < fr_max) + (*frag)->fr_max = fr_max; + + /* This is the last segment */ + if (!mff) + (*frag)->fr_flags |= PFFRAG_SEENLAST; + + /* Check if we are completely reassembled */ + if (((*frag)->fr_flags & PFFRAG_SEENLAST) && + LIST_FIRST(&(*frag)->fr_cache)->fr_off == 0 && + LIST_FIRST(&(*frag)->fr_cache)->fr_end == (*frag)->fr_max) { + /* Remove from fragment queue */ + DPFPRINTF(("frag6cache[%u]: done 0-%d\n", + ntohl(fh->ip6f_ident), (*frag)->fr_max)); + pf_free_fragment(*frag); + *frag = NULL; + } + + return (m); + + no_mem: + *nomem = 1; + + /* Still need to pay attention to !IP_MF */ + if (!mff && *frag != NULL) + (*frag)->fr_flags |= PFFRAG_SEENLAST; + + m_freem(m); + return (NULL); + + drop_fragment: + + /* Still need to pay attention to !IP_MF */ + if (!mff && *frag != NULL) + (*frag)->fr_flags |= PFFRAG_SEENLAST; + + if (drop) { + /* This fragment has been deemed bad. Don't reass */ + if (((*frag)->fr_flags & PFFRAG_DROP) == 0) + DPFPRINTF(("frag6cache[%u]: dropping overall fragment\n", + ntohl(fh->ip6f_ident))); + (*frag)->fr_flags |= PFFRAG_DROP; + } + + m_freem(m); + return (NULL); +} + int -pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, +pf_normalize_ip(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; struct pf_frent *frent; struct pf_fragment *frag = NULL; - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; int mff = (ntohs(h->ip_off) & IP_MF); int hlen = h->ip_hl << 2; u_int16_t fragoff = (ntohs(h->ip_off) & IP_OFFMASK) << 3; u_int16_t fr_max; int ip_len; int ip_off; + int asd = 0; + struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -900,7 +1755,15 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, (struct pf_addr *)&h->ip_dst.s_addr, AF_INET, r->dst.neg, NULL)) r = r->skip[PF_SKIP_DST_ADDR].ptr; - else + else { + if (r->anchor == NULL) + break; + else + pf_step_into_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL); + } + if (r == NULL && pf_step_out_of_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL)) break; } @@ -959,17 +1822,27 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) { /* Fully buffer all of the fragments */ - frag = pf_find_fragment(h, &pf_frag_tree); - + frag = pf_find_fragment_by_ipv4_header(h, &pf_frag_tree); /* Check if we saw the last fragment already */ if (frag != NULL && (frag->fr_flags & PFFRAG_SEENLAST) && fr_max > frag->fr_max) goto bad; + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + /* Get an entry for the fragment queue */ frent = pool_get(&pf_frent_pl, PR_NOWAIT); if (frent == NULL) { REASON_SET(reason, PFRES_MEMORY); + m_freem(m); return (PF_DROP); } pf_nfrents++; @@ -977,34 +1850,41 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, frent->fr_m = m; /* Might return a completely reassembled mbuf, or NULL */ - DPFPRINTF(("reass frag %d @ %d-%d\n", h->ip_id, fragoff, - fr_max)); - *m0 = m = pf_reassemble(m0, &frag, frent, mff); + DPFPRINTF(("reass IPv4 frag %d @ %d-%d\n", ntohs(h->ip_id), + fragoff, fr_max)); + m = pf_reassemble(m, &frag, frent, mff); if (m == NULL) return (PF_DROP); + VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); + /* use mtag from concatenated mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); -#ifdef DIAGNOSTIC + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous +#if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(1)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - *m0 = NULL; + m = NULL; goto no_mem; } } #endif - if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) - goto drop; +#endif h = mtod(m, struct ip *); + + if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) + goto drop; } else { /* non-buffering fragment cache (drops or masks overlaps) */ int nomem = 0; - if (dir == PF_OUT && (pd->pf_mtag->flags & PF_TAG_FRAGCACHE)) { + if (dir == PF_OUT && (pd->pf_mtag->pftag_flags & PF_TAG_FRAGCACHE)) { /* * Already passed the fragment cache in the * input direction. If we continued, it would @@ -1013,7 +1893,7 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, goto fragment_pass; } - frag = pf_find_fragment(h, &pf_cache_tree); + frag = pf_find_fragment_by_ipv4_header(h, &pf_cache_tree); /* Check if we saw the last fragment already */ if (frag != NULL && (frag->fr_flags & PFFRAG_SEENLAST) && @@ -1023,31 +1903,49 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason, goto bad; } - *m0 = m = pf_fragcache(m0, h, &frag, mff, + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + goto bad; + } + + VERIFY(!pbuf_is_valid(pbuf)); + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip *); + + m = pf_fragcache(&m, h, &frag, mff, (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); if (m == NULL) { + // Note: pf_fragcache() has already m_freem'd the mbuf if (nomem) goto no_mem; goto drop; } + VERIFY(m->m_flags & M_PKTHDR); + pbuf_init_mbuf(pbuf, m, ifp); + /* use mtag from copied and trimmed mbuf chain */ - pd->pf_mtag = pf_find_mtag(m); -#ifdef DIAGNOSTIC + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); +#if 0 +// SCW: This check is superfluous +#if DIAGNOSTIC if (pd->pf_mtag == NULL) { printf("%s: pf_find_mtag returned NULL(2)\n", __func__); if ((pd->pf_mtag = pf_get_mtag(m)) == NULL) { m_freem(m); - *m0 = NULL; + m = NULL; goto no_mem; } } +#endif #endif if (dir == PF_IN) - pd->pf_mtag->flags |= PF_TAG_FRAGCACHE; + pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; if (frag != NULL && (frag->fr_flags & PFFRAG_DROP)) goto drop; + goto fragment_pass; } @@ -1067,14 +1965,16 @@ no_fragment: h->ip_ttl = r->min_ttl; h->ip_sum = pf_cksum_fixup(h->ip_sum, ip_ttl, h->ip_ttl, 0); } -#if RANDOM_IP_ID if (r->rule_flag & PFRULE_RANDOMID) { - u_int16_t ip_id = h->ip_id; + u_int16_t oip_id = h->ip_id; - h->ip_id = ip_randomid(); - h->ip_sum = pf_cksum_fixup(h->ip_sum, ip_id, h->ip_id, 0); + if (rfc6864 && IP_OFF_IS_ATOMIC(ntohs(h->ip_off))) { + h->ip_id = 0; + } else { + h->ip_id = ip_randomid(); + } + h->ip_sum = pf_cksum_fixup(h->ip_sum, oip_id, h->ip_id, 0); } -#endif /* RANDOM_IP_ID */ if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) pd->flags |= PFDESC_IP_REAS; @@ -1094,40 +1994,40 @@ fragment_pass: no_mem: REASON_SET(reason, PFRES_MEMORY); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); drop: REASON_SET(reason, PFRES_NORM); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); bad: - DPFPRINTF(("dropping bad fragment\n")); + DPFPRINTF(("dropping bad IPv4 fragment\n")); /* Free associated fragments */ if (frag != NULL) pf_free_fragment(frag); REASON_SET(reason, PFRES_FRAG); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, *reason, r, NULL, NULL, pd); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #if INET6 int -pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, +pf_normalize_ip6(pbuf_t *pbuf, int dir, struct pfi_kif *kif, u_short *reason, struct pf_pdesc *pd) { - struct mbuf *m = *m0; + struct mbuf *m; struct pf_rule *r; - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; int off; struct ip6_ext ext; /* adi XXX */ @@ -1142,6 +2042,13 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, u_int16_t fragoff = 0; u_int8_t proto; int terminal; + struct pf_frent *frent; + struct pf_fragment *pff = NULL; + int mff = 0, rh_cnt = 0; + u_int16_t fr_max; + int asd = 0; + struct pf_ruleset *ruleset = NULL; + struct ifnet *ifp = pbuf->pb_ifp; r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1157,14 +2064,22 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, r = r->skip[PF_SKIP_PROTO].ptr; #endif else if (PF_MISMATCHAW(&r->src.addr, - (struct pf_addr *)&h->ip6_src, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_src, AF_INET6, r->src.neg, kif)) r = r->skip[PF_SKIP_SRC_ADDR].ptr; else if (PF_MISMATCHAW(&r->dst.addr, - (struct pf_addr *)&h->ip6_dst, AF_INET6, + (struct pf_addr *)(uintptr_t)&h->ip6_dst, AF_INET6, r->dst.neg, NULL)) r = r->skip[PF_SKIP_DST_ADDR].ptr; - else + else { + if (r->anchor == NULL) + break; + else + pf_step_into_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL); + } + if (r == NULL && pf_step_out_of_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL)) break; } @@ -1176,36 +2091,38 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, } /* Check for illegal packets */ - if ((int)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + IPV6_MAXPACKET) < + pbuf->pb_packet_len) goto drop; off = sizeof (struct ip6_hdr); proto = h->ip6_nxt; terminal = 0; do { + pd->proto = proto; switch (proto) { case IPPROTO_FRAGMENT: goto fragment; - break; case IPPROTO_AH: case IPPROTO_ROUTING: case IPPROTO_DSTOPTS: - if (!pf_pull_hdr(m, off, &ext, sizeof (ext), NULL, + if (!pf_pull_hdr(pbuf, off, &ext, sizeof (ext), NULL, NULL, AF_INET6)) goto shortpkt; -#ifndef NO_APPLE_EXTENSIONS /* * + * Multiple routing headers not allowed. * Routing header type zero considered harmful. */ if (proto == IPPROTO_ROUTING) { const struct ip6_rthdr *rh = (const struct ip6_rthdr *)&ext; + if (rh_cnt++) + goto drop; if (rh->ip6r_type == IPV6_RTHDR_TYPE_0) goto drop; } else -#endif if (proto == IPPROTO_AH) off += (ext.ip6e_len + 2) * 4; else @@ -1274,7 +2191,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif, plen = ntohs(h->ip6_plen); if (plen == 0) goto drop; - if ((int)(sizeof (struct ip6_hdr) + plen) > m->m_pkthdr.len) + if ((uint32_t)(sizeof (struct ip6_hdr) + plen) > pbuf->pb_packet_len) goto shortpkt; /* Enforce a minimum ttl, may cause endless packet loops */ @@ -1288,56 +2205,158 @@ fragment: goto drop; plen = ntohs(h->ip6_plen); - if (!pf_pull_hdr(m, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) + if (!pf_pull_hdr(pbuf, off, &frag, sizeof (frag), NULL, NULL, AF_INET6)) goto shortpkt; fragoff = ntohs(frag.ip6f_offlg & IP6F_OFF_MASK); - if (fragoff + (plen - off - sizeof (frag)) > IPV6_MAXPACKET) - goto badfrag; + pd->proto = frag.ip6f_nxt; + mff = ntohs(frag.ip6f_offlg & IP6F_MORE_FRAG); + off += sizeof frag; + if (fragoff + (plen - off) > IPV6_MAXPACKET) + goto badfrag; + + fr_max = fragoff + plen - (off - sizeof(struct ip6_hdr)); +// XXX SCW: mbuf-specific +// DPFPRINTF(("0x%llx IPv6 frag plen %u mff %d off %u fragoff %u " +// "fr_max %u\n", (uint64_t)VM_KERNEL_ADDRPERM(m), plen, mff, off, +// fragoff, fr_max)); + + if ((r->rule_flag & (PFRULE_FRAGCROP|PFRULE_FRAGDROP)) == 0) { + /* Fully buffer all of the fragments */ + pd->flags |= PFDESC_IP_REAS; + + pff = pf_find_fragment_by_ipv6_header(h, &frag, + &pf_frag_tree); + + /* Check if we saw the last fragment already */ + if (pff != NULL && (pff->fr_flags & PFFRAG_SEENLAST) && + fr_max > pff->fr_max) + goto badfrag; + + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + + /* Get an entry for the fragment queue */ + frent = pool_get(&pf_frent_pl, PR_NOWAIT); + if (frent == NULL) { + REASON_SET(reason, PFRES_MEMORY); + return (PF_DROP); + } + + pf_nfrents++; + frent->fr_ip6 = h; + frent->fr_m = m; + frent->fr_ip6f_opt = frag; + frent->fr_ip6f_hlen = off; + + /* Might return a completely reassembled mbuf, or NULL */ + DPFPRINTF(("reass IPv6 frag %d @ %d-%d\n", + ntohl(frag.ip6f_ident), fragoff, fr_max)); + m = pf_reassemble6(&m, &pff, frent, mff); + + if (m == NULL) + return (PF_DROP); + + pbuf_init_mbuf(pbuf, m, ifp); + h = pbuf->pb_data; + + if (pff != NULL && (pff->fr_flags & PFFRAG_DROP)) + goto drop; + } + else if (dir == PF_IN || !(pd->pf_mtag->pftag_flags & PF_TAG_FRAGCACHE)) { + /* non-buffering fragment cache (overlaps: see RFC 5722) */ + int nomem = 0; + + pff = pf_find_fragment_by_ipv6_header(h, &frag, + &pf_cache_tree); + + /* Check if we saw the last fragment already */ + if (pff != NULL && (pff->fr_flags & PFFRAG_SEENLAST) && + fr_max > pff->fr_max) { + if (r->rule_flag & PFRULE_FRAGDROP) + pff->fr_flags |= PFFRAG_DROP; + goto badfrag; + } + + if ((m = pbuf_to_mbuf(pbuf, TRUE)) == NULL) { + goto no_mem; + } - /* do something about it */ - /* remember to set pd->flags |= PFDESC_IP_REAS */ + /* Restore iph pointer after pbuf_to_mbuf() */ + h = mtod(m, struct ip6_hdr *); + + m = pf_frag6cache(&m, h, &frag, &pff, off, mff, + (r->rule_flag & PFRULE_FRAGDROP) ? 1 : 0, &nomem); + if (m == NULL) { + // Note: pf_frag6cache() has already m_freem'd the mbuf + if (nomem) + goto no_mem; + goto drop; + } + + pbuf_init_mbuf(pbuf, m, ifp); + pd->pf_mtag = pf_find_mtag_pbuf(pbuf); + h = pbuf->pb_data; + + if (dir == PF_IN) + pd->pf_mtag->pftag_flags |= PF_TAG_FRAGCACHE; + + if (pff != NULL && (pff->fr_flags & PFFRAG_DROP)) + goto drop; + } + + /* Enforce a minimum ttl, may cause endless packet loops */ + if (r->min_ttl && h->ip6_hlim < r->min_ttl) + h->ip6_hlim = r->min_ttl; return (PF_PASS); -shortpkt: + no_mem: + REASON_SET(reason, PFRES_MEMORY); + goto dropout; + + shortpkt: REASON_SET(reason, PFRES_SHORT); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, - NULL, NULL, pd); - return (PF_DROP); - -drop: + goto dropout; + + drop: REASON_SET(reason, PFRES_NORM); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, - NULL, NULL, pd); - return (PF_DROP); - -badfrag: + goto dropout; + + badfrag: + DPFPRINTF(("dropping bad IPv6 fragment\n")); REASON_SET(reason, PFRES_FRAG); - if (r != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, - NULL, NULL, pd); + goto dropout; + + dropout: + if (pff != NULL) + pf_free_fragment(pff); + if (r != NULL && r->log && pbuf_is_valid(pbuf)) + PFLOG_PACKET(kif, h, pbuf, AF_INET6, dir, *reason, r, NULL, NULL, pd); return (PF_DROP); } #endif /* INET6 */ int -pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, +pf_normalize_tcp(int dir, struct pfi_kif *kif, pbuf_t *pbuf, int ipoff, int off, void *h, struct pf_pdesc *pd) { #pragma unused(ipoff, h) struct pf_rule *r, *rm = NULL; struct tcphdr *th = pd->hdr.tcp; int rewrite = 0; + int asd = 0; u_short reason; u_int8_t flags; sa_family_t af = pd->af; -#ifndef NO_APPLE_EXTENSIONS + struct pf_ruleset *ruleset = NULL; union pf_state_xport sxport, dxport; sxport.port = th->th_sport; dxport.port = th->th_dport; -#endif r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr); while (r != NULL) { @@ -1353,35 +2372,33 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, else if (PF_MISMATCHAW(&r->src.addr, pd->src, af, r->src.neg, kif)) r = r->skip[PF_SKIP_SRC_ADDR].ptr; -#ifndef NO_APPLE_EXTENSIONS else if (r->src.xport.range.op && !pf_match_xport(r->src.xport.range.op, r->proto_variant, &r->src.xport, &sxport)) -#else - else if (r->src.port_op && !pf_match_port(r->src.port_op, - r->src.port[0], r->src.port[1], th->th_sport)) -#endif r = r->skip[PF_SKIP_SRC_PORT].ptr; else if (PF_MISMATCHAW(&r->dst.addr, pd->dst, af, r->dst.neg, NULL)) r = r->skip[PF_SKIP_DST_ADDR].ptr; -#ifndef NO_APPLE_EXTENSIONS else if (r->dst.xport.range.op && !pf_match_xport(r->dst.xport.range.op, r->proto_variant, &r->dst.xport, &dxport)) -#else - else if (r->dst.port_op && !pf_match_port(r->dst.port_op, - r->dst.port[0], r->dst.port[1], th->th_dport)) -#endif r = r->skip[PF_SKIP_DST_PORT].ptr; else if (r->os_fingerprint != PF_OSFP_ANY && - !pf_osfp_match(pf_osfp_fingerprint(pd, m, off, th), + !pf_osfp_match(pf_osfp_fingerprint(pd, pbuf, off, th), r->os_fingerprint)) r = TAILQ_NEXT(r, entries); else { - rm = r; - break; + if (r->anchor == NULL) { + rm = r; + break; + } else { + pf_step_into_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL); + } } + if (r == NULL && pf_step_out_of_anchor(&asd, &ruleset, + PF_RULESET_SCRUB, &r, NULL, NULL)) + break; } if (rm == NULL || rm->action == PF_NOSCRUB) @@ -1439,49 +2456,39 @@ pf_normalize_tcp(int dir, struct pfi_kif *kif, struct mbuf *m, int ipoff, } /* copy back packet headers if we sanitized */ -#ifndef NO_APPLE_EXTENSIONS /* Process options */ if (r->max_mss) { - int rv = pf_normalize_tcpopt(r, dir, kif, pd, m, th, off, + int rv = pf_normalize_tcpopt(r, dir, kif, pd, pbuf, th, off, &rewrite); if (rv == PF_DROP) return rv; - m = pd->mp; + pbuf = pd->mp; } if (rewrite) { - struct mbuf *mw = pf_lazy_makewritable(pd, m, - off + sizeof (*th)); - if (!mw) { + if (pf_lazy_makewritable(pd, pbuf, + off + sizeof (*th)) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } - m_copyback(mw, off, sizeof (*th), th); + pbuf_copy_back(pbuf, off, sizeof (*th), th); } -#else - /* Process options */ - if (r->max_mss && pf_normalize_tcpopt(r, m, th, off, pd->af)) - rewrite = 1; - - if (rewrite) - m_copyback(m, off, sizeof (*th), th); -#endif return (PF_PASS); tcp_drop: REASON_SET(&reason, PFRES_NORM); if (rm != NULL && r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, r, NULL, NULL, pd); + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, NULL, NULL, pd); return (PF_DROP); } int -pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_init(pbuf_t *pbuf, int off, struct pf_pdesc *pd, struct tcphdr *th, struct pf_state_peer *src, struct pf_state_peer *dst) { #pragma unused(dst) @@ -1499,14 +2506,14 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, switch (pd->af) { #if INET case AF_INET: { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip_ttl; break; } #endif /* INET */ #if INET6 case AF_INET6: { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; src->scrub->pfss_ttl = h->ip6_hlim; break; } @@ -1523,7 +2530,7 @@ pf_normalize_tcp_init(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && src->scrub && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -1576,12 +2583,12 @@ pf_normalize_tcp_cleanup(struct pf_state *state) } int -pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, +pf_normalize_tcp_stateful(pbuf_t *pbuf, int off, struct pf_pdesc *pd, u_short *reason, struct tcphdr *th, struct pf_state *state, struct pf_state_peer *src, struct pf_state_peer *dst, int *writeback) { struct timeval uptime; - u_int32_t tsval, tsecr; + u_int32_t tsval = 0, tsecr = 0; u_int tsval_from_last; u_int8_t hdr[60]; u_int8_t *opt; @@ -1599,7 +2606,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET case AF_INET: { if (src->scrub) { - struct ip *h = mtod(m, struct ip *); + struct ip *h = pbuf->pb_data; if (h->ip_ttl > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip_ttl; h->ip_ttl = src->scrub->pfss_ttl; @@ -1610,7 +2617,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, #if INET6 case AF_INET6: { if (src->scrub) { - struct ip6_hdr *h = mtod(m, struct ip6_hdr *); + struct ip6_hdr *h = pbuf->pb_data; if (h->ip6_hlim > src->scrub->pfss_ttl) src->scrub->pfss_ttl = h->ip6_hlim; h->ip6_hlim = src->scrub->pfss_ttl; @@ -1623,7 +2630,7 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, if (th->th_off > (sizeof (struct tcphdr) >> 2) && ((src->scrub && (src->scrub->pfss_flags & PFSS_TIMESTAMP)) || (dst->scrub && (dst->scrub->pfss_flags & PFSS_TIMESTAMP))) && - pf_pull_hdr(m, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { + pf_pull_hdr(pbuf, off, hdr, th->th_off << 2, NULL, NULL, pd->af)) { /* Diddle with TCP options */ int hlen; opt = hdr + sizeof (struct tcphdr); @@ -1691,22 +2698,15 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, } if (copyback) { /* Copyback the options, caller copys back header */ -#ifndef NO_APPLE_EXTENSIONS int optoff = off + sizeof (*th); int optlen = (th->th_off << 2) - sizeof (*th); - m = pf_lazy_makewritable(pd, m, optoff + optlen); - if (!m) { + if (pf_lazy_makewritable(pd, pbuf, optoff + optlen) == + NULL) { REASON_SET(reason, PFRES_MEMORY); return PF_DROP; } *writeback = optoff + optlen; - m_copyback(m, optoff, optlen, hdr + sizeof (*th)); -#else - *writeback = 1; - m_copyback(m, off + sizeof (struct tcphdr), - (th->th_off << 2) - sizeof (struct tcphdr), hdr + - sizeof (struct tcphdr)); -#endif + pbuf_copy_back(pbuf, optoff, optlen, hdr + sizeof(*th)); } } @@ -1982,20 +2982,13 @@ pf_normalize_tcp_stateful(struct mbuf *m, int off, struct pf_pdesc *pd, return (0); } -#ifndef NO_APPLE_EXTENSIONS static int pf_normalize_tcpopt(struct pf_rule *r, int dir, struct pfi_kif *kif, - struct pf_pdesc *pd, struct mbuf *m, struct tcphdr *th, int off, + struct pf_pdesc *pd, pbuf_t *pbuf, struct tcphdr *th, int off, int *rewrptr) { #pragma unused(dir, kif) sa_family_t af = pd->af; -#else -static int -pf_normalize_tcpopt(struct pf_rule *r, struct mbuf *m, struct tcphdr *th, - int off, sa_family_t af) -{ -#endif u_int16_t *mss; int thoff; int opt, cnt, optlen = 0; @@ -2006,15 +2999,9 @@ pf_normalize_tcpopt(struct pf_rule *r, struct mbuf *m, struct tcphdr *th, thoff = th->th_off << 2; cnt = thoff - sizeof (struct tcphdr); -#ifndef NO_APPLE_MODIFICATIONS - if (cnt > 0 && !pf_pull_hdr(m, off + sizeof (*th), opts, cnt, + if (cnt > 0 && !pf_pull_hdr(pbuf, off + sizeof (*th), opts, cnt, NULL, NULL, af)) return PF_DROP; -#else - if (cnt > 0 && !pf_pull_hdr(m, off + sizeof (*th), opts, cnt, - NULL, NULL, af)) - return (rewrite); -#endif for (; cnt > 0; cnt -= optlen, optp += optlen) { opt = optp[0]; @@ -2031,22 +3018,17 @@ pf_normalize_tcpopt(struct pf_rule *r, struct mbuf *m, struct tcphdr *th, } switch (opt) { case TCPOPT_MAXSEG: - mss = (u_int16_t *)(optp + 2); + mss = (u_int16_t *)(void *)(optp + 2); if ((ntohs(*mss)) > r->max_mss) { -#ifndef NO_APPLE_MODIFICATIONS /* * * Only do the TCP checksum fixup if delayed * checksum calculation will not be performed. */ - if (m->m_pkthdr.rcvif || - !(m->m_pkthdr.csum_flags & CSUM_TCP)) + if (pbuf->pb_ifp || + !(*pbuf->pb_csum_flags & CSUM_TCP)) th->th_sum = pf_cksum_fixup(th->th_sum, *mss, htons(r->max_mss), 0); -#else - th->th_sum = pf_cksum_fixup(th->th_sum, - *mss, htons(r->max_mss), 0); -#endif *mss = htons(r->max_mss); rewrite = 1; } @@ -2056,30 +3038,23 @@ pf_normalize_tcpopt(struct pf_rule *r, struct mbuf *m, struct tcphdr *th, } } -#ifndef NO_APPLE_MODIFICATIONS if (rewrite) { - struct mbuf *mw; u_short reason; - mw = pf_lazy_makewritable(pd, pd->mp, - off + sizeof (*th) + thoff); - if (!mw) { + VERIFY(pbuf == pd->mp); + + if (pf_lazy_makewritable(pd, pd->mp, + off + sizeof (*th) + thoff) == NULL) { REASON_SET(&reason, PFRES_MEMORY); if (r->log) - PFLOG_PACKET(kif, h, m, AF_INET, dir, reason, + PFLOG_PACKET(kif, h, pbuf, AF_INET, dir, reason, r, 0, 0, pd); return PF_DROP; } *rewrptr = 1; - m_copyback(mw, off + sizeof (*th), thoff - sizeof (*th), opts); + pbuf_copy_back(pd->mp, off + sizeof (*th), thoff - sizeof (*th), opts); } return PF_PASS; -#else - if (rewrite) - m_copyback(m, off + sizeof (*th), thoff - sizeof (*th), opts); - - return (rewrite); -#endif }