]> git.saurik.com Git - apple/xnu.git/blobdiff - bsd/netinet6/esp_input.c
xnu-1228.0.2.tar.gz
[apple/xnu.git] / bsd / netinet6 / esp_input.c
index 3d5e88900029c0a97f3d5053e94792c074134ea8..1286b439f7f685764e519f8212ffa4a62cf38d12 100644 (file)
@@ -48,8 +48,8 @@
 
 #include <net/if.h>
 #include <net/route.h>
-#include <net/netisr.h>
 #include <kern/cpu_number.h>
+#include <kern/locks.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
@@ -58,6 +58,7 @@
 #include <netinet/in_var.h>
 #include <netinet/ip_ecn.h>
 #include <netinet/in_pcb.h>
+#include <netinet/udp.h>
 #if INET6
 #include <netinet6/ip6_ecn.h>
 #endif
 #include <netkey/keydb.h>
 #include <netkey/key_debug.h>
 
+#include <net/kpi_protocol.h>
+#include <netinet/kpi_ipfilter_var.h>
 
 #include <net/net_osdep.h>
 
+#include <sys/kdebug.h>
+#define DBG_LAYER_BEG          NETDBG_CODE(DBG_NETIPSEC, 1)
+#define DBG_LAYER_END          NETDBG_CODE(DBG_NETIPSEC, 3)
+#define DBG_FNC_ESPIN          NETDBG_CODE(DBG_NETIPSEC, (6 << 8))
+#define DBG_FNC_DECRYPT                NETDBG_CODE(DBG_NETIPSEC, (7 << 8))
 #define IPLEN_FLIPPED
 
+extern lck_mtx_t  *sadb_mutex;
+
 #if INET
 extern struct protosw inetsw[];
 
@@ -104,6 +114,7 @@ esp4_input(m, off)
        int off;
 {
        struct ip *ip;
+       struct ip6_hdr *ip6;
        struct esp *esp;
        struct esptail esptail;
        u_int32_t spi;
@@ -114,13 +125,14 @@ esp4_input(m, off)
        int ivlen;
        size_t hlen;
        size_t esplen;
-       int s;
+       sa_family_t     ifamily;
 
+       KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_START, 0,0,0,0,0);
        /* sanity check for alignment. */
        if (off % 4 != 0 || m->m_pkthdr.len % 4 != 0) {
                ipseclog((LOG_ERR, "IPv4 ESP input: packet alignment problem "
                        "(off=%d, pktlen=%d)\n", off, m->m_pkthdr.len));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -129,7 +141,7 @@ esp4_input(m, off)
                if (!m) {
                        ipseclog((LOG_DEBUG,
                            "IPv4 ESP input: can't pullup in esp4_input\n"));
-                       ipsecstat.in_inval++;
+                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                        goto bad;
                }
        }
@@ -151,7 +163,7 @@ esp4_input(m, off)
                ipseclog((LOG_WARNING,
                    "IPv4 ESP input: no key association found for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsecstat.in_nosa++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_nosa);
                goto bad;
        }
        KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -161,7 +173,7 @@ esp4_input(m, off)
                ipseclog((LOG_DEBUG,
                    "IPv4 ESP input: non-mature/dying SA found for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsecstat.in_badspi++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
                goto bad;
        }
        algo = esp_algorithm_lookup(sav->alg_enc);
@@ -169,7 +181,7 @@ esp4_input(m, off)
                ipseclog((LOG_DEBUG, "IPv4 ESP input: "
                    "unsupported encryption algorithm for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsecstat.in_badspi++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
                goto bad;
        }
 
@@ -178,7 +190,7 @@ esp4_input(m, off)
        if (ivlen < 0) {
                ipseclog((LOG_ERR, "inproper ivlen in IPv4 ESP input: %s %s\n",
                    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -196,7 +208,7 @@ esp4_input(m, off)
        if (ipsec_chkreplay(ntohl(((struct newesp *)esp)->esp_seq), sav))
                ; /*okey*/
        else {
-               ipsecstat.in_espreplay++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_espreplay);
                ipseclog((LOG_WARNING,
                    "replay packet in IPv4 ESP input: %s %s\n",
                    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
@@ -214,11 +226,15 @@ esp4_input(m, off)
        if (!sumalgo)
                goto noreplaycheck;
        siz = (((*sumalgo->sumsiz)(sav) + 3) & ~(4 - 1));
+       if (m->m_pkthdr.len < off + ESPMAXLEN + siz) {
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+               goto bad;
+       }
        if (AH_MAXSUMSIZE < siz) {
                ipseclog((LOG_DEBUG,
                    "internal error: AH_MAXSUMSIZE must be larger than %lu\n",
                    (u_long)siz));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -227,14 +243,14 @@ esp4_input(m, off)
        if (esp_auth(m, off, m->m_pkthdr.len - off - siz, sav, sum)) {
                ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
                    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-               ipsecstat.in_espauthfail++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail);
                goto bad;
        }
 
        if (bcmp(sum0, sum, siz) != 0) {
                ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
                    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-               ipsecstat.in_espauthfail++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail);
                goto bad;
        }
 
@@ -247,7 +263,7 @@ esp4_input(m, off)
        ip->ip_len = htons(ntohs(ip->ip_len) - siz);
 #endif
        m->m_flags |= M_AUTHIPDGM;
-       ipsecstat.in_espauthsucc++;
+       IPSEC_STAT_INCREMENT(ipsecstat.in_espauthsucc);
     }
 
        /*
@@ -255,7 +271,7 @@ esp4_input(m, off)
         */
        if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
                if (ipsec_updatereplay(ntohl(((struct newesp *)esp)->esp_seq), sav)) {
-                       ipsecstat.in_espreplay++;
+                       IPSEC_STAT_INCREMENT(ipsecstat.in_espreplay);
                        goto bad;
                }
        }
@@ -277,7 +293,7 @@ noreplaycheck:
        if (m->m_pkthdr.len < off + esplen + ivlen + sizeof(esptail)) {
                ipseclog((LOG_WARNING,
                    "IPv4 ESP input: packet too short\n"));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -286,7 +302,7 @@ noreplaycheck:
                if (!m) {
                        ipseclog((LOG_DEBUG,
                            "IPv4 ESP input: can't pullup in esp4_input\n"));
-                       ipsecstat.in_inval++;
+                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                        goto bad;
                }
        }
@@ -295,7 +311,7 @@ noreplaycheck:
         * pre-compute and cache intermediate key
         */
        if (esp_schedule(algo, sav) != 0) {
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -304,15 +320,18 @@ noreplaycheck:
         */
        if (!algo->decrypt)
                panic("internal error: no decrypt function");
+       KERNEL_DEBUG(DBG_FNC_DECRYPT | DBG_FUNC_START, 0,0,0,0,0);
        if ((*algo->decrypt)(m, off, sav, algo, ivlen)) {
                /* m is already freed */
                m = NULL;
                ipseclog((LOG_ERR, "decrypt fail in IPv4 ESP input: %s\n",
                    ipsec_logsastr(sav)));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+               KERNEL_DEBUG(DBG_FNC_DECRYPT | DBG_FUNC_END, 1,0,0,0,0);
                goto bad;
        }
-       ipsecstat.in_esphist[sav->alg_enc]++;
+       KERNEL_DEBUG(DBG_FNC_DECRYPT | DBG_FUNC_END, 2,0,0,0,0);
+       IPSEC_STAT_INCREMENT(ipsecstat.in_esphist[sav->alg_enc]);
 
        m->m_flags |= M_DECRYPTED;
 
@@ -329,7 +348,7 @@ noreplaycheck:
                ipseclog((LOG_WARNING,
                    "bad pad length in IPv4 ESP input: %s %s\n",
                    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-               ipsecstat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
                goto bad;
        }
 
@@ -343,7 +362,7 @@ noreplaycheck:
 #endif
 
        /* was it transmitted over the IPsec tunnel SA? */
-       if (ipsec4_tunnel_validate(m, off + esplen + ivlen, nxt, sav)) {
+       if (ipsec4_tunnel_validate(m, off + esplen + ivlen, nxt, sav, &ifamily)) {
                /*
                 * strip off all the headers that precedes ESP header.
                 *      IP4 xx ESP IP4' payload -> IP4' payload
@@ -355,51 +374,76 @@ noreplaycheck:
 
                tos = ip->ip_tos;
                m_adj(m, off + esplen + ivlen);
-               if (m->m_len < sizeof(*ip)) {
-                       m = m_pullup(m, sizeof(*ip));
-                       if (!m) {
-                               ipsecstat.in_inval++;
-                               goto bad;
+               if (ifamily == AF_INET) {
+                       if (m->m_len < sizeof(*ip)) {
+                               m = m_pullup(m, sizeof(*ip));
+                               if (!m) {
+                                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                                       goto bad;
+                               }
                        }
-               }
-               ip = mtod(m, struct ip *);
-               /* ECN consideration. */
-               ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
-               if (!key_checktunnelsanity(sav, AF_INET,
+                       ip = mtod(m, struct ip *);
+                       /* ECN consideration. */
+                       ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
+                       if (!key_checktunnelsanity(sav, AF_INET,
                            (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst)) {
-                       ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
-                           "in IPv4 ESP input: %s %s\n",
+                               ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
+                           "in ESP input: %s %s\n",
                            ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-                       ipsecstat.in_inval++;
-                       goto bad;
-               }
+                               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                               goto bad;
+                       }
+#if INET6
+               } else if (ifamily == AF_INET6) {
+#ifndef PULLDOWN_TEST
+                       /*
+                        * m_pullup is prohibited in KAME IPv6 input processing
+                        * but there's no other way!
+                        */
+#else
+                       /* okay to pullup in m_pulldown style */
+#endif
+                       if (m->m_len < sizeof(*ip6)) {
+                               m = m_pullup(m, sizeof(*ip6));
+                               if (!m) {
+                                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                                       goto bad;
+                               }
+                       }
 
-#if 0 /* XXX should call ipfw rather than ipsec_in_reject, shouldn't it ? */
-               /* drop it if it does not match the default policy */
-               if (ipsec4_in_reject(m, NULL)) {
-                       ipsecstat.in_polvio++;
+                       ip6 = mtod(m, struct ip6_hdr *);
+
+                       /* ECN consideration. */
+                       /* XXX To be fixed later if needed */
+                       //  ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
+
+                       if (!key_checktunnelsanity(sav, AF_INET6,
+                           (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst)) {
+                               ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
+                           "in ESP input: %s %s\n",
+                           ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
+                               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                               goto bad;
+                       }               
+#endif /* INET6 */
+               } else {
+                       ipseclog((LOG_ERR, "ipsec tunnel unsupported address family "
+                                 "in ESP input\n"));
                        goto bad;
                }
-#endif
 
                key_sa_recordxfer(sav, m);
                if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0 ||
                    ipsec_addhist(m, IPPROTO_IPV4, 0) != 0) {
-                       ipsecstat.in_nomem++;
+                       IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
                        goto bad;
                }
-
-               s = splimp();
-               if (IF_QFULL(&ipintrq)) {
-                       ipsecstat.in_inval++;
-                       splx(s);
-                       goto bad;
-               }
-               IF_ENQUEUE(&ipintrq, m);
-               m = NULL;
-               schednetisr(NETISR_IP); /*can be skipped but to make sure*/
-               splx(s);
+               
+               /* Clear the csum flags, they can't be valid for the inner headers */
+               m->m_pkthdr.csum_flags = 0;
+               proto_input(ifamily == AF_INET ? PF_INET : PF_INET6, m);
                nxt = IPPROTO_DONE;
+               KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 2,0,0,0,0);
        } else {
                /*
                 * strip off ESP header and IV.
@@ -426,17 +470,63 @@ noreplaycheck:
 
                key_sa_recordxfer(sav, m);
                if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0) {
-                       ipsecstat.in_nomem++;
+                       IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
                        goto bad;
                }
+               
+               /*
+                * Set the csum valid flag, if we authenticated the
+                * packet, the payload shouldn't be corrupt unless
+                * it was corrupted before being signed on the other
+                * side.
+                */
+               if (nxt == IPPROTO_TCP || nxt == IPPROTO_UDP) {
+                       m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+                       m->m_pkthdr.csum_data = 0xFFFF;
+               }
 
                if (nxt != IPPROTO_DONE) {
                        if ((ip_protox[nxt]->pr_flags & PR_LASTHDR) != 0 &&
                            ipsec4_in_reject(m, NULL)) {
-                               ipsecstat.in_polvio++;
+                               IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
                                goto bad;
                        }
-                       (*ip_protox[nxt]->pr_input)(m, off);
+                       KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 3,0,0,0,0);
+                       
+                       /* translate encapsulated UDP port ? */
+                       if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0)  {
+                               struct udphdr   *udp;
+                               
+                               if (nxt != IPPROTO_UDP) {       /* not UPD packet - drop it */
+                                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                                       goto bad;
+                               }
+                                       
+                               if (m->m_len < off + sizeof(struct udphdr)) {
+                                       m = m_pullup(m, off + sizeof(struct udphdr));
+                                       if (!m) {
+                                               ipseclog((LOG_DEBUG,
+                                                       "IPv4 ESP input: can't pullup UDP header in esp4_input\n"));
+                                               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                                               goto bad;
+                                       }
+                                       ip = mtod(m, struct ip *);
+                               }
+                               udp = (struct udphdr *)(((u_int8_t *)ip) + off);
+                       
+                               lck_mtx_lock(sadb_mutex);
+                               if (sav->natt_encapsulated_src_port == 0) {     
+                                       sav->natt_encapsulated_src_port = udp->uh_sport;
+                               } else if (sav->natt_encapsulated_src_port != udp->uh_sport) {  /* something wrong */
+                                       IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+                                       lck_mtx_unlock(sadb_mutex);
+                                       goto bad;
+                               }
+                               lck_mtx_unlock(sadb_mutex);
+                               udp->uh_sport = htons(sav->remote_ike_port);
+                               udp->uh_sum = 0;
+                       }
+                       ip_proto_dispatch_in(m, off, nxt, 0);
                } else
                        m_freem(m);
                m = NULL;
@@ -445,28 +535,29 @@ noreplaycheck:
        if (sav) {
                KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
                        printf("DP esp4_input call free SA:%p\n", sav));
-               key_freesav(sav);
+               key_freesav(sav, KEY_SADB_UNLOCKED);
        }
-       ipsecstat.in_success++;
+       IPSEC_STAT_INCREMENT(ipsecstat.in_success);
        return;
 
 bad:
        if (sav) {
                KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
                        printf("DP esp4_input call free SA:%p\n", sav));
-               key_freesav(sav);
+               key_freesav(sav, KEY_SADB_UNLOCKED);
        }
        if (m)
                m_freem(m);
+       KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 4,0,0,0,0);
        return;
 }
 #endif /* INET */
 
 #if INET6
 int
-esp6_input(mp, offp, proto)
+esp6_input(mp, offp)
        struct mbuf **mp;
-       int *offp, proto;
+       int *offp;
 {
        struct mbuf *m = *mp;
        int off = *offp;
@@ -480,23 +571,22 @@ esp6_input(mp, offp, proto)
        const struct esp_algorithm *algo;
        int ivlen;
        size_t esplen;
-       int s;
 
        /* sanity check for alignment. */
        if (off % 4 != 0 || m->m_pkthdr.len % 4 != 0) {
                ipseclog((LOG_ERR, "IPv6 ESP input: packet alignment problem "
                        "(off=%d, pktlen=%d)\n", off, m->m_pkthdr.len));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
 #ifndef PULLDOWN_TEST
-       IP6_EXTHDR_CHECK(m, off, ESPMAXLEN, IPPROTO_DONE);
+       IP6_EXTHDR_CHECK(m, off, ESPMAXLEN, {return IPPROTO_DONE;});
        esp = (struct esp *)(mtod(m, caddr_t) + off);
 #else
        IP6_EXTHDR_GET(esp, struct esp *, m, off, ESPMAXLEN);
        if (esp == NULL) {
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                return IPPROTO_DONE;
        }
 #endif
@@ -505,7 +595,7 @@ esp6_input(mp, offp, proto)
        if (ntohs(ip6->ip6_plen) == 0) {
                ipseclog((LOG_ERR, "IPv6 ESP input: "
                    "ESP with IPv6 jumbogram is not supported.\n"));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
@@ -518,7 +608,7 @@ esp6_input(mp, offp, proto)
                ipseclog((LOG_WARNING,
                    "IPv6 ESP input: no key association found for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsec6stat.in_nosa++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_nosa);
                goto bad;
        }
        KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -528,7 +618,7 @@ esp6_input(mp, offp, proto)
                ipseclog((LOG_DEBUG,
                    "IPv6 ESP input: non-mature/dying SA found for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsec6stat.in_badspi++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
                goto bad;
        }
        algo = esp_algorithm_lookup(sav->alg_enc);
@@ -536,7 +626,7 @@ esp6_input(mp, offp, proto)
                ipseclog((LOG_DEBUG, "IPv6 ESP input: "
                    "unsupported encryption algorithm for spi %u\n",
                    (u_int32_t)ntohl(spi)));
-               ipsec6stat.in_badspi++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
                goto bad;
        }
 
@@ -545,7 +635,7 @@ esp6_input(mp, offp, proto)
        if (ivlen < 0) {
                ipseclog((LOG_ERR, "inproper ivlen in IPv6 ESP input: %s %s\n",
                    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-               ipsec6stat.in_badspi++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
                goto bad;
        }
 
@@ -563,7 +653,7 @@ esp6_input(mp, offp, proto)
        if (ipsec_chkreplay(ntohl(((struct newesp *)esp)->esp_seq), sav))
                ; /*okey*/
        else {
-               ipsec6stat.in_espreplay++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_espreplay);
                ipseclog((LOG_WARNING,
                    "replay packet in IPv6 ESP input: %s %s\n",
                    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
@@ -581,11 +671,15 @@ esp6_input(mp, offp, proto)
        if (!sumalgo)
                goto noreplaycheck;
        siz = (((*sumalgo->sumsiz)(sav) + 3) & ~(4 - 1));
+       if (m->m_pkthdr.len < off + ESPMAXLEN + siz) {
+               IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+               goto bad;
+       }
        if (AH_MAXSUMSIZE < siz) {
                ipseclog((LOG_DEBUG,
                    "internal error: AH_MAXSUMSIZE must be larger than %lu\n",
                    (u_long)siz));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
@@ -594,14 +688,14 @@ esp6_input(mp, offp, proto)
        if (esp_auth(m, off, m->m_pkthdr.len - off - siz, sav, sum)) {
                ipseclog((LOG_WARNING, "auth fail in IPv6 ESP input: %s %s\n",
                    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-               ipsec6stat.in_espauthfail++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthfail);
                goto bad;
        }
 
        if (bcmp(sum0, sum, siz) != 0) {
                ipseclog((LOG_WARNING, "auth fail in IPv6 ESP input: %s %s\n",
                    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-               ipsec6stat.in_espauthfail++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthfail);
                goto bad;
        }
 
@@ -611,7 +705,7 @@ esp6_input(mp, offp, proto)
        ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - siz);
 
        m->m_flags |= M_AUTHIPDGM;
-       ipsec6stat.in_espauthsucc++;
+       IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthsucc);
     }
 
        /*
@@ -619,7 +713,7 @@ esp6_input(mp, offp, proto)
         */
        if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
                if (ipsec_updatereplay(ntohl(((struct newesp *)esp)->esp_seq), sav)) {
-                       ipsec6stat.in_espreplay++;
+                       IPSEC_STAT_INCREMENT(ipsec6stat.in_espreplay);
                        goto bad;
                }
        }
@@ -641,16 +735,16 @@ noreplaycheck:
        if (m->m_pkthdr.len < off + esplen + ivlen + sizeof(esptail)) {
                ipseclog((LOG_WARNING,
                    "IPv6 ESP input: packet too short\n"));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
 #ifndef PULLDOWN_TEST
-       IP6_EXTHDR_CHECK(m, off, esplen + ivlen, IPPROTO_DONE); /*XXX*/
+       IP6_EXTHDR_CHECK(m, off, esplen + ivlen, return IPPROTO_DONE);  /*XXX*/
 #else
        IP6_EXTHDR_GET(esp, struct esp *, m, off, esplen + ivlen);
        if (esp == NULL) {
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                m = NULL;
                goto bad;
        }
@@ -661,7 +755,7 @@ noreplaycheck:
         * pre-compute and cache intermediate key
         */
        if (esp_schedule(algo, sav) != 0) {
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
@@ -675,10 +769,10 @@ noreplaycheck:
                m = NULL;
                ipseclog((LOG_ERR, "decrypt fail in IPv6 ESP input: %s\n",
                    ipsec_logsastr(sav)));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
-       ipsec6stat.in_esphist[sav->alg_enc]++;
+       IPSEC_STAT_INCREMENT(ipsec6stat.in_esphist[sav->alg_enc]);
 
        m->m_flags |= M_DECRYPTED;
 
@@ -695,7 +789,7 @@ noreplaycheck:
                ipseclog((LOG_WARNING,
                    "bad pad length in IPv6 ESP input: %s %s\n",
                    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-               ipsec6stat.in_inval++;
+               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                goto bad;
        }
 
@@ -727,7 +821,7 @@ noreplaycheck:
 #endif
                        m = m_pullup(m, sizeof(*ip6));
                        if (!m) {
-                               ipsec6stat.in_inval++;
+                               IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                                goto bad;
                        }
                }
@@ -740,35 +834,17 @@ noreplaycheck:
                            "in IPv6 ESP input: %s %s\n",
                            ipsec6_logpacketstr(ip6, spi),
                            ipsec_logsastr(sav)));
-                       ipsec6stat.in_inval++;
+                       IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
                        goto bad;
                }
 
-#if 0 /* XXX should call ipfw rather than ipsec_in_reject, shouldn't it ? */
-               /* drop it if it does not match the default policy */
-               if (ipsec6_in_reject(m, NULL)) {
-                       ipsec6stat.in_polvio++;
-                       goto bad;
-               }
-#endif
-
                key_sa_recordxfer(sav, m);
                if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0 || 
                    ipsec_addhist(m, IPPROTO_IPV6, 0) != 0) {
-                       ipsec6stat.in_nomem++;
-                       goto bad;
-               }
-
-               s = splimp();
-               if (IF_QFULL(&ip6intrq)) {
-                       ipsec6stat.in_inval++;
-                       splx(s);
+                       IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
                        goto bad;
                }
-               IF_ENQUEUE(&ip6intrq, m);
-               m = NULL;
-               schednetisr(NETISR_IPV6); /*can be skipped but to make sure*/
-               splx(s);
+               proto_input(PF_INET6, m);
                nxt = IPPROTO_DONE;
        } else {
                /*
@@ -806,9 +882,9 @@ noreplaycheck:
                                goto bad;
                        }
                        m_adj(n, stripsiz);
-                       m_cat(m, n);
                        /* m_cat does not update m_pkthdr.len */
                        m->m_pkthdr.len += n->m_pkthdr.len;
+                       m_cat(m, n);
                }
 
 #ifndef PULLDOWN_TEST
@@ -822,7 +898,7 @@ noreplaycheck:
                        struct mbuf *n = NULL;
                        int maxlen;
 
-                       MGETHDR(n, M_DONTWAIT, MT_HEADER);
+                       MGETHDR(n, M_DONTWAIT, MT_HEADER);      /* MAC-OK */
                        maxlen = MHLEN;
                        if (n)
                                M_COPY_PKTHDR(n, m);
@@ -847,10 +923,10 @@ noreplaycheck:
                                m_freem(m);
                        } else {
                                m_copydata(m, 0, maxlen, mtod(n, caddr_t));
-                               m_adj(m, maxlen);
                                n->m_len = maxlen;
                                n->m_pkthdr.len = m->m_pkthdr.len;
                                n->m_next = m;
+                               m_adj(m, maxlen);
                                m->m_flags &= ~M_PKTHDR;
                        }
                        m = n;
@@ -862,7 +938,7 @@ noreplaycheck:
 
                key_sa_recordxfer(sav, m);
                if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0) {
-                       ipsec6stat.in_nomem++;
+                       IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
                        goto bad;
                }
        }
@@ -873,16 +949,16 @@ noreplaycheck:
        if (sav) {
                KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
                        printf("DP esp6_input call free SA:%p\n", sav));
-               key_freesav(sav);
+               key_freesav(sav, KEY_SADB_UNLOCKED);
        }
-       ipsec6stat.in_success++;
+       IPSEC_STAT_INCREMENT(ipsec6stat.in_success);
        return nxt;
 
 bad:
        if (sav) {
                KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
                        printf("DP esp6_input call free SA:%p\n", sav));
-               key_freesav(sav);
+               key_freesav(sav, KEY_SADB_UNLOCKED);
        }
        if (m)
                m_freem(m);
@@ -902,7 +978,7 @@ esp6_ctlinput(cmd, sa, d)
        struct ip6_hdr *ip6;
        struct mbuf *m;
        int off;
-       struct sockaddr_in6 sa6_src, sa6_dst;
+       struct sockaddr_in6 *sa6_src, *sa6_dst;
 
        if (sa->sa_family != AF_INET6 ||
            sa->sa_len != sizeof(struct sockaddr_in6))
@@ -966,15 +1042,17 @@ esp6_ctlinput(cmd, sa, d)
                         * Check to see if we have a valid SA corresponding to
                         * the address in the ICMP message payload.
                         */
+                       sa6_src = ip6cp->ip6c_src;
+                       sa6_dst = (struct sockaddr_in6 *)sa;
                        sav = key_allocsa(AF_INET6,
-                                         (caddr_t)&sa6_src.sin6_addr,
-                                         (caddr_t)&sa6_dst, IPPROTO_ESP,
-                                         espp->esp_spi);
+                                         (caddr_t)&sa6_src->sin6_addr,
+                                         (caddr_t)&sa6_dst->sin6_addr,
+                                         IPPROTO_ESP, espp->esp_spi);
                        if (sav) {
                                if (sav->state == SADB_SASTATE_MATURE ||
                                    sav->state == SADB_SASTATE_DYING)
                                        valid++;
-                               key_freesav(sav);
+                               key_freesav(sav, KEY_SADB_LOCKED);
                        }
 
                        /* XXX Further validation? */