xnu-4570.31.3.tar.gz

diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c
index 1867151e9a9288089b39384a52946b3e739a7f56..aa2ed85ad7c4edfb1ff968b45191644f266ff6fd 100644 (file)
--- a/bsd/kern/bsd_stubs.c
+++ b/bsd/kern/bsd_stubs.c
@@ -64,9 +64,9 @@ kmem_mb_alloc(vm_map_t mbmap, int size, int physContig, kern_return_t *err)
        kern_return_t kr = KERN_SUCCESS;
 
        if (!physContig)
        kern_return_t kr = KERN_SUCCESS;
 
        if (!physContig)

-               kr = kernel_memory_allocate(mbmap, &addr, size, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);
+               kr = kernel_memory_allocate(mbmap, &addr, size, 0, KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);

        else
        else

-               kr = kmem_alloc_contig(mbmap, &addr, size, PAGE_MASK, 0xfffff, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);
+               kr = kmem_alloc_contig(mbmap, &addr, size, PAGE_MASK, 0xfffff, 0, KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);

 
        if (kr != KERN_SUCCESS)
                addr = 0;
 
        if (kr != KERN_SUCCESS)
                addr = 0;

diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c
index cda6f10460cb3ec766e24d2291d0d1ee8afc32f1..099201dacdbf6559d6640cb2fe1cd04ebf4351fd 100644 (file)
--- a/bsd/kern/kern_control.c
+++ b/bsd/kern/kern_control.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 1999-2015 Apple Inc. All rights reserved.
+ * Copyright (c) 1999-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -72,6 +72,7 @@ struct kctl {
        u_int32_t               sendbufsize;    /* request more than the default buffer size */
 
        /* Dispatch functions */
        u_int32_t               sendbufsize;    /* request more than the default buffer size */
 
        /* Dispatch functions */

+       ctl_bind_func           bind;           /* Prepare contact */

        ctl_connect_func        connect;        /* Make contact */
        ctl_disconnect_func     disconnect;     /* Break contact */
        ctl_send_func           send;           /* Send data to nke */
        ctl_connect_func        connect;        /* Make contact */
        ctl_disconnect_func     disconnect;     /* Break contact */
        ctl_send_func           send;           /* Send data to nke */


@@ -90,7 +91,7 @@ struct ctl_cb {
        struct socket           *so;            /* controlling socket */
        struct kctl             *kctl;          /* back pointer to controller */
        void                    *userdata;
        struct socket           *so;            /* controlling socket */
        struct kctl             *kctl;          /* back pointer to controller */
        void                    *userdata;

-       u_int32_t               unit;
+       struct sockaddr_ctl     sac;

        u_int32_t               usecount;
 };
 
        u_int32_t               usecount;
 };
 


@@ -125,6 +126,7 @@ TAILQ_HEAD(kctl_list, kctl)         ctl_head;
 static int ctl_attach(struct socket *, int, struct proc *);
 static int ctl_detach(struct socket *);
 static int ctl_sofreelastref(struct socket *so);
 static int ctl_attach(struct socket *, int, struct proc *);
 static int ctl_detach(struct socket *);
 static int ctl_sofreelastref(struct socket *so);

+static int ctl_bind(struct socket *, struct sockaddr *, struct proc *);

 static int ctl_connect(struct socket *, struct sockaddr *, struct proc *);
 static int ctl_disconnect(struct socket *);
 static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
 static int ctl_connect(struct socket *, struct sockaddr *, struct proc *);
 static int ctl_disconnect(struct socket *);
 static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,


@@ -151,6 +153,7 @@ static lck_mtx_t * ctl_getlock(struct socket *, int);
 
 static struct pr_usrreqs ctl_usrreqs = {
        .pru_attach =           ctl_attach,
 
 static struct pr_usrreqs ctl_usrreqs = {
        .pru_attach =           ctl_attach,

+       .pru_bind =             ctl_bind,

        .pru_connect =          ctl_connect,
        .pru_control =          ctl_ioctl,
        .pru_detach =           ctl_detach,
        .pru_connect =          ctl_connect,
        .pru_control =          ctl_ioctl,
        .pru_detach =           ctl_detach,


@@ -354,28 +357,46 @@ ctl_detach(struct socket *so)
        if (kcb == 0)
                return (0);
 
        if (kcb == 0)
                return (0);
 

+       if (kcb->kctl != NULL && kcb->kctl->bind != NULL &&
+           kcb->userdata != NULL && !(so->so_state & SS_ISCONNECTED)) {
+               // The unit was bound, but not connected
+               // Invoke the disconnected call to cleanup
+               if (kcb->kctl->disconnect != NULL) {
+                       socket_unlock(so, 0);
+                       (*kcb->kctl->disconnect)(kcb->kctl->kctlref,
+                           kcb->sac.sc_unit, kcb->userdata);
+                       socket_lock(so, 0);
+               }
+       }
+

        soisdisconnected(so);
        so->so_flags |= SOF_PCBCLEARING;
        return (0);
 }
 
 static int
        soisdisconnected(so);
        so->so_flags |= SOF_PCBCLEARING;
        return (0);
 }
 
 static int

-ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
+ctl_setup_kctl(struct socket *so, struct sockaddr *nam, struct proc *p)

 {
 {

-#pragma unused(p)
-       struct kctl             *kctl;
-       int                     error = 0;
+       struct kctl *kctl = NULL;
+       int error = 0;

        struct sockaddr_ctl     sa;
        struct sockaddr_ctl     sa;

-       struct ctl_cb           *kcb = (struct ctl_cb *)so->so_pcb;
-       struct ctl_cb           *kcb_next = NULL;
-       u_quad_t                sbmaxsize;
-       u_int32_t               recvbufsize, sendbufsize;
+       struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
+       struct ctl_cb *kcb_next = NULL;
+       u_quad_t sbmaxsize;
+       u_int32_t recvbufsize, sendbufsize;

 
 

-       if (kcb == 0)
-               panic("ctl_connect so_pcb null\n");
+       if (kcb == 0) {
+               panic("ctl_setup_kctl so_pcb null\n");
+       }
+
+       if (kcb->kctl != NULL) {
+               // Already set up, skip
+               return (0);
+       }

 
 

-       if (nam->sa_len !=  sizeof(struct sockaddr_ctl))
+       if (nam->sa_len != sizeof(struct sockaddr_ctl)) {

                return (EINVAL);
                return (EINVAL);

+       }

 
        bcopy(nam, &sa, sizeof(struct sockaddr_ctl));
 
 
        bcopy(nam, &sa, sizeof(struct sockaddr_ctl));
 


@@ -387,12 +408,12 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
        }
 
        if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
        }
 
        if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&

-                       (so->so_type != SOCK_STREAM)) ||
+                (so->so_type != SOCK_STREAM)) ||

                (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
                (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&

-                       (so->so_type != SOCK_DGRAM))) {
-               lck_mtx_unlock(ctl_mtx);
-               return (EPROTOTYPE);
-       }
+                (so->so_type != SOCK_DGRAM))) {
+                       lck_mtx_unlock(ctl_mtx);
+                       return (EPROTOTYPE);
+               }

 
        if (kctl->flags & CTL_FLAG_PRIVILEGED) {
                if (p == 0) {
 
        if (kctl->flags & CTL_FLAG_PRIVILEGED) {
                if (p == 0) {


@@ -412,16 +433,17 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
                }
        } else {
                /* Find an unused ID, assumes control IDs are in order */
                }
        } else {
                /* Find an unused ID, assumes control IDs are in order */

-               u_int32_t       unit = 1;
+               u_int32_t unit = 1;

 
                TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {
 
                TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {

-                       if (kcb_next->unit > unit) {
+                       if (kcb_next->sac.sc_unit > unit) {

                                /* Found a gap, lets fill it in */
                                break;
                        }
                                /* Found a gap, lets fill it in */
                                break;
                        }

-                       unit = kcb_next->unit + 1;
-                       if (unit == ctl_maxunit)
+                       unit = kcb_next->sac.sc_unit + 1;
+                       if (unit == ctl_maxunit) {

                                break;
                                break;

+                       }

                }
 
                if (unit == ctl_maxunit) {
                }
 
                if (unit == ctl_maxunit) {


@@ -432,7 +454,7 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
                sa.sc_unit = unit;
        }
 
                sa.sc_unit = unit;
        }
 

-       kcb->unit = sa.sc_unit;
+       bcopy(&sa, &kcb->sac, sizeof(struct sockaddr_ctl));

        kcb->kctl = kctl;
        if (kcb_next != NULL) {
                TAILQ_INSERT_BEFORE(kcb_next, kcb, next);
        kcb->kctl = kctl;
        if (kcb_next != NULL) {
                TAILQ_INSERT_BEFORE(kcb_next, kcb, next);


@@ -450,36 +472,102 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
         */
        sbmaxsize = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES);
 
         */
        sbmaxsize = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES);
 

-       if (kctl->sendbufsize > sbmaxsize)
+       if (kctl->sendbufsize > sbmaxsize) {

                sendbufsize = sbmaxsize;
                sendbufsize = sbmaxsize;

-       else
+       } else {

                sendbufsize = kctl->sendbufsize;
                sendbufsize = kctl->sendbufsize;

+       }

 
 

-       if (kctl->recvbufsize > sbmaxsize)
+       if (kctl->recvbufsize > sbmaxsize) {

                recvbufsize = sbmaxsize;
                recvbufsize = sbmaxsize;

-       else
+       } else {

                recvbufsize = kctl->recvbufsize;
                recvbufsize = kctl->recvbufsize;

+       }

 
        error = soreserve(so, sendbufsize, recvbufsize);
        if (error) {
                if (ctl_debug)
                        printf("%s - soreserve(%llx, %u, %u) error %d\n",
 
        error = soreserve(so, sendbufsize, recvbufsize);
        if (error) {
                if (ctl_debug)
                        printf("%s - soreserve(%llx, %u, %u) error %d\n",

-                           __func__, (uint64_t)VM_KERNEL_ADDRPERM(so),
-                           sendbufsize, recvbufsize, error);
+                                  __func__, (uint64_t)VM_KERNEL_ADDRPERM(so),
+                                  sendbufsize, recvbufsize, error);

                goto done;
        }
                goto done;
        }

-       soisconnecting(so);
+
+done:
+       if (error) {
+               soisdisconnected(so);
+               lck_mtx_lock(ctl_mtx);
+               TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
+               kcb->kctl = NULL;
+               kcb->sac.sc_unit = 0;
+               kctlstat.kcs_pcbcount--;
+               kctlstat.kcs_gencnt++;
+               kctlstat.kcs_conn_fail++;
+               lck_mtx_unlock(ctl_mtx);
+       }
+       return (error);
+}
+
+static int
+ctl_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
+{
+       int error = 0;
+       struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
+
+       if (kcb == NULL) {
+               panic("ctl_bind so_pcb null\n");
+       }
+
+       error = ctl_setup_kctl(so, nam, p);
+       if (error) {
+               return (error);
+       }
+
+       if (kcb->kctl == NULL) {
+               panic("ctl_bind kctl null\n");
+       }
+
+       if (kcb->kctl->bind == NULL) {
+               return (EINVAL);
+       }

 
        socket_unlock(so, 0);
 
        socket_unlock(so, 0);

-       error = (*kctl->connect)(kctl->kctlref, &sa, &kcb->userdata);
+       error = (*kcb->kctl->bind)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);

        socket_lock(so, 0);
        socket_lock(so, 0);

-       if (error)
-               goto end;

 
 

+       return (error);
+}
+
+static int
+ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
+{
+       int error = 0;
+       struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
+
+       if (kcb == NULL) {
+               panic("ctl_connect so_pcb null\n");
+       }
+
+       error = ctl_setup_kctl(so, nam, p);
+       if (error) {
+               return (error);
+       }
+
+       if (kcb->kctl == NULL) {
+               panic("ctl_connect kctl null\n");
+       }
+
+       soisconnecting(so);
+       socket_unlock(so, 0);
+       error = (*kcb->kctl->connect)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
+       socket_lock(so, 0);
+       if (error) {
+               goto end;
+       }

        soisconnected(so);
 
 end:
        soisconnected(so);
 
 end:

-       if (error && kctl->disconnect) {
+       if (error && kcb->kctl->disconnect) {

                /*
                 * XXX Make sure we Don't check the return value
                 * of disconnect here.
                /*
                 * XXX Make sure we Don't check the return value
                 * of disconnect here.


@@ -490,16 +578,15 @@ end:
                 * ipsec/utun_ctl_disconnect.
                 */
                socket_unlock(so, 0);
                 * ipsec/utun_ctl_disconnect.
                 */
                socket_unlock(so, 0);

-               (*kctl->disconnect)(kctl->kctlref, kcb->unit, kcb->userdata);
+               (*kcb->kctl->disconnect)(kcb->kctl->kctlref, kcb->sac.sc_unit, kcb->userdata);

                socket_lock(so, 0);
        }
                socket_lock(so, 0);
        }

-done:

        if (error) {
                soisdisconnected(so);
                lck_mtx_lock(ctl_mtx);
        if (error) {
                soisdisconnected(so);
                lck_mtx_lock(ctl_mtx);

-               kcb->kctl = 0;
-               kcb->unit = 0;
-               TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
+               TAILQ_REMOVE(&kcb->kctl->kcb_head, kcb, next);
+               kcb->kctl = NULL;
+               kcb->sac.sc_unit = 0;

                kctlstat.kcs_pcbcount--;
                kctlstat.kcs_gencnt++;
                kctlstat.kcs_conn_fail++;
                kctlstat.kcs_pcbcount--;
                kctlstat.kcs_gencnt++;
                kctlstat.kcs_conn_fail++;


@@ -518,7 +605,7 @@ ctl_disconnect(struct socket *so)
 
                if (kctl && kctl->disconnect) {
                        socket_unlock(so, 0);
 
                if (kctl && kctl->disconnect) {
                        socket_unlock(so, 0);

-                       (*kctl->disconnect)(kctl->kctlref, kcb->unit,
+                       (*kctl->disconnect)(kctl->kctlref, kcb->sac.sc_unit,

                            kcb->userdata);
                        socket_lock(so, 0);
                }
                            kcb->userdata);
                        socket_lock(so, 0);
                }


@@ -528,7 +615,7 @@ ctl_disconnect(struct socket *so)
                socket_unlock(so, 0);
                lck_mtx_lock(ctl_mtx);
                kcb->kctl = 0;
                socket_unlock(so, 0);
                lck_mtx_lock(ctl_mtx);
                kcb->kctl = 0;

-               kcb->unit = 0;
+               kcb->sac.sc_unit = 0;

                while (kcb->usecount != 0) {
                        msleep(&kcb->usecount, ctl_mtx, 0, "kcb->usecount", 0);
                }
                while (kcb->usecount != 0) {
                        msleep(&kcb->usecount, ctl_mtx, 0, "kcb->usecount", 0);
                }


@@ -559,7 +646,7 @@ ctl_peeraddr(struct socket *so, struct sockaddr **nam)
        sc.sc_family = AF_SYSTEM;
        sc.ss_sysaddr = AF_SYS_CONTROL;
        sc.sc_id =  kctl->id;
        sc.sc_family = AF_SYSTEM;
        sc.ss_sysaddr = AF_SYS_CONTROL;
        sc.sc_id =  kctl->id;

-       sc.sc_unit = kcb->unit;
+       sc.sc_unit = kcb->sac.sc_unit;

 
        *nam = dup_sockaddr((struct sockaddr *)&sc, 1);
 
 
        *nam = dup_sockaddr((struct sockaddr *)&sc, 1);
 


@@ -609,7 +696,7 @@ ctl_usr_rcvd(struct socket *so, int flags)
 
        if (kctl->rcvd) {
                socket_unlock(so, 0);
 
        if (kctl->rcvd) {
                socket_unlock(so, 0);

-               (*kctl->rcvd)(kctl->kctlref, kcb->unit, kcb->userdata, flags);
+               (*kctl->rcvd)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, flags);

                socket_lock(so, 0);
        }
 
                socket_lock(so, 0);
        }
 


@@ -640,7 +727,7 @@ ctl_send(struct socket *so, int flags, struct mbuf *m,
        if (error == 0 && kctl->send) {
                so_tc_update_stats(m, so, m_get_service_class(m));
                socket_unlock(so, 0);
        if (error == 0 && kctl->send) {
                so_tc_update_stats(m, so, m_get_service_class(m));
                socket_unlock(so, 0);

-               error = (*kctl->send)(kctl->kctlref, kcb->unit, kcb->userdata,
+               error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata,

                    m, flags);
                socket_lock(so, 0);
        } else {
                    m, flags);
                socket_lock(so, 0);
        } else {


@@ -678,7 +765,7 @@ ctl_send_list(struct socket *so, int flags, struct mbuf *m,
                        so_tc_update_stats(nxt, so, m_get_service_class(nxt));
 
                socket_unlock(so, 0);
                        so_tc_update_stats(nxt, so, m_get_service_class(nxt));
 
                socket_unlock(so, 0);

-               error = (*kctl->send_list)(kctl->kctlref, kcb->unit,
+               error = (*kctl->send_list)(kctl->kctlref, kcb->sac.sc_unit,

                    kcb->userdata, m, flags);
                socket_lock(so, 0);
        } else if (error == 0 && kctl->send) {
                    kcb->userdata, m, flags);
                socket_lock(so, 0);
        } else if (error == 0 && kctl->send) {


@@ -688,7 +775,7 @@ ctl_send_list(struct socket *so, int flags, struct mbuf *m,
                        m->m_nextpkt = NULL;
                        so_tc_update_stats(m, so, m_get_service_class(m));
                        socket_unlock(so, 0);
                        m->m_nextpkt = NULL;
                        so_tc_update_stats(m, so, m_get_service_class(m));
                        socket_unlock(so, 0);

-                       error = (*kctl->send)(kctl->kctlref, kcb->unit,
+                       error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit,

                            kcb->userdata, m, flags);
                        socket_lock(so, 0);
                        m = nextpkt;
                            kcb->userdata, m, flags);
                        socket_lock(so, 0);
                        m = nextpkt;


@@ -1089,7 +1176,7 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt)
        struct ctl_cb   *kcb = (struct ctl_cb *)so->so_pcb;
        struct kctl     *kctl;
        int     error = 0;
        struct ctl_cb   *kcb = (struct ctl_cb *)so->so_pcb;
        struct kctl     *kctl;
        int     error = 0;

-       void    *data;
+       void    *data = NULL;

        size_t  len;
 
        if (sopt->sopt_level != SYSPROTO_CONTROL) {
        size_t  len;
 
        if (sopt->sopt_level != SYSPROTO_CONTROL) {


@@ -1106,11 +1193,9 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt)
                case SOPT_SET:
                        if (kctl->setopt == NULL)
                                return (ENOTSUP);
                case SOPT_SET:
                        if (kctl->setopt == NULL)
                                return (ENOTSUP);

-                       if (sopt->sopt_valsize == 0) {
-                               data = NULL;
-                       } else {
+                       if (sopt->sopt_valsize != 0) {

                                MALLOC(data, void *, sopt->sopt_valsize, M_TEMP,
                                MALLOC(data, void *, sopt->sopt_valsize, M_TEMP,

-                                       M_WAITOK);
+                                       M_WAITOK | M_ZERO);

                                if (data == NULL)
                                        return (ENOMEM);
                                error = sooptcopyin(sopt, data,
                                if (data == NULL)
                                        return (ENOMEM);
                                error = sooptcopyin(sopt, data,


@@ -1119,20 +1204,22 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt)
                        if (error == 0) {
                                socket_unlock(so, 0);
                                error = (*kctl->setopt)(kctl->kctlref,
                        if (error == 0) {
                                socket_unlock(so, 0);
                                error = (*kctl->setopt)(kctl->kctlref,

-                                   kcb->unit, kcb->userdata, sopt->sopt_name,
+                                   kcb->sac.sc_unit, kcb->userdata, sopt->sopt_name,

                                    data, sopt->sopt_valsize);
                                socket_lock(so, 0);
                        }
                                    data, sopt->sopt_valsize);
                                socket_lock(so, 0);
                        }

-                       FREE(data, M_TEMP);
+
+                       if (data != NULL)
+                               FREE(data, M_TEMP);

                        break;
 
                case SOPT_GET:
                        if (kctl->getopt == NULL)
                                return (ENOTSUP);
                        break;
 
                case SOPT_GET:
                        if (kctl->getopt == NULL)
                                return (ENOTSUP);

-                       data = NULL;
+

                        if (sopt->sopt_valsize && sopt->sopt_val) {
                                MALLOC(data, void *, sopt->sopt_valsize, M_TEMP,
                        if (sopt->sopt_valsize && sopt->sopt_val) {
                                MALLOC(data, void *, sopt->sopt_valsize, M_TEMP,

-                                       M_WAITOK);
+                                       M_WAITOK | M_ZERO);

                                if (data == NULL)
                                        return (ENOMEM);
                                /*
                                if (data == NULL)
                                        return (ENOMEM);
                                /*


@@ -1142,22 +1229,25 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt)
                                error = sooptcopyin(sopt, data,
                                        sopt->sopt_valsize, sopt->sopt_valsize);
                        }
                                error = sooptcopyin(sopt, data,
                                        sopt->sopt_valsize, sopt->sopt_valsize);
                        }

-                       len = sopt->sopt_valsize;
-                       socket_unlock(so, 0);
-                       error = (*kctl->getopt)(kctl->kctlref, kcb->unit,
-                                       kcb->userdata, sopt->sopt_name,
-                                               data, &len);
-                       if (data != NULL && len > sopt->sopt_valsize)
-                               panic_plain("ctl_ctloutput: ctl %s returned "
-                                       "len (%lu) > sopt_valsize (%lu)\n",
-                                               kcb->kctl->name, len,
-                                               sopt->sopt_valsize);
-                       socket_lock(so, 0);
+

                        if (error == 0) {
                        if (error == 0) {

-                               if (data != NULL)
-                                       error = sooptcopyout(sopt, data, len);
-                               else
-                                       sopt->sopt_valsize = len;
+                               len = sopt->sopt_valsize;
+                               socket_unlock(so, 0);
+                               error = (*kctl->getopt)(kctl->kctlref, kcb->sac.sc_unit,
+                                               kcb->userdata, sopt->sopt_name,
+                                               data, &len);
+                               if (data != NULL && len > sopt->sopt_valsize)
+                                       panic_plain("ctl_ctloutput: ctl %s returned "
+                                           "len (%lu) > sopt_valsize (%lu)\n",
+                                           kcb->kctl->name, len,
+                                           sopt->sopt_valsize);
+                               socket_lock(so, 0);
+                               if (error == 0) {
+                                       if (data != NULL)
+                                               error = sooptcopyout(sopt, data, len);
+                                       else
+                                               sopt->sopt_valsize = len;
+                               }

                        }
                        if (data != NULL)
                                FREE(data, M_TEMP);
                        }
                        if (data != NULL)
                                FREE(data, M_TEMP);


@@ -1496,6 +1586,7 @@ ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
                kctl->recvbufsize = userkctl->ctl_recvsize;
        }
 
                kctl->recvbufsize = userkctl->ctl_recvsize;
        }
 

+       kctl->bind = userkctl->ctl_bind;

        kctl->connect = userkctl->ctl_connect;
        kctl->disconnect = userkctl->ctl_disconnect;
        kctl->send = userkctl->ctl_send;
        kctl->connect = userkctl->ctl_connect;
        kctl->disconnect = userkctl->ctl_disconnect;
        kctl->send = userkctl->ctl_send;


@@ -1643,7 +1734,7 @@ kcb_find(struct kctl *kctl, u_int32_t unit)
        lck_mtx_assert(ctl_mtx, LCK_MTX_ASSERT_OWNED);
 
        TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
        lck_mtx_assert(ctl_mtx, LCK_MTX_ASSERT_OWNED);
 
        TAILQ_FOREACH(kcb, &kctl->kcb_head, next)

-               if (kcb->unit == unit)
+               if (kcb->sac.sc_unit == unit)

                        return (kcb);
 
        return (NULL);
                        return (kcb);
 
        return (NULL);


@@ -2014,7 +2105,7 @@ kctl_pcblist SYSCTL_HANDLER_ARGS
 
                        xk->xkp_len = sizeof(struct xkctlpcb);
                        xk->xkp_kind = XSO_KCB;
 
                        xk->xkp_len = sizeof(struct xkctlpcb);
                        xk->xkp_kind = XSO_KCB;

-                       xk->xkp_unit = kcb->unit;
+                       xk->xkp_unit = kcb->sac.sc_unit;

                        xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb);
                        xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl);
                        xk->xkp_kctlid = kctl->id;
                        xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb);
                        xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl);
                        xk->xkp_kctlid = kctl->id;


@@ -2099,6 +2190,6 @@ kctl_fill_socketinfo(struct socket *so, struct socket_info *si)
        kcsi->kcsi_flags = kctl->flags;
        kcsi->kcsi_recvbufsize = kctl->recvbufsize;
        kcsi->kcsi_sendbufsize = kctl->sendbufsize;
        kcsi->kcsi_flags = kctl->flags;
        kcsi->kcsi_recvbufsize = kctl->recvbufsize;
        kcsi->kcsi_sendbufsize = kctl->sendbufsize;

-       kcsi->kcsi_unit = kcb->unit;
+       kcsi->kcsi_unit = kcb->sac.sc_unit;

        strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
 }
        strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
 }

diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c
index f64bef436b145a5d12446f9c6199ae7f39ec6cc4..12885133e1b9c755ecc3674bec9e83499843d8ba 100644 (file)
--- a/bsd/kern/kern_event.c
+++ b/bsd/kern/kern_event.c
@@ -382,6 +382,7 @@ SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = {
 extern const struct filterops pipe_rfiltops;
 extern const struct filterops pipe_wfiltops;
 extern const struct filterops ptsd_kqops;
 extern const struct filterops pipe_rfiltops;
 extern const struct filterops pipe_wfiltops;
 extern const struct filterops ptsd_kqops;

+extern const struct filterops ptmx_kqops;

 extern const struct filterops soread_filtops;
 extern const struct filterops sowrite_filtops;
 extern const struct filterops sock_filtops;
 extern const struct filterops soread_filtops;
 extern const struct filterops sowrite_filtops;
 extern const struct filterops sock_filtops;


@@ -449,7 +450,8 @@ SECURITY_READ_ONLY_EARLY(static struct filterops *) sysfilt_ops[EVFILTID_MAX] =
        [EVFILTID_NECP_FD]                              = &necp_fd_rfiltops,
        [EVFILTID_FSEVENT]                              = &fsevent_filtops,
        [EVFILTID_VN]                                   = &vnode_filtops,
        [EVFILTID_NECP_FD]                              = &necp_fd_rfiltops,
        [EVFILTID_FSEVENT]                              = &fsevent_filtops,
        [EVFILTID_VN]                                   = &vnode_filtops,

-       [EVFILTID_TTY]                                  = &tty_filtops
+       [EVFILTID_TTY]                                  = &tty_filtops,
+       [EVFILTID_PTMX]                                 = &ptmx_kqops,

 };
 
 /* waitq prepost callback */
 };
 
 /* waitq prepost callback */


@@ -2107,7 +2109,8 @@ filt_wldebounce(
 
 #if DEBUG || DEVELOPMENT
                if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && !(kev->flags & EV_DELETE)) {
 
 #if DEBUG || DEVELOPMENT
                if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && !(kev->flags & EV_DELETE)) {

-                       if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0) {
+                       if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0 &&
+                                       (udata >> 48) != 0 && (udata >> 48) != 0xffff) {

                                panic("kevent: workloop %#016llx is not enqueued "
                                                "(kev:%p dq_state:%#016llx)", kev->udata, kev, udata);
                        }
                                panic("kevent: workloop %#016llx is not enqueued "
                                                "(kev:%p dq_state:%#016llx)", kev->udata, kev, udata);
                        }


@@ -2807,7 +2810,8 @@ filt_wlprocess(
                        uint64_t val;
                        if (addr && task_is_active(t) && !task_is_halting(t) &&
                                        copyin_word(addr, &val, sizeof(val)) == 0 &&
                        uint64_t val;
                        if (addr && task_is_active(t) && !task_is_halting(t) &&
                                        copyin_word(addr, &val, sizeof(val)) == 0 &&

-                                       val && (val & DISPATCH_QUEUE_ENQUEUED) == 0) {
+                                       val && (val & DISPATCH_QUEUE_ENQUEUED) == 0 &&
+                                       (val >> 48) != 0 && (val >> 48) != 0xffff) {

                                panic("kevent: workloop %#016llx is not enqueued "
                                                "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)",
                                                kn->kn_udata, kn, val,
                                panic("kevent: workloop %#016llx is not enqueued "
                                                "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)",
                                                kn->kn_udata, kn, val,

diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c
index df43f2013bec20407bf001cf4a9db579281c632a..3ccb12119150b7bc2eab719167fca3253c62eda8 100644 (file)
--- a/bsd/kern/kern_exec.c
+++ b/bsd/kern/kern_exec.c
@@ -1320,12 +1320,6 @@ badtoolate:
        }
        
 done:
        }
        
 done:

-       if (!spawn) {
-               /* notify only if it has not failed due to FP Key error */
-               if ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0)
-                       proc_knote(p, NOTE_EXEC);
-       }
-

        if (load_result.threadstate) {
                kfree(load_result.threadstate, load_result.threadstate_sz);
                load_result.threadstate = NULL;
        if (load_result.threadstate) {
                kfree(load_result.threadstate, load_result.threadstate_sz);
                load_result.threadstate = NULL;


@@ -1991,7 +1985,8 @@ spawn_copyin_macpolicyinfo(const struct user__posix_spawn_args_desc *px_args, _p
        if ((error = copyin(px_args->mac_extensions, psmx, px_args->mac_extensions_size)) != 0)
                goto bad;
 
        if ((error = copyin(px_args->mac_extensions, psmx, px_args->mac_extensions_size)) != 0)
                goto bad;
 

-       if (PS_MAC_EXTENSIONS_SIZE(psmx->psmx_count) > px_args->mac_extensions_size) {
+       size_t extsize = PS_MAC_EXTENSIONS_SIZE(psmx->psmx_count);
+       if (extsize == 0 || extsize > px_args->mac_extensions_size) {

                error = EINVAL;
                goto bad;
        }
                error = EINVAL;
                goto bad;
        }


@@ -2304,8 +2299,9 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval)
                if (px_args.file_actions_size != 0) {
                        /* Limit file_actions to allowed number of open files */
                        int maxfa = (p->p_limit ? p->p_rlimit[RLIMIT_NOFILE].rlim_cur : NOFILE);
                if (px_args.file_actions_size != 0) {
                        /* Limit file_actions to allowed number of open files */
                        int maxfa = (p->p_limit ? p->p_rlimit[RLIMIT_NOFILE].rlim_cur : NOFILE);

+                       size_t maxfa_size = PSF_ACTIONS_SIZE(maxfa);

                        if (px_args.file_actions_size < PSF_ACTIONS_SIZE(1) ||
                        if (px_args.file_actions_size < PSF_ACTIONS_SIZE(1) ||

-                               px_args.file_actions_size > PSF_ACTIONS_SIZE(maxfa)) {
+                           maxfa_size == 0 || px_args.file_actions_size > maxfa_size) {

                                error = EINVAL;
                                goto bad;
                        }
                                error = EINVAL;
                                goto bad;
                        }


@@ -2321,7 +2317,8 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval)
                                goto bad;
 
                        /* Verify that the action count matches the struct size */
                                goto bad;
 
                        /* Verify that the action count matches the struct size */

-                       if (PSF_ACTIONS_SIZE(px_sfap->psfa_act_count) != px_args.file_actions_size) {
+                       size_t psfsize = PSF_ACTIONS_SIZE(px_sfap->psfa_act_count);
+                       if (psfsize == 0 || psfsize != px_args.file_actions_size) {

                                error = EINVAL;
                                goto bad;
                        }
                                error = EINVAL;
                                goto bad;
                        }


@@ -2347,7 +2344,8 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval)
                                goto bad;
 
                        /* Verify that the action count matches the struct size */
                                goto bad;
 
                        /* Verify that the action count matches the struct size */

-                       if (PS_PORT_ACTIONS_SIZE(px_spap->pspa_count) != px_args.port_actions_size) {
+                       size_t pasize = PS_PORT_ACTIONS_SIZE(px_spap->pspa_count);
+                       if (pasize == 0 || pasize != px_args.port_actions_size) {

                                error = EINVAL;
                                goto bad;
                        }
                                error = EINVAL;
                                goto bad;
                        }


@@ -2944,12 +2942,13 @@ bad:
 
                /* flag the 'fork' has occurred */
                proc_knote(p->p_pptr, NOTE_FORK | p->p_pid);
 
                /* flag the 'fork' has occurred */
                proc_knote(p->p_pptr, NOTE_FORK | p->p_pid);

-               /* then flag exec has occurred */
-               /* notify only if it has not failed due to FP Key error */
-               if ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0)
-                       proc_knote(p, NOTE_EXEC);

        }
 
        }
 

+       /* flag exec has occurred, notify only if it has not failed due to FP Key error */
+       if (!error && ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0))
+               proc_knote(p, NOTE_EXEC);
+
+

        if (error == 0) {
                /*
                 * We need to initialize the bank context behind the protection of
        if (error == 0) {
                /*
                 * We need to initialize the bank context behind the protection of


@@ -3522,7 +3521,12 @@ __mac_execve(proc_t p, struct __mac_execve_args *uap, int32_t *retval)
 
                exec_resettextvp(p, imgp);
                error = check_for_signature(p, imgp);
 
                exec_resettextvp(p, imgp);
                error = check_for_signature(p, imgp);

-       }       
+       }
+
+       /* flag exec has occurred, notify only if it has not failed due to FP Key error */
+       if (exec_done && ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0))
+               proc_knote(p, NOTE_EXEC);
+

        if (imgp->ip_vp != NULLVP)
                vnode_put(imgp->ip_vp);
        if (imgp->ip_scriptvp != NULLVP)
        if (imgp->ip_vp != NULLVP)
                vnode_put(imgp->ip_vp);
        if (imgp->ip_scriptvp != NULLVP)

diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c
index b8f5def79fd308f346eed191d51d1dbae0a4759e..fdbee03d10c298ee814d5bd7eb2f9da12b7e4915 100644 (file)
--- a/bsd/kern/kern_exit.c
+++ b/bsd/kern/kern_exit.c
@@ -2668,6 +2668,9 @@ vfork_exit_internal(proc_t p, int rv, int forceexit)
 __private_extern__  void 
 munge_user64_rusage(struct rusage *a_rusage_p, struct user64_rusage *a_user_rusage_p)
 {
 __private_extern__  void 
 munge_user64_rusage(struct rusage *a_rusage_p, struct user64_rusage *a_user_rusage_p)
 {

+       /* Zero-out struct so that padding is cleared */
+       bzero(a_user_rusage_p, sizeof(struct user64_rusage));
+

        /* timeval changes size, so utime and stime need special handling */
        a_user_rusage_p->ru_utime.tv_sec = a_rusage_p->ru_utime.tv_sec;
        a_user_rusage_p->ru_utime.tv_usec = a_rusage_p->ru_utime.tv_usec;
        /* timeval changes size, so utime and stime need special handling */
        a_user_rusage_p->ru_utime.tv_sec = a_rusage_p->ru_utime.tv_sec;
        a_user_rusage_p->ru_utime.tv_usec = a_rusage_p->ru_utime.tv_usec;

diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c
index 74a80dc55f7e843138d9794026dc58e9de4204e3..54e431d0556de2354a2ed1975806a179e0d0e7ad 100644 (file)
--- a/bsd/kern/kern_memorystatus.c
+++ b/bsd/kern/kern_memorystatus.c
@@ -4123,11 +4123,87 @@ memorystatus_get_task_memory_region_count(task_t task, uint64_t *count)
        *count = get_task_memory_region_count(task);
 }
 
        *count = get_task_memory_region_count(task);
 }
 

+#if DEVELOPMENT || DEBUG
+
+/*
+ * Sysctl only used to test memorystatus_allowed_vm_map_fork() path.
+ *   set a new pidwatch value
+ *     or
+ *   get the current pidwatch value
+ */
+
+uint64_t memorystatus_vm_map_fork_pidwatch_val = 0;
+#define MEMORYSTATUS_VM_MAP_FORK_ALLOWED     0x100000000
+#define MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED 0x200000000
+
+static int sysctl_memorystatus_vm_map_fork_pidwatch SYSCTL_HANDLER_ARGS {
+#pragma unused(oidp, arg1, arg2)
+
+        uint64_t new_value = 0;
+       uint64_t old_value = 0;
+        int error = 0;
+
+       /*
+        * The pid is held in the low 32 bits.
+        * The 'allowed' flags are in the upper 32 bits.
+        */
+       old_value = memorystatus_vm_map_fork_pidwatch_val;
+
+        error = sysctl_io_number(req, old_value, sizeof(old_value), &new_value, NULL);
+
+        if (error || !req->newptr) {
+               /*
+                * No new value passed in.
+                */
+               return(error);
+       }
+
+       /*
+        * A new pid was passed in via req->newptr.
+        * Ignore any attempt to set the higher order bits.
+        */
+       memorystatus_vm_map_fork_pidwatch_val = new_value & 0xFFFFFFFF;
+       printf("memorystatus: pidwatch old_value = 0x%llx, new_value = 0x%llx \n", old_value, new_value);
+
+        return(error);
+}
+
+SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_map_fork_pidwatch, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED| CTLFLAG_MASKED,
+            0, 0, sysctl_memorystatus_vm_map_fork_pidwatch, "Q", "get/set pid watched for in vm_map_fork");
+
+
+#define SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task)                                                 \
+MACRO_BEGIN                                                                                    \
+if (memorystatus_vm_map_fork_pidwatch_val != 0) {                                              \
+       proc_t p = get_bsdtask_info(task);                                                      \
+       if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) {               \
+               memorystatus_vm_map_fork_pidwatch_val |= MEMORYSTATUS_VM_MAP_FORK_ALLOWED;      \
+       }                                                                                       \
+}                                                                                              \
+MACRO_END
+
+#define SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task)                                             \
+MACRO_BEGIN                                                                                    \
+if (memorystatus_vm_map_fork_pidwatch_val != 0) {                                              \
+       proc_t p = get_bsdtask_info(task);                                                      \
+       if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) {               \
+               memorystatus_vm_map_fork_pidwatch_val |= MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED;  \
+       }                                                                                       \
+}                                                                                              \
+MACRO_END
+
+#else /* DEVELOPMENT || DEBUG */
+
+#define SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task)
+#define SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task)
+
+#endif /* DEVELOPMENT || DEBUG */
+

 /*
  * Called during EXC_RESOURCE handling when a process exceeds a soft
  * memory limit.  This is the corpse fork path and here we decide if
  * vm_map_fork will be allowed when creating the corpse.
 /*
  * Called during EXC_RESOURCE handling when a process exceeds a soft
  * memory limit.  This is the corpse fork path and here we decide if
  * vm_map_fork will be allowed when creating the corpse.

- * The current task is suspended.
+ * The task being considered is suspended.

  *
  * By default, a vm_map_fork is allowed to proceed.
  *
  *
  * By default, a vm_map_fork is allowed to proceed.
  *


@@ -4156,6 +4232,7 @@ memorystatus_allowed_vm_map_fork(__unused task_t task)
        uint64_t max_allowed_bytes = 0;
 
        if (max_task_footprint_mb == 0) {
        uint64_t max_allowed_bytes = 0;
 
        if (max_task_footprint_mb == 0) {

+               SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task);

                return (is_allowed);
        }
 
                return (is_allowed);
        }
 


@@ -4172,14 +4249,17 @@ memorystatus_allowed_vm_map_fork(__unused task_t task)
        }
 
        if (footprint_in_bytes <= max_allowed_bytes) {
        }
 
        if (footprint_in_bytes <= max_allowed_bytes) {

+               SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task);

                return (is_allowed);
        } else {
                printf("memorystatus disallowed vm_map_fork %lld  %lld\n", footprint_in_bytes, max_allowed_bytes);
                return (is_allowed);
        } else {
                printf("memorystatus disallowed vm_map_fork %lld  %lld\n", footprint_in_bytes, max_allowed_bytes);

+               SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task);

                return (!is_allowed);
        }
 
 #else /* CONFIG_EMBEDDED */
 
                return (!is_allowed);
        }
 
 #else /* CONFIG_EMBEDDED */
 

+       SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task);

        return (is_allowed);
 
 #endif /* CONFIG_EMBEDDED */
        return (is_allowed);
 
 #endif /* CONFIG_EMBEDDED */

diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c
index dd57cd72238d8cf905bf50f7cb6e365f66388625..d3a9e2c060fb397e25cfdc225fc3fa8f47f357a8 100644 (file)
--- a/bsd/kern/kern_mib.c
+++ b/bsd/kern/kern_mib.c
@@ -594,8 +594,7 @@ sysctl_mib_init(void)
        watchpoint_flag = arm_debug_info()->num_watchpoint_pairs;
        breakpoint_flag = arm_debug_info()->num_breakpoint_pairs;
 
        watchpoint_flag = arm_debug_info()->num_watchpoint_pairs;
        breakpoint_flag = arm_debug_info()->num_breakpoint_pairs;
 

-       arm_mvfp_info_t *mvfp_info;
-       mvfp_info = arm_mvfp_info();
+       arm_mvfp_info_t *mvfp_info = arm_mvfp_info();

        gNeon = mvfp_info->neon;
        gNeonHpfp = mvfp_info->neon_hpfp;
 
        gNeon = mvfp_info->neon;
        gNeonHpfp = mvfp_info->neon_hpfp;
 

diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c
index c599a4bc7e033551197db59d5b686ee7902f0276..83bd75cd2d8dffa88363cacf6860269c23ac2226 100644 (file)
--- a/bsd/kern/kern_proc.c
+++ b/bsd/kern/kern_proc.c
@@ -163,7 +163,9 @@ extern int cs_enforcement_enable;
 #endif
 #if CONFIG_COREDUMP
 /* Name to give to core files */
 #endif
 #if CONFIG_COREDUMP
 /* Name to give to core files */

-#if CONFIG_EMBEDDED
+#if defined(XNU_TARGET_OS_BRIDGE)
+__XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/internal/%N.core"};
+#elif CONFIG_EMBEDDED

 __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/cores/%N.core"};
 #else
 __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"};
 __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/cores/%N.core"};
 #else
 __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"};

diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c
index af90ce9fffecc85c22fc79deb67ffd94d4b2f3ed..63bcc0443e4d26625ecf997e432aa56b1f216eaf 100644 (file)
--- a/bsd/kern/kern_sysctl.c
+++ b/bsd/kern/kern_sysctl.c
@@ -1622,8 +1622,13 @@ SYSCTL_STRING(_kern, KERN_VERSION, version,
 SYSCTL_STRING(_kern, OID_AUTO, uuid, 
                CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 
                &kernel_uuid_string[0], 0, "");
 SYSCTL_STRING(_kern, OID_AUTO, uuid, 
                CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, 
                &kernel_uuid_string[0], 0, "");

-

 #if DEBUG
 #if DEBUG

+#ifndef DKPR
+#define DKPR 1
+#endif
+#endif
+
+#if DKPR

 int debug_kprint_syscall = 0;
 char debug_kprint_syscall_process[MAXCOMLEN+1];
 
 int debug_kprint_syscall = 0;
 char debug_kprint_syscall_process[MAXCOMLEN+1];
 

diff --git a/bsd/kern/kern_xxx.c b/bsd/kern/kern_xxx.c
index c4674fc33f7ff2bf53921ef490d348c826ab185b..c922c1641894ee97851121a48153fbb060e48531 100644 (file)
--- a/bsd/kern/kern_xxx.c
+++ b/bsd/kern/kern_xxx.c
@@ -91,7 +91,7 @@ int psem_cache_purge_all(proc_t p);
 int
 reboot(struct proc *p, struct reboot_args *uap, __unused int32_t *retval)
 {
 int
 reboot(struct proc *p, struct reboot_args *uap, __unused int32_t *retval)
 {

-       char message[128];
+       char message[256];

        int error=0;
        size_t dummy=0;
 #if CONFIG_MACF
        int error=0;
        size_t dummy=0;
 #if CONFIG_MACF

diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c
index 169eba5c5ef34b77982b8fa2954f885d68cb9d11..1fe7878123144c4056e1609702b8f42c17dd97f1 100644 (file)
--- a/bsd/kern/posix_shm.c
+++ b/bsd/kern/posix_shm.c
@@ -904,11 +904,16 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct
                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }
                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }

-       if ((off_t)user_size > pinfo->pshm_length) {
+       if (user_size > (vm_map_size_t)pinfo->pshm_length) {

                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }
                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }

-       if ((off_t)(user_size + file_pos) > pinfo->pshm_length) {
+       vm_map_size_t end_pos = 0;
+       if (os_add_overflow(user_size, file_pos, &end_pos)) {
+               PSHM_SUBSYS_UNLOCK();
+               return(EINVAL);
+       }
+       if (end_pos > (vm_map_size_t)pinfo->pshm_length) {

                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }
                PSHM_SUBSYS_UNLOCK();
                return(EINVAL);
        }

diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c
index 9efb0b1ca1300cdd861f2fc384b40f40eed4c2f1..1f88ae54ef06d0a6779bf402220a24efc2f1c66f 100644 (file)
--- a/bsd/kern/proc_info.c
+++ b/bsd/kern/proc_info.c
@@ -2472,11 +2472,14 @@ proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *re
                count = buffersize / sizeof(uint64_t);
                if (count > MAX_UPTRS) {
                        count = MAX_UPTRS;
                count = buffersize / sizeof(uint64_t);
                if (count > MAX_UPTRS) {
                        count = MAX_UPTRS;

-                       buffersize = count * sizeof(uint64_t);

                }
                if (count > 0) {
                }
                if (count > 0) {

+                       buffersize = count * sizeof(uint64_t);

                        kbuf = kalloc(buffersize);
                        kbuf = kalloc(buffersize);

+                       bzero(kbuf, buffersize);

                        assert(kbuf != NULL);
                        assert(kbuf != NULL);

+               } else {
+                       buffersize = 0;

                }
        } else {
                buffersize = 0;
                }
        } else {
                buffersize = 0;

diff --git a/bsd/kern/process_policy.c b/bsd/kern/process_policy.c
index 0a0b43e66817ab17ce01e2e18db3415a0f472558..8c2b5b230ed69e694b901d99e440cda2545b09b8 100644 (file)
--- a/bsd/kern/process_policy.c
+++ b/bsd/kern/process_policy.c
@@ -252,7 +252,7 @@ handle_lowresource(__unused int scope, int action, __unused int policy, int poli
 static int 
 handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t target_threadid)
 {
 static int 
 handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t target_threadid)
 {

-       proc_policy_cpuusage_attr_t     cpuattr;
+       proc_policy_cpuusage_attr_t     cpuattr = { };

 #if CONFIG_MACF || !CONFIG_EMBEDDED
        proc_t                          curp = current_proc();
 #endif
 #if CONFIG_MACF || !CONFIG_EMBEDDED
        proc_t                          curp = current_proc();
 #endif

diff --git a/bsd/kern/sys_coalition.c b/bsd/kern/sys_coalition.c
index 85100962e492ca887851901d6c01d109bcc948ca..5cb3daa6b6e5773b3bc6023df4c9fdbc8885f256 100644 (file)
--- a/bsd/kern/sys_coalition.c
+++ b/bsd/kern/sys_coalition.c
@@ -484,6 +484,11 @@ static int sysctl_coalition_get_pid_list SYSCTL_HANDLER_ARGS
 out:
        proc_rele(tproc);
 
 out:
        proc_rele(tproc);
 

+       if (npids < 0) {
+               /* npids is a negative errno */
+               return -npids;
+       }
+

        if (npids == 0)
                return ENOENT;
 
        if (npids == 0)
                return ENOENT;
 

diff --git a/bsd/kern/sys_reason.c b/bsd/kern/sys_reason.c
index 16ea4cfe49b25d9fc902516ccd73a36a7bc2b6aa..ea260cdc356bf0d4b024a1d0f314bfb3faaae219 100644 (file)
--- a/bsd/kern/sys_reason.c
+++ b/bsd/kern/sys_reason.c
@@ -278,7 +278,9 @@ os_reason_ref(os_reason_t cur_reason)
        lck_mtx_lock(&cur_reason->osr_lock);
 
        assert(cur_reason->osr_refcount > 0);
        lck_mtx_lock(&cur_reason->osr_lock);
 
        assert(cur_reason->osr_refcount > 0);

-       cur_reason->osr_refcount++;
+       if (os_add_overflow(cur_reason->osr_refcount, 1, &cur_reason->osr_refcount)) {
+               panic("os reason refcount overflow");
+       }

 
        lck_mtx_unlock(&cur_reason->osr_lock);
 
 
        lck_mtx_unlock(&cur_reason->osr_lock);
 


@@ -298,7 +300,9 @@ os_reason_free(os_reason_t cur_reason)
 
        lck_mtx_lock(&cur_reason->osr_lock);
 
 
        lck_mtx_lock(&cur_reason->osr_lock);
 

-       assert(cur_reason->osr_refcount > 0);
+       if (cur_reason->osr_refcount == 0) {
+               panic("os_reason_free called on reason with zero refcount");
+       }

 
        cur_reason->osr_refcount--;
        if (cur_reason->osr_refcount != 0) {
 
        cur_reason->osr_refcount--;
        if (cur_reason->osr_refcount != 0) {

diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master
index f18cc91210ce751cb5664b59b334ceefa0aaf114..fd4edb968d55422c932b407b9856667746044c3b 100644 (file)
--- a/bsd/kern/syscalls.master
+++ b/bsd/kern/syscalls.master
@@ -102,7 +102,7 @@
 52     AUE_SIGPENDING  ALL     { int sigpending(struct sigvec *osv); } 
 53     AUE_SIGALTSTACK ALL     { int sigaltstack(struct sigaltstack *nss, struct sigaltstack *oss) NO_SYSCALL_STUB ; } 
 54     AUE_IOCTL       ALL     { int ioctl(int fd, u_long com, caddr_t data) NO_SYSCALL_STUB; } 
 52     AUE_SIGPENDING  ALL     { int sigpending(struct sigvec *osv); } 
 53     AUE_SIGALTSTACK ALL     { int sigaltstack(struct sigaltstack *nss, struct sigaltstack *oss) NO_SYSCALL_STUB ; } 
 54     AUE_IOCTL       ALL     { int ioctl(int fd, u_long com, caddr_t data) NO_SYSCALL_STUB; } 

-55     AUE_REBOOT      ALL     { int reboot(int opt, char *command); } 
+55     AUE_REBOOT      ALL     { int reboot(int opt, char *command) NO_SYSCALL_STUB; }

 56     AUE_REVOKE      ALL     { int revoke(char *path); } 
 57     AUE_SYMLINK     ALL     { int symlink(char *path, char *link); } 
 58     AUE_READLINK    ALL     { int readlink(char *path, char *buf, int count); } 
 56     AUE_REVOKE      ALL     { int revoke(char *path); } 
 57     AUE_SYMLINK     ALL     { int symlink(char *path, char *link); } 
 58     AUE_READLINK    ALL     { int readlink(char *path, char *buf, int count); } 

diff --git a/bsd/kern/trace_codes b/bsd/kern/trace_codes
index 981ab7f57d40c7797b6729d172c2b3f5956c1b3b..2e174231c2b7e687da63c4a1dfe41d1aa539b557 100644 (file)
--- a/bsd/kern/trace_codes
+++ b/bsd/kern/trace_codes
@@ -348,6 +348,8 @@
 0x1800000      MACH_CLOCK_EPOCH_CHANGE
 0x1800004      MACH_CLOCK_BRIDGE_RCV_TS
 0x1800008      MACH_CLOCK_BRIDGE_REMOTE_TIME
 0x1800000      MACH_CLOCK_EPOCH_CHANGE
 0x1800004      MACH_CLOCK_BRIDGE_RCV_TS
 0x1800008      MACH_CLOCK_BRIDGE_REMOTE_TIME

+0x180000c      MACH_CLOCK_BRIDGE_RESET_TS
+0x1800010      MACH_CLOCK_BRIDGE_TS_PARAMS

 0x1900000      MP_TLB_FLUSH
 0x1900004      MP_CPUS_CALL
 0x1900008      MP_CPUS_CALL_LOCAL
 0x1900000      MP_TLB_FLUSH
 0x1900004      MP_CPUS_CALL
 0x1900008      MP_CPUS_CALL_LOCAL

diff --git a/bsd/kern/tty_dev.c b/bsd/kern/tty_dev.c
index 87f3cd7ee6a5724eb4e940bdde1fb2e28547ee9e..dac57968b64558c5f9c25cc4cb69e83b94fe9fca 100644 (file)
--- a/bsd/kern/tty_dev.c
+++ b/bsd/kern/tty_dev.c
@@ -496,10 +496,12 @@ ptcwakeup(struct tty *tp, int flag)
        if (flag & FREAD) {
                selwakeup(&pti->pt_selr);
                wakeup(TSA_PTC_READ(tp));
        if (flag & FREAD) {
                selwakeup(&pti->pt_selr);
                wakeup(TSA_PTC_READ(tp));

+               KNOTE(&pti->pt_selr.si_note, 1);

        }
        if (flag & FWRITE) {
                selwakeup(&pti->pt_selw);
                wakeup(TSA_PTC_WRITE(tp));
        }
        if (flag & FWRITE) {
                selwakeup(&pti->pt_selw);
                wakeup(TSA_PTC_WRITE(tp));

+               KNOTE(&pti->pt_selw.si_note, 1);

        }
 }
 
        }
 }
 


@@ -776,10 +778,10 @@ ptcselect(dev_t dev, int rw, void *wql, proc_t p)
                }
                /* FALLTHROUGH */
 
                }
                /* FALLTHROUGH */
 

-       case 0:                                 /* exceptional */
+       case 0: /* exceptional */

                if ((tp->t_state&TS_ISOPEN) &&
                if ((tp->t_state&TS_ISOPEN) &&

-                   ((pti->pt_flags & PF_PKT && pti->pt_send) ||
-                    (pti->pt_flags & PF_UCNTL && pti->pt_ucntl))) {
+                               (((pti->pt_flags & PF_PKT) && pti->pt_send) ||
+                                ((pti->pt_flags & PF_UCNTL) && pti->pt_ucntl))) {

                        retval = 1;
                        break;
                }
                        retval = 1;
                        break;
                }


@@ -790,21 +792,21 @@ ptcselect(dev_t dev, int rw, void *wql, proc_t p)
        case FWRITE:
                if (tp->t_state&TS_ISOPEN) {
                        if (pti->pt_flags & PF_REMOTE) {
        case FWRITE:
                if (tp->t_state&TS_ISOPEN) {
                        if (pti->pt_flags & PF_REMOTE) {

-                           if (tp->t_canq.c_cc == 0) {
-                               retval = (driver->fix_7828447) ? (TTYHOG - 1) : 1;
-                               break;
+                               if (tp->t_canq.c_cc == 0) {
+                                       retval = (driver->fix_7828447) ? (TTYHOG - 1) : 1;
+                                       break;

                            }
                        } else {
                            }
                        } else {

-                           retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc);
-                           if (retval > 0) {
-                                   retval = (driver->fix_7828447) ? retval : 1;
-                                   break;
-                           }
-                           if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) {
-                                   retval = 1;
-                                   break;
-                           }
-                           retval = 0;
+                               retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc);
+                               if (retval > 0) {
+                                       retval = (driver->fix_7828447) ? retval : 1;
+                                       break;
+                               }
+                               if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) {
+                                       retval = 1;
+                                       break;
+                               }
+                               retval = 0;

                        }
                }
                selrecord(p, &pti->pt_selw, wql);
                        }
                }
                selrecord(p, &pti->pt_selw, wql);

diff --git a/bsd/kern/tty_ptmx.c b/bsd/kern/tty_ptmx.c
index cf89b93ebc1f5d8eeaf5bcfcd0e49727535e5c16..aa583d857b7eb29f8371fabbd3d9ccf6dd1764c6 100644 (file)
--- a/bsd/kern/tty_ptmx.c
+++ b/bsd/kern/tty_ptmx.c
@@ -142,7 +142,7 @@ static struct cdevsw ptsd_cdev = {
 /*
  * System-wide limit on the max number of cloned ptys
  */
 /*
  * System-wide limit on the max number of cloned ptys
  */

-#define        PTMX_MAX_DEFAULT        127     /* 128 entries */
+#define        PTMX_MAX_DEFAULT        511     /* 512 entries */

 #define        PTMX_MAX_HARD           999     /* 1000 entries, due to PTSD_TEMPLATE */
 
 static int ptmx_max = PTMX_MAX_DEFAULT;        /* default # of clones we allow */
 #define        PTMX_MAX_HARD           999     /* 1000 entries, due to PTSD_TEMPLATE */
 
 static int ptmx_max = PTMX_MAX_DEFAULT;        /* default # of clones we allow */


@@ -695,3 +695,247 @@ ptsd_revoke_knotes(__unused int minor, struct tty *tp)
 
        tty_unlock(tp);
 }
 
        tty_unlock(tp);
 }

+
+/*
+ * kevent filter routines for the master side of a pty, a ptmx.
+ *
+ * Stuff the ptmx_ioctl structure into the hook for ptmx knotes.  Use the
+ * embedded tty's lock for synchronization.
+ */
+
+int ptmx_kqfilter(dev_t dev, struct knote *kn);
+static void ptmx_kqops_detach(struct knote *);
+static int ptmx_kqops_event(struct knote *, long);
+static int ptmx_kqops_touch(struct knote *kn, struct kevent_internal_s *kev);
+static int ptmx_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev);
+static int ptmx_kqops_common(struct knote *kn, struct ptmx_ioctl *pti, struct tty *tp);
+
+SECURITY_READ_ONLY_EARLY(struct filterops) ptmx_kqops = {
+       .f_isfd = 1,
+       /* attach is handled by ptmx_kqfilter -- the dev node must be passed in */
+       .f_detach = ptmx_kqops_detach,
+       .f_event = ptmx_kqops_event,
+       .f_touch = ptmx_kqops_touch,
+       .f_process = ptmx_kqops_process,
+};
+
+static struct ptmx_ioctl *
+ptmx_knote_ioctl(struct knote *kn)
+{
+       return (struct ptmx_ioctl *)kn->kn_hook;
+}
+
+static struct tty *
+ptmx_knote_tty(struct knote *kn)
+{
+       struct ptmx_ioctl *pti = kn->kn_hook;
+       return pti->pt_tty;
+}
+
+int
+ptmx_kqfilter(dev_t dev, struct knote *kn)
+{
+       struct tty *tp = NULL;
+       struct ptmx_ioctl *pti = NULL;
+       int ret;
+
+       /* make sure we're talking about the right device type */
+       if (cdevsw[major(dev)].d_open != ptcopen) {
+               knote_set_error(kn, ENODEV);
+               return 0;
+       }
+
+       if ((pti = ptmx_get_ioctl(minor(dev), 0)) == NULL) {
+               knote_set_error(kn, ENXIO);
+               return 0;
+       }
+
+       tp = pti->pt_tty;
+       tty_lock(tp);
+
+       kn->kn_filtid = EVFILTID_PTMX;
+       kn->kn_hook = pti;
+
+       /*
+        * Attach to the ptmx's selinfo structures.  This is the major difference
+        * to the ptsd filtops, which use the selinfo structures in the tty
+        * structure.
+        */
+       switch (kn->kn_filter) {
+       case EVFILT_READ:
+               KNOTE_ATTACH(&pti->pt_selr.si_note, kn);
+               break;
+       case EVFILT_WRITE:
+               KNOTE_ATTACH(&pti->pt_selw.si_note, kn);
+               break;
+       default:
+               panic("ptmx kevent: unexpected filter: %d, kn = %p, tty = %p",
+                               kn->kn_filter, kn, tp);
+               break;
+       }
+
+       /* capture current event state */
+       ret = ptmx_kqops_common(kn, pti, tp);
+
+       /* take a reference on the TTY */
+       ttyhold(tp);
+       tty_unlock(tp);
+
+       return ret;
+}
+
+static void
+ptmx_kqops_detach(struct knote *kn)
+{
+       struct ptmx_ioctl *pti = kn->kn_hook;
+       struct tty *tp = pti->pt_tty;
+
+       assert(tp != NULL);
+
+       tty_lock(tp);
+
+       switch (kn->kn_filter) {
+       case EVFILT_READ:
+               KNOTE_DETACH(&pti->pt_selr.si_note, kn);
+               break;
+
+       case EVFILT_WRITE:
+               KNOTE_DETACH(&pti->pt_selw.si_note, kn);
+               break;
+
+       default:
+               panic("invalid knote %p detach, filter: %d", kn, kn->kn_filter);
+               break;
+       }
+
+       kn->kn_hook = NULL;
+       tty_unlock(tp);
+
+       ttyfree(tp);
+}
+
+static int
+ptmx_kqops_common(struct knote *kn, struct ptmx_ioctl *pti, struct tty *tp)
+{
+       int retval = 0;
+
+       TTY_LOCK_OWNED(tp);
+
+       /* disconnects should force a wakeup (EOF) */
+       if (!(tp->t_state & TS_CONNECTED)) {
+               return 1;
+       }
+
+       switch (kn->kn_filter) {
+       case EVFILT_READ:
+               /* there's data on the TTY and it's not stopped */
+               if (tp->t_outq.c_cc && !(tp->t_state & TS_TTSTOP)) {
+                       retval = tp->t_outq.c_cc;
+               } else if (((pti->pt_flags & PF_PKT) && pti->pt_send) ||
+                               ((pti->pt_flags & PF_UCNTL) && pti->pt_ucntl)) {
+                       retval = 1;
+               }
+               break;
+
+       case EVFILT_WRITE:
+               if (pti->pt_flags & PF_REMOTE) {
+                       if (tp->t_canq.c_cc == 0) {
+                               retval = TTYHOG - 1;
+                       }
+               } else {
+                       retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc);
+                       if (tp->t_canq.c_cc == 0 && (tp->t_lflag & ICANON)) {
+                               retval = 1;
+                       }
+                       if (retval < 0) {
+                               retval = 0;
+                       }
+               }
+               break;
+
+       default:
+               panic("ptmx kevent: unexpected filter: %d, kn = %p, tty = %p",
+                               kn->kn_filter, kn, tp);
+               break;
+       }
+
+       if (tp->t_state & TS_ZOMBIE) {
+               kn->kn_flags |= EV_EOF;
+               retval = 1;
+       }
+
+       return retval;
+}
+
+static int
+ptmx_kqops_event(struct knote *kn, long hint)
+{
+       struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn);
+       struct tty *tp = ptmx_knote_tty(kn);
+       int ret;
+       bool revoked = hint & NOTE_REVOKE;
+       hint &= ~NOTE_REVOKE;
+
+       if (!hint) {
+               tty_lock(tp);
+       }
+
+       if (revoked) {
+               kn->kn_flags |= EV_EOF | EV_ONESHOT;
+               ret = 1;
+       } else {
+               ret = ptmx_kqops_common(kn, pti, tp);
+       }
+
+       if (!hint) {
+               tty_unlock(tp);
+       }
+
+       return ret;
+}
+
+static int
+ptmx_kqops_touch(struct knote *kn, struct kevent_internal_s *kev)
+{
+       struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn);
+       struct tty *tp = ptmx_knote_tty(kn);
+       int ret;
+
+       tty_lock(tp);
+
+       /* accept new kevent state */
+       kn->kn_sfflags = kev->fflags;
+       kn->kn_sdata = kev->data;
+       if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) {
+               kn->kn_udata = kev->udata;
+       }
+
+       /* recapture fired state of knote */
+       ret = ptmx_kqops_common(kn, pti, tp);
+
+       tty_unlock(tp);
+
+       return ret;
+}
+
+static int
+ptmx_kqops_process(struct knote *kn, __unused struct filt_process_s *data,
+               struct kevent_internal_s *kev)
+{
+       struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn);
+       struct tty *tp = ptmx_knote_tty(kn);
+       int ret;
+
+       tty_lock(tp);
+       ret = ptmx_kqops_common(kn, pti, tp);
+       if (ret) {
+               *kev = kn->kn_kevent;
+               if (kn->kn_flags & EV_CLEAR) {
+                       kn->kn_fflags = 0;
+                       kn->kn_data = 0;
+               }
+       }
+       tty_unlock(tp);
+
+       return ret;
+}

diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c
index 57915610ceffa2d8566598e5312bed7c6d01bbc6..ca8fde048190926b5037695bcbfc361cedeca871 100644 (file)
--- a/bsd/kern/uipc_mbuf.c
+++ b/bsd/kern/uipc_mbuf.c
@@ -582,6 +582,18 @@ static lck_attr_t *mleak_lock_attr;
 static lck_grp_t *mleak_lock_grp;
 static lck_grp_attr_t *mleak_lock_grp_attr;
 
 static lck_grp_t *mleak_lock_grp;
 static lck_grp_attr_t *mleak_lock_grp_attr;
 

+/* *Failed* large allocations. */
+struct mtracelarge {
+       uint64_t        size;
+       uint64_t        depth;
+       uintptr_t       addr[MLEAK_STACK_DEPTH];
+};
+
+#define MTRACELARGE_NUM_TRACES         5
+static struct mtracelarge mtracelarge_table[MTRACELARGE_NUM_TRACES];
+
+static void mtracelarge_register(size_t size);
+

 /* Lock to protect the completion callback table */
 static lck_grp_attr_t *mbuf_tx_compl_tbl_lck_grp_attr = NULL;
 static lck_attr_t *mbuf_tx_compl_tbl_lck_attr = NULL;
 /* Lock to protect the completion callback table */
 static lck_grp_attr_t *mbuf_tx_compl_tbl_lck_grp_attr = NULL;
 static lck_attr_t *mbuf_tx_compl_tbl_lck_attr = NULL;


@@ -681,7 +693,7 @@ boolean_t mb_peak_firstreport = FALSE;
 static struct timeval mb_wdtstart;     /* watchdog start timestamp */
 static char *mbuf_dump_buf;
 
 static struct timeval mb_wdtstart;     /* watchdog start timestamp */
 static char *mbuf_dump_buf;
 

-#define        MBUF_DUMP_BUF_SIZE      3072
+#define        MBUF_DUMP_BUF_SIZE      4096

 
 /*
  * mbuf watchdog is enabled by default on embedded platforms.  It is
 
 /*
  * mbuf watchdog is enabled by default on embedded platforms.  It is


@@ -2954,12 +2966,14 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
                        /* Try for 1 page if failed */
                        size = PAGE_SIZE;
                        page = kmem_mb_alloc(mb_map, size, 0, &error);
                        /* Try for 1 page if failed */
                        size = PAGE_SIZE;
                        page = kmem_mb_alloc(mb_map, size, 0, &error);

+                       if (page == 0) {
+                               m_vm_error_stats(&mb_kmem_one_failed,
+                                   &mb_kmem_one_failed_ts,
+                                   NULL, size, error);
+                       }

                }
 
                if (page == 0) {
                }
 
                if (page == 0) {

-                       m_vm_error_stats(&mb_kmem_one_failed,
-                           &mb_kmem_one_failed_ts,
-                           NULL, size, error);

                        lck_mtx_lock(mbuf_mlock);
                        goto out;
                }
                        lck_mtx_lock(mbuf_mlock);
                        goto out;
                }


@@ -3108,6 +3122,8 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
 out:
        LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
 
 out:
        LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
 

+       mtracelarge_register(size);
+

        /* We're done; let others enter */
        mb_clalloc_busy = FALSE;
        if (mb_clalloc_waiters > 0) {
        /* We're done; let others enter */
        mb_clalloc_busy = FALSE;
        if (mb_clalloc_waiters > 0) {


@@ -7324,7 +7340,7 @@ mleak_update_stats()
        mltr = &mleak_stat->ml_trace[0];
        bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
        for (i = 0; i < MLEAK_NUM_TRACES; i++) {
        mltr = &mleak_stat->ml_trace[0];
        bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
        for (i = 0; i < MLEAK_NUM_TRACES; i++) {

-       int j;
+               int j;

 
                if (mleak_top_trace[i] == NULL ||
                    mleak_top_trace[i]->allocs == 0)
 
                if (mleak_top_trace[i] == NULL ||
                    mleak_top_trace[i]->allocs == 0)


@@ -7386,7 +7402,7 @@ mbuf_dump(void)
        mb_class_stat_t *sp;
        mleak_trace_stat_t *mltr;
        char *c = mbuf_dump_buf;
        mb_class_stat_t *sp;
        mleak_trace_stat_t *mltr;
        char *c = mbuf_dump_buf;

-       int i, k, clen = MBUF_DUMP_BUF_SIZE;
+       int i, j, k, clen = MBUF_DUMP_BUF_SIZE;

 
        mbuf_dump_buf[0] = '\0';
 
 
        mbuf_dump_buf[0] = '\0';
 


@@ -7531,6 +7547,32 @@ mbuf_dump(void)
                MBUF_DUMP_BUF_CHK();
        }
 
                MBUF_DUMP_BUF_CHK();
        }
 

+       k = snprintf(c, clen, "\nlargest allocation failure backtraces:\n");
+       MBUF_DUMP_BUF_CHK();
+
+       for (j = 0; j < MTRACELARGE_NUM_TRACES; j++) {
+               struct mtracelarge *trace = &mtracelarge_table[j];
+               if (trace->size == 0 || trace->depth == 0)
+                       continue;
+               k = snprintf(c, clen, "size %llu: < ", trace->size);
+               MBUF_DUMP_BUF_CHK();
+               for (i = 0; i < trace->depth; i++) {
+                       if (mleak_stat->ml_isaddr64) {
+                               k = snprintf(c, clen, "0x%0llx ",
+                                   (uint64_t)VM_KERNEL_UNSLIDE(
+                                           trace->addr[i]));
+                       } else {
+                               k = snprintf(c, clen,
+                                   "0x%08x ",
+                                   (uint32_t)VM_KERNEL_UNSLIDE(
+                                           trace->addr[i]));
+                       }
+                       MBUF_DUMP_BUF_CHK();
+               }
+               k = snprintf(c, clen, ">\n");
+               MBUF_DUMP_BUF_CHK();
+       }
+

        /* mbuf leak detection statistics */
        mleak_update_stats();
 
        /* mbuf leak detection statistics */
        mleak_update_stats();
 


@@ -7575,7 +7617,6 @@ mbuf_dump(void)
        MBUF_DUMP_BUF_CHK();
 
        for (i = 0; i < MLEAK_STACK_DEPTH; i++) {
        MBUF_DUMP_BUF_CHK();
 
        for (i = 0; i < MLEAK_STACK_DEPTH; i++) {

-               int j;

                k = snprintf(c, clen, "%2d: ", (i + 1));
                MBUF_DUMP_BUF_CHK();
                for (j = 0; j < mleak_stat->ml_cnt; j++) {
                k = snprintf(c, clen, "%2d: ", (i + 1));
                MBUF_DUMP_BUF_CHK();
                for (j = 0; j < mleak_stat->ml_cnt; j++) {


@@ -8160,8 +8201,12 @@ m_drain_force_sysctl SYSCTL_HANDLER_ARGS
        err = sysctl_handle_int(oidp, &val, 0, req);
        if (err != 0 || req->newptr == USER_ADDR_NULL)
                return (err);
        err = sysctl_handle_int(oidp, &val, 0, req);
        if (err != 0 || req->newptr == USER_ADDR_NULL)
                return (err);

-       if (val)
+       if (val) {
+               lck_mtx_lock(mbuf_mlock);
+               printf("%s\n", mbuf_dump());
+               lck_mtx_unlock(mbuf_mlock);

                m_drain();
                m_drain();

+       }

 
        return (err);
 }
 
        return (err);
 }


@@ -8237,6 +8282,36 @@ mbtest SYSCTL_HANDLER_ARGS
 }
 #endif
 
 }
 #endif
 

+
+static void
+mtracelarge_register(size_t size)
+{
+       int i;
+       struct mtracelarge *trace;
+       uintptr_t bt[MLEAK_STACK_DEPTH];
+       unsigned int depth;
+
+       depth = backtrace(bt, MLEAK_STACK_DEPTH);
+       /* Check if this entry is already on the list. */
+       for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
+               trace = &mtracelarge_table[i];
+               if (trace->size == size && trace->depth == depth &&
+                   memcmp(bt, trace->addr, depth * sizeof(uintptr_t)) == 0) {
+                       return;
+               }
+
+       }
+       for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
+               trace = &mtracelarge_table[i];
+               if (size > trace->size) {
+                       trace->depth = depth;
+                       memcpy(trace->addr, bt, depth * sizeof(uintptr_t));
+                       trace->size = size;
+                       break;
+               }
+       }
+}
+

 SYSCTL_DECL(_kern_ipc);
 #if DEBUG || DEVELOPMENT
 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbtest,
 SYSCTL_DECL(_kern_ipc);
 #if DEBUG || DEVELOPMENT
 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbtest,

diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c
index 08ec21e9773b9248c78f0e5e2be1e28997cc24b3..ce89864cc3337d1ed2a03b044bd3931522feb673 100644 (file)
--- a/bsd/kern/uipc_socket2.c
+++ b/bsd/kern/uipc_socket2.c
@@ -1421,7 +1421,11 @@ sbappendstream_rcvdemux(struct socket *so, struct mbuf *m, uint32_t seqnum,
 {
        int ret = 0;
 
 {
        int ret = 0;
 

-       if ((m != NULL) && (m_pktlen(m) <= 0)) {
+       if ((m != NULL) &&
+           m_pktlen(m) <= 0 &&
+           !((so->so_flags & SOF_MP_SUBFLOW) &&
+             (m->m_flags & M_PKTHDR) &&
+             (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN))) {

                m_freem(m);
                return (ret);
        }
                m_freem(m);
                return (ret);
        }


@@ -1453,8 +1457,14 @@ sbappendmptcpstream_rcv(struct sockbuf *sb, struct mbuf *m)
 
        if (m == NULL || m_pktlen(m) == 0 || (sb->sb_flags & SB_DROP) ||
            (so->so_state & SS_CANTRCVMORE)) {
 
        if (m == NULL || m_pktlen(m) == 0 || (sb->sb_flags & SB_DROP) ||
            (so->so_state & SS_CANTRCVMORE)) {

-               if (m != NULL)
+               if (m && m_pktlen(m) == 0 &&
+                   (m->m_flags & M_PKTHDR) &&
+                   (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
+                       mptcp_input(tptomptp(sototcpcb(so))->mpt_mpte, m);
+                       return (1);
+               } else if (m != NULL) {

                        m_freem(m);
                        m_freem(m);

+               }

                return (0);
        }
        /* the socket is not closed, so SOF_MP_SUBFLOW must be set */
                return (0);
        }
        /* the socket is not closed, so SOF_MP_SUBFLOW must be set */

diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c
index 73c6ba447ee10ab7011307905f49868c1f880574..f51596c3766b1cf2843d23619fac1a1a54945a44 100644 (file)
--- a/bsd/kern/uipc_syscalls.c
+++ b/bsd/kern/uipc_syscalls.c
@@ -90,6 +90,7 @@
 #include <kern/task.h>
 #include <sys/priv.h>
 #include <sys/sysctl.h>
 #include <kern/task.h>
 #include <sys/priv.h>
 #include <sys/sysctl.h>

+#include <sys/sys_domain.h>

 
 #include <security/audit/audit.h>
 
 
 #include <security/audit/audit.h>
 


@@ -328,7 +329,8 @@ bind(__unused proc_t p, struct bind_args *uap, __unused int32_t *retval)
                goto out;
        AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa);
 #if CONFIG_MACF_SOCKET_SUBSET
                goto out;
        AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa);
 #if CONFIG_MACF_SOCKET_SUBSET

-       if ((error = mac_socket_check_bind(kauth_cred_get(), so, sa)) == 0)
+       if ((sa != NULL && sa->sa_family == AF_SYSTEM) ||
+               (error = mac_socket_check_bind(kauth_cred_get(), so, sa)) == 0)

                error = sobindlock(so, sa, 1);  /* will lock socket */
 #else
                error = sobindlock(so, sa, 1);  /* will lock socket */
                error = sobindlock(so, sa, 1);  /* will lock socket */
 #else
                error = sobindlock(so, sa, 1);  /* will lock socket */

diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c
index 625e9736fb7b87cf4a1e7dadd1d11e36e152bf00..11c3ac25e08a83dac9d00c78120647bdd9159c49 100644 (file)
--- a/bsd/miscfs/specfs/spec_vnops.c
+++ b/bsd/miscfs/specfs/spec_vnops.c
@@ -104,6 +104,7 @@ extern dev_t        chrtoblk(dev_t dev);
 extern boolean_t       iskmemdev(dev_t dev);
 extern int     bpfkqfilter(dev_t dev, struct knote *kn);
 extern int ptsd_kqfilter(dev_t, struct knote *);
 extern boolean_t       iskmemdev(dev_t dev);
 extern int     bpfkqfilter(dev_t dev, struct knote *kn);
 extern int ptsd_kqfilter(dev_t, struct knote *);

+extern int ptmx_kqfilter(dev_t, struct knote *);

 
 struct vnode *speclisth[SPECHSZ];
 
 
 struct vnode *speclisth[SPECHSZ];
 


@@ -768,9 +769,10 @@ spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_internal_s *kev)
        if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
                kn->kn_filtid = EVFILTID_PTSD;
                return ptsd_kqfilter(dev, kn);
        if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
                kn->kn_filtid = EVFILTID_PTSD;
                return ptsd_kqfilter(dev, kn);

-       } else if (cdevsw[major(dev)].d_type == D_TTY &&
-                  !(cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) &&
-                  kn->kn_vnode_kqok) {
+       } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
+               kn->kn_filtid = EVFILTID_PTMX;
+               return ptmx_kqfilter(dev, kn);
+       } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) {

                /*
                 * TTYs from drivers that use struct ttys use their own filter
                 * routines.  The PTC driver doesn't use the tty for character
                /*
                 * TTYs from drivers that use struct ttys use their own filter
                 * routines.  The PTC driver doesn't use the tty for character

diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c
index 3bf795149be228b4fb141e9937b0957c30cc5014..1448fc02b576f73643ea03e8a02f33e57e38b323 100644 (file)
--- a/bsd/net/bpf.c
+++ b/bsd/net/bpf.c
@@ -1979,6 +1979,11 @@ bpf_setif(struct bpf_d *d, ifnet_t theywant)
                 */
                if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP))
                        continue;
                 */
                if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP))
                        continue;

+               /*
+                * Skip the coprocessor interface
+                */
+               if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp))
+                       continue;

                /*
                 * We found the requested interface.
                 * Allocate the packet buffers.
                /*
                 * We found the requested interface.
                 * Allocate the packet buffers.

diff --git a/bsd/net/classq/classq_fq_codel.c b/bsd/net/classq/classq_fq_codel.c
index 3710ad112cc9a1ef44eac433df3b2f3c4e76e8f1..f762add426f9cb96a64e70ff838af47217d61fe4 100644 (file)
--- a/bsd/net/classq/classq_fq_codel.c
+++ b/bsd/net/classq/classq_fq_codel.c
@@ -253,6 +253,7 @@ fq_addq(fq_if_t *fqs, pktsched_pkt_t *pkt, fq_if_classq_t *fq_cl)
                        if (fqs->fqs_large_flow == NULL) {
                                droptype = DTYPE_FORCED;
                                fq_cl->fcl_stat.fcl_drop_overflow++;
                        if (fqs->fqs_large_flow == NULL) {
                                droptype = DTYPE_FORCED;
                                fq_cl->fcl_stat.fcl_drop_overflow++;

+                               ret = CLASSQEQ_DROP;

 
                                /*
                                 * if this fq was freshly created and there
 
                                /*
                                 * if this fq was freshly created and there

diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c
index 50e85b2742365d0c9737c5bb0ddcc8a9aa86449f..9219ab940d39e0e77dd72fee2b556b7f61197a7b 100644 (file)
--- a/bsd/net/dlil.c
+++ b/bsd/net/dlil.c
@@ -2723,7 +2723,7 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head,
 
 
 static void
 
 
 static void

-ifnet_start_common(struct ifnet *ifp, int resetfc)
+ifnet_start_common(struct ifnet *ifp, boolean_t resetfc)

 {
        if (!(ifp->if_eflags & IFEF_TXSTART))
                return;
 {
        if (!(ifp->if_eflags & IFEF_TXSTART))
                return;


@@ -2731,7 +2731,8 @@ ifnet_start_common(struct ifnet *ifp, int resetfc)
         * If the starter thread is inactive, signal it to do work,
         * unless the interface is being flow controlled from below,
         * e.g. a virtual interface being flow controlled by a real
         * If the starter thread is inactive, signal it to do work,
         * unless the interface is being flow controlled from below,
         * e.g. a virtual interface being flow controlled by a real

-        * network interface beneath it.
+        * network interface beneath it, or it's been disabled via
+        * a call to ifnet_disable_output().

         */
        lck_mtx_lock_spin(&ifp->if_start_lock);
        if (resetfc) {
         */
        lck_mtx_lock_spin(&ifp->if_start_lock);
        if (resetfc) {


@@ -2754,7 +2755,7 @@ ifnet_start_common(struct ifnet *ifp, int resetfc)
 void
 ifnet_start(struct ifnet *ifp)
 {
 void
 ifnet_start(struct ifnet *ifp)
 {

-       ifnet_start_common(ifp, 0);
+       ifnet_start_common(ifp, FALSE);

 }
 
 static void
 }
 
 static void


@@ -2868,9 +2869,14 @@ ifnet_start_thread_fn(void *v, wait_result_t w)
 
                        lck_mtx_lock_spin(&ifp->if_start_lock);
 
 
                        lck_mtx_lock_spin(&ifp->if_start_lock);
 

-                       /* if there's no pending request, we're done */
-                       if (req == ifp->if_start_req)
+                       /*
+                        * If there's no pending request or if the
+                        * interface has been disabled, we're done.
+                        */
+                       if (req == ifp->if_start_req ||
+                           (ifp->if_start_flags & IFSF_FLOW_CONTROLLED)) {

                                break;
                                break;

+                       }

                }
 
                ifp->if_start_req = 0;
                }
 
                ifp->if_start_req = 0;


@@ -3041,8 +3047,9 @@ ifnet_poll_thread_fn(void *v, wait_result_t w)
                        lck_mtx_lock_spin(&ifp->if_poll_lock);
 
                        /* if there's no pending request, we're done */
                        lck_mtx_lock_spin(&ifp->if_poll_lock);
 
                        /* if there's no pending request, we're done */

-                       if (req == ifp->if_poll_req)
+                       if (req == ifp->if_poll_req) {

                                break;
                                break;

+                       }

                }
                ifp->if_poll_req = 0;
                ifp->if_poll_active = 0;
                }
                ifp->if_poll_req = 0;
                ifp->if_poll_active = 0;


@@ -6076,8 +6083,14 @@ ifnet_detach(ifnet_t ifp)
        ifp->if_refflags |= IFRF_DETACHING;
        lck_mtx_unlock(&ifp->if_ref_lock);
 
        ifp->if_refflags |= IFRF_DETACHING;
        lck_mtx_unlock(&ifp->if_ref_lock);
 

-       if (dlil_verbose)
+       if (dlil_verbose) {

                printf("%s: detaching\n", if_name(ifp));
                printf("%s: detaching\n", if_name(ifp));

+       }
+
+       /* clean up flow control entry object if there's any */
+       if (ifp->if_eflags & IFEF_TXSTART) {
+               ifnet_flowadv(ifp->if_flowhash);
+       }

 
        /* Reset ECN enable/disable flags */
        ifp->if_eflags &= ~IFEF_ECN_DISABLE;
 
        /* Reset ECN enable/disable flags */
        ifp->if_eflags &= ~IFEF_ECN_DISABLE;


@@ -6438,6 +6451,7 @@ ifnet_detach_final(struct ifnet *ifp)
                        wakeup_one((caddr_t)&inp->input_waiting);
                }
                lck_mtx_unlock(&inp->input_lck);
                        wakeup_one((caddr_t)&inp->input_waiting);
                }
                lck_mtx_unlock(&inp->input_lck);

+               ifnet_lock_done(ifp);

 
                /* wait for the input thread to terminate */
                lck_mtx_lock_spin(&inp->input_lck);
 
                /* wait for the input thread to terminate */
                lck_mtx_lock_spin(&inp->input_lck);


@@ -6447,6 +6461,7 @@ ifnet_detach_final(struct ifnet *ifp)
                            (PZERO - 1) | PSPIN, inp->input_name, NULL);
                }
                lck_mtx_unlock(&inp->input_lck);
                            (PZERO - 1) | PSPIN, inp->input_name, NULL);
                }
                lck_mtx_unlock(&inp->input_lck);

+               ifnet_lock_exclusive(ifp);

 
                /* clean-up input thread state */
                dlil_clean_threading_info(inp);
 
                /* clean-up input thread state */
                dlil_clean_threading_info(inp);


@@ -7834,7 +7849,7 @@ ifnet_enable_output(struct ifnet *ifp)
                return (ENXIO);
        }
 
                return (ENXIO);
        }
 

-       ifnet_start_common(ifp, 1);
+       ifnet_start_common(ifp, TRUE);

        return (0);
 }
 
        return (0);
 }
 


@@ -7904,7 +7919,7 @@ ifnet_fc_add(struct ifnet *ifp)
        /* become regular mutex */
        lck_mtx_convert_spin(&ifnet_fc_lock);
 
        /* become regular mutex */
        lck_mtx_convert_spin(&ifnet_fc_lock);
 

-       ifce = zalloc_noblock(ifnet_fc_zone);
+       ifce = zalloc(ifnet_fc_zone);

        if (ifce == NULL) {
                /* memory allocation failed */
                lck_mtx_unlock(&ifnet_fc_lock);
        if (ifce == NULL) {
                /* memory allocation failed */
                lck_mtx_unlock(&ifnet_fc_lock);

diff --git a/bsd/net/if.c b/bsd/net/if.c
index 53df43d54fc2bd26f02a1eacb935632871ecbd90..6b00d6cad9a1cb8d5d277c1f00a2ef9986e0487b 100644 (file)
--- a/bsd/net/if.c
+++ b/bsd/net/if.c
@@ -243,6 +243,8 @@ static uint32_t if_verbose = 0;
 SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_verbose,
     CTLFLAG_RW | CTLFLAG_LOCKED, &if_verbose, 0, "");
 
 SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_verbose,
     CTLFLAG_RW | CTLFLAG_LOCKED, &if_verbose, 0, "");
 

+boolean_t intcoproc_unrestricted;
+

 /* Eventhandler context for interface events */
 struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt;
 
 /* Eventhandler context for interface events */
 struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt;
 


@@ -270,6 +272,9 @@ ifa_init(void)
 
        lck_mtx_init(&ifma_trash_lock, ifa_mtx_grp, ifa_mtx_attr);
        TAILQ_INIT(&ifma_trash_head);
 
        lck_mtx_init(&ifma_trash_lock, ifa_mtx_grp, ifa_mtx_attr);
        TAILQ_INIT(&ifma_trash_head);

+
+       PE_parse_boot_argn("intcoproc_unrestricted", &intcoproc_unrestricted,
+           sizeof (intcoproc_unrestricted));

 }
 
 /*
 }
 
 /*


@@ -2302,6 +2307,88 @@ ifioctl_nat64prefix(struct ifnet *ifp, u_long cmd, caddr_t data)
 #endif
 
 
 #endif
 
 

+/*
+ * List the ioctl()s we can perform on restricted INTCOPROC interfaces.
+ */
+static bool
+ifioctl_restrict_intcoproc(unsigned long cmd, const char *ifname,
+    struct ifnet *ifp, struct proc *p)
+{
+
+       if (intcoproc_unrestricted == TRUE) {
+               return (false);
+       }
+       if (proc_pid(p) == 0) {
+               return (false);
+       }
+       if (ifname) {
+               ifp = ifunit(ifname);
+       }
+       if (ifp == NULL) {
+               return (false);
+       }
+       if (!IFNET_IS_INTCOPROC(ifp)) {
+               return (false);
+       }
+       switch (cmd) {
+       case SIOCGIFBRDADDR:
+       case SIOCGIFCONF32:
+       case SIOCGIFCONF64:
+       case SIOCGIFFLAGS:
+       case SIOCGIFEFLAGS:
+       case SIOCGIFCAP:
+       case SIOCGIFMAC:
+       case SIOCGIFMETRIC:
+       case SIOCGIFMTU:
+       case SIOCGIFPHYS:
+       case SIOCGIFTYPE:
+       case SIOCGIFFUNCTIONALTYPE:
+       case SIOCGIFPSRCADDR:
+       case SIOCGIFPDSTADDR:
+       case SIOCGIFGENERIC:
+       case SIOCGIFDEVMTU:
+       case SIOCGIFVLAN:
+       case SIOCGIFBOND:
+       case SIOCGIFWAKEFLAGS:
+       case SIOCGIFGETRTREFCNT:
+       case SIOCGIFOPPORTUNISTIC:
+       case SIOCGIFLINKQUALITYMETRIC:
+       case SIOCGIFLOG:
+       case SIOCGIFDELEGATE:
+       case SIOCGIFEXPENSIVE:
+       case SIOCGIFINTERFACESTATE:
+       case SIOCGIFPROBECONNECTIVITY:
+       case SIOCGIFTIMESTAMPENABLED:
+       case SIOCGECNMODE:
+       case SIOCGQOSMARKINGMODE:
+       case SIOCGQOSMARKINGENABLED:
+       case SIOCGIFLOWINTERNET:
+       case SIOCGIFSTATUS:
+       case SIOCGIFMEDIA32:
+       case SIOCGIFMEDIA64:
+       case SIOCGIFDESC:
+       case SIOCGIFLINKPARAMS:
+       case SIOCGIFQUEUESTATS:
+       case SIOCGIFTHROTTLE:
+       case SIOCGIFAGENTIDS32:
+       case SIOCGIFAGENTIDS64:
+       case SIOCGIFNETSIGNATURE:
+       case SIOCGIFINFO_IN6:
+       case SIOCGIFAFLAG_IN6:
+       case SIOCGNBRINFO_IN6:
+       case SIOCGIFALIFETIME_IN6:
+       case SIOCGIFNETMASK_IN6:
+               return (false);
+       default:
+#if (DEBUG || DEVELOPMENT)
+               printf("%s: cmd 0x%lx not allowed (pid %u)\n",
+                   __func__, cmd, proc_pid(p));
+#endif
+               return (true);
+       }
+       return (false);
+}
+

 /*
  * Interface ioctls.
  *
 /*
  * Interface ioctls.
  *


@@ -2434,6 +2521,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
                bcopy(data, &ifr, sizeof (ifr));
                ifr.ifr_name[IFNAMSIZ - 1] = '\0';
                bcopy(&ifr.ifr_name, ifname, IFNAMSIZ);
                bcopy(data, &ifr, sizeof (ifr));
                ifr.ifr_name[IFNAMSIZ - 1] = '\0';
                bcopy(&ifr.ifr_name, ifname, IFNAMSIZ);

+               if (ifioctl_restrict_intcoproc(cmd, ifname, NULL, p) == true) {
+                       error = EPERM;
+                       goto done;
+               }

                error = ifioctl_ifreq(so, cmd, &ifr, p);
                bcopy(&ifr, data, sizeof (ifr));
                goto done;
                error = ifioctl_ifreq(so, cmd, &ifr, p);
                bcopy(&ifr, data, sizeof (ifr));
                goto done;


@@ -2551,6 +2642,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
                goto done;
        }
 
                goto done;
        }
 

+       if (ifioctl_restrict_intcoproc(cmd, NULL, ifp, p) == true) {
+               error = EPERM;
+               goto done;
+       }

        switch (cmd) {
        case SIOCSIFPHYADDR:                    /* struct {if,in_}aliasreq */
 #if INET6
        switch (cmd) {
        case SIOCSIFPHYADDR:                    /* struct {if,in_}aliasreq */
 #if INET6

diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c
index 202f6c092904bd5a042592e9b51bf14fd9fcb8eb..204a04c0e9a92d2d7a6490174096ca120551848d 100644 (file)
--- a/bsd/net/if_bond.c
+++ b/bsd/net/if_bond.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2004-2014 Apple Inc. All rights reserved.
+ * Copyright (c) 2004-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -1957,6 +1957,10 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
     bondport_ref               p;
     int                                progress = 0;
 
     bondport_ref               p;
     int                                progress = 0;
 

+    if (IFNET_IS_INTCOPROC(port_ifp)) {
+       return (EINVAL);
+    }
+

     /* pre-allocate space for new port */
     p = bondport_create(port_ifp, 0x8000, 1, 0, &error);
     if (p == NULL) {
     /* pre-allocate space for new port */
     p = bondport_create(port_ifp, 0x8000, 1, 0, &error);
     if (p == NULL) {

diff --git a/bsd/net/if_bridge.c b/bsd/net/if_bridge.c
index 76de0c2bbdd5c56ef2aabc7cea8ea0ab464d4f70..133d9af30dffa5938a2432d7cfb4f090bddfe940 100644 (file)
--- a/bsd/net/if_bridge.c
+++ b/bsd/net/if_bridge.c
@@ -2313,6 +2313,10 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg)
        if (ifs->if_ioctl == NULL)      /* must be supported */
                return (EINVAL);
 
        if (ifs->if_ioctl == NULL)      /* must be supported */
                return (EINVAL);
 

+       if (IFNET_IS_INTCOPROC(ifs)) {
+               return (EINVAL);
+       }
+

        if (bsd_mode) {
                /* If it's in the span list, it can't be a member. */
                TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next)
        if (bsd_mode) {
                /* If it's in the span list, it can't be a member. */
                TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next)


@@ -3104,6 +3108,10 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg)
        if (ifs == NULL)
                return (ENOENT);
 
        if (ifs == NULL)
                return (ENOENT);
 

+       if (IFNET_IS_INTCOPROC(ifs)) {
+               return (EINVAL);
+       }
+

        TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next)
                if (ifs == bif->bif_ifp)
                        return (EBUSY);
        TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next)
                if (ifs == bif->bif_ifp)
                        return (EBUSY);

diff --git a/bsd/net/if_ipsec.c b/bsd/net/if_ipsec.c
index 8ffac41fe2e6caa6d5b3d94c2f92d074493a843f..4c330d6396c00e9584b2f1b4e8f1dc128cc10291 100644 (file)
--- a/bsd/net/if_ipsec.c
+++ b/bsd/net/if_ipsec.c
@@ -61,6 +61,8 @@ extern int net_qos_policy_restrict_avapps;
 extern unsigned int if_enable_netagent;
 
 /* Kernel Control functions */
 extern unsigned int if_enable_netagent;
 
 /* Kernel Control functions */

+static errno_t ipsec_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
+                                                          void **unitinfo);

 static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
                                                                  void **unitinfo);
 static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
 static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
                                                                  void **unitinfo);
 static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,


@@ -73,9 +75,7 @@ static errno_t        ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unit
                                                                 int opt, void *data, size_t len);
 
 /* Network Interface functions */
                                                                 int opt, void *data, size_t len);
 
 /* Network Interface functions */

-#if !IPSEC_NEXUS

 static void     ipsec_start(ifnet_t    interface);
 static void     ipsec_start(ifnet_t    interface);

-#endif // !IPSEC_NEXUS

 static errno_t ipsec_output(ifnet_t interface, mbuf_t data);
 static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header,
                                                        protocol_family_t *protocol);
 static errno_t ipsec_output(ifnet_t interface, mbuf_t data);
 static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header,
                                                        protocol_family_t *protocol);


@@ -110,7 +110,7 @@ SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG
 
 #define IPSEC_IF_VERIFY(_e)            if (unlikely(if_ipsec_verify_interface_creation)) { VERIFY(_e); }
 
 
 #define IPSEC_IF_VERIFY(_e)            if (unlikely(if_ipsec_verify_interface_creation)) { VERIFY(_e); }
 

-#define IPSEC_IF_DEFAULT_SLOT_SIZE 4096
+#define IPSEC_IF_DEFAULT_SLOT_SIZE 2048

 #define IPSEC_IF_DEFAULT_RING_SIZE 64
 #define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64
 #define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128
 #define IPSEC_IF_DEFAULT_RING_SIZE 64
 #define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64
 #define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128


@@ -118,6 +118,9 @@ SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG
 #define IPSEC_IF_MIN_RING_SIZE 16
 #define IPSEC_IF_MAX_RING_SIZE 1024
 
 #define IPSEC_IF_MIN_RING_SIZE 16
 #define IPSEC_IF_MAX_RING_SIZE 1024
 

+#define IPSEC_IF_MIN_SLOT_SIZE 1024
+#define IPSEC_IF_MAX_SLOT_SIZE 4096
+

 static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;


@@ -168,7 +171,6 @@ struct ipsec_pcb {
        char                            ipsec_unique_name[IFXNAMSIZ];
        // PCB lock protects state fields, like ipsec_kpipe_enabled
        decl_lck_rw_data(, ipsec_pcb_lock);
        char                            ipsec_unique_name[IFXNAMSIZ];
        // PCB lock protects state fields, like ipsec_kpipe_enabled
        decl_lck_rw_data(, ipsec_pcb_lock);

-       bool                            ipsec_output_disabled;

 
 #if IPSEC_NEXUS
        lck_mtx_t                       ipsec_input_chain_lock;
 
 #if IPSEC_NEXUS
        lck_mtx_t                       ipsec_input_chain_lock;


@@ -186,6 +188,13 @@ struct ipsec_pcb {
        void *                          ipsec_netif_rxring;
        void *                          ipsec_netif_txring;
        uint64_t                        ipsec_netif_txring_size;
        void *                          ipsec_netif_rxring;
        void *                          ipsec_netif_txring;
        uint64_t                        ipsec_netif_txring_size;

+
+       u_int32_t                       ipsec_slot_size;
+       u_int32_t                       ipsec_netif_ring_size;
+       u_int32_t                       ipsec_tx_fsw_ring_size;
+       u_int32_t                       ipsec_rx_fsw_ring_size;
+       bool                            ipsec_use_netif;
+

 #endif // IPSEC_NEXUS
 };
 
 #endif // IPSEC_NEXUS
 };
 


@@ -298,6 +307,7 @@ ipsec_register_control(void)
        kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */
        kern_ctl.ctl_sendsize = 64 * 1024;
        kern_ctl.ctl_recvsize = 64 * 1024;
        kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */
        kern_ctl.ctl_sendsize = 64 * 1024;
        kern_ctl.ctl_recvsize = 64 * 1024;

+       kern_ctl.ctl_bind = ipsec_ctl_bind;

        kern_ctl.ctl_connect = ipsec_ctl_connect;
        kern_ctl.ctl_disconnect = ipsec_ctl_disconnect;
        kern_ctl.ctl_send = ipsec_ctl_send;
        kern_ctl.ctl_connect = ipsec_ctl_connect;
        kern_ctl.ctl_disconnect = ipsec_ctl_disconnect;
        kern_ctl.ctl_send = ipsec_ctl_send;


@@ -734,7 +744,7 @@ ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
 
                length = MIN(kern_packet_get_data_length(tx_ph),
                bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
 
                length = MIN(kern_packet_get_data_length(tx_ph),

-                                        IPSEC_IF_DEFAULT_SLOT_SIZE);
+                                        pcb->ipsec_slot_size);

 
                // Increment TX stats
                tx_ring_stats.kcrsi_slots_transferred++;
 
                // Increment TX stats
                tx_ring_stats.kcrsi_slots_transferred++;


@@ -828,13 +838,10 @@ ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                (void)kern_channel_reclaim(tx_ring);
        }
 
                (void)kern_channel_reclaim(tx_ring);
        }
 

-       if (pcb->ipsec_output_disabled) {
-               errno_t error = ifnet_enable_output(pcb->ipsec_ifp);
-               if (error != 0) {
-                       printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
-               } else {
-                       pcb->ipsec_output_disabled = false;
-               }
+       /* always reenable output */
+       errno_t error = ifnet_enable_output(pcb->ipsec_ifp);
+       if (error != 0) {
+               printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);

        }
 
        // Unlock first, then exit ring
        }
 
        // Unlock first, then exit ring


@@ -985,7 +992,7 @@ ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
 
                length = MIN(kern_packet_get_data_length(tx_ph),
                bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0);
 
                length = MIN(kern_packet_get_data_length(tx_ph),

-                                        IPSEC_IF_DEFAULT_SLOT_SIZE);
+                                        pcb->ipsec_slot_size);

 
                if (length > 0) {
                        errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);
 
                if (length > 0) {
                        errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data);


@@ -1056,19 +1063,23 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
 {
 #pragma unused(nxprov)
        struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);
 {
 #pragma unused(nxprov)
        struct ipsec_pcb *pcb = kern_nexus_get_context(nexus);

-
-       lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
-

        boolean_t more = false;
        errno_t rc = 0;
        boolean_t more = false;
        errno_t rc = 0;

-       do {
-               rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more);
-               if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
-                       printf("%s, tx refill failed %d\n", __func__, rc);
-               }
-       } while ((rc == 0) && more);

 
 

-       if (pcb->ipsec_kpipe_enabled && !pcb->ipsec_output_disabled) {
+       /*
+        * Refill and sync the ring; we may be racing against another thread doing
+        * an RX sync that also wants to do kr_enter(), and so use the blocking
+        * variant here.
+        */
+       rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
+       if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
+               printf("%s, tx refill failed %d\n", __func__, rc);
+       }
+
+       (void) kr_enter(ring, TRUE);
+       lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
+
+       if (pcb->ipsec_kpipe_enabled) {

                uint32_t tx_available = kern_channel_available_slot_count(ring);
                if (pcb->ipsec_netif_txring_size > 0 &&
                        tx_available >= pcb->ipsec_netif_txring_size - 1) {
                uint32_t tx_available = kern_channel_available_slot_count(ring);
                if (pcb->ipsec_netif_txring_size > 0 &&
                        tx_available >= pcb->ipsec_netif_txring_size - 1) {


@@ -1076,14 +1087,11 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                        errno_t error = ifnet_disable_output(pcb->ipsec_ifp);
                        if (error != 0) {
                                printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
                        errno_t error = ifnet_disable_output(pcb->ipsec_ifp);
                        if (error != 0) {
                                printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);

-                       } else {
-                               pcb->ipsec_output_disabled = true;

                        }
                }
        }
 
                        }
                }
        }
 

-       if (pcb->ipsec_kpipe_enabled &&
-               (((rc != 0) && (rc != EAGAIN)) || pcb->ipsec_output_disabled)) {
+       if (pcb->ipsec_kpipe_enabled) {

                kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring;
 
                // Unlock while calling notify
                kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring;
 
                // Unlock while calling notify


@@ -1092,11 +1100,12 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                if (rx_ring != NULL) {
                        kern_channel_notify(rx_ring, 0);
                }
                if (rx_ring != NULL) {
                        kern_channel_notify(rx_ring, 0);
                }

-               lck_rw_lock_shared(&pcb->ipsec_pcb_lock);

        } else {
                lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
        }
 
        } else {
                lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
        }
 

+       kr_exit(ring);
+

        return (0);
 }
 
        return (0);
 }
 


@@ -1145,8 +1154,6 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);

-                       printf("ipsec_netif_sync_rx %s: failed to allocate packet\n",
-                                  pcb->ipsec_ifp->if_xname);

                        lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
                        break;
                }
                        lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
                        break;
                }


@@ -1366,8 +1373,6 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);

-                       printf("ipsec_netif_sync_rx %s: failed to allocate packet\n",
-                                  pcb->ipsec_ifp->if_xname);

                        break;
                }
 
                        break;
                }
 


@@ -1388,7 +1393,7 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                tx_baddr += kern_buflet_get_data_offset(tx_buf);
 
                length = MIN(kern_packet_get_data_length(tx_ph),
                tx_baddr += kern_buflet_get_data_offset(tx_buf);
 
                length = MIN(kern_packet_get_data_length(tx_ph),

-                                        IPSEC_IF_DEFAULT_SLOT_SIZE);
+                                        pcb->ipsec_slot_size);

 
                // Increment TX stats
                tx_ring_stats.kcrsi_slots_transferred++;
 
                // Increment TX stats
                tx_ring_stats.kcrsi_slots_transferred++;


@@ -1595,12 +1600,12 @@ ipsec_nexus_ifattach(struct ipsec_pcb *pcb,
                goto failed;
        }
 
                goto failed;
        }
 

-       uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
+       uint64_t slot_buffer_size = pcb->ipsec_slot_size;

        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for netif nexus to limit memory usage
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for netif nexus to limit memory usage

-       uint64_t ring_size = if_ipsec_ring_size;
+       uint64_t ring_size = pcb->ipsec_netif_ring_size;

        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);


@@ -1712,7 +1717,8 @@ ipsec_nexus_detach(ipsec_nx_t nx)
 }
 
 static errno_t
 }
 
 static errno_t

-ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
+ipsec_create_fs_provider_and_instance(struct ipsec_pcb *pcb,
+                                                                         uint32_t subtype, const char *type_name,

                                                                          const char *ifname,
                                                                          uuid_t *provider, uuid_t *instance)
 {
                                                                          const char *ifname,
                                                                          uuid_t *provider, uuid_t *instance)
 {


@@ -1743,15 +1749,15 @@ ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
        VERIFY(err == 0);
 
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
        VERIFY(err == 0);
 

-       uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
+       uint64_t slot_buffer_size = pcb->ipsec_slot_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.

-       uint64_t tx_ring_size = if_ipsec_tx_fsw_ring_size;
+       uint64_t tx_ring_size = pcb->ipsec_tx_fsw_ring_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
        VERIFY(err == 0);

-       uint64_t rx_ring_size = if_ipsec_rx_fsw_ring_size;
+       uint64_t rx_ring_size = pcb->ipsec_rx_fsw_ring_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
        VERIFY(err == 0);
 
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
        VERIFY(err == 0);
 


@@ -1798,7 +1804,8 @@ ipsec_multistack_attach(struct ipsec_pcb *pcb)
        ipsec_nx_t nx = &pcb->ipsec_nx;
 
        // Allocate multistack flowswitch
        ipsec_nx_t nx = &pcb->ipsec_nx;
 
        // Allocate multistack flowswitch

-       err = ipsec_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
+       err = ipsec_create_fs_provider_and_instance(pcb,
+                                                                                               NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,

                                                                                                "multistack",
                                                                                                pcb->ipsec_ifp->if_xname,
                                                                                                &nx->ms_provider,
                                                                                                "multistack",
                                                                                                pcb->ipsec_ifp->if_xname,
                                                                                                &nx->ms_provider,


@@ -2063,27 +2070,26 @@ done:
 /* Kernel control functions */
 
 static inline void
 /* Kernel control functions */
 
 static inline void

-ipsec_free_pcb(struct ipsec_pcb *pcb)
+ipsec_free_pcb(struct ipsec_pcb *pcb, bool in_list)

 {
 #if IPSEC_NEXUS
        mbuf_freem_list(pcb->ipsec_input_chain);
        lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp);
 #endif // IPSEC_NEXUS
        lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp);
 {
 #if IPSEC_NEXUS
        mbuf_freem_list(pcb->ipsec_input_chain);
        lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp);
 #endif // IPSEC_NEXUS
        lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp);

-       lck_mtx_lock(&ipsec_lock);
-       TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain);
-       lck_mtx_unlock(&ipsec_lock);
+       if (in_list) {
+               lck_mtx_lock(&ipsec_lock);
+               TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain);
+               lck_mtx_unlock(&ipsec_lock);
+       }

        zfree(ipsec_pcb_zone, pcb);
 }
 
 static errno_t
        zfree(ipsec_pcb_zone, pcb);
 }
 
 static errno_t

-ipsec_ctl_connect(kern_ctl_ref kctlref,
-                                 struct sockaddr_ctl *sac,
-                                 void **unitinfo)
+ipsec_ctl_bind(kern_ctl_ref kctlref,
+                          struct sockaddr_ctl *sac,
+                          void **unitinfo)

 {
 {

-       struct ifnet_init_eparams ipsec_init = {};
-       errno_t result = 0;
-       

        struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone);
        memset(pcb, 0, sizeof(*pcb));
 
        struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone);
        memset(pcb, 0, sizeof(*pcb));
 


@@ -2093,6 +2099,36 @@ ipsec_ctl_connect(kern_ctl_ref kctlref,
        pcb->ipsec_unit = sac->sc_unit;
        pcb->ipsec_output_service_class = MBUF_SC_OAM;
 
        pcb->ipsec_unit = sac->sc_unit;
        pcb->ipsec_output_service_class = MBUF_SC_OAM;
 

+#if IPSEC_NEXUS
+       pcb->ipsec_use_netif = false;
+       pcb->ipsec_slot_size = IPSEC_IF_DEFAULT_SLOT_SIZE;
+       pcb->ipsec_netif_ring_size = IPSEC_IF_DEFAULT_RING_SIZE;
+       pcb->ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE;
+       pcb->ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE;
+#endif // IPSEC_NEXUS
+
+       lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr);
+#if IPSEC_NEXUS
+       lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr);
+#endif // IPSEC_NEXUS
+
+       return (0);
+}
+
+static errno_t
+ipsec_ctl_connect(kern_ctl_ref kctlref,
+                                 struct sockaddr_ctl *sac,
+                                 void **unitinfo)
+{
+       struct ifnet_init_eparams ipsec_init = {};
+       errno_t result = 0;
+
+       if (*unitinfo == NULL) {
+               (void)ipsec_ctl_bind(kctlref, sac, unitinfo);
+       }
+
+       struct ipsec_pcb *pcb = *unitinfo;
+

        lck_mtx_lock(&ipsec_lock);
 
        /* Find some open interface id */
        lck_mtx_lock(&ipsec_lock);
 
        /* Find some open interface id */


@@ -2134,22 +2170,20 @@ ipsec_ctl_connect(kern_ctl_ref kctlref,
        snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1);
        printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name);
 
        snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1);
        printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name);
 

-       lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr);
-#if IPSEC_NEXUS
-       lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr);
-#endif // IPSEC_NEXUS
-

        /* Create the interface */
        bzero(&ipsec_init, sizeof(ipsec_init));
        ipsec_init.ver = IFNET_INIT_CURRENT_VERSION;
        ipsec_init.len = sizeof (ipsec_init);
 
 #if IPSEC_NEXUS
        /* Create the interface */
        bzero(&ipsec_init, sizeof(ipsec_init));
        ipsec_init.ver = IFNET_INIT_CURRENT_VERSION;
        ipsec_init.len = sizeof (ipsec_init);
 
 #if IPSEC_NEXUS

-       ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
-#else // IPSEC_NEXUS
-       ipsec_init.flags = IFNET_INIT_NX_NOAUTO;
-       ipsec_init.start = ipsec_start;
+       if (pcb->ipsec_use_netif) {
+               ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
+       } else

 #endif // IPSEC_NEXUS
 #endif // IPSEC_NEXUS

+       {
+               ipsec_init.flags = IFNET_INIT_NX_NOAUTO;
+               ipsec_init.start = ipsec_start;
+       }

        ipsec_init.name = "ipsec";
        ipsec_init.unit = pcb->ipsec_unit - 1;
        ipsec_init.uniqueid = pcb->ipsec_unique_name;
        ipsec_init.name = "ipsec";
        ipsec_init.unit = pcb->ipsec_unit - 1;
        ipsec_init.uniqueid = pcb->ipsec_unique_name;


@@ -2165,44 +2199,49 @@ ipsec_ctl_connect(kern_ctl_ref kctlref,
        ipsec_init.detach = ipsec_detached;
 
 #if IPSEC_NEXUS
        ipsec_init.detach = ipsec_detached;
 
 #if IPSEC_NEXUS

-       result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp);
-       if (result != 0) {
-               printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result);
-               ipsec_free_pcb(pcb);
-               *unitinfo = NULL;
-               return result;
-       }
-
-       result = ipsec_multistack_attach(pcb);
-       if (result != 0) {
-               printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result);
-               *unitinfo = NULL;
-               return result;
-       }
+       if (pcb->ipsec_use_netif) {
+               result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp);
+               if (result != 0) {
+                       printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result);
+                       ipsec_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return result;
+               }

 
 

-#else // IPSEC_NEXUS
-       result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp);
-       if (result != 0) {
-               printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result);
-               ipsec_free_pcb(pcb);
-               *unitinfo = NULL;
-               return result;
-       }
-       ipsec_ifnet_set_attrs(pcb->ipsec_ifp);
+               result = ipsec_multistack_attach(pcb);
+               if (result != 0) {
+                       printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result);
+                       *unitinfo = NULL;
+                       return result;
+               }

 
 

-       /* Attach the interface */
-       result = ifnet_attach(pcb->ipsec_ifp, NULL);
-       if (result != 0) {
-               printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result);
-               ifnet_release(pcb->ipsec_ifp);
-               ipsec_free_pcb(pcb);
-               *unitinfo = NULL;
-               return (result);
-       }
+               /* Attach to bpf */
+               bpfattach(pcb->ipsec_ifp, DLT_RAW, 0);
+       } else

 #endif // IPSEC_NEXUS
 #endif // IPSEC_NEXUS

+       {
+               result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp);
+               if (result != 0) {
+                       printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result);
+                       ipsec_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return result;
+               }
+               ipsec_ifnet_set_attrs(pcb->ipsec_ifp);
+
+               /* Attach the interface */
+               result = ifnet_attach(pcb->ipsec_ifp, NULL);
+               if (result != 0) {
+                       printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result);
+                       ifnet_release(pcb->ipsec_ifp);
+                       ipsec_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return (result);
+               }

 
 

-       /* Attach to bpf */
-       bpfattach(pcb->ipsec_ifp, DLT_RAW, 0);
+               /* Attach to bpf */
+               bpfattach(pcb->ipsec_ifp, DLT_NULL, 0);
+       }

 
        /* The interfaces resoures allocated, mark it as running */
        ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING);
 
        /* The interfaces resoures allocated, mark it as running */
        ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING);


@@ -2388,46 +2427,84 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref      kctlref,
        pcb->ipsec_kpipe_enabled = FALSE;
 #endif // IPSEC_NEXUS
 
        pcb->ipsec_kpipe_enabled = FALSE;
 #endif // IPSEC_NEXUS
 

-       ifp = pcb->ipsec_ifp;
-       VERIFY(ifp != NULL);

        pcb->ipsec_ctlref = NULL;
 
        pcb->ipsec_ctlref = NULL;
 

-       /*
-        * Quiesce the interface and flush any pending outbound packets.
-        */
-       if_down(ifp);
+       ifp = pcb->ipsec_ifp;
+       if (ifp != NULL) {
+#if IPSEC_NEXUS
+               if (pcb->ipsec_netif_nexus != NULL) {
+                       /*
+                        * Quiesce the interface and flush any pending outbound packets.
+                        */
+                       if_down(ifp);
+
+                       /* Increment refcnt, but detach interface */
+                       ifnet_incr_iorefcnt(ifp);
+                       if ((result = ifnet_detach(ifp)) != 0) {
+                               panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
+                               /* NOT REACHED */
+                       }

 
 

-       /* Increment refcnt, but detach interface */
-       ifnet_incr_iorefcnt(ifp);
-       if ((result = ifnet_detach(ifp)) != 0) {
-               panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
-               /* NOT REACHED */
-       }
+                       /*
+                        * We want to do everything in our power to ensure that the interface
+                        * really goes away when the socket is closed. We must remove IP/IPv6
+                        * addresses and detach the protocols. Finally, we can remove and
+                        * release the interface.
+                        */
+                       key_delsp_for_ipsec_if(ifp);

 
 

-       /*
-        * We want to do everything in our power to ensure that the interface
-        * really goes away when the socket is closed. We must remove IP/IPv6
-        * addresses and detach the protocols. Finally, we can remove and
-        * release the interface.
-        */
-       key_delsp_for_ipsec_if(ifp);
-    
-       ipsec_cleanup_family(ifp, AF_INET);
-       ipsec_cleanup_family(ifp, AF_INET6);
+                       ipsec_cleanup_family(ifp, AF_INET);
+                       ipsec_cleanup_family(ifp, AF_INET6);

 
 

-       lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
+                       lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
+
+                       if (!uuid_is_null(kpipe_uuid)) {
+                               if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) {
+                                       ipsec_unregister_kernel_pipe_nexus();
+                               }
+                       }
+                       ipsec_nexus_detach(&pcb->ipsec_nx);
+
+                       /* Decrement refcnt to finish detaching and freeing */
+                       ifnet_decr_iorefcnt(ifp);
+               } else
+#endif // IPSEC_NEXUS
+               {
+                       lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);

 
 #if IPSEC_NEXUS
 
 #if IPSEC_NEXUS

-       if (!uuid_is_null(kpipe_uuid)) {
-               if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) {
-                       ipsec_unregister_kernel_pipe_nexus();
-               }
-       }
-       ipsec_nexus_detach(&pcb->ipsec_nx);
+                       if (!uuid_is_null(kpipe_uuid)) {
+                               if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) {
+                                       ipsec_unregister_kernel_pipe_nexus();
+                               }
+                       }

 #endif // IPSEC_NEXUS
 
 #endif // IPSEC_NEXUS
 

-       /* Decrement refcnt to finish detaching and freeing */
-       ifnet_decr_iorefcnt(ifp);
+                       /*
+                        * We want to do everything in our power to ensure that the interface
+                        * really goes away when the socket is closed. We must remove IP/IPv6
+                        * addresses and detach the protocols. Finally, we can remove and
+                        * release the interface.
+                        */
+                       key_delsp_for_ipsec_if(ifp);
+
+                       ipsec_cleanup_family(ifp, AF_INET);
+                       ipsec_cleanup_family(ifp, AF_INET6);
+
+                       /*
+                        * Detach now; ipsec_detach() will be called asynchronously once
+                        * the I/O reference count drops to 0.  There we will invoke
+                        * ifnet_release().
+                        */
+                       if ((result = ifnet_detach(ifp)) != 0) {
+                               printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result);
+                       }
+               }
+       } else {
+               // Bound, but not connected
+               lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock);
+               ipsec_free_pcb(pcb, false);
+       }

        
        return 0;
 }
        
        return 0;
 }


@@ -2469,10 +2546,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref  kctlref,
        
        switch (opt) {
                case IPSEC_OPT_FLAGS:
        
        switch (opt) {
                case IPSEC_OPT_FLAGS:

-                       if (len != sizeof(u_int32_t))
+                       if (len != sizeof(u_int32_t)) {

                                result = EMSGSIZE;
                                result = EMSGSIZE;

-                       else
+                       } else {

                                pcb->ipsec_flags = *(u_int32_t *)data;
                                pcb->ipsec_flags = *(u_int32_t *)data;

+                       }

                        break;
                        
                case IPSEC_OPT_EXT_IFDATA_STATS:
                        break;
                        
                case IPSEC_OPT_EXT_IFDATA_STATS:


@@ -2480,6 +2558,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
                        break;
                        
                        pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
                        break;
                        


@@ -2491,6 +2574,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EINVAL;
                                break;
                        }
                                result = EINVAL;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (!pcb->ipsec_ext_ifdata_stats) {
                                result = EINVAL;
                                break;
                        if (!pcb->ipsec_ext_ifdata_stats) {
                                result = EINVAL;
                                break;


@@ -2512,6 +2600,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (len != 0) {   /* if len==0, del_ifp will be NULL causing the delegate to be removed */
                                bcopy(data, name, len);
                                name[len] = 0;
                        if (len != 0) {   /* if len==0, del_ifp will be NULL causing the delegate to be removed */
                                bcopy(data, name, len);
                                name[len] = 0;


@@ -2534,6 +2627,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data);
                        if (output_service_class == MBUF_SC_UNSPEC) {
                                pcb->ipsec_output_service_class = MBUF_SC_OAM;
                        mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data);
                        if (output_service_class == MBUF_SC_UNSPEC) {
                                pcb->ipsec_output_service_class = MBUF_SC_OAM;


@@ -2552,6 +2650,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (*(int *)data) {
                                result = ipsec_enable_channel(pcb, current_proc());
                        } else {
                        if (*(int *)data) {
                                result = ipsec_enable_channel(pcb, current_proc());
                        } else {


@@ -2565,6 +2668,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->ipsec_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (!if_enable_netagent) {
                                result = ENOTSUP;
                                break;
                        if (!if_enable_netagent) {
                                result = ENOTSUP;
                                break;


@@ -2597,6 +2705,92 @@ ipsec_ctl_setopt(__unused kern_ctl_ref   kctlref,
                        }
                        break;
                }
                        }
                        break;
                }

+               case IPSEC_OPT_ENABLE_NETIF: {
+                       if (len != sizeof(int)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->ipsec_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       pcb->ipsec_use_netif = true;
+                       break;
+               }
+               case IPSEC_OPT_SLOT_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->ipsec_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t slot_size = *(u_int32_t *)data;
+                       if (slot_size < IPSEC_IF_MIN_SLOT_SIZE ||
+                               slot_size > IPSEC_IF_MAX_SLOT_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->ipsec_slot_size = slot_size;
+                       break;
+               }
+               case IPSEC_OPT_NETIF_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->ipsec_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
+                               ring_size > IPSEC_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->ipsec_netif_ring_size = ring_size;
+                       break;
+               }
+               case IPSEC_OPT_TX_FSW_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->ipsec_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
+                               ring_size > IPSEC_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->ipsec_tx_fsw_ring_size = ring_size;
+                       break;
+               }
+               case IPSEC_OPT_RX_FSW_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->ipsec_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < IPSEC_IF_MIN_RING_SIZE ||
+                               ring_size > IPSEC_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->ipsec_rx_fsw_ring_size = ring_size;
+                       break;
+               }
+

 #endif // IPSEC_NEXUS
                        
                default:
 #endif // IPSEC_NEXUS
                        
                default:


@@ -2641,6 +2835,11 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref,
                        if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) {
                                result = EMSGSIZE;
                        } else {
                        if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) {
                                result = EMSGSIZE;
                        } else {

+                               if (pcb->ipsec_ifp == NULL) {
+                                       // Only can get after connecting
+                                       result = EINVAL;
+                                       break;
+                               }

                                *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1;
                        }
                        break;
                                *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1;
                        }
                        break;


@@ -2677,6 +2876,39 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref,
                        }
                        break;
                }
                        }
                        break;
                }

+               case IPSEC_OPT_SLOT_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->ipsec_slot_size;
+                       }
+                       break;
+               }
+               case IPSEC_OPT_NETIF_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->ipsec_netif_ring_size;
+                       }
+                       break;
+               }
+               case IPSEC_OPT_TX_FSW_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->ipsec_tx_fsw_ring_size;
+                       }
+                       break;
+               }
+               case IPSEC_OPT_RX_FSW_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->ipsec_rx_fsw_ring_size;
+                       }
+                       break;
+               }
+

 #endif // IPSEC_NEXUS
 
                default: {
 #endif // IPSEC_NEXUS
 
                default: {


@@ -2721,6 +2953,14 @@ ipsec_output(ifnet_t interface,
        
     switch (ip_version) {
                case 4: {
        
     switch (ip_version) {
                case 4: {

+#if IPSEC_NEXUS
+                       if (!pcb->ipsec_use_netif)
+#endif // IPSEC_NEXUS
+                       {
+                               int af = AF_INET;
+                               bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
+                       }
+

             /* Apply encryption */
             memset(&ipsec_state, 0, sizeof(ipsec_state));
             ipsec_state.m = data;
             /* Apply encryption */
             memset(&ipsec_state, 0, sizeof(ipsec_state));
             ipsec_state.m = data;


@@ -2785,6 +3025,14 @@ ipsec_output(ifnet_t interface,
             goto done;
                }
                case 6: {
             goto done;
                }
                case 6: {

+#if IPSEC_NEXUS
+                       if (!pcb->ipsec_use_netif)
+#endif // IPSEC_NEXUS
+                       {
+                               int af = AF_INET6;
+                               bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af));
+                       }
+

             data = ipsec6_splithdr(data);
                        if (data == NULL) {
                                printf("ipsec_output: ipsec6_splithdr returned NULL\n");
             data = ipsec6_splithdr(data);
                        if (data == NULL) {
                                printf("ipsec_output: ipsec6_splithdr returned NULL\n");


@@ -2863,7 +3111,6 @@ ipsec_output_err:
        goto done;
 }
 
        goto done;
 }
 

-#if !IPSEC_NEXUS

 static void
 ipsec_start(ifnet_t    interface)
 {
 static void
 ipsec_start(ifnet_t    interface)
 {


@@ -2878,7 +3125,6 @@ ipsec_start(ifnet_t       interface)
                        break;
        }
 }
                        break;
        }
 }

-#endif // !IPSEC_NEXUS

 
 /* Network Interface functions */
 static errno_t
 
 /* Network Interface functions */
 static errno_t


@@ -2944,19 +3190,26 @@ ipsec_ioctl(ifnet_t interface,
                        u_long command,
                        void *data)
 {
                        u_long command,
                        void *data)
 {

+       struct ipsec_pcb *pcb = ifnet_softc(interface);

        errno_t result = 0;
        
        switch(command) {
        errno_t result = 0;
        
        switch(command) {

-               case SIOCSIFMTU:
+               case SIOCSIFMTU: {

 #if IPSEC_NEXUS
 #if IPSEC_NEXUS

-                       // Make sure we can fit packets in the channel buffers
-                       if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > IPSEC_IF_DEFAULT_SLOT_SIZE) {
-                               ifnet_set_mtu(interface, IPSEC_IF_DEFAULT_SLOT_SIZE);
-                               break;
-                       }
+                       if (pcb->ipsec_use_netif) {
+                               // Make sure we can fit packets in the channel buffers
+                               if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > pcb->ipsec_slot_size) {
+                                       result = EINVAL;
+                               } else {
+                                       ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
+                               }
+                       } else

 #endif // IPSEC_NEXUS
 #endif // IPSEC_NEXUS

-                       ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
+                       {
+                               ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
+                       }

                        break;
                        break;

+               }

                        
                case SIOCSIFFLAGS:
                        /* ifioctl() takes care of it */
                        
                case SIOCSIFFLAGS:
                        /* ifioctl() takes care of it */


@@ -2974,7 +3227,7 @@ ipsec_detached(ifnet_t interface)
 {
        struct ipsec_pcb *pcb = ifnet_softc(interface);
        (void)ifnet_release(interface);
 {
        struct ipsec_pcb *pcb = ifnet_softc(interface);
        (void)ifnet_release(interface);

-       ipsec_free_pcb(pcb);
+       ipsec_free_pcb(pcb, true);

 }
 
 /* Protocol Handlers */
 }
 
 /* Protocol Handlers */


@@ -2986,6 +3239,21 @@ ipsec_proto_input(ifnet_t interface,
                                  __unused char *frame_header)
 {
        mbuf_pkthdr_setrcvif(m, interface);
                                  __unused char *frame_header)
 {
        mbuf_pkthdr_setrcvif(m, interface);

+
+#if IPSEC_NEXUS
+       struct ipsec_pcb *pcb = ifnet_softc(interface);
+       if (!pcb->ipsec_use_netif)
+#endif // IPSEC_NEXUS
+       {
+               uint32_t af = 0;
+               struct ip *ip = mtod(m, struct ip *);
+               if (ip->ip_v == 4) {
+                       af = AF_INET;
+               } else if (ip->ip_v == 6) {
+                       af = AF_INET6;
+               }
+               bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af));
+       }

        pktap_input(interface, protocol, m, NULL);
 
        if (proto_input(protocol, m) != 0) {
        pktap_input(interface, protocol, m, NULL);
 
        if (proto_input(protocol, m) != 0) {


@@ -3032,51 +3300,49 @@ ipsec_attach_proto(ifnet_t                              interface,
        return result;
 }
 
        return result;
 }
 

-#if IPSEC_NEXUS

 errno_t
 ipsec_inject_inbound_packet(ifnet_t    interface,
                                                        mbuf_t packet)
 {
        struct ipsec_pcb *pcb = ifnet_softc(interface);
 
 errno_t
 ipsec_inject_inbound_packet(ifnet_t    interface,
                                                        mbuf_t packet)
 {
        struct ipsec_pcb *pcb = ifnet_softc(interface);
 

-       lck_rw_lock_shared(&pcb->ipsec_pcb_lock);
+#if IPSEC_NEXUS
+       if (pcb->ipsec_use_netif) {
+               lck_rw_lock_shared(&pcb->ipsec_pcb_lock);

 
 

-       lck_mtx_lock(&pcb->ipsec_input_chain_lock);
-       if (pcb->ipsec_input_chain != NULL) {
-               pcb->ipsec_input_chain_last->m_nextpkt = packet;
-       } else {
-               pcb->ipsec_input_chain = packet;
-       }
-       while (packet->m_nextpkt) {
-               VERIFY(packet != packet->m_nextpkt);
-               packet = packet->m_nextpkt;
-       }
-       pcb->ipsec_input_chain_last = packet;
-       lck_mtx_unlock(&pcb->ipsec_input_chain_lock);
+               lck_mtx_lock(&pcb->ipsec_input_chain_lock);
+               if (pcb->ipsec_input_chain != NULL) {
+                       pcb->ipsec_input_chain_last->m_nextpkt = packet;
+               } else {
+                       pcb->ipsec_input_chain = packet;
+               }
+               while (packet->m_nextpkt) {
+                       VERIFY(packet != packet->m_nextpkt);
+                       packet = packet->m_nextpkt;
+               }
+               pcb->ipsec_input_chain_last = packet;
+               lck_mtx_unlock(&pcb->ipsec_input_chain_lock);

 
 

-       kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring;
-       lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);
+               kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring;
+               lck_rw_unlock_shared(&pcb->ipsec_pcb_lock);

 
 

-       if (rx_ring != NULL) {
-               kern_channel_notify(rx_ring, 0);
-       }
+               if (rx_ring != NULL) {
+                       kern_channel_notify(rx_ring, 0);
+               }

 
 

-       return (0);
-}
-#else // IPSEC_NEXUS
-errno_t
-ipsec_inject_inbound_packet(ifnet_t    interface,
-                                                       mbuf_t packet)
-{
-       errno_t error;
-       protocol_family_t protocol;
-       if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) {
-               return error;
-       }
+               return (0);
+       } else
+#endif // IPSEC_NEXUS
+       {
+               errno_t error;
+               protocol_family_t protocol;
+               if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) {
+                       return error;
+               }

 
 

-       return ipsec_proto_input(interface, protocol, packet, NULL);
+               return ipsec_proto_input(interface, protocol, packet, NULL);
+       }

 }
 }

-#endif // IPSEC_NEXUS

 
 void
 ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family)
 
 void
 ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family)

diff --git a/bsd/net/if_ipsec.h b/bsd/net/if_ipsec.h
index 159ef2036b63721f6954676f0aff8c6d2874059c..93429c1adf511681bb356e56e9cc7d6bd60036f8 100644 (file)
--- a/bsd/net/if_ipsec.h
+++ b/bsd/net/if_ipsec.h
@@ -72,6 +72,12 @@ void ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa
 #define IPSEC_OPT_ENABLE_FLOWSWITCH                            10      /* enable a flowswitch nexus that clients can use */
 #define IPSEC_OPT_INPUT_FRAG_SIZE                              11      /* set the maximum size of input packets before fragmenting as a uint32_t */
 
 #define IPSEC_OPT_ENABLE_FLOWSWITCH                            10      /* enable a flowswitch nexus that clients can use */
 #define IPSEC_OPT_INPUT_FRAG_SIZE                              11      /* set the maximum size of input packets before fragmenting as a uint32_t */
 

+#define IPSEC_OPT_ENABLE_NETIF                                 12              /* Must be set before connecting */
+#define IPSEC_OPT_SLOT_SIZE                                            13              /* Must be set before connecting */
+#define IPSEC_OPT_NETIF_RING_SIZE                              14              /* Must be set before connecting */
+#define IPSEC_OPT_TX_FSW_RING_SIZE                             15              /* Must be set before connecting */
+#define IPSEC_OPT_RX_FSW_RING_SIZE                             16              /* Must be set before connecting */
+

 /*
  * ipsec stats parameter structure
  */
 /*
  * ipsec stats parameter structure
  */

diff --git a/bsd/net/if_loop.c b/bsd/net/if_loop.c
index 00a8345e3a8fe7a8536f9d47fe802e144b58edbd..cc2089cb629fd9c515b490d0917f456aac34c1f2 100644 (file)
--- a/bsd/net/if_loop.c
+++ b/bsd/net/if_loop.c
@@ -698,6 +698,13 @@ loopattach(void)
                    __func__, result);
                /* NOTREACHED */
        }
                    __func__, result);
                /* NOTREACHED */
        }

+       /*
+        * Disable ECN on loopback as ECN serves no purpose and otherwise
+        * TCP connections are subject to heuristics like SYN retransmits on RST
+        */
+       lo_ifp->if_eflags &= ~IFEF_ECN_ENABLE;
+       lo_ifp->if_eflags |= IFEF_ECN_DISABLE;
+

        bpfattach(lo_ifp, DLT_NULL, sizeof (u_int32_t));
 }
 
        bpfattach(lo_ifp, DLT_NULL, sizeof (u_int32_t));
 }
 

diff --git a/bsd/net/if_utun.c b/bsd/net/if_utun.c
index 5e01509dfad8c961497fbe20acc8b97b29b1ac0e..ffaa1124c9e82e4326fd578ea9f9866df5691842 100644 (file)
--- a/bsd/net/if_utun.c
+++ b/bsd/net/if_utun.c
@@ -96,7 +96,6 @@ struct utun_pcb {
        // Input chain lock protects the list of input mbufs
        // The input chain lock must be taken AFTER the PCB lock if both are held
        lck_mtx_t               utun_input_chain_lock;
        // Input chain lock protects the list of input mbufs
        // The input chain lock must be taken AFTER the PCB lock if both are held
        lck_mtx_t               utun_input_chain_lock;

-       bool                    utun_output_disabled;

 
 #if UTUN_NEXUS
        struct utun_nx  utun_nx;
 
 #if UTUN_NEXUS
        struct utun_nx  utun_nx;


@@ -109,10 +108,18 @@ struct utun_pcb {
        void *                  utun_netif_rxring;
        void *                  utun_netif_txring;
        uint64_t                utun_netif_txring_size;
        void *                  utun_netif_rxring;
        void *                  utun_netif_txring;
        uint64_t                utun_netif_txring_size;

+
+       u_int32_t               utun_slot_size;
+       u_int32_t               utun_netif_ring_size;
+       u_int32_t               utun_tx_fsw_ring_size;
+       u_int32_t               utun_rx_fsw_ring_size;
+       bool                    utun_use_netif;

 #endif // UTUN_NEXUS
 };
 
 /* Kernel Control functions */
 #endif // UTUN_NEXUS
 };
 
 /* Kernel Control functions */

+static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
+                                                         void **unitinfo);

 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
                                                                 void **unitinfo);
 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,
 static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac,
                                                                 void **unitinfo);
 static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit,


@@ -127,12 +134,10 @@ static void               utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
                                                                int flags);
 
 /* Network Interface functions */
                                                                int flags);
 
 /* Network Interface functions */

-#if !UTUN_NEXUS

 static void     utun_start(ifnet_t interface);
 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
                                                        const struct sockaddr *dest, const char *desk_linkaddr,
                                                        const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);
 static void     utun_start(ifnet_t interface);
 static errno_t utun_framer(ifnet_t interface, mbuf_t *packet,
                                                        const struct sockaddr *dest, const char *desk_linkaddr,
                                                        const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len);

-#endif // !UTUN_NEXUS

 static errno_t utun_output(ifnet_t interface, mbuf_t data);
 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
                                                   protocol_family_t *protocol);
 static errno_t utun_output(ifnet_t interface, mbuf_t data);
 static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header,
                                                   protocol_family_t *protocol);


@@ -150,11 +155,11 @@ static errno_t    utun_proto_input(ifnet_t interface, protocol_family_t protocol,
 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol, 
                                         mbuf_t *packet, const struct sockaddr *dest, void *route,
                                         char *frame_type, char *link_layer_dest);
 static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol, 
                                         mbuf_t *packet, const struct sockaddr *dest, void *route,
                                         char *frame_type, char *link_layer_dest);

-static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m);
+static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m);

 
 #if UTUN_NEXUS
 
 
 #if UTUN_NEXUS
 

-#define UTUN_IF_DEFAULT_SLOT_SIZE 4096
+#define UTUN_IF_DEFAULT_SLOT_SIZE 2048

 #define UTUN_IF_DEFAULT_RING_SIZE 64
 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128
 #define UTUN_IF_DEFAULT_RING_SIZE 64
 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64
 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128


@@ -163,6 +168,9 @@ static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m);
 #define UTUN_IF_MIN_RING_SIZE 16
 #define UTUN_IF_MAX_RING_SIZE 1024
 
 #define UTUN_IF_MIN_RING_SIZE 16
 #define UTUN_IF_MAX_RING_SIZE 1024
 

+#define UTUN_IF_MIN_SLOT_SIZE 1024
+#define UTUN_IF_MAX_SLOT_SIZE 4096
+

 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS;
 static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS;


@@ -427,7 +435,7 @@ utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                tx_length += UTUN_HEADER_SIZE(pcb);
                tx_baddr += tx_offset;
 
                tx_length += UTUN_HEADER_SIZE(pcb);
                tx_baddr += tx_offset;
 

-               length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE);
+               length = MIN(tx_length, pcb->utun_slot_size);

 
                // Copy in family
                memcpy(tx_baddr, &af, sizeof(af));
 
                // Copy in family
                memcpy(tx_baddr, &af, sizeof(af));


@@ -492,20 +500,23 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
 {
 #pragma unused(nxprov)
        struct utun_pcb *pcb = kern_nexus_get_context(nexus);
 {
 #pragma unused(nxprov)
        struct utun_pcb *pcb = kern_nexus_get_context(nexus);

-
-       lck_rw_lock_shared(&pcb->utun_pcb_lock);
-

        boolean_t more = false;
        errno_t rc = 0;
        boolean_t more = false;
        errno_t rc = 0;

-       do {
-               /* Refill and sync the ring */
-               rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more);
-               if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
-                       printf("%s, tx refill failed %d\n", __func__, rc);
-               }
-       } while ((rc == 0) && more);

 
 

-       if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) {
+       /*
+        * Refill and sync the ring; we may be racing against another thread doing
+        * an RX sync that also wants to do kr_enter(), and so use the blocking
+        * variant here.
+        */
+       rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more);
+       if (rc != 0 && rc != EAGAIN && rc != EBUSY) {
+               printf("%s, tx refill failed %d\n", __func__, rc);
+       }
+
+       (void) kr_enter(ring, TRUE);
+       lck_rw_lock_shared(&pcb->utun_pcb_lock);
+
+       if (pcb->utun_kpipe_enabled) {

                uint32_t tx_available = kern_channel_available_slot_count(ring);
                if (pcb->utun_netif_txring_size > 0 &&
                        tx_available >= pcb->utun_netif_txring_size - 1) {
                uint32_t tx_available = kern_channel_available_slot_count(ring);
                if (pcb->utun_netif_txring_size > 0 &&
                        tx_available >= pcb->utun_netif_txring_size - 1) {


@@ -513,14 +524,11 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                        errno_t error = ifnet_disable_output(pcb->utun_ifp);
                        if (error != 0) {
                                printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);
                        errno_t error = ifnet_disable_output(pcb->utun_ifp);
                        if (error != 0) {
                                printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error);

-                       } else {
-                               pcb->utun_output_disabled = true;

                        }
                }
        }
 
                        }
                }
        }
 

-       if (pcb->utun_kpipe_enabled &&
-               (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) {
+       if (pcb->utun_kpipe_enabled) {

                kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
 
                // Unlock while calling notify
                kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring;
 
                // Unlock while calling notify


@@ -533,6 +541,8 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                lck_rw_unlock_shared(&pcb->utun_pcb_lock);
        }
 
                lck_rw_unlock_shared(&pcb->utun_pcb_lock);
        }
 

+       kr_exit(ring);
+

        return (0);
 }
 
        return (0);
 }
 


@@ -581,8 +591,6 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);

-                       printf("utun_netif_sync_rx %s: failed to allocate packet\n",
-                                  pcb->utun_ifp->if_xname);

                        lck_mtx_unlock(&pcb->utun_input_chain_lock);
                        break;
                }
                        lck_mtx_unlock(&pcb->utun_input_chain_lock);
                        break;
                }


@@ -706,8 +714,6 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);
                if (unlikely(error != 0)) {
                        STATS_INC(nifs, NETIF_STATS_NOMEM_PKT);
                        STATS_INC(nifs, NETIF_STATS_DROPPED);

-                       printf("utun_netif_sync_rx %s: failed to allocate packet\n",
-                                  pcb->utun_ifp->if_xname);

                        break;
                }
 
                        break;
                }
 


@@ -731,7 +737,7 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                }
 
                size_t length = MIN(tx_length - header_offset,
                }
 
                size_t length = MIN(tx_length - header_offset,

-                                                       UTUN_IF_DEFAULT_SLOT_SIZE);
+                                                       pcb->utun_slot_size);

 
                tx_ring_stats.kcrsi_slots_transferred++;
                tx_ring_stats.kcrsi_bytes_transferred += length;
 
                tx_ring_stats.kcrsi_slots_transferred++;
                tx_ring_stats.kcrsi_bytes_transferred += length;


@@ -832,12 +838,12 @@ utun_nexus_ifattach(struct utun_pcb *pcb,
                goto failed;
        }
 
                goto failed;
        }
 

-       uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
+       uint64_t slot_buffer_size = pcb->utun_slot_size;

        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for netif nexus to limit memory usage
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for netif nexus to limit memory usage

-       uint64_t ring_size = if_utun_ring_size;
+       uint64_t ring_size = pcb->utun_netif_ring_size;

        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size);


@@ -947,7 +953,8 @@ utun_nexus_detach(utun_nx_t nx)
 }
 
 static errno_t
 }
 
 static errno_t

-utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
+utun_create_fs_provider_and_instance(struct utun_pcb *pcb,
+                                                                        uint32_t subtype, const char *type_name,

                                                                         const char *ifname,
                                                                         uuid_t *provider, uuid_t *instance)
 {
                                                                         const char *ifname,
                                                                         uuid_t *provider, uuid_t *instance)
 {


@@ -976,15 +983,15 @@ utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name,
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
        VERIFY(err == 0);
 
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype);
        VERIFY(err == 0);
 

-       uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE;
+       uint64_t slot_buffer_size = pcb->utun_slot_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size);
        VERIFY(err == 0);
 
        // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif.

-       uint64_t tx_ring_size = if_utun_tx_fsw_ring_size;
+       uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
        VERIFY(err == 0);
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size);
        VERIFY(err == 0);

-       uint64_t rx_ring_size = if_utun_rx_fsw_ring_size;
+       uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size;

        err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
        VERIFY(err == 0);
 
        err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size);
        VERIFY(err == 0);
 


@@ -1029,7 +1036,8 @@ utun_multistack_attach(struct utun_pcb *pcb)
        utun_nx_t nx = &pcb->utun_nx;
 
        // Allocate multistack flowswitch
        utun_nx_t nx = &pcb->utun_nx;
 
        // Allocate multistack flowswitch

-       err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,
+       err = utun_create_fs_provider_and_instance(pcb,
+                                                                                          NEXUS_EXTENSION_FSW_TYPE_MULTISTACK,

                                                                                           "multistack",
                                                                                           pcb->utun_ifp->if_xname,
                                                                                           &nx->ms_provider,
                                                                                           "multistack",
                                                                                           pcb->utun_ifp->if_xname,
                                                                                           &nx->ms_provider,


@@ -1258,7 +1266,7 @@ utun_enable_channel(struct utun_pcb *pcb, struct proc *proc)
         * Make sure we can fit packets in the channel buffers and
         * Allow an extra 4 bytes for the protocol number header in the channel
         */
         * Make sure we can fit packets in the channel buffers and
         * Allow an extra 4 bytes for the protocol number header in the channel
         */

-       if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) {
+       if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) {

                result = EOPNOTSUPP;
                goto done;
        }
                result = EOPNOTSUPP;
                goto done;
        }


@@ -1331,6 +1339,7 @@ utun_register_control(void)
        kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
        kern_ctl.ctl_sendsize = 512 * 1024;
        kern_ctl.ctl_recvsize = 512 * 1024;
        kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */
        kern_ctl.ctl_sendsize = 512 * 1024;
        kern_ctl.ctl_recvsize = 512 * 1024;

+       kern_ctl.ctl_bind = utun_ctl_bind;

        kern_ctl.ctl_connect = utun_ctl_connect;
        kern_ctl.ctl_disconnect = utun_ctl_disconnect;
        kern_ctl.ctl_send = utun_ctl_send;
        kern_ctl.ctl_connect = utun_ctl_connect;
        kern_ctl.ctl_disconnect = utun_ctl_disconnect;
        kern_ctl.ctl_send = utun_ctl_send;


@@ -1367,9 +1376,7 @@ utun_register_control(void)
        utun_lck_grp_attr = lck_grp_attr_alloc_init();
        utun_lck_grp = lck_grp_alloc_init("utun",  utun_lck_grp_attr);
 
        utun_lck_grp_attr = lck_grp_attr_alloc_init();
        utun_lck_grp = lck_grp_alloc_init("utun",  utun_lck_grp_attr);
 

-#if UTUN_NEXUS

        lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);
        lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr);

-#endif // UTUN_NEXUS

 
        return 0;
 }
 
        return 0;
 }


@@ -1377,27 +1384,26 @@ utun_register_control(void)
 /* Kernel control functions */
 
 static inline void
 /* Kernel control functions */
 
 static inline void

-utun_free_pcb(struct utun_pcb *pcb)
+utun_free_pcb(struct utun_pcb *pcb, bool in_list)

 {
 #ifdef UTUN_NEXUS
        mbuf_freem_list(pcb->utun_input_chain);
        lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
 #endif // UTUN_NEXUS
        lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);
 {
 #ifdef UTUN_NEXUS
        mbuf_freem_list(pcb->utun_input_chain);
        lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp);
 #endif // UTUN_NEXUS
        lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp);

-       lck_mtx_lock(&utun_lock);
-       TAILQ_REMOVE(&utun_head, pcb, utun_chain);
-       lck_mtx_unlock(&utun_lock);
+       if (in_list) {
+               lck_mtx_lock(&utun_lock);
+               TAILQ_REMOVE(&utun_head, pcb, utun_chain);
+               lck_mtx_unlock(&utun_lock);
+       }

        zfree(utun_pcb_zone, pcb);
 }
 
 static errno_t
        zfree(utun_pcb_zone, pcb);
 }
 
 static errno_t

-utun_ctl_connect(kern_ctl_ref kctlref,
-                                struct sockaddr_ctl *sac,
-                                void **unitinfo)
+utun_ctl_bind(kern_ctl_ref kctlref,
+                         struct sockaddr_ctl *sac,
+                         void **unitinfo)

 {
 {

-       struct ifnet_init_eparams utun_init = {};
-       errno_t result = 0;
-       

        struct utun_pcb *pcb = zalloc(utun_pcb_zone);
        memset(pcb, 0, sizeof(*pcb));
 
        struct utun_pcb *pcb = zalloc(utun_pcb_zone);
        memset(pcb, 0, sizeof(*pcb));
 


@@ -1406,9 +1412,34 @@ utun_ctl_connect(kern_ctl_ref kctlref,
        pcb->utun_unit = sac->sc_unit;
        pcb->utun_max_pending_packets = 1;
 
        pcb->utun_unit = sac->sc_unit;
        pcb->utun_max_pending_packets = 1;
 

+#if UTUN_NEXUS
+       pcb->utun_use_netif = false;
+       pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE;
+       pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE;
+       pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE;
+       pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE;
+#endif // UTUN_NEXUS
+

        lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
        lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
 
        lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr);
        lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr);
 

+       return (0);
+}
+
+static errno_t
+utun_ctl_connect(kern_ctl_ref kctlref,
+                                struct sockaddr_ctl *sac,
+                                void **unitinfo)
+{
+       struct ifnet_init_eparams utun_init = {};
+       errno_t result = 0;
+
+       if (*unitinfo == NULL) {
+               (void)utun_ctl_bind(kctlref, sac, unitinfo);
+       }
+       
+       struct utun_pcb *pcb = *unitinfo;
+

        lck_mtx_lock(&utun_lock);
 
        /* Find some open interface id */
        lck_mtx_lock(&utun_lock);
 
        /* Find some open interface id */


@@ -1456,13 +1487,16 @@ utun_ctl_connect(kern_ctl_ref kctlref,
        utun_init.len = sizeof (utun_init);
 
 #if UTUN_NEXUS
        utun_init.len = sizeof (utun_init);
 
 #if UTUN_NEXUS

-       utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
-       utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
-#else // UTUN_NEXUS
-       utun_init.flags = IFNET_INIT_NX_NOAUTO;
-       utun_init.start = utun_start;
-       utun_init.framer_extended = utun_framer;
+       if (pcb->utun_use_netif) {
+               utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO);
+               utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE;
+       } else

 #endif // UTUN_NEXUS
 #endif // UTUN_NEXUS

+       {
+               utun_init.flags = IFNET_INIT_NX_NOAUTO;
+               utun_init.start = utun_start;
+               utun_init.framer_extended = utun_framer;
+       }

        utun_init.name = "utun";
        utun_init.unit = pcb->utun_unit - 1;
        utun_init.uniqueid = pcb->utun_unique_name;
        utun_init.name = "utun";
        utun_init.unit = pcb->utun_unit - 1;
        utun_init.uniqueid = pcb->utun_unique_name;


@@ -1478,62 +1512,68 @@ utun_ctl_connect(kern_ctl_ref kctlref,
        utun_init.detach = utun_detached;
 
 #if UTUN_NEXUS
        utun_init.detach = utun_detached;
 
 #if UTUN_NEXUS

-       result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
-       if (result != 0) {
-               printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
-               utun_free_pcb(pcb);
-               *unitinfo = NULL;
-               return result;
-       }
+       if (pcb->utun_use_netif) {
+               result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp);
+               if (result != 0) {
+                       printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result);
+                       utun_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return result;
+               }

 
 

-       result = utun_multistack_attach(pcb);
-       if (result != 0) {
-               printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
-               *unitinfo = NULL;
-               return result;
-       }
+               result = utun_multistack_attach(pcb);
+               if (result != 0) {
+                       printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result);
+                       *unitinfo = NULL;
+                       return result;
+               }

 
 

-#else // UTUN_NEXUS
-       /*
-        * Upon success, this holds an ifnet reference which we will
-        * release via ifnet_release() at final detach time.
-        */
-       result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
-       if (result != 0) {
-               printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
-               utun_free_pcb(pcb);
-               *unitinfo = NULL;
-               return result;
-       }
-       
-       /* Set flags and additional information. */
-       ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
-       ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);
+               /* Attach to bpf */
+               bpfattach(pcb->utun_ifp, DLT_RAW, 0);
+       } else
+#endif // UTUN_NEXUS
+       {
+               /*
+                * Upon success, this holds an ifnet reference which we will
+                * release via ifnet_release() at final detach time.
+                */
+               result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp);
+               if (result != 0) {
+                       printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result);
+                       utun_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return result;
+               }

 
 

-       /* The interface must generate its own IPv6 LinkLocal address,
-        * if possible following the recommendation of RFC2472 to the 64bit interface ID
-        */
-       ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
-       
-       /* Reset the stats in case as the interface may have been recycled */
-       struct ifnet_stats_param stats;
-       bzero(&stats, sizeof(struct ifnet_stats_param));
-       ifnet_set_stat(pcb->utun_ifp, &stats);
+               /* Set flags and additional information. */
+               ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU);
+               ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff);

 
 

-       /* Attach the interface */
-       result = ifnet_attach(pcb->utun_ifp, NULL);
-       if (result != 0) {
-               printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
-               /* Release reference now since attach failed */
-               ifnet_release(pcb->utun_ifp);
-               utun_free_pcb(pcb);
-               *unitinfo = NULL;
-               return (result);
+               /* The interface must generate its own IPv6 LinkLocal address,
+                * if possible following the recommendation of RFC2472 to the 64bit interface ID
+                */
+               ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL);
+
+               /* Reset the stats in case as the interface may have been recycled */
+               struct ifnet_stats_param stats;
+               bzero(&stats, sizeof(struct ifnet_stats_param));
+               ifnet_set_stat(pcb->utun_ifp, &stats);
+
+               /* Attach the interface */
+               result = ifnet_attach(pcb->utun_ifp, NULL);
+               if (result != 0) {
+                       printf("utun_ctl_connect - ifnet_attach failed: %d\n", result);
+                       /* Release reference now since attach failed */
+                       ifnet_release(pcb->utun_ifp);
+                       utun_free_pcb(pcb, true);
+                       *unitinfo = NULL;
+                       return (result);
+               }
+
+               /* Attach to bpf */
+               bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));

        }
        }

-#endif // UTUN_NEXUS

 
 

-       /* Attach to bpf */
-       bpfattach(pcb->utun_ifp, DLT_RAW, 0);

        /* The interfaces resoures allocated, mark it as running */
        ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
 
        /* The interfaces resoures allocated, mark it as running */
        ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING);
 


@@ -1715,43 +1755,80 @@ utun_ctl_disconnect(__unused kern_ctl_ref kctlref,
        pcb->utun_kpipe_enabled = FALSE;
 #endif // UTUN_NEXUS
 
        pcb->utun_kpipe_enabled = FALSE;
 #endif // UTUN_NEXUS
 

-       ifp = pcb->utun_ifp;
-       VERIFY(ifp != NULL);

        pcb->utun_ctlref = NULL;
 
        pcb->utun_ctlref = NULL;
 

-       /*
-        * Quiesce the interface and flush any pending outbound packets.
-        */
-       if_down(ifp);
+       ifp = pcb->utun_ifp;
+       if (ifp != NULL) {
+#if UTUN_NEXUS
+               // Tell the nexus to stop all rings
+               if (pcb->utun_netif_nexus != NULL) {
+                       /*
+                        * Quiesce the interface and flush any pending outbound packets.
+                        */
+                       if_down(ifp);
+
+                       /* Increment refcnt, but detach interface */
+                       ifnet_incr_iorefcnt(ifp);
+                       if ((result = ifnet_detach(ifp)) != 0) {
+                               panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
+                       }

 
 

-       /* Increment refcnt, but detach interface */
-       ifnet_incr_iorefcnt(ifp);
-       if ((result = ifnet_detach(ifp)) != 0) {
-               panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
-       }
+                       /*
+                        * We want to do everything in our power to ensure that the interface
+                        * really goes away when the socket is closed. We must remove IP/IPv6
+                        * addresses and detach the protocols. Finally, we can remove and
+                        * release the interface.
+                        */
+                       utun_cleanup_family(ifp, AF_INET);
+                       utun_cleanup_family(ifp, AF_INET6);

 
 

-       /*
-        * We want to do everything in our power to ensure that the interface
-        * really goes away when the socket is closed. We must remove IP/IPv6
-        * addresses and detach the protocols. Finally, we can remove and
-        * release the interface.
-        */
-       utun_cleanup_family(ifp, AF_INET);
-       utun_cleanup_family(ifp, AF_INET6);
+                       lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);

 
 

-       lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
+                       if (!uuid_is_null(kpipe_uuid)) {
+                               if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
+                                       utun_unregister_kernel_pipe_nexus();
+                               }
+                       }
+                       utun_nexus_detach(&pcb->utun_nx);
+
+                       /* Decrement refcnt to finish detaching and freeing */
+                       ifnet_decr_iorefcnt(ifp);
+               } else
+#endif // UTUN_NEXUS
+               {
+                       lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);

 
 #if UTUN_NEXUS
 
 #if UTUN_NEXUS

-       if (!uuid_is_null(kpipe_uuid)) {
-               if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
-                       utun_unregister_kernel_pipe_nexus();
-               }
-       }
-       utun_nexus_detach(&pcb->utun_nx);
+                       if (!uuid_is_null(kpipe_uuid)) {
+                               if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) {
+                                       utun_unregister_kernel_pipe_nexus();
+                               }
+                       }

 #endif // UTUN_NEXUS
 
 #endif // UTUN_NEXUS
 

-       /* Decrement refcnt to finish detaching and freeing */
-       ifnet_decr_iorefcnt(ifp);
+                       /*
+                        * We want to do everything in our power to ensure that the interface
+                        * really goes away when the socket is closed. We must remove IP/IPv6
+                        * addresses and detach the protocols. Finally, we can remove and
+                        * release the interface.
+                        */
+                       utun_cleanup_family(ifp, AF_INET);
+                       utun_cleanup_family(ifp, AF_INET6);
+
+                       /*
+                        * Detach now; utun_detach() will be called asynchronously once
+                        * the I/O reference count drops to 0.  There we will invoke
+                        * ifnet_release().
+                        */
+                       if ((result = ifnet_detach(ifp)) != 0) {
+                               printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result);
+                       }
+               }
+       } else {
+               // Bound, but not connected
+               lck_rw_unlock_exclusive(&pcb->utun_pcb_lock);
+               utun_free_pcb(pcb, false);
+       }

        
        return 0;
 }
        
        return 0;
 }


@@ -1802,7 +1879,25 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                        if (len != sizeof(u_int32_t)) {
                                result = EMSGSIZE;
                        } else {
                        if (len != sizeof(u_int32_t)) {
                                result = EMSGSIZE;
                        } else {

-                               pcb->utun_flags = *(u_int32_t *)data;
+                               if (pcb->utun_ifp == NULL) {
+                                       // Only can set after connecting
+                                       result = EINVAL;
+                                       break;
+                               }
+#if UTUN_NEXUS
+                               if (pcb->utun_use_netif) {
+                                       pcb->utun_flags = *(u_int32_t *)data;
+                               } else
+#endif // UTUN_NEXUS
+                               {
+                                       u_int32_t old_flags = pcb->utun_flags;
+                                       pcb->utun_flags = *(u_int32_t *)data;
+                                       if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) {
+                                               // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf
+                                               bpfdetach(pcb->utun_ifp);
+                                               bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb));
+                                       }
+                               }

                        }
                        break;
 
                        }
                        break;
 


@@ -1811,6 +1906,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->utun_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
                        break;
                        
                        pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0;
                        break;
                        


@@ -1822,6 +1922,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                                result = EINVAL;
                                break;
                        }
                                result = EINVAL;
                                break;
                        }

+                       if (pcb->utun_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (!pcb->utun_ext_ifdata_stats) {
                                result = EINVAL;
                                break;
                        if (!pcb->utun_ext_ifdata_stats) {
                                result = EINVAL;
                                break;


@@ -1842,6 +1947,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->utun_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (len != 0) {    /* if len==0, del_ifp will be NULL causing the delegate to be removed */
                                bcopy(data, name, len);
                                name[len] = 0;
                        if (len != 0) {    /* if len==0, del_ifp will be NULL causing the delegate to be removed */
                                bcopy(data, name, len);
                                name[len] = 0;


@@ -1874,6 +1984,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->utun_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (*(int *)data) {
                                result = utun_enable_channel(pcb, current_proc());
                        } else {
                        if (*(int *)data) {
                                result = utun_enable_channel(pcb, current_proc());
                        } else {


@@ -1886,6 +2001,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                                result = EMSGSIZE;
                                break;
                        }
                                result = EMSGSIZE;
                                break;
                        }

+                       if (pcb->utun_ifp == NULL) {
+                               // Only can set after connecting
+                               result = EINVAL;
+                               break;
+                       }

                        if (!if_enable_netagent) {
                                result = ENOTSUP;
                                break;
                        if (!if_enable_netagent) {
                                result = ENOTSUP;
                                break;


@@ -1902,6 +2022,91 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref,
                        }
                        break;
                }
                        }
                        break;
                }

+               case UTUN_OPT_ENABLE_NETIF: {
+                       if (len != sizeof(int)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->utun_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       pcb->utun_use_netif = true;
+                       break;
+               }
+               case UTUN_OPT_SLOT_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->utun_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t slot_size = *(u_int32_t *)data;
+                       if (slot_size < UTUN_IF_MIN_SLOT_SIZE ||
+                               slot_size > UTUN_IF_MAX_SLOT_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->utun_slot_size = slot_size;
+                       break;
+               }
+               case UTUN_OPT_NETIF_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->utun_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < UTUN_IF_MIN_RING_SIZE ||
+                               ring_size > UTUN_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->utun_netif_ring_size = ring_size;
+                       break;
+               }
+               case UTUN_OPT_TX_FSW_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->utun_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < UTUN_IF_MIN_RING_SIZE ||
+                               ring_size > UTUN_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->utun_tx_fsw_ring_size = ring_size;
+                       break;
+               }
+               case UTUN_OPT_RX_FSW_RING_SIZE: {
+                       if (len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                               break;
+                       }
+                       if (pcb->utun_ifp != NULL) {
+                               // Only can set before connecting
+                               result = EINVAL;
+                               break;
+                       }
+                       u_int32_t ring_size = *(u_int32_t *)data;
+                       if (ring_size < UTUN_IF_MIN_RING_SIZE ||
+                               ring_size > UTUN_IF_MAX_RING_SIZE) {
+                               return (EINVAL);
+                       }
+                       pcb->utun_rx_fsw_ring_size = ring_size;
+                       break;
+               }

 #endif // UTUN_NEXUS
                default: {
                        result = ENOPROTOOPT;
 #endif // UTUN_NEXUS
                default: {
                        result = ENOPROTOOPT;


@@ -1944,6 +2149,11 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref,
                        if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
                                result = EMSGSIZE;
                        } else {
                        if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) {
                                result = EMSGSIZE;
                        } else {

+                               if (pcb->utun_ifp == NULL) {
+                                       // Only can get after connecting
+                                       result = EINVAL;
+                                       break;
+                               }

                                *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
                        }
                        break;
                                *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1;
                        }
                        break;


@@ -1958,7 +2168,7 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref,
                }
 
 #if UTUN_NEXUS
                }
 
 #if UTUN_NEXUS

-               case UTUN_OPT_GET_CHANNEL_UUID:
+               case UTUN_OPT_GET_CHANNEL_UUID: {

                        lck_rw_lock_shared(&pcb->utun_pcb_lock);
                        if (uuid_is_null(pcb->utun_kpipe_uuid)) {
                                result = ENXIO;
                        lck_rw_lock_shared(&pcb->utun_pcb_lock);
                        if (uuid_is_null(pcb->utun_kpipe_uuid)) {
                                result = ENXIO;


@@ -1969,6 +2179,39 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref,
                        }
                        lck_rw_unlock_shared(&pcb->utun_pcb_lock);
                        break;
                        }
                        lck_rw_unlock_shared(&pcb->utun_pcb_lock);
                        break;

+               }
+               case UTUN_OPT_SLOT_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->utun_slot_size;
+                       }
+                       break;
+               }
+               case UTUN_OPT_NETIF_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->utun_netif_ring_size;
+                       }
+                       break;
+               }
+               case UTUN_OPT_TX_FSW_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size;
+                       }
+                       break;
+               }
+               case UTUN_OPT_RX_FSW_RING_SIZE: {
+                       if (*len != sizeof(u_int32_t)) {
+                               result = EMSGSIZE;
+                       } else {
+                               *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size;
+                       }
+                       break;
+               }

 #endif // UTUN_NEXUS
 
                default:
 #endif // UTUN_NEXUS
 
                default:


@@ -2011,7 +2254,6 @@ utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags)
 }
 
 /* Network Interface functions */
 }
 
 /* Network Interface functions */

-#if !UTUN_NEXUS

 static void
 utun_start(ifnet_t interface)
 {
 static void
 utun_start(ifnet_t interface)
 {


@@ -2074,7 +2316,6 @@ utun_start(ifnet_t interface)
                }
        }
 }
                }
        }
 }

-#endif // !UTUN_NEXUS

 
 static errno_t
 utun_output(ifnet_t    interface,
 
 static errno_t
 utun_output(ifnet_t    interface,


@@ -2085,6 +2326,15 @@ utun_output(ifnet_t      interface,
 
        VERIFY(interface == pcb->utun_ifp);
 
 
        VERIFY(interface == pcb->utun_ifp);
 

+#if UTUN_NEXUS
+       if (!pcb->utun_use_netif)
+#endif // UTUN_NEXUS
+       {
+               if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) {
+                       bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
+               }
+       }
+

        if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
                /* flush data */
                mbuf_freem(data);
        if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) {
                /* flush data */
                mbuf_freem(data);


@@ -2107,13 +2357,21 @@ utun_output(ifnet_t     interface,
                if (result != 0) {
                        mbuf_freem(data);
                        printf("utun_output - ctl_enqueuembuf failed: %d\n", result);
                if (result != 0) {
                        mbuf_freem(data);
                        printf("utun_output - ctl_enqueuembuf failed: %d\n", result);

-#if !UTUN_NEXUS
-                       ifnet_stat_increment_out(interface, 0, 0, 1);
+#if UTUN_NEXUS
+                       if (!pcb->utun_use_netif)
+#endif // UTUN_NEXUS
+                       {
+                               ifnet_stat_increment_out(interface, 0, 0, 1);
+                       }

                } else {
                } else {

-                       if (!pcb->utun_ext_ifdata_stats) {
-                               ifnet_stat_increment_out(interface, 1, length, 0);
+#if UTUN_NEXUS
+                       if (!pcb->utun_use_netif)
+#endif // UTUN_NEXUS
+                       {
+                               if (!pcb->utun_ext_ifdata_stats) {
+                                       ifnet_stat_increment_out(interface, 1, length, 0);
+                               }

                        }
                        }

-#endif // !UTUN_NEXUS

                }
        } else {
                mbuf_freem(data);
                }
        } else {
                mbuf_freem(data);


@@ -2128,7 +2386,7 @@ utun_demux(__unused ifnet_t interface,
                   __unused char *frame_header,
                   protocol_family_t *protocol)
 {
                   __unused char *frame_header,
                   protocol_family_t *protocol)
 {

-       
+       struct utun_pcb *pcb = ifnet_softc(interface);

        struct ip *ip;
        u_int ip_version;
 
        struct ip *ip;
        u_int ip_version;
 


@@ -2136,28 +2394,35 @@ utun_demux(__unused ifnet_t interface,
                data = mbuf_next(data);
        }
 
                data = mbuf_next(data);
        }
 

-       if (data == NULL)
+       if (data == NULL) {

                return ENOENT;
                return ENOENT;

+       }

 
 

-       ip = mtod(data, struct ip *);
-       ip_version = ip->ip_v;
-
-       switch(ip_version) {
-               case 4:
-                       *protocol = PF_INET;
-                       return 0;
-               case 6:
-                       *protocol = PF_INET6;
-                       return 0;
-               default:
-                       *protocol = 0;
-                       break;
+#if UTUN_NEXUS
+       if (pcb->utun_use_netif) {
+               ip = mtod(data, struct ip *);
+               ip_version = ip->ip_v;
+
+               switch(ip_version) {
+                       case 4:
+                               *protocol = PF_INET;
+                               return 0;
+                       case 6:
+                               *protocol = PF_INET6;
+                               return 0;
+                       default:
+                               *protocol = 0;
+                               break;
+               }
+       } else
+#endif // UTUN_NEXUS
+       {
+               *protocol = *(u_int32_t *)mbuf_data(data);

        }
 
        return 0;
 }
 
        }
 
        return 0;
 }
 

-#if !UTUN_NEXUS

 static errno_t
 utun_framer(ifnet_t interface,
                        mbuf_t *packet,
 static errno_t
 utun_framer(ifnet_t interface,
                        mbuf_t *packet,


@@ -2192,7 +2457,6 @@ utun_framer(ifnet_t interface,
 
     return 0;
 }
 
     return 0;
 }

-#endif // !UTUN_NEXUS

 
 static errno_t
 utun_add_proto(__unused ifnet_t interface,
 
 static errno_t
 utun_add_proto(__unused ifnet_t interface,


@@ -2224,22 +2488,27 @@ utun_ioctl(ifnet_t interface,
                   u_long command,
                   void *data)
 {
                   u_long command,
                   void *data)
 {

+       struct utun_pcb *pcb = ifnet_softc(interface);

        errno_t result = 0;
        
        switch(command) {
        errno_t result = 0;
        
        switch(command) {

-               case SIOCSIFMTU:
+               case SIOCSIFMTU: {

 #if UTUN_NEXUS
 #if UTUN_NEXUS

-               {
-                       // Make sure we can fit packets in the channel buffers
-                       // Allow for the headroom in the slot
-                       if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) {
-                               ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE);
-                               break;
-                       }
-               }
+                       if (pcb->utun_use_netif) {
+                               // Make sure we can fit packets in the channel buffers
+                               // Allow for the headroom in the slot
+                               if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) {
+                                       result = EINVAL;
+                               } else {
+                                       ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu);
+                               }
+                       } else

 #endif // UTUN_NEXUS
 #endif // UTUN_NEXUS

-                       ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
+                       {
+                               ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu);
+                       }

                        break;
                        break;

+               }

                        
                case SIOCSIFFLAGS:
                        /* ifioctl() takes care of it */
                        
                case SIOCSIFFLAGS:
                        /* ifioctl() takes care of it */


@@ -2257,7 +2526,7 @@ utun_detached(ifnet_t interface)
 {
        struct utun_pcb *pcb = ifnet_softc(interface);
        (void)ifnet_release(interface);
 {
        struct utun_pcb *pcb = ifnet_softc(interface);
        (void)ifnet_release(interface);

-       utun_free_pcb(pcb);
+       utun_free_pcb(pcb, true);

 }
 
 /* Protocol Handlers */
 }
 
 /* Protocol Handlers */


@@ -2268,13 +2537,28 @@ utun_proto_input(__unused ifnet_t interface,
                                 mbuf_t m,
                                 __unused char *frame_header)
 {
                                 mbuf_t m,
                                 __unused char *frame_header)
 {

+       struct utun_pcb *pcb = ifnet_softc(interface);
+#if UTUN_NEXUS
+       if (!pcb->utun_use_netif)
+#endif // UTUN_NEXUS
+       {
+               mbuf_adj(m, UTUN_HEADER_SIZE(pcb));
+       }

        if (proto_input(protocol, m) != 0) {
                m_freem(m);
        if (proto_input(protocol, m) != 0) {
                m_freem(m);

-#if !UTUN_NEXUS
-               ifnet_stat_increment_in(interface, 0, 0, 1);
+#if UTUN_NEXUS
+               if (!pcb->utun_use_netif)
+#endif // UTUN_NEXUS
+               {
+                       ifnet_stat_increment_in(interface, 0, 0, 1);
+               }

        } else {
        } else {

-               ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0);
+#if UTUN_NEXUS
+               if (!pcb->utun_use_netif)

 #endif // UTUN_NEXUS
 #endif // UTUN_NEXUS

+               {
+                       ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0);
+               }

        }
        
        return 0;
        }
        
        return 0;


@@ -2312,75 +2596,67 @@ utun_attach_proto(ifnet_t interface,
        return result;
 }
 
        return result;
 }
 

-#if UTUN_NEXUS

 static errno_t
 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
 {
 static errno_t
 utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet)
 {

-       lck_rw_lock_shared(&pcb->utun_pcb_lock);
-
-       lck_mtx_lock(&pcb->utun_input_chain_lock);
-       if (pcb->utun_input_chain != NULL) {
-               pcb->utun_input_chain_last->m_nextpkt = packet;
-       } else {
-               pcb->utun_input_chain = packet;
-       }
-       while (packet->m_nextpkt) {
-               VERIFY(packet != packet->m_nextpkt);
-               packet = packet->m_nextpkt;
-       }
-       pcb->utun_input_chain_last = packet;
-       lck_mtx_unlock(&pcb->utun_input_chain_lock);
-
-       kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
-       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+#if UTUN_NEXUS
+       if (pcb->utun_use_netif) {
+               lck_rw_lock_shared(&pcb->utun_pcb_lock);

 
 

-       if (rx_ring != NULL) {
-               kern_channel_notify(rx_ring, 0);
-       }
+               lck_mtx_lock(&pcb->utun_input_chain_lock);
+               if (pcb->utun_input_chain != NULL) {
+                       pcb->utun_input_chain_last->m_nextpkt = packet;
+               } else {
+                       pcb->utun_input_chain = packet;
+               }
+               while (packet->m_nextpkt) {
+                       VERIFY(packet != packet->m_nextpkt);
+                       packet = packet->m_nextpkt;
+               }
+               pcb->utun_input_chain_last = packet;
+               lck_mtx_unlock(&pcb->utun_input_chain_lock);

 
 

-       return (0);
-}
-#else
-static errno_t
-utun_pkt_input (struct utun_pcb *pcb, mbuf_t m)
-{
-       errno_t result;
-       protocol_family_t protocol = 0;
+               kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
+               lck_rw_unlock_shared(&pcb->utun_pcb_lock);

 
 

-       mbuf_pkthdr_setrcvif(m, pcb->utun_ifp);
+               if (rx_ring != NULL) {
+                       kern_channel_notify(rx_ring, 0);
+               }

 
 

-       if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb))  {
-               protocol = *(u_int32_t *)mbuf_data(m);
+               return (0);
+       } else
+#endif // IPSEC_NEXUS
+       {
+               mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp);

 
 

-               bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0);
-       }
-       if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
-               /* flush data */
-               mbuf_freem(m);
-               return 0;
-       }
+               if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb))  {
+                       bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0);
+               }
+               if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) {
+                       /* flush data */
+                       mbuf_freem(packet);
+                       return 0;
+               }

 
 

-       if (!pcb->utun_ext_ifdata_stats) {
-               struct ifnet_stat_increment_param       incs;
+               errno_t result = 0;
+               if (!pcb->utun_ext_ifdata_stats) {
+                       struct ifnet_stat_increment_param incs = {};
+                       incs.packets_in = 1;
+                       incs.bytes_in = mbuf_pkthdr_len(packet);
+                       result = ifnet_input(pcb->utun_ifp, packet, &incs);
+               } else {
+                       result = ifnet_input(pcb->utun_ifp, packet, NULL);
+               }
+               if (result != 0) {
+                       ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);

 
 

-               bzero(&incs, sizeof(incs));
-               incs.packets_in = 1;
-               incs.bytes_in = mbuf_pkthdr_len(m);
-               result = ifnet_input(pcb->utun_ifp, m, &incs);
-       } else {
-               result = ifnet_input(pcb->utun_ifp, m, NULL);
-       }
-       if (result != 0) {
-               ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1);
+                       printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
+                       mbuf_freem(packet);
+               }

 
 

-               printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result);
-               mbuf_freem(m);
+               return (0);

        }
        }

-
-       return 0;

 }
 }

-#endif // UTUN_NEXUS
-

 
 #if UTUN_NEXUS
 
 
 #if UTUN_NEXUS
 


@@ -2535,18 +2811,80 @@ utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                return 0;
        }
 
                return 0;
        }
 

-       kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
-       if (tx_slot == NULL) {
-               // Nothing to write, bail
+       if (pcb->utun_use_netif) {
+               kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
+               if (tx_slot == NULL) {
+                       // Nothing to write, bail
+                       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+                       return 0;
+               }
+
+               // Signal the netif ring to read
+               kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
+               lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+               if (rx_ring != NULL) {
+                       kern_channel_notify(rx_ring, 0);
+               }
+       } else {

                lck_rw_unlock_shared(&pcb->utun_pcb_lock);
                lck_rw_unlock_shared(&pcb->utun_pcb_lock);

-               return 0;
-       }

 
 

-       // Signal the netif ring to read
-       kern_channel_ring_t rx_ring = pcb->utun_netif_rxring;
-       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
-       if (rx_ring != NULL) {
-               kern_channel_notify(rx_ring, 0);
+               struct ifnet_stat_increment_param incs = {};
+               struct kern_channel_ring_stat_increment tx_ring_stats = {};
+               MBUFQ_HEAD(mbufq) mbq;
+               MBUFQ_INIT(&mbq);
+               kern_channel_slot_t tx_pslot = NULL;
+               kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
+               while (tx_slot != NULL) {
+                       kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
+
+                       // Advance TX ring
+                       tx_pslot = tx_slot;
+                       tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
+
+                       if (tx_ph == 0) {
+                               continue;
+                       }
+
+                       kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
+                       VERIFY(tx_buf != NULL);
+                       uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
+                       VERIFY(tx_baddr != 0);
+                       tx_baddr += kern_buflet_get_data_offset(tx_buf);
+
+                       size_t length = MIN(kern_packet_get_data_length(tx_ph),
+                                                               pcb->utun_slot_size);
+
+                       mbuf_t data = NULL;
+                       if (length >= UTUN_HEADER_SIZE(pcb) &&
+                               !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) {
+                               errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data);
+                               VERIFY(0 == error);
+                               error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK);
+                               VERIFY(0 == error);
+                               /*
+                                * The userland ABI requires the first four bytes have
+                                * the protocol family in network byte order: swap them
+                                */
+                               *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data));
+                               mbuf_pkthdr_setrcvif(data, pcb->utun_ifp);
+                               bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0);
+                               incs.packets_in++;
+                               incs.bytes_in += length;
+                               MBUFQ_ENQUEUE(&mbq, data);
+                       }
+               }
+               if (tx_pslot) {
+                       kern_channel_advance_slot(tx_ring, tx_pslot);
+                       tx_ring_stats.kcrsi_slots_transferred = incs.packets_in;
+                       tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in;
+                       kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
+                       (void) kern_channel_reclaim(tx_ring);
+               }
+               if (!MBUFQ_EMPTY(&mbq)) {
+                       (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq),
+                                                                               MBUFQ_LAST(&mbq), &incs);
+                       MBUFQ_INIT(&mbq);
+               }

        }
 
        return 0;
        }
 
        return 0;


@@ -2559,7 +2897,7 @@ utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
 #pragma unused(nxprov)
 #pragma unused(flags)
        struct utun_pcb *pcb = kern_nexus_get_context(nexus);
 #pragma unused(nxprov)
 #pragma unused(flags)
        struct utun_pcb *pcb = kern_nexus_get_context(nexus);

-       struct kern_channel_ring_stat_increment rx_ring_stats;
+       struct kern_channel_ring_stat_increment rx_ring_stats = {};

 
        lck_rw_lock_shared(&pcb->utun_pcb_lock);
 
 
        lck_rw_lock_shared(&pcb->utun_pcb_lock);
 


@@ -2578,185 +2916,276 @@ utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
                return 0;
        }
 
                return 0;
        }
 

-       kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
-       if (tx_ring == NULL ||
-               pcb->utun_netif_nexus == NULL) {
-               // Net-If TX ring not set up yet, nothing to read
-               lck_rw_unlock_shared(&pcb->utun_pcb_lock);
-               return 0;
-       }
-
-       struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;
-
-       // Unlock utun before entering ring
-       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
-
-       (void)kr_enter(tx_ring, TRUE);
+       if (pcb->utun_use_netif) {
+               kern_channel_ring_t tx_ring = pcb->utun_netif_txring;
+               if (tx_ring == NULL ||
+                       pcb->utun_netif_nexus == NULL) {
+                       // Net-If TX ring not set up yet, nothing to read
+                       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+                       return 0;
+               }

 
 

-       // Lock again after entering and validate
-       lck_rw_lock_shared(&pcb->utun_pcb_lock);
-       if (tx_ring != pcb->utun_netif_txring) {
-               // Ring no longer valid
-               // Unlock first, then exit ring
-               lck_rw_unlock_shared(&pcb->utun_pcb_lock);
-               kr_exit(tx_ring);
-               return 0;
-       }
+               struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats;

 
 

-       struct kern_channel_ring_stat_increment tx_ring_stats;
-       bzero(&tx_ring_stats, sizeof(tx_ring_stats));
-       kern_channel_slot_t tx_pslot = NULL;
-       kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
-       if (tx_slot == NULL) {
-               // Nothing to read, don't bother signalling
-               // Unlock first, then exit ring
+               // Unlock utun before entering ring

                lck_rw_unlock_shared(&pcb->utun_pcb_lock);
                lck_rw_unlock_shared(&pcb->utun_pcb_lock);

-               kr_exit(tx_ring);
-               return 0;
-       }
-
-       struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
-       VERIFY(rx_pp != NULL);
-       bzero(&rx_ring_stats, sizeof(rx_ring_stats));
-       kern_channel_slot_t rx_pslot = NULL;
-       kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);

 
 

-       while (rx_slot != NULL && tx_slot != NULL) {
-               size_t length;
-               kern_buflet_t rx_buf;
-               void *rx_baddr;
+               (void)kr_enter(tx_ring, TRUE);

 
 

-               kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);
-
-               // Advance TX ring
-               tx_pslot = tx_slot;
-               tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
-
-               /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
-               if (tx_ph == 0) {
-                       continue;
+               // Lock again after entering and validate
+               lck_rw_lock_shared(&pcb->utun_pcb_lock);
+               if (tx_ring != pcb->utun_netif_txring) {
+                       // Ring no longer valid
+                       // Unlock first, then exit ring
+                       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+                       kr_exit(tx_ring);
+                       return 0;

                }
 
                }
 

-               // Allocate rx packet
-               kern_packet_t rx_ph = 0;
-               errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
-               if (unlikely(error != 0)) {
-                       printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
-                                  pcb->utun_ifp->if_xname);
-                       break;
+               struct kern_channel_ring_stat_increment tx_ring_stats;
+               bzero(&tx_ring_stats, sizeof(tx_ring_stats));
+               kern_channel_slot_t tx_pslot = NULL;
+               kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
+               if (tx_slot == NULL) {
+                       // Nothing to read, don't bother signalling
+                       // Unlock first, then exit ring
+                       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+                       kr_exit(tx_ring);
+                       return 0;

                }
 
                }
 

-               kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
-               VERIFY(tx_buf != NULL);
-               uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
-               VERIFY(tx_baddr != NULL);
-               tx_baddr += kern_buflet_get_data_offset(tx_buf);
+               struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
+               VERIFY(rx_pp != NULL);
+               kern_channel_slot_t rx_pslot = NULL;
+               kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);

 
 

-               bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
-
-               length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
-                                        UTUN_IF_DEFAULT_SLOT_SIZE);
+               while (rx_slot != NULL && tx_slot != NULL) {
+                       size_t length;
+                       kern_buflet_t rx_buf;
+                       void *rx_baddr;

 
 

-               tx_ring_stats.kcrsi_slots_transferred++;
-               tx_ring_stats.kcrsi_bytes_transferred += length;
+                       kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot);

 
 

-               if (length < UTUN_HEADER_SIZE(pcb) ||
-                   length > UTUN_IF_DEFAULT_SLOT_SIZE ||
-                   length > rx_pp->pp_buflet_size ||
-                   (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
-                       /* flush data */
-                       kern_pbufpool_free(rx_pp, rx_ph);
-                       printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
-                                  pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
-                       STATS_INC(nifs, NETIF_STATS_BADLEN);
-                       STATS_INC(nifs, NETIF_STATS_DROPPED);
-                       continue;
-               }
+                       // Advance TX ring
+                       tx_pslot = tx_slot;
+                       tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);

 
 

-               /* fillout packet */
-               rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
-               VERIFY(rx_buf != NULL);
-               rx_baddr = kern_buflet_get_object_address(rx_buf);
-               VERIFY(rx_baddr != NULL);
+                       /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */
+                       if (tx_ph == 0) {
+                               continue;
+                       }

 
 

-               // Find family
-               uint32_t af = 0;
-               uint8_t vhl = *(uint8_t *)(tx_baddr);
-               u_int ip_version = (vhl >> 4);
-               switch (ip_version) {
-                       case 4: {
-                               af = AF_INET;
+                       // Allocate rx packet
+                       kern_packet_t rx_ph = 0;
+                       errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
+                       if (unlikely(error != 0)) {
+                               printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
+                                          pcb->utun_ifp->if_xname);

                                break;
                        }
                                break;
                        }

-                       case 6: {
-                               af = AF_INET6;
-                               break;
+
+                       kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL);
+                       VERIFY(tx_buf != NULL);
+                       uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf);
+                       VERIFY(tx_baddr != NULL);
+                       tx_baddr += kern_buflet_get_data_offset(tx_buf);
+
+                       bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0);
+
+                       length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb),
+                                                pcb->utun_slot_size);
+
+                       tx_ring_stats.kcrsi_slots_transferred++;
+                       tx_ring_stats.kcrsi_bytes_transferred += length;
+
+                       if (length < UTUN_HEADER_SIZE(pcb) ||
+                               length > pcb->utun_slot_size ||
+                               length > rx_pp->pp_buflet_size ||
+                               (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
+                               /* flush data */
+                               kern_pbufpool_free(rx_pp, rx_ph);
+                               printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n",
+                                          pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb));
+                               STATS_INC(nifs, NETIF_STATS_BADLEN);
+                               STATS_INC(nifs, NETIF_STATS_DROPPED);
+                               continue;

                        }
                        }

-                       default: {
-                               printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
-                                          pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
-                               break;
+
+                       /* fillout packet */
+                       rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
+                       VERIFY(rx_buf != NULL);
+                       rx_baddr = kern_buflet_get_object_address(rx_buf);
+                       VERIFY(rx_baddr != NULL);
+
+                       // Find family
+                       uint32_t af = 0;
+                       uint8_t vhl = *(uint8_t *)(tx_baddr);
+                       u_int ip_version = (vhl >> 4);
+                       switch (ip_version) {
+                               case 4: {
+                                       af = AF_INET;
+                                       break;
+                               }
+                               case 6: {
+                                       af = AF_INET6;
+                                       break;
+                               }
+                               default: {
+                                       printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n",
+                                                  pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb));
+                                       break;
+                               }

                        }
                        }

-               }

 
 

-               // Copy header
-               af = htonl(af);
-               memcpy((void *)rx_baddr, &af, sizeof(af));
-               if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
-                       kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
-               }
+                       // Copy header
+                       af = htonl(af);
+                       memcpy((void *)rx_baddr, &af, sizeof(af));
+                       if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) {
+                               kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af)));
+                       }

 
 

-               // Copy data from tx to rx
-               memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
-               kern_packet_clear_flow_uuid(rx_ph); // zero flow id
+                       // Copy data from tx to rx
+                       memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb));
+                       kern_packet_clear_flow_uuid(rx_ph); // zero flow id

 
 

-               /* finalize and attach the packet */
-               error = kern_buflet_set_data_offset(rx_buf, 0);
-               VERIFY(error == 0);
-               error = kern_buflet_set_data_length(rx_buf, length);
-               VERIFY(error == 0);
-               error = kern_packet_finalize(rx_ph);
-               VERIFY(error == 0);
-               error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
-               VERIFY(error == 0);
+                       /* finalize and attach the packet */
+                       error = kern_buflet_set_data_offset(rx_buf, 0);
+                       VERIFY(error == 0);
+                       error = kern_buflet_set_data_length(rx_buf, length);
+                       VERIFY(error == 0);
+                       error = kern_packet_finalize(rx_ph);
+                       VERIFY(error == 0);
+                       error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
+                       VERIFY(error == 0);

 
 

-               STATS_INC(nifs, NETIF_STATS_TXPKTS);
-               STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);
+                       STATS_INC(nifs, NETIF_STATS_TXPKTS);
+                       STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT);

 
 

-               rx_ring_stats.kcrsi_slots_transferred++;
-               rx_ring_stats.kcrsi_bytes_transferred += length;
+                       rx_ring_stats.kcrsi_slots_transferred++;
+                       rx_ring_stats.kcrsi_bytes_transferred += length;

 
 

-               rx_pslot = rx_slot;
-               rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
-       }
+                       rx_pslot = rx_slot;
+                       rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
+               }

 
 

-       if (rx_pslot) {
-               kern_channel_advance_slot(rx_ring, rx_pslot);
-               kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
-       }
+               if (rx_pslot) {
+                       kern_channel_advance_slot(rx_ring, rx_pslot);
+                       kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats);
+               }

 
 

-       if (tx_pslot) {
-               kern_channel_advance_slot(tx_ring, tx_pslot);
-               kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
-               (void)kern_channel_reclaim(tx_ring);
-       }
+               if (tx_pslot) {
+                       kern_channel_advance_slot(tx_ring, tx_pslot);
+                       kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats);
+                       (void)kern_channel_reclaim(tx_ring);
+               }

 
 

-       if (pcb->utun_output_disabled) {
+               /* just like utun_ctl_rcvd(), always reenable output */

                errno_t error = ifnet_enable_output(pcb->utun_ifp);
                if (error != 0) {
                        printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);
                errno_t error = ifnet_enable_output(pcb->utun_ifp);
                if (error != 0) {
                        printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error);

-               } else {
-                       pcb->utun_output_disabled = false;

                }
                }

-       }

 
 

-       // Unlock first, then exit ring
-       lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+               // Unlock first, then exit ring
+               lck_rw_unlock_shared(&pcb->utun_pcb_lock);

 
 

-       if (tx_pslot != NULL) {
-               kern_channel_notify(tx_ring, 0);
+               if (tx_pslot != NULL) {
+                       kern_channel_notify(tx_ring, 0);
+               }
+               kr_exit(tx_ring);
+       } else {
+               lck_rw_unlock_shared(&pcb->utun_pcb_lock);
+
+               uint32_t mb_cnt = 0;
+               uint32_t mb_len = 0;
+               struct mbuf *mb_head = NULL;
+               struct mbuf *mb_tail = NULL;
+
+               if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head,
+                                                               &mb_tail, &mb_cnt, &mb_len) != 0) {
+                       return 0;
+               }
+               VERIFY(mb_cnt <= avail);
+
+               struct kern_pbufpool *rx_pp = rx_ring->ckr_pp;
+               VERIFY(rx_pp != NULL);
+               kern_channel_slot_t rx_pslot = NULL;
+               kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
+               while (rx_slot) {
+                       size_t length = 0;
+                       mbuf_t data = NULL;
+                       if ((data = mb_head) == NULL) {
+                               VERIFY(mb_cnt == 0);
+                               break;
+                       }
+                       mb_head = mbuf_nextpkt(mb_head);
+                       mbuf_setnextpkt(data, NULL);
+                       VERIFY(mb_cnt != 0);
+                       --mb_cnt;
+                       length = mbuf_pkthdr_len(data);
+                       if (length < UTUN_HEADER_SIZE(pcb) ||
+                               length > pcb->utun_slot_size ||
+                               (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) {
+                               /* flush data */
+                               mbuf_freem(data);
+                               continue;
+                       }
+                       bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0);
+
+                       // Allocate rx packet
+                       kern_packet_t rx_ph = 0;
+                       errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph);
+                       if (unlikely(error != 0)) {
+                               printf("utun_kpipe_sync_rx %s: failed to allocate packet\n",
+                                          pcb->utun_ifp->if_xname);
+                               break;
+                       }
+
+                       /*
+                        * The ABI requires the protocol in network byte order
+                        */
+                       *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data));
+
+                       // Fillout rx packet
+                       kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL);
+                       VERIFY(rx_buf != NULL);
+                       void *rx_baddr = kern_buflet_get_object_address(rx_buf);
+                       VERIFY(rx_baddr != NULL);
+
+                       // Copy-in data from mbuf to buflet
+                       mbuf_copydata(data, 0, length, (void *)rx_baddr);
+                       kern_packet_clear_flow_uuid(rx_ph);     // Zero flow id
+
+                       // Finalize and attach the packet
+                       error = kern_buflet_set_data_offset(rx_buf, 0);
+                       VERIFY(error == 0);
+                       error = kern_buflet_set_data_length(rx_buf, length);
+                       VERIFY(error == 0);
+                       error = kern_packet_finalize(rx_ph);
+                       VERIFY(error == 0);
+                       error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph);
+                       VERIFY(error == 0);
+
+                       rx_ring_stats.kcrsi_slots_transferred++;
+                       rx_ring_stats.kcrsi_bytes_transferred += length;
+
+                       if (!pcb->utun_ext_ifdata_stats) {
+                               ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0);
+                       }
+
+                       mbuf_freem(data);
+
+                       rx_pslot = rx_slot;
+                       rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
+               }
+               if (rx_pslot) {
+                       kern_channel_advance_slot(rx_ring, rx_pslot);
+                       kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats);
+               }
+               if (mb_head != NULL) {
+                       VERIFY(mb_cnt != 0);
+                       mbuf_freem_list(mb_head);
+               }

        }
        }

-       kr_exit(tx_ring);

 
        return 0;
 }
 
        return 0;
 }

diff --git a/bsd/net/if_utun.h b/bsd/net/if_utun.h
index 00892415432e59218c6b3ec1dde59d0b02eeecf1..4e9868a8302c56868b9f3a5c9e1c4869378b1ab8 100644 (file)
--- a/bsd/net/if_utun.h
+++ b/bsd/net/if_utun.h
@@ -60,6 +60,13 @@ errno_t utun_register_control(void);
 #define UTUN_OPT_ENABLE_CHANNEL                                17
 #define UTUN_OPT_GET_CHANNEL_UUID                      18
 #define UTUN_OPT_ENABLE_FLOWSWITCH                     19
 #define UTUN_OPT_ENABLE_CHANNEL                                17
 #define UTUN_OPT_GET_CHANNEL_UUID                      18
 #define UTUN_OPT_ENABLE_FLOWSWITCH                     19

+
+#define UTUN_OPT_ENABLE_NETIF                          20              /* Must be set before connecting */
+#define UTUN_OPT_SLOT_SIZE                                     21              /* Must be set before connecting */
+#define UTUN_OPT_NETIF_RING_SIZE                       22              /* Must be set before connecting */
+#define UTUN_OPT_TX_FSW_RING_SIZE                      23              /* Must be set before connecting */
+#define UTUN_OPT_RX_FSW_RING_SIZE                      24              /* Must be set before connecting */
+

 /*
  * Flags for by UTUN_OPT_FLAGS 
  */
 /*
  * Flags for by UTUN_OPT_FLAGS 
  */

diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h
index 0541857f064d46ea04691cbb80969860a067e7ee..60aa09b8fef9e1ef9a38d383f00c1c6ca0871499 100644 (file)
--- a/bsd/net/if_var.h
+++ b/bsd/net/if_var.h
@@ -754,6 +754,8 @@ LIST_HEAD(ifmultihead, ifmultiaddr);
 TAILQ_HEAD(tailq_head, tqdummy);
 TAILQ_HEAD(ifnet_filter_head, ifnet_filter);
 TAILQ_HEAD(ddesc_head_name, dlil_demux_desc);
 TAILQ_HEAD(tailq_head, tqdummy);
 TAILQ_HEAD(ifnet_filter_head, ifnet_filter);
 TAILQ_HEAD(ddesc_head_name, dlil_demux_desc);

+
+extern boolean_t intcoproc_unrestricted;

 #endif /* BSD_KERNEL_PRIVATE */
 
 #ifdef PRIVATE
 #endif /* BSD_KERNEL_PRIVATE */
 
 #ifdef PRIVATE

diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c
index 737ce40ada3d8b8feae52297645e458ea11c1932..5c3535e68fc79838ae0de3ebd076f832550c2c34 100644 (file)
--- a/bsd/net/if_vlan.c
+++ b/bsd/net/if_vlan.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2003-2016 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -1835,6 +1835,11 @@ vlan_ioctl(ifnet_t ifp, u_long cmd, void * data)
                error = ENXIO;
                break;
            }
                error = ENXIO;
                break;
            }

+           if (IFNET_IS_INTCOPROC(p)) {
+               error = EINVAL;
+               break;
+           }
+

            /* can't do VLAN over anything but ethernet or ethernet aggregate */
            if (ifnet_type(p) != IFT_ETHER 
                && ifnet_type(p) != IFT_IEEE8023ADLAG) {
            /* can't do VLAN over anything but ethernet or ethernet aggregate */
            if (ifnet_type(p) != IFT_ETHER 
                && ifnet_type(p) != IFT_IEEE8023ADLAG) {

diff --git a/bsd/net/necp_client.c b/bsd/net/necp_client.c
index 0cb79a5b741d0a9a83b94b5365f5d9c355bc0a84..1588ac17738892995ef0a525ef96fe656d87692f 100644 (file)
--- a/bsd/net/necp_client.c
+++ b/bsd/net/necp_client.c
@@ -4706,8 +4706,18 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt)
        u_int8_t *buffer = NULL;
        u_int8_t *cursor = NULL;
        size_t valsize = 0;
        u_int8_t *buffer = NULL;
        u_int8_t *cursor = NULL;
        size_t valsize = 0;

-       struct inpcb *inp = sotoinpcb(so);
+       struct inpcb *inp = NULL;

 
 

+       if ((SOCK_DOM(so) != PF_INET
+#if INET6
+                && SOCK_DOM(so) != PF_INET6
+#endif
+                )) {
+               error = EINVAL;
+               goto done;
+       }
+
+       inp = sotoinpcb(so);

        if (inp->inp_necp_attributes.inp_domain != NULL) {
                valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_domain);
        }
        if (inp->inp_necp_attributes.inp_domain != NULL) {
                valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_domain);
        }

diff --git a/bsd/net/packet_mangler.c b/bsd/net/packet_mangler.c
index d6c8b9dbda0d52bec0e9f9ce7c6d97eb670d5e4d..2efa88cc2d15e17a83c2dd9643f794021546f166 100644 (file)
--- a/bsd/net/packet_mangler.c
+++ b/bsd/net/packet_mangler.c
@@ -955,7 +955,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u
                                        }
                                }
 
                                        }
                                }
 

-                               while (tcp_optlen) {
+                               while (tcp_optlen > 0) {

                                        if (tcp_opt_buf[i] == 0x1) {
                                                PKT_MNGLR_LOG(LOG_INFO, "Skipping NOP\n");
                                                tcp_optlen--;
                                        if (tcp_opt_buf[i] == 0x1) {
                                                PKT_MNGLR_LOG(LOG_INFO, "Skipping NOP\n");
                                                tcp_optlen--;


@@ -968,7 +968,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u
                                                continue;
                                        } else if (tcp_opt_buf[i] == TCP_OPT_MULTIPATH_TCP) {
                                                int j = 0;
                                                continue;
                                        } else if (tcp_opt_buf[i] == TCP_OPT_MULTIPATH_TCP) {
                                                int j = 0;

-                                               int mptcpoptlen = tcp_opt_buf[i+1];
+                                               unsigned char mptcpoptlen = tcp_opt_buf[i+1];

                                                uint8_t sbtver = tcp_opt_buf[i+MPTCP_SBT_VER_OFFSET];
                                                uint8_t subtype = sbtver >> 4;
 
                                                uint8_t sbtver = tcp_opt_buf[i+MPTCP_SBT_VER_OFFSET];
                                                uint8_t subtype = sbtver >> 4;
 


@@ -984,7 +984,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u
                                                }
 
                                                PKT_MNGLR_LOG(LOG_INFO, "Got MPTCP option %x\n", tcp_opt_buf[i]);
                                                }
 
                                                PKT_MNGLR_LOG(LOG_INFO, "Got MPTCP option %x\n", tcp_opt_buf[i]);

-                                               for (; j < mptcpoptlen; j++) {
+                                               for (; j < mptcpoptlen && j < tcp_optlen; j++) {

                                                        if (p_pkt_mnglr->proto_action_mask &
                                                            PKT_MNGLR_TCP_ACT_NOP_MPTCP) {
                                                                tcp_opt_buf[i+j] = 0x1;
                                                        if (p_pkt_mnglr->proto_action_mask &
                                                            PKT_MNGLR_TCP_ACT_NOP_MPTCP) {
                                                                tcp_opt_buf[i+j] = 0x1;

diff --git a/bsd/net/pf_ioctl.c b/bsd/net/pf_ioctl.c
index 05bb21b565cab8a720fd4d787122f4409d4fcecb..62a475f9d2b4416c315083ceadfb16242f43aed4 100644 (file)
--- a/bsd/net/pf_ioctl.c
+++ b/bsd/net/pf_ioctl.c
@@ -4341,7 +4341,7 @@ pf_af_hook(struct ifnet *ifp, struct mbuf **mppn, struct mbuf **mp,
        struct ifnet * pf_ifp = ifp;
 
        /* Always allow traffic on co-processor interfaces. */
        struct ifnet * pf_ifp = ifp;
 
        /* Always allow traffic on co-processor interfaces. */

-       if (ifp && IFNET_IS_INTCOPROC(ifp))
+       if (!intcoproc_unrestricted && ifp && IFNET_IS_INTCOPROC(ifp))

                return (0);
 
        marks = net_thread_marks_push(NET_THREAD_HELD_PF);
                return (0);
 
        marks = net_thread_marks_push(NET_THREAD_HELD_PF);

diff --git a/bsd/net/pktap.c b/bsd/net/pktap.c
index 6608e9df3929fa5213ce3227a73a292beb925edd..66616a73f11b636c55daeb0c5f4655a13bdf4f9c 100644 (file)
--- a/bsd/net/pktap.c
+++ b/bsd/net/pktap.c
@@ -994,6 +994,13 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m,
        void (*bpf_tap_func)(ifnet_t, u_int32_t, mbuf_t, void *, size_t) =
                outgoing ? bpf_tap_out : bpf_tap_in;
 
        void (*bpf_tap_func)(ifnet_t, u_int32_t, mbuf_t, void *, size_t) =
                outgoing ? bpf_tap_out : bpf_tap_in;
 

+
+       /*
+        * Skip the coprocessor interface
+        */
+       if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp))
+               return;
+

        lck_rw_lock_shared(pktap_lck_rw);
 
        /*
        lck_rw_lock_shared(pktap_lck_rw);
 
        /*

diff --git a/bsd/netinet/flow_divert.c b/bsd/netinet/flow_divert.c
index c5e974aab4c29237893db2969355b993d6e7a26a..b73a8617273fc4f0435c661be102de1d8282f549 100644 (file)
--- a/bsd/netinet/flow_divert.c
+++ b/bsd/netinet/flow_divert.c
@@ -1070,6 +1070,9 @@ flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr
                }
        } else {
                FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");
                }
        } else {
                FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity");

+               if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) {
+                       error = 0;
+               }

        }
 
        if (src_proc != PROC_NULL) {
        }
 
        if (src_proc != PROC_NULL) {


@@ -3383,11 +3386,13 @@ flow_divert_token_set(struct socket *so, struct sockopt *sopt)
 
        error = soopt_getm(sopt, &token);
        if (error) {
 
        error = soopt_getm(sopt, &token);
        if (error) {

+               token = NULL;

                goto done;
        }
 
        error = soopt_mcopyin(sopt, token);
        if (error) {
                goto done;
        }
 
        error = soopt_mcopyin(sopt, token);
        if (error) {

+               token = NULL;

                goto done;
        }
 
                goto done;
        }
 

diff --git a/bsd/netinet/in_pcb.c b/bsd/netinet/in_pcb.c
index 3371c71b3ace215b8e75839f6b1ed977e23ddff3..b3ff42cf90b101295b28317ea168d0746afc85c1 100644 (file)
--- a/bsd/netinet/in_pcb.c
+++ b/bsd/netinet/in_pcb.c
@@ -131,7 +131,6 @@ static u_int16_t inpcb_timeout_run = 0;     /* INPCB timer is scheduled to run */
 static boolean_t inpcb_garbage_collecting = FALSE; /* gc timer is scheduled */
 static boolean_t inpcb_ticking = FALSE;                /* "slow" timer is scheduled */
 static boolean_t inpcb_fast_timer_on = FALSE;
 static boolean_t inpcb_garbage_collecting = FALSE; /* gc timer is scheduled */
 static boolean_t inpcb_ticking = FALSE;                /* "slow" timer is scheduled */
 static boolean_t inpcb_fast_timer_on = FALSE;

-static boolean_t intcoproc_unrestricted = FALSE;

 
 extern char *proc_best_name(proc_t);
 
 
 extern char *proc_best_name(proc_t);
 


@@ -308,9 +307,6 @@ in_pcbinit(void)
        RB_INIT(&inp_fc_tree);
        bzero(&key_inp, sizeof(key_inp));
        lck_mtx_unlock(&inp_fc_lck);
        RB_INIT(&inp_fc_tree);
        bzero(&key_inp, sizeof(key_inp));
        lck_mtx_unlock(&inp_fc_lck);

-
-       PE_parse_boot_argn("intcoproc_unrestricted", &intcoproc_unrestricted,
-           sizeof (intcoproc_unrestricted));

 }
 
 #define        INPCB_HAVE_TIMER_REQ(req)       (((req).intimer_lazy > 0) || \
 }
 
 #define        INPCB_HAVE_TIMER_REQ(req)       (((req).intimer_lazy > 0) || \

diff --git a/bsd/netinet/ip_icmp.c b/bsd/netinet/ip_icmp.c
index 3d7ecafe3516d30b575e4d22bd966e7965a03fd4..ab1b0452e1efbdf8d588da1e6cb294589674576d 100644 (file)
--- a/bsd/netinet/ip_icmp.c
+++ b/bsd/netinet/ip_icmp.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -538,9 +538,17 @@ icmp_input(struct mbuf *m, int hlen)
                 * notification to TCP layer.
                 */
                ctlfunc = ip_protox[icp->icmp_ip.ip_p]->pr_ctlinput;
                 * notification to TCP layer.
                 */
                ctlfunc = ip_protox[icp->icmp_ip.ip_p]->pr_ctlinput;

-               if (ctlfunc)
+
+               if (ctlfunc) {
+                       LCK_MTX_ASSERT(inet_domain_mutex, LCK_MTX_ASSERT_OWNED);
+
+                       lck_mtx_unlock(inet_domain_mutex);
+

                        (*ctlfunc)(code, (struct sockaddr *)&icmpsrc,
                                   (void *)&icp->icmp_ip, m->m_pkthdr.rcvif);
                        (*ctlfunc)(code, (struct sockaddr *)&icmpsrc,
                                   (void *)&icp->icmp_ip, m->m_pkthdr.rcvif);

+
+                       lck_mtx_lock(inet_domain_mutex);
+               }

                break;
 
        badcode:
                break;
 
        badcode:

diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c
index 4e4c270408ff4506fe26a1d69c77d7b6baa3d032..1298ab28eb73dc83b6673e5a14902fa38a5be146 100644 (file)
--- a/bsd/netinet/ip_input.c
+++ b/bsd/netinet/ip_input.c
@@ -4117,7 +4117,7 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop)
                        if (sav != NULL) {
                                lck_mtx_lock(sadb_mutex);
                                if (sav->sah != NULL) {
                        if (sav != NULL) {
                                lck_mtx_lock(sadb_mutex);
                                if (sav->sah != NULL) {

-                                       ro = &sav->sah->sa_route;
+                                       ro = (struct route *)&sav->sah->sa_route;

                                        if (ro->ro_rt != NULL) {
                                                RT_LOCK(ro->ro_rt);
                                                if (ro->ro_rt->rt_ifp != NULL) {
                                        if (ro->ro_rt != NULL) {
                                                RT_LOCK(ro->ro_rt);
                                                if (ro->ro_rt->rt_ifp != NULL) {

diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c
index 16aff3c65bd1d73ad7b243f2f90cc7b04553d26e..f5b51ac5251a76661d22467733c9cd2a2218578e 100644 (file)
--- a/bsd/netinet/ip_output.c
+++ b/bsd/netinet/ip_output.c
@@ -1341,7 +1341,7 @@ sendit:
        if (flags & IP_ROUTETOIF) {
                bzero(&ipsec_state.ro, sizeof (ipsec_state.ro));
        } else {
        if (flags & IP_ROUTETOIF) {
                bzero(&ipsec_state.ro, sizeof (ipsec_state.ro));
        } else {

-               route_copyout(&ipsec_state.ro, ro, sizeof (ipsec_state.ro));
+               route_copyout((struct route *)&ipsec_state.ro, ro, sizeof (struct route));

        }
        ipsec_state.dst = SA(dst);
 
        }
        ipsec_state.dst = SA(dst);
 


@@ -1389,10 +1389,10 @@ sendit:
                 */
                if (ipsec_state.tunneled) {
                        flags &= ~IP_ROUTETOIF;
                 */
                if (ipsec_state.tunneled) {
                        flags &= ~IP_ROUTETOIF;

-                       ro = &ipsec_state.ro;
+                       ro = (struct route *)&ipsec_state.ro;

                }
        } else {
                }
        } else {

-               ro = &ipsec_state.ro;
+               ro = (struct route *)&ipsec_state.ro;

        }
        dst = SIN(ipsec_state.dst);
        if (error) {
        }
        dst = SIN(ipsec_state.dst);
        if (error) {

diff --git a/bsd/netinet/mptcp.c b/bsd/netinet/mptcp.c
index 5d901a9da48770ce22fd5a5fb867c4fb291712b7..8cf437f64795a6865b477fe05616ae64599d4743 100644 (file)
--- a/bsd/netinet/mptcp.c
+++ b/bsd/netinet/mptcp.c
@@ -173,7 +173,7 @@ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED,
  * Static declarations
  */
 static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t,
  * Static declarations
  */
 static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t,

-                                uint32_t, uint16_t, uint16_t);
+                                uint32_t, uint16_t, uint16_t, uint16_t);

 
 static int
 mptcp_reass_present(struct socket *mp_so)
 
 static int
 mptcp_reass_present(struct socket *mp_so)


@@ -181,16 +181,17 @@ mptcp_reass_present(struct socket *mp_so)
        struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb;
        struct tseg_qent *q;
        int dowakeup = 0;
        struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb;
        struct tseg_qent *q;
        int dowakeup = 0;

+       int flags = 0;

 
        /*
         * Present data to user, advancing rcv_nxt through
         * completed sequence space.
         */
        if (mp_tp->mpt_state < MPTCPS_ESTABLISHED)
 
        /*
         * Present data to user, advancing rcv_nxt through
         * completed sequence space.
         */
        if (mp_tp->mpt_state < MPTCPS_ESTABLISHED)

-               return (0);
+               return (flags);

        q = LIST_FIRST(&mp_tp->mpt_segq);
        if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt)
        q = LIST_FIRST(&mp_tp->mpt_segq);
        if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt)

-               return (0);
+               return (flags);

 
        /*
         * If there is already another thread doing reassembly for this
 
        /*
         * If there is already another thread doing reassembly for this


@@ -198,7 +199,7 @@ mptcp_reass_present(struct socket *mp_so)
         * (radar 16316196)
         */
        if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG)
         * (radar 16316196)
         */
        if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG)

-               return (0);
+               return (flags);

 
        mp_tp->mpt_flags |= MPTCPF_REASS_INPROG;
 
 
        mp_tp->mpt_flags |= MPTCPF_REASS_INPROG;
 


@@ -208,7 +209,8 @@ mptcp_reass_present(struct socket *mp_so)
                if (mp_so->so_state & SS_CANTRCVMORE) {
                        m_freem(q->tqe_m);
                } else {
                if (mp_so->so_state & SS_CANTRCVMORE) {
                        m_freem(q->tqe_m);
                } else {

-                       if (sbappendstream(&mp_so->so_rcv, q->tqe_m))
+                       flags = !!(q->tqe_m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);
+                       if (sbappendstream_rcvdemux(mp_so, q->tqe_m, 0, 0))

                                dowakeup = 1;
                }
                zfree(tcp_reass_zone, q);
                                dowakeup = 1;
                }
                zfree(tcp_reass_zone, q);


@@ -219,7 +221,7 @@ mptcp_reass_present(struct socket *mp_so)
 
        if (dowakeup)
                sorwakeup(mp_so); /* done with socket lock held */
 
        if (dowakeup)
                sorwakeup(mp_so); /* done with socket lock held */

-       return (0);
+       return (flags);

 
 }
 
 
 }
 


@@ -376,21 +378,40 @@ mptcp_input(struct mptses *mpte, struct mbuf *m)
         * In the degraded fallback case, data is accepted without DSS map
         */
        if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {
         * In the degraded fallback case, data is accepted without DSS map
         */
        if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {

+               struct mbuf *iter;
+               int mb_dfin = 0;

 fallback:
                mptcp_sbrcv_grow(mp_tp);
 
 fallback:
                mptcp_sbrcv_grow(mp_tp);
 

+               for (iter = m; iter; iter = iter->m_next) {
+                       if ((iter->m_flags & M_PKTHDR) &&
+                           (iter->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
+                               mb_dfin = 1;
+                               break;
+                       }
+               }
+

                /*
                 * assume degraded flow as this may be the first packet
                 * without DSS, and the subflow state is not updated yet.
                 */
                /*
                 * assume degraded flow as this may be the first packet
                 * without DSS, and the subflow state is not updated yet.
                 */

-               if (sbappendstream(&mp_so->so_rcv, m))
+               if (sbappendstream_rcvdemux(mp_so, m, 0, 0))

                        sorwakeup(mp_so);
                        sorwakeup(mp_so);

+

                DTRACE_MPTCP5(receive__degraded, struct mbuf *, m,
                    struct socket *, mp_so,
                    struct sockbuf *, &mp_so->so_rcv,
                    struct sockbuf *, &mp_so->so_snd,
                    struct mptses *, mpte);
                count = mp_so->so_rcv.sb_cc - count;
                DTRACE_MPTCP5(receive__degraded, struct mbuf *, m,
                    struct socket *, mp_so,
                    struct sockbuf *, &mp_so->so_rcv,
                    struct sockbuf *, &mp_so->so_snd,
                    struct mptses *, mpte);
                count = mp_so->so_rcv.sb_cc - count;

+
+               mp_tp->mpt_rcvnxt += count;
+
+               if (mb_dfin) {
+                       mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
+                       socantrcvmore(mp_so);
+               }
+

                mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__,
                    count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
                return;
                mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__,
                    count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
                return;


@@ -400,6 +421,7 @@ fallback:
                u_int64_t mb_dsn;
                int32_t mb_datalen;
                int64_t todrop;
                u_int64_t mb_dsn;
                int32_t mb_datalen;
                int64_t todrop;

+               int mb_dfin = 0;

 
                /* If fallback occurs, mbufs will not have PKTF_MPTCP set */
                if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP))
 
                /* If fallback occurs, mbufs will not have PKTF_MPTCP set */
                if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP))


@@ -450,14 +472,21 @@ fallback:
                                m_adj(m, -todrop);
                                mb_datalen -= todrop;
                        }
                                m_adj(m, -todrop);
                                mb_datalen -= todrop;
                        }

+
+                       /*
+                        * We drop from the right edge of the mbuf, thus the
+                        * DATA_FIN is dropped as well
+                        */
+                       m->m_pkthdr.pkt_flags &= ~PKTF_MPTCP_DFIN;

                }
 
                if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) ||
                    !LIST_EMPTY(&mp_tp->mpt_segq)) {
                }
 
                if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) ||
                    !LIST_EMPTY(&mp_tp->mpt_segq)) {

-                       mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m);
+                       mb_dfin = mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m);

 
                        goto next;
                }
 
                        goto next;
                }

+               mb_dfin = !!(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);

 
                if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) {
                        if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen),
 
                if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) {
                        if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen),


@@ -485,7 +514,7 @@ fallback:
 
                mptcp_sbrcv_grow(mp_tp);
 
 
                mptcp_sbrcv_grow(mp_tp);
 

-               if (sbappendstream(&mp_so->so_rcv, m))
+               if (sbappendstream_rcvdemux(mp_so, m, 0, 0))

                        wakeup = 1;
 
                DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so,
                        wakeup = 1;
 
                DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so,


@@ -502,6 +531,10 @@ fallback:
                mp_tp->mpt_rcvnxt += count;
 
 next:
                mp_tp->mpt_rcvnxt += count;
 
 next:

+               if (mb_dfin) {
+                       mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
+                       socantrcvmore(mp_so);
+               }

                m = save;
                prev = save = NULL;
                count = mp_so->so_rcv.sb_cc;
                m = save;
                prev = save = NULL;
                count = mp_so->so_rcv.sb_cc;


@@ -1076,19 +1109,13 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp,
                mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
                return;
        }
                mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
                return;
        }

-               mptcplog((LOG_DEBUG,
-                   "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__,
-                   seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt),
-                   MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);
+       mptcplog((LOG_DEBUG,
+           "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__,
+           seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt),
+           MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE);

 
 

-       /* Process a Data FIN packet , handled in mptcp_do_fin_opt */
-       if ((seqn == 0) && (mdss_data_len == 1)) {
-               mptcplog((LOG_INFO, "%s: Data FIN in %s state \n", __func__,
-                   mptcp_state_to_str(mp_tp->mpt_state)),
-                   MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);
-               return;
-       }

        mptcp_notify_mpready(tp->t_inpcb->inp_socket);
        mptcp_notify_mpready(tp->t_inpcb->inp_socket);

+

        tp->t_rcv_map.mpt_dsn = full_dsn;
        tp->t_rcv_map.mpt_sseq = seqn;
        tp->t_rcv_map.mpt_len = mdss_data_len;
        tp->t_rcv_map.mpt_dsn = full_dsn;
        tp->t_rcv_map.mpt_sseq = seqn;
        tp->t_rcv_map.mpt_len = mdss_data_len;


@@ -1123,9 +1150,10 @@ mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m,
 }
 
 int
 }
 
 int

-mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen)
+mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
+                   int drop_hdrlen)

 {
 {

-       mptcp_insert_rmap(tp, m);
+       mptcp_insert_rmap(tp, m, th);

        if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m,
            drop_hdrlen) != 0)
                return -1;
        if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m,
            drop_hdrlen) != 0)
                return -1;


@@ -1142,11 +1170,11 @@ mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen)
 
 int
 mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,
 
 int
 mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,

-                   uint32_t sseq, uint16_t dlen, uint16_t csum)
+                   uint32_t sseq, uint16_t dlen, uint16_t csum, uint16_t dfin)

 {
        uint16_t mptcp_csum;
 
 {
        uint16_t mptcp_csum;
 

-       mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum);
+       mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum, dfin);

        if (mptcp_csum) {
                tp->t_mpflags |= TMPF_SND_MPFAIL;
                mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
        if (mptcp_csum) {
                tp->t_mpflags |= TMPF_SND_MPFAIL;
                mptcp_notify_mpfail(tp->t_inpcb->inp_socket);


@@ -1159,9 +1187,10 @@ mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,
 
 static uint16_t
 mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq,
 
 static uint16_t
 mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq,

-                uint16_t dlen, uint16_t csum)
+                uint16_t dlen, uint16_t csum, uint16_t dfin)

 {
        struct mptcb *mp_tp = tptomptp(tp);
 {
        struct mptcb *mp_tp = tptomptp(tp);

+       uint16_t real_len = dlen - dfin;

        uint32_t sum = 0;
 
        if (mp_tp == NULL)
        uint32_t sum = 0;
 
        if (mp_tp == NULL)


@@ -1177,11 +1206,12 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq,
         * The remote side may send a packet with fewer bytes than the
         * claimed DSS checksum length.
         */
         * The remote side may send a packet with fewer bytes than the
         * claimed DSS checksum length.
         */

-       if ((int)m_length2(m, NULL) < dlen)
+       if ((int)m_length2(m, NULL) < real_len) {

                return (0xffff);
                return (0xffff);

+       }

 
 

-       if (dlen != 0)
-               sum = m_sum16(m, 0, dlen);
+       if (real_len != 0)
+               sum = m_sum16(m, 0, real_len);

 
        sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum);
        ADDCARRY(sum);
 
        sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum);
        ADDCARRY(sum);

diff --git a/bsd/netinet/mptcp_opt.c b/bsd/netinet/mptcp_opt.c
index 8aa8a9a4e38344f2227bdbe094d2935c6aad63f1..40007851af41d3cf77522ce00f27dc707479c9ff 100644 (file)
--- a/bsd/netinet/mptcp_opt.c
+++ b/bsd/netinet/mptcp_opt.c
@@ -1366,33 +1366,6 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp, struct tcphdr *th)
        }
 }
 
        }
 }
 

-static void
-mptcp_do_fin_opt(struct tcpcb *tp)
-{
-       struct mptcb *mp_tp = tptomptp(tp);
-
-       if (!(tp->t_mpflags & TMPF_RECV_DFIN)) {
-               if (mp_tp != NULL) {
-                       mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
-
-                       if (tp->t_inpcb->inp_socket != NULL) {
-                               soevent(tp->t_inpcb->inp_socket,
-                                   SO_FILT_HINT_LOCKED |
-                                   SO_FILT_HINT_MPCANTRCVMORE);
-                       }
-
-               }
-               tp->t_mpflags |= TMPF_RECV_DFIN;
-       }
-
-       tp->t_mpflags |= TMPF_MPTCP_ACKNOW;
-       /*
-        * Since this is a data level FIN, TCP needs to be explicitly told
-        * to send back an ACK on which the Data ACK is piggybacked.
-        */
-       tp->t_flags |= TF_ACKNOW;
-}
-

 static void
 mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen)
 {
 static void
 mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen)
 {


@@ -1408,9 +1381,8 @@ mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen)
                struct mptcp_dss_copt *dss_rsp = (struct mptcp_dss_copt *)cp;
 
                if (dss_rsp->mdss_subtype == MPO_DSS) {
                struct mptcp_dss_copt *dss_rsp = (struct mptcp_dss_copt *)cp;
 
                if (dss_rsp->mdss_subtype == MPO_DSS) {

-                       if (dss_rsp->mdss_flags & MDSS_F) {
-                               mptcp_do_fin_opt(tp);
-                       }
+                       if (dss_rsp->mdss_flags & MDSS_F)
+                               tp->t_rcv_map.mpt_dfin = 1;

 
                        mptcp_do_dss_opt_meat(cp, tp, th);
                }
 
                        mptcp_do_dss_opt_meat(cp, tp, th);
                }

diff --git a/bsd/netinet/mptcp_subr.c b/bsd/netinet/mptcp_subr.c
index 19e128687e4f1a3dd622b19f161367e90a333b42..590fa2c465855603654fbc249900f1f7e31cfaa4 100644 (file)
--- a/bsd/netinet/mptcp_subr.c
+++ b/bsd/netinet/mptcp_subr.c
@@ -1733,7 +1733,7 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
        SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1");
 
        while (m != NULL) {
        SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1");
 
        while (m != NULL) {

-               int dlen = 0;
+               int dlen = 0, dfin = 0, error_out = 0;

                struct mbuf *start = m;
                uint64_t dsn;
                uint32_t sseq;
                struct mbuf *start = m;
                uint64_t dsn;
                uint32_t sseq;


@@ -1749,7 +1749,7 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
                        csum = m->m_pkthdr.mp_csum;
                } else {
                        /* We did fallback */
                        csum = m->m_pkthdr.mp_csum;
                } else {
                        /* We did fallback */

-                       mptcp_adj_rmap(so, m, 0);
+                       mptcp_adj_rmap(so, m, 0, 0, 0, 0);

 
                        sbfree(&so->so_rcv, m);
 
 
                        sbfree(&so->so_rcv, m);
 


@@ -1770,10 +1770,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
                        continue;
                }
 
                        continue;
                }
 

+               if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)
+                       dfin = 1;
+

                /*
                 * Check if the full mapping is now present
                 */
                /*
                 * Check if the full mapping is now present
                 */

-               if ((int)so->so_rcv.sb_cc < dlen) {
+               if ((int)so->so_rcv.sb_cc < dlen - dfin) {

                        mptcplog((LOG_INFO, "%s not enough data (%u) need %u\n",
                                  __func__, so->so_rcv.sb_cc, dlen),
                                 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);
                        mptcplog((LOG_INFO, "%s not enough data (%u) need %u\n",
                                  __func__, so->so_rcv.sb_cc, dlen),
                                 MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG);


@@ -1785,7 +1788,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
 
                /* Now, get the full mapping */
                while (dlen > 0) {
 
                /* Now, get the full mapping */
                while (dlen > 0) {

-                       mptcp_adj_rmap(so, m, orig_dlen - dlen);
+                       if (mptcp_adj_rmap(so, m, orig_dlen - dlen, dsn, sseq, orig_dlen)) {
+                               error_out = 1;
+                               error = EIO;
+                               dlen = 0;
+                               soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST);
+                               break;
+                       }

 
                        dlen -= m->m_len;
                        sbfree(&so->so_rcv, m);
 
                        dlen -= m->m_len;
                        sbfree(&so->so_rcv, m);


@@ -1797,6 +1806,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
                                *mp = NULL;
                        }
 
                                *mp = NULL;
                        }
 

+                       if (dlen - dfin == 0)
+                               dlen = 0;
+

                        VERIFY(dlen <= 0 || m);
                }
 
                        VERIFY(dlen <= 0 || m);
                }
 


@@ -1808,7 +1820,11 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa,
                        SB_EMPTY_FIXUP(&so->so_rcv);
                }
 
                        SB_EMPTY_FIXUP(&so->so_rcv);
                }
 

-               if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum)) {
+               if (error_out)
+                       goto release;
+
+
+               if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum, dfin)) {

                        error = EIO;
                        *mp0 = NULL;
                        goto release;
                        error = EIO;
                        *mp0 = NULL;
                        goto release;


@@ -4118,16 +4134,8 @@ mptcp_subflow_workloop(struct mptses *mpte)
        }
 
        if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) {
        }
 
        if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) {

-               struct mptcb *mp_tp = mpte->mpte_mptcb;
-

                VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED);
 
                VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED);
 

-               if (mpsofilt_hint_mask & SO_FILT_HINT_CANTRCVMORE) {
-                       mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
-                       socantrcvmore(mp_so);
-                       mpsofilt_hint_mask &= ~SO_FILT_HINT_CANTRCVMORE;
-               }
-

                soevent(mp_so, mpsofilt_hint_mask);
        }
 
                soevent(mp_so, mpsofilt_hint_mask);
        }
 


@@ -4778,31 +4786,49 @@ mptcp_output_getm_dsnmap64(struct socket *so, int off, uint64_t *dsn,
  * with mptcp_adj_rmap()
  */
 void
  * with mptcp_adj_rmap()
  */
 void

-mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m)
+mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th)

 {
 {

+       VERIFY(m->m_flags & M_PKTHDR);

        VERIFY(!(m->m_pkthdr.pkt_flags & PKTF_MPTCP));
 
        if (tp->t_mpflags & TMPF_EMBED_DSN) {
        VERIFY(!(m->m_pkthdr.pkt_flags & PKTF_MPTCP));
 
        if (tp->t_mpflags & TMPF_EMBED_DSN) {

-               VERIFY(m->m_flags & M_PKTHDR);

                m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn;
                m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq;
                m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len;
                m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum;
                m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn;
                m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq;
                m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len;
                m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum;

+               if (tp->t_rcv_map.mpt_dfin)
+                       m->m_pkthdr.pkt_flags |= PKTF_MPTCP_DFIN;
+

                m->m_pkthdr.pkt_flags |= PKTF_MPTCP;
                m->m_pkthdr.pkt_flags |= PKTF_MPTCP;

+

                tp->t_mpflags &= ~TMPF_EMBED_DSN;
                tp->t_mpflags |= TMPF_MPTCP_ACKNOW;
                tp->t_mpflags &= ~TMPF_EMBED_DSN;
                tp->t_mpflags |= TMPF_MPTCP_ACKNOW;

+       } else if (tp->t_mpflags & TMPF_TCP_FALLBACK) {
+               if (th->th_flags & TH_FIN)
+                       m->m_pkthdr.pkt_flags |= PKTF_MPTCP_DFIN;

        }
 }
 
        }
 }
 

-void
-mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off)
+int
+mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off, uint64_t dsn,
+              uint32_t rseq, uint16_t dlen)

 {
        struct mptsub *mpts = sototcpcb(so)->t_mpsub;
 
        if (m_pktlen(m) == 0)
 {
        struct mptsub *mpts = sototcpcb(so)->t_mpsub;
 
        if (m_pktlen(m) == 0)

-               return;
+               return (0);

 
        if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) {
 
        if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) {

+               if (off && (dsn != m->m_pkthdr.mp_dsn ||
+                           rseq != m->m_pkthdr.mp_rseq ||
+                           dlen != m->m_pkthdr.mp_rlen)) {
+                       mptcplog((LOG_ERR, "%s: Received incorrect second mapping: %llu - %llu , %u - %u, %u - %u\n",
+                                 __func__, dsn, m->m_pkthdr.mp_dsn,
+                                 rseq, m->m_pkthdr.mp_rseq,
+                                 dlen, m->m_pkthdr.mp_rlen),
+                                MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR);
+                       return (-1);
+               }

                m->m_pkthdr.mp_dsn += off;
                m->m_pkthdr.mp_rseq += off;
                m->m_pkthdr.mp_rlen = m->m_pkthdr.len;
                m->m_pkthdr.mp_dsn += off;
                m->m_pkthdr.mp_rseq += off;
                m->m_pkthdr.mp_rlen = m->m_pkthdr.len;


@@ -4817,7 +4843,7 @@ mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off)
 
        mpts->mpts_flags |= MPTSF_CONFIRMED;
 
 
        mpts->mpts_flags |= MPTSF_CONFIRMED;
 

-       return;
+       return (0);

 }
 
 /*
 }
 
 /*

diff --git a/bsd/netinet/mptcp_var.h b/bsd/netinet/mptcp_var.h
index 2d1f99061048bf913bab4796a923a4c9b9de23be..1ff155f549e102c8706c14846baaef3d89dd6030 100644 (file)
--- a/bsd/netinet/mptcp_var.h
+++ b/bsd/netinet/mptcp_var.h
@@ -223,7 +223,7 @@ static inline int
 mptcp_subflow_cwnd_space(struct socket *so)
 {
        struct tcpcb *tp = sototcpcb(so);
 mptcp_subflow_cwnd_space(struct socket *so)
 {
        struct tcpcb *tp = sototcpcb(so);

-       int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (tp->snd_nxt - tp->snd_una);
+       int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (so->so_snd.sb_cc);

 
        return (min(cwnd, sbspace(&so->so_snd)));
 }
 
        return (min(cwnd, sbspace(&so->so_snd)));
 }


@@ -653,7 +653,8 @@ extern void mptcp_unset_cellicon(void);
 extern void mptcp_reset_rexmit_state(struct tcpcb *tp);
 extern void mptcp_reset_keepalive(struct tcpcb *tp);
 extern int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,
 extern void mptcp_reset_rexmit_state(struct tcpcb *tp);
 extern void mptcp_reset_keepalive(struct tcpcb *tp);
 extern int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,

-                              uint32_t sseq, uint16_t dlen, uint16_t csum);
+                              uint32_t sseq, uint16_t dlen, uint16_t csum,
+                              uint16_t dfin);

 __END_DECLS
 
 #endif /* BSD_KERNEL_PRIVATE */
 __END_DECLS
 
 #endif /* BSD_KERNEL_PRIVATE */

diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c
index 741aa00d5a7a622b0a036e230d3c35dde44794b6..139984932fbaaefa771de6cb9578256063356185 100644 (file)
--- a/bsd/netinet/tcp_input.c
+++ b/bsd/netinet/tcp_input.c
@@ -2741,18 +2741,19 @@ findpcb:
         */
        if (tp->t_state != TCPS_LISTEN && optp) {
                tcp_dooptions(tp, optp, optlen, th, &to);
         */
        if (tp->t_state != TCPS_LISTEN && optp) {
                tcp_dooptions(tp, optp, optlen, th, &to);

+       }

 #if MPTCP
 #if MPTCP

-               if (mptcp_input_preproc(tp, m, drop_hdrlen) != 0) {
-                       tp->t_flags |= TF_ACKNOW;
-                       (void) tcp_output(tp);
-                       tcp_check_timer_state(tp);
-                       socket_unlock(so, 1);
-                       KERNEL_DEBUG(DBG_FNC_TCP_INPUT |
-                           DBG_FUNC_END,0,0,0,0,0);
-                       return;
-               }
-#endif /* MPTCP */
+       if (tp->t_state != TCPS_LISTEN && (so->so_flags & SOF_MP_SUBFLOW) &&
+           mptcp_input_preproc(tp, m, th, drop_hdrlen) != 0) {
+               tp->t_flags |= TF_ACKNOW;
+               (void) tcp_output(tp);
+               tcp_check_timer_state(tp);
+               socket_unlock(so, 1);
+               KERNEL_DEBUG(DBG_FNC_TCP_INPUT |
+                   DBG_FUNC_END,0,0,0,0,0);
+               return;

        }
        }

+#endif /* MPTCP */

        if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
                if (!(thflags & TH_ACK) ||
                    (SEQ_GT(th->th_ack, tp->iss) &&
        if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
                if (!(thflags & TH_ACK) ||
                    (SEQ_GT(th->th_ack, tp->iss) &&


@@ -4940,7 +4941,14 @@ dodata:
 
                }
        } else {
 
                }
        } else {

-               m_freem(m);
+               if ((so->so_flags & SOF_MP_SUBFLOW) && tlen == 0 &&
+                   (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
+                       m_adj(m, drop_hdrlen);  /* delayed header drop */
+                       mptcp_input(tptomptp(tp)->mpt_mpte, m);
+                       tp->t_flags |= TF_ACKNOW;
+               } else {
+                       m_freem(m);
+               }

                thflags &= ~TH_FIN;
        }
 
                thflags &= ~TH_FIN;
        }
 

diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c
index 1be5b6a8085824911b02f5cd1c65bd0c17f01807..2d6d4d1bf4de20a9b036cf111df1b3748a54ba78 100644 (file)
--- a/bsd/netinet/tcp_subr.c
+++ b/bsd/netinet/tcp_subr.c
@@ -401,8 +401,6 @@ get_tcp_str_size(void)
        return (sizeof(struct tcpcb));
 }
 
        return (sizeof(struct tcpcb));
 }
 

-int    tcp_freeq(struct tcpcb *tp);
-

 static int scale_to_powerof2(int size);
 
 /*
 static int scale_to_powerof2(int size);
 
 /*

diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c
index f410382edd8644242e424217d5d95ba715b1bfb0..2c861120e8e0f011eb442a405b1f7b9cbbd02abf 100644 (file)
--- a/bsd/netinet/tcp_timer.c
+++ b/bsd/netinet/tcp_timer.c
@@ -1040,7 +1040,7 @@ retransmit_packet:
                    (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
                    !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
                    ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||
                    (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
                    !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
                    ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||

-                    tp->t_rxtshift > 2)) {
+                    tp->t_rxtshift > 4)) {

                        /*
                         * For regular retransmissions, a first one is being
                         * done for tail-loss probe.
                        /*
                         * For regular retransmissions, a first one is being
                         * done for tail-loss probe.


@@ -1062,7 +1062,7 @@ retransmit_packet:
 
                if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
                    (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&
 
                if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
                    (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&

-                   tp->t_rxtshift > 1) {
+                   tp->t_rxtshift > 3) {

                        if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
                                tcp_heuristic_tfo_middlebox(tp);
 
                        if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
                                tcp_heuristic_tfo_middlebox(tp);
 

diff --git a/bsd/netinet/tcp_usrreq.c b/bsd/netinet/tcp_usrreq.c
index 0f0dc353e72587433a6878fb4b5ff7539e1549dc..9d8fa7efbd5c622da36d4af90929766e61fc1692 100644 (file)
--- a/bsd/netinet/tcp_usrreq.c
+++ b/bsd/netinet/tcp_usrreq.c
@@ -2719,6 +2719,9 @@ tcp_disconnect(struct tcpcb *tp)
 {
        struct socket *so = tp->t_inpcb->inp_socket;
 
 {
        struct socket *so = tp->t_inpcb->inp_socket;
 

+       if (so->so_rcv.sb_cc != 0 || tp->t_reassqlen != 0)
+               return tcp_drop(tp, 0);
+

        if (tp->t_state < TCPS_ESTABLISHED)
                tp = tcp_close(tp);
        else if ((so->so_options & SO_LINGER) && so->so_linger == 0)
        if (tp->t_state < TCPS_ESTABLISHED)
                tp = tcp_close(tp);
        else if ((so->so_options & SO_LINGER) && so->so_linger == 0)

diff --git a/bsd/netinet/tcp_var.h b/bsd/netinet/tcp_var.h
index 4a5e1d3b4ce991245470a5ec46a8f1b583f4700d..4fde35c904f95d0be2d3bfbb0078c7ee6425a01b 100644 (file)
--- a/bsd/netinet/tcp_var.h
+++ b/bsd/netinet/tcp_var.h
@@ -216,6 +216,7 @@ struct mpt_dsn_map {
        uint32_t                mpt_sseq;       /* relative subflow # */
        uint16_t                mpt_len;        /* length of mapping */
        uint16_t                mpt_csum;       /* checksum value if on */
        uint32_t                mpt_sseq;       /* relative subflow # */
        uint16_t                mpt_len;        /* length of mapping */
        uint16_t                mpt_csum;       /* checksum value if on */

+       uint8_t                 mpt_dfin;       /* It's a DATA_FIN */

 };
 #define tcp6cb         tcpcb  /* for KAME src sync over BSD*'s */
 
 };
 #define tcp6cb         tcpcb  /* for KAME src sync over BSD*'s */
 


@@ -1634,6 +1635,7 @@ extern boolean_t tfo_enabled(const struct tcpcb *tp);
 extern void tcp_disable_tfo(struct tcpcb *tp);
 extern void tcp_tfo_gen_cookie(struct inpcb *inp, u_char *out, size_t blk_size);
 #define        TCP_FASTOPEN_KEYLEN 16
 extern void tcp_disable_tfo(struct tcpcb *tp);
 extern void tcp_tfo_gen_cookie(struct inpcb *inp, u_char *out, size_t blk_size);
 #define        TCP_FASTOPEN_KEYLEN 16

+extern int tcp_freeq(struct tcpcb *tp);

 extern errno_t tcp_notify_ack_id_valid(struct tcpcb *, struct socket *, u_int32_t);
 extern errno_t tcp_add_notify_ack_marker(struct tcpcb *, u_int32_t);
 extern void tcp_notify_ack_free(struct tcpcb *);
 extern errno_t tcp_notify_ack_id_valid(struct tcpcb *, struct socket *, u_int32_t);
 extern errno_t tcp_add_notify_ack_marker(struct tcpcb *, u_int32_t);
 extern void tcp_notify_ack_free(struct tcpcb *);


@@ -1648,11 +1650,12 @@ extern int get_tcp_inp_list(struct inpcb **, int, inp_gen_t);
 extern bool tcp_notify_ack_active(struct socket *so);
 
 #if MPTCP
 extern bool tcp_notify_ack_active(struct socket *so);
 
 #if MPTCP

-extern int mptcp_input_preproc(struct tcpcb *, struct mbuf *, int);
+extern int mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m,
+                              struct tcphdr *th, int drop_hdrlen);

 extern uint32_t mptcp_output_csum(struct mbuf *m, uint64_t dss_val,
                                  uint32_t sseq, uint16_t dlen);
 extern int mptcp_adj_mss(struct tcpcb *, boolean_t);
 extern uint32_t mptcp_output_csum(struct mbuf *m, uint64_t dss_val,
                                  uint32_t sseq, uint16_t dlen);
 extern int mptcp_adj_mss(struct tcpcb *, boolean_t);

-extern void mptcp_insert_rmap(struct tcpcb *, struct mbuf *);
+extern void mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th);

 #endif
 
 __private_extern__ void tcp_update_stats_per_flow(
 #endif
 
 __private_extern__ void tcp_update_stats_per_flow(

diff --git a/bsd/netinet6/icmp6.c b/bsd/netinet6/icmp6.c
index 1580cae9d47c304150cd032e577b7ce0df7cbb52..f80a4154aa04abf5d0cda74d66ac9aa9335a7a0c 100644 (file)
--- a/bsd/netinet6/icmp6.c
+++ b/bsd/netinet6/icmp6.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -1133,8 +1133,14 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code)
 
                ctlfunc = ip6_protox[nxt]->pr_ctlinput;
                if (ctlfunc) {
 
                ctlfunc = ip6_protox[nxt]->pr_ctlinput;
                if (ctlfunc) {

+                       LCK_MTX_ASSERT(inet6_domain_mutex, LCK_MTX_ASSERT_OWNED);
+
+                       lck_mtx_unlock(inet6_domain_mutex);
+

                        (void) (*ctlfunc)(code, (struct sockaddr *)&icmp6dst,
                            &ip6cp, m->m_pkthdr.rcvif);
                        (void) (*ctlfunc)(code, (struct sockaddr *)&icmp6dst,
                            &ip6cp, m->m_pkthdr.rcvif);

+
+                       lck_mtx_lock(inet6_domain_mutex);

                }
        }
        return(0);
                }
        }
        return(0);

diff --git a/bsd/netinet6/in6_proto.c b/bsd/netinet6/in6_proto.c
index 34f8f754a6ac4c12d77197d52a8af2b621980811..e71714e69a380949c96b53b158c6572c5849fa6e 100644 (file)
--- a/bsd/netinet6/in6_proto.c
+++ b/bsd/netinet6/in6_proto.c
@@ -477,7 +477,7 @@ u_int32_t   rip6_sendspace = RIPV6SNDQ;
 u_int32_t      rip6_recvspace = RIPV6RCVQ;
 
 /* ICMPV6 parameters */
 u_int32_t      rip6_recvspace = RIPV6RCVQ;
 
 /* ICMPV6 parameters */

-int    icmp6_rediraccept = 0;          /* accept and process redirects */
+int    icmp6_rediraccept = 1;          /* accept and process redirects */

 int    icmp6_redirtimeout = 10 * 60;   /* 10 minutes */
 int    icmp6errppslim = 500;           /* 500 packets per second */
 int    icmp6rappslim = 10;             /* 10 packets per second */
 int    icmp6_redirtimeout = 10 * 60;   /* 10 minutes */
 int    icmp6errppslim = 500;           /* 500 packets per second */
 int    icmp6rappslim = 10;             /* 10 packets per second */

diff --git a/bsd/netinet6/ip6_output.c b/bsd/netinet6/ip6_output.c
index 5e541f14c53ec5568926e4dac028e35e26b72a78..73a66159e4109fa15b81ef42626f1351ede9a37f 100644 (file)
--- a/bsd/netinet6/ip6_output.c
+++ b/bsd/netinet6/ip6_output.c
@@ -1055,8 +1055,8 @@ skip_ipsec:
                exthdrs.ip6e_ip6 = m;
 
                ipsec_state.m = m;
                exthdrs.ip6e_ip6 = m;
 
                ipsec_state.m = m;

-               route_copyout(&ipsec_state.ro, (struct route *)ro,
-                   sizeof (ipsec_state.ro));
+               route_copyout((struct route *)&ipsec_state.ro, (struct route *)ro,
+                   sizeof (struct route_in6));

                ipsec_state.dst = SA(dst);
 
                /* So that we can see packets inside the tunnel */
                ipsec_state.dst = SA(dst);
 
                /* So that we can see packets inside the tunnel */

diff --git a/bsd/netinet6/ipsec.c b/bsd/netinet6/ipsec.c
index 895d63e7781ab7cd0b9c8be88e98019700e899b0..1ba0680113a8fc26581b4169ec09637d142b0b8d 100644 (file)
--- a/bsd/netinet6/ipsec.c
+++ b/bsd/netinet6/ipsec.c
@@ -2498,7 +2498,7 @@ ipsec6_update_routecache_and_output(
        struct secasvar *sav)
 {
        struct sockaddr_in6* dst6;
        struct secasvar *sav)
 {
        struct sockaddr_in6* dst6;

-       struct route *ro6;
+       struct route_in6 *ro6;

        struct ip6_hdr *ip6;
        errno_t error = 0;
 
        struct ip6_hdr *ip6;
        errno_t error = 0;
 


@@ -2530,7 +2530,7 @@ ipsec6_update_routecache_and_output(
                dst6->sin6_family = AF_INET6;
                dst6->sin6_len = sizeof(*dst6);
                dst6->sin6_addr = ip6->ip6_dst;
                dst6->sin6_family = AF_INET6;
                dst6->sin6_len = sizeof(*dst6);
                dst6->sin6_addr = ip6->ip6_dst;

-               rtalloc_scoped(ro6, sav->sah->outgoing_if);
+               rtalloc_scoped((struct route *)ro6, sav->sah->outgoing_if);

                if (ro6->ro_rt) {
                        RT_LOCK(ro6->ro_rt);
                }
                if (ro6->ro_rt) {
                        RT_LOCK(ro6->ro_rt);
                }


@@ -2557,7 +2557,7 @@ ipsec6_update_routecache_and_output(
                dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway;
        RT_UNLOCK(ro6->ro_rt);
        ROUTE_RELEASE(&state->ro);
                dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway;
        RT_UNLOCK(ro6->ro_rt);
        ROUTE_RELEASE(&state->ro);

-       route_copyout(&state->ro, ro6, sizeof(state->ro));
+       route_copyout((struct route *)&state->ro, (struct route *)ro6, sizeof(struct route_in6));

        state->dst = (struct sockaddr *)dst6;
        state->tunneled = 6;
        // release sadb_mutex, after updating sah's route cache                                                                                                                          
        state->dst = (struct sockaddr *)dst6;
        state->tunneled = 6;
        // release sadb_mutex, after updating sah's route cache                                                                                                                          


@@ -3162,7 +3162,7 @@ ipsec4_output_internal(struct ipsec_output_state *state, struct secasvar *sav)
 
                        // grab sadb_mutex, before updating sah's route cache
                        lck_mtx_lock(sadb_mutex);
 
                        // grab sadb_mutex, before updating sah's route cache
                        lck_mtx_lock(sadb_mutex);

-                       ro4= &sav->sah->sa_route;
+                       ro4= (struct route *)&sav->sah->sa_route;

                        dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst;
                        if (ro4->ro_rt != NULL) {
                                RT_LOCK(ro4->ro_rt);
                        dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst;
                        if (ro4->ro_rt != NULL) {
                                RT_LOCK(ro4->ro_rt);


@@ -3201,7 +3201,7 @@ ipsec4_output_internal(struct ipsec_output_state *state, struct secasvar *sav)
                                dst4 = (struct sockaddr_in *)(void *)ro4->ro_rt->rt_gateway;
                        RT_UNLOCK(ro4->ro_rt);
                        ROUTE_RELEASE(&state->ro);
                                dst4 = (struct sockaddr_in *)(void *)ro4->ro_rt->rt_gateway;
                        RT_UNLOCK(ro4->ro_rt);
                        ROUTE_RELEASE(&state->ro);

-                       route_copyout(&state->ro, ro4, sizeof(state->ro));
+                       route_copyout((struct route *)&state->ro, ro4, sizeof(struct route));

                        state->dst = (struct sockaddr *)dst4;
                        state->tunneled = 4;
                        // release sadb_mutex, after updating sah's route cache                                                                                                    
                        state->dst = (struct sockaddr *)dst4;
                        state->tunneled = 4;
                        // release sadb_mutex, after updating sah's route cache                                                                                                    


@@ -3644,7 +3644,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
        int error = 0;
        int plen;
        struct sockaddr_in6* dst6;
        int error = 0;
        int plen;
        struct sockaddr_in6* dst6;

-       struct route *ro6;
+       struct route_in6 *ro6;

        
        /* validity check */
        if (sav == NULL || sav->sah == NULL || sav->sah->saidx.mode != IPSEC_MODE_TUNNEL) {
        
        /* validity check */
        if (sav == NULL || sav->sah == NULL || sav->sah->saidx.mode != IPSEC_MODE_TUNNEL) {


@@ -3706,7 +3706,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
                        
                        // grab sadb_mutex, to update sah's route cache and get a local copy of it
                        lck_mtx_lock(sadb_mutex);
                        
                        // grab sadb_mutex, to update sah's route cache and get a local copy of it
                        lck_mtx_lock(sadb_mutex);

-                       ro4 = &sav->sah->sa_route;
+                       ro4 = (struct route *)&sav->sah->sa_route;

                        dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst;
                        if (ro4->ro_rt) {
                                RT_LOCK(ro4->ro_rt);
                        dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst;
                        if (ro4->ro_rt) {
                                RT_LOCK(ro4->ro_rt);


@@ -3724,7 +3724,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
                        } else {
                                RT_UNLOCK(ro4->ro_rt);
                        }
                        } else {
                                RT_UNLOCK(ro4->ro_rt);
                        }

-                       route_copyout(&ro4_copy, ro4, sizeof(ro4_copy));
+                       route_copyout(&ro4_copy, ro4, sizeof(struct route));

                        // release sadb_mutex, after updating sah's route cache and getting a local copy
                        lck_mtx_unlock(sadb_mutex);
                        state->m = ipsec4_splithdr(state->m);
                        // release sadb_mutex, after updating sah's route cache and getting a local copy
                        lck_mtx_unlock(sadb_mutex);
                        state->m = ipsec4_splithdr(state->m);


@@ -3789,7 +3789,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
                        state->m = NULL;
                        // grab sadb_mutex, to synchronize the sah's route cache with the local copy
                        lck_mtx_lock(sadb_mutex);
                        state->m = NULL;
                        // grab sadb_mutex, to synchronize the sah's route cache with the local copy
                        lck_mtx_lock(sadb_mutex);

-                       route_copyin(&ro4_copy, ro4, sizeof(ro4_copy));
+                       route_copyin(&ro4_copy, ro4, sizeof(struct route));

                        lck_mtx_unlock(sadb_mutex);
                        if (error != 0)
                                goto bad;
                        lck_mtx_unlock(sadb_mutex);
                        if (error != 0)
                                goto bad;


@@ -3821,7 +3821,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
                        dst6->sin6_family = AF_INET6;
                        dst6->sin6_len = sizeof(*dst6);
                        dst6->sin6_addr = ip6->ip6_dst;
                        dst6->sin6_family = AF_INET6;
                        dst6->sin6_len = sizeof(*dst6);
                        dst6->sin6_addr = ip6->ip6_dst;

-                       rtalloc_scoped(ro6, sav->sah->outgoing_if);
+                       rtalloc_scoped((struct route *)ro6, sav->sah->outgoing_if);

                        if (ro6->ro_rt) {
                                RT_LOCK(ro6->ro_rt);
                        }
                        if (ro6->ro_rt) {
                                RT_LOCK(ro6->ro_rt);
                        }


@@ -3848,7 +3848,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar
                        dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway;
                RT_UNLOCK(ro6->ro_rt);
                ROUTE_RELEASE(&state->ro);
                        dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway;
                RT_UNLOCK(ro6->ro_rt);
                ROUTE_RELEASE(&state->ro);

-               route_copyout(&state->ro, ro6, sizeof(state->ro));
+               route_copyout((struct route *)&state->ro, (struct route *)ro6, sizeof(struct route_in6));

                state->dst = (struct sockaddr *)dst6;
                state->tunneled = 6;
                // release sadb_mutex, after updating sah's route cache
                state->dst = (struct sockaddr *)dst6;
                state->tunneled = 6;
                // release sadb_mutex, after updating sah's route cache


@@ -4743,7 +4743,7 @@ ipsec_send_natt_keepalive(
            rt_key(sav->sah->sa_route.ro_rt)->sa_family != AF_INET)
                ROUTE_RELEASE(&sav->sah->sa_route);
 
            rt_key(sav->sah->sa_route.ro_rt)->sa_family != AF_INET)
                ROUTE_RELEASE(&sav->sah->sa_route);
 

-       route_copyout(&ro, &sav->sah->sa_route, sizeof(ro));
+       route_copyout(&ro, (struct route *)&sav->sah->sa_route, sizeof(struct route));

        lck_mtx_unlock(sadb_mutex);
        
        necp_mark_packet_as_keepalive(m, TRUE);
        lck_mtx_unlock(sadb_mutex);
        
        necp_mark_packet_as_keepalive(m, TRUE);


@@ -4752,7 +4752,7 @@ ipsec_send_natt_keepalive(
 
        // grab sadb_mutex, to synchronize the sah's route cache with the local copy
        lck_mtx_lock(sadb_mutex);
 
        // grab sadb_mutex, to synchronize the sah's route cache with the local copy
        lck_mtx_lock(sadb_mutex);

-       route_copyin(&ro, &sav->sah->sa_route, sizeof(ro));
+       route_copyin(&ro, (struct route *)&sav->sah->sa_route, sizeof(struct route));

        lck_mtx_unlock(sadb_mutex);
        if (error == 0) {
                sav->natt_last_activity = natt_now;
        lck_mtx_unlock(sadb_mutex);
        if (error == 0) {
                sav->natt_last_activity = natt_now;

diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h
index 8d0e6dd51e0c19581afb0c723e173a1b4bc0cfc7..e016082475d024cf25ad3711ef7fba150450666b 100644 (file)
--- a/bsd/netinet6/ipsec.h
+++ b/bsd/netinet6/ipsec.h
@@ -301,7 +301,7 @@ struct ipsecstat {
 struct ipsec_output_state {
        int tunneled;
        struct mbuf *m;
 struct ipsec_output_state {
        int tunneled;
        struct mbuf *m;

-       struct route ro;
+       struct route_in6 ro;

        struct sockaddr *dst;
        u_int outgoing_if;
 };
        struct sockaddr *dst;
        u_int outgoing_if;
 };

diff --git a/bsd/netinet6/nd6.c b/bsd/netinet6/nd6.c
index 0a16cdb14475c1eb4b7559488475270fc665e94e..495a3b3044c8bb8824bcab9a0e6e92cc607511ac 100644 (file)
--- a/bsd/netinet6/nd6.c
+++ b/bsd/netinet6/nd6.c
@@ -1310,9 +1310,13 @@ addrloop:
 
                        /*
                         * Only enqueue the Deprecated event when the address just
 
                        /*
                         * Only enqueue the Deprecated event when the address just

-                        * becomes deprecated
+                        * becomes deprecated.
+                        * Keep it limited to the stable address it is common for
+                        * older temporary addresses to get deprecated while we generate
+                        * new ones.

                         */
                         */

-                       if((oldflags & IN6_IFF_DEPRECATED) == 0) {
+                       if((oldflags & IN6_IFF_DEPRECATED) == 0 &&
+                           (ia6->ia6_flags & IN6_IFF_TEMPORARY) == 0) {

                                in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DEPRECATED,
                                    ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr,
                                    0);
                                in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DEPRECATED,
                                    ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr,
                                    0);

diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c
index d8e8a1b395f71e63b47ba22c16361378a52ac24c..a7935a5f20202faf1c7b2e0a7093191fa08af72d 100644 (file)
--- a/bsd/netkey/key.c
+++ b/bsd/netkey/key.c
@@ -9920,7 +9920,7 @@ key_sa_routechange(
        
        lck_mtx_lock(sadb_mutex);
        LIST_FOREACH(sah, &sahtree, chain) {
        
        lck_mtx_lock(sadb_mutex);
        LIST_FOREACH(sah, &sahtree, chain) {

-               ro = &sah->sa_route;
+               ro = (struct route *)&sah->sa_route;

                if (ro->ro_rt && dst->sa_len == ro->ro_dst.sa_len
                        && bcmp(dst, &ro->ro_dst, dst->sa_len) == 0) {
                        ROUTE_RELEASE(ro);
                if (ro->ro_rt && dst->sa_len == ro->ro_dst.sa_len
                        && bcmp(dst, &ro->ro_dst, dst->sa_len) == 0) {
                        ROUTE_RELEASE(ro);

diff --git a/bsd/netkey/keydb.h b/bsd/netkey/keydb.h
index c62dc964c68fe953efdcd97d595ab206a9aeb20b..8e053fbfffa2e07b20360d54172faa4ed8601bc9 100644 (file)
--- a/bsd/netkey/keydb.h
+++ b/bsd/netkey/keydb.h
@@ -67,7 +67,7 @@ struct secashead {
                                        /* SA chain */
                                        /* The first of this list is newer SA */
 
                                        /* SA chain */
                                        /* The first of this list is newer SA */
 

-       struct route sa_route;          /* route cache */
+       struct route_in6 sa_route;              /* route cache */

 };
 
 /* Security Association */
 };
 
 /* Security Association */

diff --git a/bsd/nfs/krpc_subr.c b/bsd/nfs/krpc_subr.c
index 53355a6e47facb702e7e0bf0b6d5b1be1dcd8b5d..20fa08bd8ff4e8138979c96489b8c73961705065 100644 (file)
--- a/bsd/nfs/krpc_subr.c
+++ b/bsd/nfs/krpc_subr.c
@@ -189,9 +189,12 @@ krpc_portmap(
                return error;
 
        rdata = mbuf_data(m);
                return error;
 
        rdata = mbuf_data(m);

-       *portp = rdata->port;

 
 

-       if (!rdata->port)
+       if (mbuf_len(m) >= sizeof(*rdata)) {
+               *portp = rdata->port;
+       }
+
+       if (mbuf_len(m) < sizeof(*rdata) || !rdata->port)

                error = EPROGUNAVAIL;
 
        mbuf_freem(m);
                error = EPROGUNAVAIL;
 
        mbuf_freem(m);

diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h
index 9cacf96c5f6abf31db832a3c13bcca37b48afb72..2277221c4e870a6464a1673bb3575fe7f47f74dc 100644 (file)
--- a/bsd/nfs/nfs.h
+++ b/bsd/nfs/nfs.h
@@ -1127,6 +1127,9 @@ struct nfsrv_descript {
 
 extern TAILQ_HEAD(nfsd_head, nfsd) nfsd_head, nfsd_queue;
 
 
 extern TAILQ_HEAD(nfsd_head, nfsd) nfsd_head, nfsd_queue;
 

+typedef int (*nfsrv_proc_t)(struct nfsrv_descript *, struct nfsrv_sock *,
+                         vfs_context_t, mbuf_t *);
+

 /* mutex for nfs server */
 extern lck_mtx_t *nfsd_mutex;
 extern int nfsd_thread_count, nfsd_thread_max;
 /* mutex for nfs server */
 extern lck_mtx_t *nfsd_mutex;
 extern int nfsd_thread_count, nfsd_thread_max;

diff --git a/bsd/nfs/nfs_serv.c b/bsd/nfs/nfs_serv.c
index 7a24596c83aecae09ef071b8cfeba570b98d3376..027d7a5d8406f9fc751b9102d48d3c55364d7cbb 100644 (file)
--- a/bsd/nfs/nfs_serv.c
+++ b/bsd/nfs/nfs_serv.c
@@ -4878,10 +4878,7 @@ nfsmout:
        return (error);
 }
 
        return (error);
 }
 

-int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
-                                   struct nfsrv_sock *slp,
-                                   vfs_context_t ctx,
-                                   mbuf_t *mrepp) = {
+const nfsrv_proc_t nfsrv_procs[NFS_NPROCS] = {

        nfsrv_null,
        nfsrv_getattr,
        nfsrv_setattr,
        nfsrv_null,
        nfsrv_getattr,
        nfsrv_setattr,

diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c
index 2a20ee2eae1f1403a254206f9ba7594e3868ef2a..2e43de4b15d05b2fafd75b7385326199b213e644 100644 (file)
--- a/bsd/nfs/nfs_syscalls.c
+++ b/bsd/nfs/nfs_syscalls.c
@@ -123,10 +123,8 @@ kern_return_t      thread_terminate(thread_t); /* XXX */
 
 #if NFSSERVER
 
 
 #if NFSSERVER
 

-extern int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
-                                           struct nfsrv_sock *slp,
-                                           vfs_context_t ctx,
-                                           mbuf_t *mrepp);
+extern const nfsrv_proc_t nfsrv_procs[NFS_NPROCS];
+

 extern int nfsrv_wg_delay;
 extern int nfsrv_wg_delay_v3;
 
 extern int nfsrv_wg_delay;
 extern int nfsrv_wg_delay_v3;
 

diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c
index 3c5f2831a9e6d6e18483d55dfe98319d8b541092..67b9befc0b7ddb596ea8c121610cbb753b72443a 100644 (file)
--- a/bsd/nfs/nfs_vfsops.c
+++ b/bsd/nfs/nfs_vfsops.c
@@ -204,7 +204,7 @@ int nfs_vfs_fhtovp(mount_t, int, unsigned char *, vnode_t *, vfs_context_t);
 int    nfs_vfs_init(struct vfsconf *);
 int    nfs_vfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
 
 int    nfs_vfs_init(struct vfsconf *);
 int    nfs_vfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
 

-struct vfsops nfs_vfsops = {
+const struct vfsops nfs_vfsops = {

        .vfs_mount       = nfs_vfs_mount,
        .vfs_start       = nfs_vfs_start,
        .vfs_unmount     = nfs_vfs_unmount,
        .vfs_mount       = nfs_vfs_mount,
        .vfs_start       = nfs_vfs_start,
        .vfs_unmount     = nfs_vfs_unmount,


@@ -237,7 +237,7 @@ int nfs3_getquota(struct nfsmount *, vfs_context_t, uid_t, int, struct dqblk *);
 int nfs4_getquota(struct nfsmount *, vfs_context_t, uid_t, int, struct dqblk *);
 #endif
 
 int nfs4_getquota(struct nfsmount *, vfs_context_t, uid_t, int, struct dqblk *);
 #endif
 

-struct nfs_funcs nfs3_funcs = {
+const struct nfs_funcs nfs3_funcs = {

        nfs3_mount,
        nfs3_update_statfs,
        nfs3_getquota,
        nfs3_mount,
        nfs3_update_statfs,
        nfs3_getquota,


@@ -258,7 +258,7 @@ struct nfs_funcs nfs3_funcs = {
        nfs3_unlock_rpc,
        nfs3_getlock_rpc
        };
        nfs3_unlock_rpc,
        nfs3_getlock_rpc
        };

-struct nfs_funcs nfs4_funcs = {
+const struct nfs_funcs nfs4_funcs = {

        nfs4_mount,
        nfs4_update_statfs,
        nfs4_getquota,
        nfs4_mount,
        nfs4_update_statfs,
        nfs4_getquota,

diff --git a/bsd/nfs/nfsm_subs.h b/bsd/nfs/nfsm_subs.h
index 69d0f78654c2e414b9b1dc119121e87567c49820..50f72c5fc6b99005679327f5f8ec3de7342f632f 100644 (file)
--- a/bsd/nfs/nfsm_subs.h
+++ b/bsd/nfs/nfsm_subs.h
@@ -144,6 +144,8 @@ int nfsm_chain_trim_data(struct nfsm_chain *, int, int *);
                VATTR_WANTED((VAP), va_change_time); \
                if ((VERS) == NFS_VER2) \
                        VATTR_WANTED((VAP), va_iosize); \
                VATTR_WANTED((VAP), va_change_time); \
                if ((VERS) == NFS_VER2) \
                        VATTR_WANTED((VAP), va_iosize); \

+               if ((VERS) == NFS_VER3) \
+                       VATTR_WANTED((VAP), va_filerev); \

        } while (0)
 
 /* Initialize a vnode_attr to retrieve pre-operation attributes for the NFS server. */
        } while (0)
 
 /* Initialize a vnode_attr to retrieve pre-operation attributes for the NFS server. */

diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h
index 78aa778ad8fd016bc93df8533ea960eea452ccdc..90b75a54879cab6a4215403611692d45b0ab79f4 100644 (file)
--- a/bsd/nfs/nfsmount.h
+++ b/bsd/nfs/nfsmount.h
@@ -267,7 +267,7 @@ struct nfsmount {
        uint32_t nm_minor_vers;         /* minor version of above */
        uint32_t nm_min_vers;           /* minimum packed version to try */
        uint32_t nm_max_vers;           /* maximum packed version to try */
        uint32_t nm_minor_vers;         /* minor version of above */
        uint32_t nm_min_vers;           /* minimum packed version to try */
        uint32_t nm_max_vers;           /* maximum packed version to try */

-       struct nfs_funcs *nm_funcs;     /* version-specific functions */
+       const struct nfs_funcs *nm_funcs;/* version-specific functions */

        kauth_cred_t nm_mcred;          /* credential used for the mount */
        mount_t nm_mountp;              /* VFS structure for this filesystem */
        nfsnode_t nm_dnp;               /* root directory nfsnode pointer */
        kauth_cred_t nm_mcred;          /* credential used for the mount */
        mount_t nm_mountp;              /* VFS structure for this filesystem */
        nfsnode_t nm_dnp;               /* root directory nfsnode pointer */

diff --git a/bsd/sys/event.h b/bsd/sys/event.h
index f10e157771aa9460b14c49637b43737b78aafed7..04385bc6c5d204199bced0ded7103ad65384b07f 100644 (file)
--- a/bsd/sys/event.h
+++ b/bsd/sys/event.h
@@ -249,8 +249,9 @@ typedef uint64_t kqueue_id_t;
 #define EVFILTID_FSEVENT           (EVFILT_SYSCOUNT + 13)
 #define EVFILTID_VN                (EVFILT_SYSCOUNT + 14)
 #define EVFILTID_TTY               (EVFILT_SYSCOUNT + 16)
 #define EVFILTID_FSEVENT           (EVFILT_SYSCOUNT + 13)
 #define EVFILTID_VN                (EVFILT_SYSCOUNT + 14)
 #define EVFILTID_TTY               (EVFILT_SYSCOUNT + 16)

+#define EVFILTID_PTMX              (EVFILT_SYSCOUNT + 17)

 
 

-#define EVFILTID_MAX               (EVFILT_SYSCOUNT + 17)
+#define EVFILTID_MAX               (EVFILT_SYSCOUNT + 18)

 
 #endif /* defined(XNU_KERNEL_PRIVATE) */
 
 
 #endif /* defined(XNU_KERNEL_PRIVATE) */
 

diff --git a/bsd/sys/kern_control.h b/bsd/sys/kern_control.h
index 8e1b514bdccdb68dca1d239e87c45a5a36d2c9f1..51324ea65260d903d480a97fa555091ef4a02a4a 100644 (file)
--- a/bsd/sys/kern_control.h
+++ b/bsd/sys/kern_control.h
@@ -417,6 +417,31 @@ typedef void (*ctl_rcvd_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitin
  */
 typedef errno_t (*ctl_send_list_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
                                        mbuf_t m, int flags);
  */
 typedef errno_t (*ctl_send_list_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo,
                                        mbuf_t m, int flags);

+
+/*!
+       @typedef ctl_bind_func
+       @discussion The ctl_bind_func is an optional function that allows the client
+               to set up their unitinfo prior to connecting.
+       @param kctlref The control ref for the kernel control the client is
+               binding to.
+       @param sac The address used to connect to this control. The field sc_unit
+               contains the unit number of the kernel control instance the client is
+               binding to. If CTL_FLAG_REG_ID_UNIT was set when the kernel control
+               was registered, sc_unit is the ctl_unit of the kern_ctl_reg structure.
+               If CTL_FLAG_REG_ID_UNIT was not set when the kernel control was
+               registered, sc_unit is the dynamically allocated unit number of
+               the new kernel control instance that is used for this connection.
+       @param unitinfo A placeholder for a pointer to the optional user-defined
+               private data associated with this kernel control instance.  This
+               opaque info will be provided to the user when the rest of the
+               callback routines are executed.  For example, it can be used
+               to pass a pointer to an instance-specific data structure in
+               order for the user to keep track of the states related to this
+               kernel control instance.
+ */
+typedef errno_t (*ctl_bind_func)(kern_ctl_ref kctlref,
+                                                                struct sockaddr_ctl *sac,
+                                                                void **unitinfo);

 #endif /* KERNEL_PRIVATE */
 
 /*!
 #endif /* KERNEL_PRIVATE */
 
 /*!


@@ -474,6 +499,7 @@ struct kern_ctl_reg
 #ifdef KERNEL_PRIVATE
     ctl_rcvd_func              ctl_rcvd;       /* Only valid if CTL_FLAG_REG_EXTENDED is set */
     ctl_send_list_func         ctl_send_list;  /* Only valid if CTL_FLAG_REG_EXTENDED is set */
 #ifdef KERNEL_PRIVATE
     ctl_rcvd_func              ctl_rcvd;       /* Only valid if CTL_FLAG_REG_EXTENDED is set */
     ctl_send_list_func         ctl_send_list;  /* Only valid if CTL_FLAG_REG_EXTENDED is set */

+       ctl_bind_func           ctl_bind;

 #endif /* KERNEL_PRIVATE */
 };
 
 #endif /* KERNEL_PRIVATE */
 };
 

diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h
index 828f9f44d83b72e85a74f747343dafcf3f7e519e..3cac39def5643196c34b69fa1254aed639a6017e 100644 (file)
--- a/bsd/sys/mbuf.h
+++ b/bsd/sys/mbuf.h
@@ -504,6 +504,7 @@ struct pkthdr {
 #define        PKTF_START_SEQ          0x8000000 /* valid start sequence */
 #define        PKTF_LAST_PKT           0x10000000 /* last packet in the flow */
 #define        PKTF_MPTCP_REINJ        0x20000000 /* Packet has been reinjected for MPTCP */
 #define        PKTF_START_SEQ          0x8000000 /* valid start sequence */
 #define        PKTF_LAST_PKT           0x10000000 /* last packet in the flow */
 #define        PKTF_MPTCP_REINJ        0x20000000 /* Packet has been reinjected for MPTCP */

+#define        PKTF_MPTCP_DFIN         0x40000000 /* Packet is a data-fin */

 
 /* flags related to flow control/advisory and identification */
 #define        PKTF_FLOW_MASK  \
 
 /* flags related to flow control/advisory and identification */
 #define        PKTF_FLOW_MASK  \

diff --git a/bsd/sys/reason.h b/bsd/sys/reason.h
index 2d81e3a41a4a5942824d16bf1c0ef379852cd0da..a7dc43f69e5bbeb24e11b401fb75ecb623cfc4ba 100644 (file)
--- a/bsd/sys/reason.h
+++ b/bsd/sys/reason.h
@@ -42,7 +42,7 @@ __BEGIN_DECLS
 
 typedef struct os_reason {
        decl_lck_mtx_data(,             osr_lock)
 
 typedef struct os_reason {
        decl_lck_mtx_data(,             osr_lock)

-       int                             osr_refcount;
+       unsigned int                    osr_refcount;

        uint32_t                        osr_namespace;
        uint64_t                        osr_code;
        uint64_t                        osr_flags;
        uint32_t                        osr_namespace;
        uint64_t                        osr_code;
        uint64_t                        osr_flags;


@@ -104,11 +104,12 @@ void os_reason_free(os_reason_t cur_reason);
 #define OS_REASON_FOUNDATION    19
 #define OS_REASON_WATCHDOG      20
 #define OS_REASON_METAL         21
 #define OS_REASON_FOUNDATION    19
 #define OS_REASON_WATCHDOG      20
 #define OS_REASON_METAL         21

+#define OS_REASON_WATCHKIT      22

 
 /*
  * Update whenever new OS_REASON namespaces are added.
  */
 
 /*
  * Update whenever new OS_REASON namespaces are added.
  */

-#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_METAL
+#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_WATCHKIT

 
 #define OS_REASON_BUFFER_MAX_SIZE 5120
 
 
 #define OS_REASON_BUFFER_MAX_SIZE 5120
 

diff --git a/bsd/sys/reboot.h b/bsd/sys/reboot.h
index 98db0e3d7873211a41e334b144f00b28f0d0c52b..717ce89dc6f30cdefdf761e1faa1de34dfc34082 100644 (file)
--- a/bsd/sys/reboot.h
+++ b/bsd/sys/reboot.h
@@ -92,6 +92,11 @@
 __BEGIN_DECLS
 /* userspace reboot control */
 int usrctl(uint32_t flags);
 __BEGIN_DECLS
 /* userspace reboot control */
 int usrctl(uint32_t flags);

+/* The normal reboot syscall. */
+int reboot(int howto);
+/* Used with RB_PANIC to panic the kernel from userspace with a message.
+ * Requires an entitlement on Release. */
+int reboot_np(int howto, const char *message);

 __END_DECLS
 #endif
 
 __END_DECLS
 #endif
 

diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h
index a8faa3c64da1f1f2dd2be8809a96b1273f7b1341..17e1e2c35a0aaf3b12475cf50d19dd7e586271d5 100644 (file)
--- a/bsd/sys/socketvar.h
+++ b/bsd/sys/socketvar.h
@@ -980,7 +980,8 @@ extern void mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len);
 extern void mptcp_preproc_sbdrop(struct socket *, struct mbuf *, unsigned int);
 extern void mptcp_postproc_sbdrop(struct mbuf *, u_int64_t, u_int32_t,
     u_int32_t);
 extern void mptcp_preproc_sbdrop(struct socket *, struct mbuf *, unsigned int);
 extern void mptcp_postproc_sbdrop(struct mbuf *, u_int64_t, u_int32_t,
     u_int32_t);

-extern void mptcp_adj_rmap(struct socket *, struct mbuf *, int);
+extern int mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off,
+                         uint64_t dsn, uint32_t rseq, uint16_t dlen);

 
 extern void netpolicy_post_msg(uint32_t, struct netpolicy_event_data *,
     uint32_t);
 
 extern void netpolicy_post_msg(uint32_t, struct netpolicy_event_data *,
     uint32_t);

diff --git a/bsd/sys/spawn_internal.h b/bsd/sys/spawn_internal.h
index 9985b3cfc666a8b2178ffbc2099604e380018fb0..29ea49d3b5d2ead53e2be2e1a0190f90bb1bfd41 100644 (file)
--- a/bsd/sys/spawn_internal.h
+++ b/bsd/sys/spawn_internal.h
@@ -48,6 +48,25 @@
 #include <mach/port.h>
 #include <mach/exception_types.h>
 #include <mach/coalition.h>    /* COALITION_NUM_TYPES */
 #include <mach/port.h>
 #include <mach/exception_types.h>
 #include <mach/coalition.h>    /* COALITION_NUM_TYPES */

+#include <os/overflow.h>
+
+/*
+ * Safely compute the size in bytes of a structure, '_type', whose last
+ * element, '_member', is a zero-sized array meant to hold 'x' bytes.
+ *
+ * If the size calculation overflows a size_t value, this macro returns 0.
+ */
+#define PS_ACTION_SIZE(x,_type,_member_type) ({ \
+       size_t _ps_count = (size_t)x; \
+       size_t _ps_size = 0; \
+       /* (count * sizeof(_member_type)) + sizeof(_type) */ \
+       if (os_mul_and_add_overflow(_ps_count, \
+                                   sizeof(_member_type), \
+                                   sizeof(_type), \
+                                   &_ps_size)) { \
+               _ps_size = 0; \
+       } \
+       _ps_size; })

 
 /*
  * Allowable posix_spawn() port action types
 
 /*
  * Allowable posix_spawn() port action types


@@ -86,7 +105,7 @@ typedef struct _posix_spawn_port_actions {
  * Returns size in bytes of a _posix_spawn_port_actions holding x elements.
  */
 #define        PS_PORT_ACTIONS_SIZE(x) \
  * Returns size in bytes of a _posix_spawn_port_actions holding x elements.
  */
 #define        PS_PORT_ACTIONS_SIZE(x) \

-       __offsetof(struct _posix_spawn_port_actions, pspa_actions[(x)])
+       PS_ACTION_SIZE(x, struct _posix_spawn_port_actions, _ps_port_action_t)

 
 #define NBINPREFS      4
 
 
 #define NBINPREFS      4
 


@@ -115,7 +134,7 @@ typedef struct _posix_spawn_mac_policy_extensions {
  * Returns size in bytes of a _posix_spawn_mac_policy_extensions holding x elements.
  */
 #define PS_MAC_EXTENSIONS_SIZE(x)     \
  * Returns size in bytes of a _posix_spawn_mac_policy_extensions holding x elements.
  */
 #define PS_MAC_EXTENSIONS_SIZE(x)     \

-        __offsetof(struct _posix_spawn_mac_policy_extensions, psmx_extensions[(x)])
+       PS_ACTION_SIZE(x, struct _posix_spawn_mac_policy_extensions, _ps_mac_policy_extension_t)

 
 #define PS_MAC_EXTENSIONS_INIT_COUNT   2
 
 
 #define PS_MAC_EXTENSIONS_INIT_COUNT   2
 


@@ -334,7 +353,7 @@ typedef struct _posix_spawn_file_actions {
  * capable of containing.
  */
 #define        PSF_ACTIONS_SIZE(x)     \
  * capable of containing.
  */
 #define        PSF_ACTIONS_SIZE(x)     \

-       __offsetof(struct _posix_spawn_file_actions, psfa_act_acts[(x)])
+       PS_ACTION_SIZE(x, struct _posix_spawn_file_actions, _psfa_action_t)

 
 /*
  * Initial count of actions in a struct _posix_spawn_file_actions after it is
 
 /*
  * Initial count of actions in a struct _posix_spawn_file_actions after it is

diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c
index 4aab3ac014479b51710142c9a6b0f5ffa24b140f..f5710e862113d8cfaedbbde0f644952f869aeb1d 100644 (file)
--- a/bsd/vfs/vfs_attrlist.c
+++ b/bsd/vfs/vfs_attrlist.c
@@ -340,7 +340,7 @@ static struct getvolattrlist_attrtab getvolattrlist_common_tab[] = {
 static struct getvolattrlist_attrtab getvolattrlist_vol_tab[] = {
        {ATTR_VOL_FSTYPE,               0,                                              sizeof(uint32_t)},
        {ATTR_VOL_SIGNATURE,            VFSATTR_BIT(f_signature),                       sizeof(uint32_t)},
 static struct getvolattrlist_attrtab getvolattrlist_vol_tab[] = {
        {ATTR_VOL_FSTYPE,               0,                                              sizeof(uint32_t)},
        {ATTR_VOL_SIGNATURE,            VFSATTR_BIT(f_signature),                       sizeof(uint32_t)},

-       {ATTR_VOL_SIZE,                 VFSATTR_BIT(f_blocks),                          sizeof(off_t)},
+       {ATTR_VOL_SIZE,                         VFSATTR_BIT(f_blocks)  |  VFSATTR_BIT(f_bsize),                         sizeof(off_t)},

        {ATTR_VOL_SPACEFREE,            VFSATTR_BIT(f_bfree) | VFSATTR_BIT(f_bsize),    sizeof(off_t)},
        {ATTR_VOL_SPACEAVAIL,           VFSATTR_BIT(f_bavail) | VFSATTR_BIT(f_bsize),   sizeof(off_t)},
        {ATTR_VOL_MINALLOCATION,        VFSATTR_BIT(f_bsize),                           sizeof(off_t)},
        {ATTR_VOL_SPACEFREE,            VFSATTR_BIT(f_bfree) | VFSATTR_BIT(f_bsize),    sizeof(off_t)},
        {ATTR_VOL_SPACEAVAIL,           VFSATTR_BIT(f_bavail) | VFSATTR_BIT(f_bsize),   sizeof(off_t)},
        {ATTR_VOL_MINALLOCATION,        VFSATTR_BIT(f_bsize),                           sizeof(off_t)},

diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c
index 7883146168b011fd714b10ca639d2ee923ef9665..2571c94d5b2b58b36c3be7e5982e85d10af6ab77 100644 (file)
--- a/bsd/vfs/vfs_syscalls.c
+++ b/bsd/vfs/vfs_syscalls.c
@@ -11993,9 +11993,11 @@ fs_snapshot(__unused proc_t p, struct fs_snapshot_args *uap,
     case SNAPSHOT_OP_REVERT:
         error = snapshot_revert(uap->dirfd, uap->name1, uap->flags, ctx);
         break;
     case SNAPSHOT_OP_REVERT:
         error = snapshot_revert(uap->dirfd, uap->name1, uap->flags, ctx);
         break;

+#if !TARGET_OS_OSX

        case SNAPSHOT_OP_ROOT:
                error = snapshot_root(uap->dirfd, uap->name1, uap->flags, ctx);
                break;
        case SNAPSHOT_OP_ROOT:
                error = snapshot_root(uap->dirfd, uap->name1, uap->flags, ctx);
                break;

+#endif /* !TARGET_OS_OSX */

        default:
                error = ENOSYS;
        }
        default:
                error = ENOSYS;
        }

diff --git a/config/MasterVersion b/config/MasterVersion
index 461cd787fc481ca9c181fa097df4bc2dc7306fea..db6ac06457fe8e5c645a9d2a0fccb595036553be 100644 (file)
--- a/config/MasterVersion
+++ b/config/MasterVersion
@@ -1,4 +1,4 @@
-17.2.0
+17.3.0

 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.
 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.

diff --git a/config/Private.arm64.exports b/config/Private.arm64.exports
index ab90073172e45e51e466743effcd78acfb529ed6..3ac76793908b0932c0fc4849ee7f447511aa72a8 100644 (file)
--- a/config/Private.arm64.exports
+++ b/config/Private.arm64.exports
@@ -4,6 +4,10 @@ _PE_consistent_debug_register
 _PE_get_default
 _PE_reboot_on_panic
 _PE_mark_hwaccess
 _PE_get_default
 _PE_reboot_on_panic
 _PE_mark_hwaccess

+_PE_smc_stashed_x86_system_state
+_PE_smc_stashed_x86_power_state
+_PE_smc_stashed_x86_efi_boot_state
+_PE_pcie_stashed_link_state

 __ZN17IONVRAMController*
 __ZTV17IONVRAMController
 _cpu_broadcast_xcall
 __ZN17IONVRAMController*
 __ZTV17IONVRAMController
 _cpu_broadcast_xcall

diff --git a/config/Unsupported.arm.exports b/config/Unsupported.arm.exports
index 6f33928e1c0f682f7d3c28cc56127b83dd467665..ee0e8db30e76040002612ab036262c8b4f6bfdb1 100644 (file)
--- a/config/Unsupported.arm.exports
+++ b/config/Unsupported.arm.exports
@@ -22,3 +22,5 @@ _ml_get_cpu_number
 _ml_get_max_cpu_number
 _ml_dbgwrap_halt_cpu_with_state
 _vm_map:_vm_map_external
 _ml_get_max_cpu_number
 _ml_dbgwrap_halt_cpu_with_state
 _vm_map:_vm_map_external

+__ZN5IORTC15getUTCTimeOfDayEPjS0_
+__ZN5IORTC15setUTCTimeOfDayEjj

diff --git a/config/Unsupported.arm64.exports b/config/Unsupported.arm64.exports
index 38e1a8c0efa7e5fdbaeaa4f16e0715921fb1c90a..883f244804a3607c98fc9f330b08ab8f05056059 100644 (file)
--- a/config/Unsupported.arm64.exports
+++ b/config/Unsupported.arm64.exports
@@ -37,4 +37,5 @@ _ml_get_max_cpu_number
 _ml_lockdown_handler_register
 _ml_dbgwrap_halt_cpu_with_state
 _vm_map:_vm_map_external
 _ml_lockdown_handler_register
 _ml_dbgwrap_halt_cpu_with_state
 _vm_map:_vm_map_external

-
+__ZN5IORTC15getUTCTimeOfDayEPmPj
+__ZN5IORTC15setUTCTimeOfDayEmj

diff --git a/config/Unsupported.exports b/config/Unsupported.exports
index 9840aa96c3b2f791f1076f44fc136db7df2109a3..7938c5da99033121e2fbc3ee65bf1a5608a3ae9a 100644 (file)
--- a/config/Unsupported.exports
+++ b/config/Unsupported.exports
@@ -188,4 +188,19 @@ _vnop_kqfilt_add_desc
 _vnop_kqfilt_remove_desc
 _vnop_makenamedstream_desc
 _vnop_removenamedstream_desc
 _vnop_kqfilt_remove_desc
 _vnop_makenamedstream_desc
 _vnop_removenamedstream_desc

-
+__ZN5IORTC10gMetaClassE
+__ZN5IORTC10superClassE
+__ZN5IORTC23getMonotonicClockOffsetEPx
+__ZN5IORTC23setMonotonicClockOffsetEx
+__ZN5IORTC29getMonotonicClockAndTimestampEPyS0_
+__ZN5IORTC9MetaClassC1Ev
+__ZN5IORTC9MetaClassC2Ev
+__ZN5IORTC9metaClassE
+__ZN5IORTCC2EPK11OSMetaClass
+__ZN5IORTCD0Ev
+__ZN5IORTCD1Ev
+__ZN5IORTCD2Ev
+__ZNK5IORTC12getMetaClassEv
+__ZNK5IORTC9MetaClass5allocEv
+__ZTV5IORTC
+__ZTVN5IORTC9MetaClassE

diff --git a/config/Unsupported.x86_64.exports b/config/Unsupported.x86_64.exports
index 57c5bb71e901596f0cdc0b29e5ad597301c182a9..c2af9c45466b4ce5cc5979730c0a39a9a13183ee 100644 (file)
--- a/config/Unsupported.x86_64.exports
+++ b/config/Unsupported.x86_64.exports
@@ -44,4 +44,13 @@ _tmrCvt
 _tsc_get_info
 _PE_state
 _vm_map
 _tsc_get_info
 _PE_state
 _vm_map

-
+__ZN5IORTC15_RESERVEDIORTC0Ev
+__ZN5IORTC15_RESERVEDIORTC1Ev
+__ZN5IORTC15_RESERVEDIORTC2Ev
+__ZN5IORTC15_RESERVEDIORTC3Ev
+__ZN5IORTC15_RESERVEDIORTC4Ev
+__ZN5IORTC15_RESERVEDIORTC5Ev
+__ZN5IORTC15_RESERVEDIORTC6Ev
+__ZN5IORTC15_RESERVEDIORTC7Ev
+__ZN5IORTC15getUTCTimeOfDayEPmPj
+__ZN5IORTC15setUTCTimeOfDayEmj

diff --git a/iokit/IOKit/IOCPU.h b/iokit/IOKit/IOCPU.h
index 25d2398de89936dac98b5432db802d1c5298ba30..1ee373c3e4c838f14306281d3f67383d6ec74b22 100644 (file)
--- a/iokit/IOKit/IOCPU.h
+++ b/iokit/IOKit/IOCPU.h
@@ -110,7 +110,7 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void);
 extern "C" kern_return_t IOCPURunPlatformActiveActions(void);
 extern "C" kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message);
 extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message);
 extern "C" kern_return_t IOCPURunPlatformActiveActions(void);
 extern "C" kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message);
 extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message);

-extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, size_t len);
+extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, uint32_t offset, uint32_t len);

 
 class IOCPUInterruptController : public IOInterruptController
 {
 
 class IOCPUInterruptController : public IOInterruptController
 {

diff --git a/iokit/IOKit/IOLocks.h b/iokit/IOKit/IOLocks.h
index 7b42679f113cea1ab03b50883b82cbc394d98648..75d327b7f5f4ec3902b037f2134ca6efce811e68 100644 (file)
--- a/iokit/IOKit/IOLocks.h
+++ b/iokit/IOKit/IOLocks.h
@@ -140,6 +140,28 @@ int        IOLockSleepDeadline( IOLock * lock, void *event,
 
 void   IOLockWakeup(IOLock * lock, void *event, bool oneThread) __DARWIN14_ALIAS(IOLockWakeup);
 
 
 void   IOLockWakeup(IOLock * lock, void *event, bool oneThread) __DARWIN14_ALIAS(IOLockWakeup);
 

+#ifdef XNU_KERNEL_PRIVATE
+/*! @enum     IOLockAssertState
+ *  @abstract Used with IOLockAssert to assert the state of a lock.
+ */
+typedef enum {
+    kIOLockAssertOwned    = LCK_ASSERT_OWNED,
+    kIOLockAssertNotOwned = LCK_ASSERT_NOTOWNED
+} IOLockAssertState;
+
+#ifdef IOLOCKS_INLINE
+#define IOLockAssert(l, type) LCK_MTX_ASSERT(l, type)
+#else
+/*! @function   IOLockAssert
+ *  @abstract   Assert that lock is either held or not held by current thread.
+ *  @discussion Call with either kIOLockAssertOwned or kIOLockAssertNotOwned.
+ *  Panics the kernel if the lock is not owned if called with kIOLockAssertOwned,
+ *  and vice-versa.
+ */
+void   IOLockAssert(IOLock * lock, IOLockAssertState type);
+#endif /* !IOLOCKS_INLINE */
+#endif /* !XNU_KERNEL_PRIVATE */
+

 #ifdef __APPLE_API_OBSOLETE
 
 /* The following API is deprecated */
 #ifdef __APPLE_API_OBSOLETE
 
 /* The following API is deprecated */


@@ -286,6 +308,30 @@ void       IORWLockWrite( IORWLock * lock);
 void   IORWLockUnlock( IORWLock * lock);
 #endif /* !IOLOCKS_INLINE */
 
 void   IORWLockUnlock( IORWLock * lock);
 #endif /* !IOLOCKS_INLINE */
 

+#ifdef XNU_KERNEL_PRIVATE
+/*! @enum     IORWLockAssertState
+ *  @abstract Used with IORWLockAssert to assert the state of a lock.
+ */
+typedef enum {
+    kIORWLockAssertRead    = LCK_RW_ASSERT_SHARED,
+    kIORWLockAssertWrite   = LCK_RW_ASSERT_EXCLUSIVE,
+    kIORWLockAssertHeld    = LCK_RW_ASSERT_HELD,
+    kIORWLockAssertNotHeld = LCK_RW_ASSERT_NOTHELD
+} IORWLockAssertState;
+
+#ifdef IOLOCKS_INLINE
+#define IORWLockAssert(l, type) LCK_RW_ASSERT(l, type)
+#else
+/*! @function   IORWLockAssert
+ *  @abstract   Assert that a reader-writer lock is either held or not held
+ *  by the current thread.
+ *  @discussion Call with a value defined by the IORWLockAssertState type.
+ *  If the specified lock is not in the state specified by the type argument,
+ *  then the kernel will panic.
+ */
+void   IORWLockAssert(IORWLock * lock, IORWLockAssertState type);
+#endif /* !IOLOCKS_INLINE */
+#endif /* !XNU_KERNEL_PRIVATE */

 
 #ifdef __APPLE_API_OBSOLETE
 
 
 #ifdef __APPLE_API_OBSOLETE
 


@@ -371,6 +417,28 @@ boolean_t IOSimpleLockTryLock( IOSimpleLock * lock );
 void IOSimpleLockUnlock( IOSimpleLock * lock );
 #endif /* !IOLOCKS_INLINE */
 
 void IOSimpleLockUnlock( IOSimpleLock * lock );
 #endif /* !IOLOCKS_INLINE */
 

+#ifdef XNU_KERNEL_PRIVATE
+/*! @enum     IOSimpleLockAssertState
+ *  @abstract Used with IOSimpleLockAssert to assert the state of a lock.
+ */
+typedef enum {
+    kIOSimpleLockAssertOwned    = LCK_ASSERT_OWNED,
+    kIOSimpleLockAssertNotOwned = LCK_ASSERT_NOTOWNED
+} IOSimpleLockAssertState;
+
+#ifdef IOLOCKS_INLINE
+#define IOSimpleLockAssert(l, type) LCK_SPIN_ASSERT(l, type)
+#else
+/*! @function   IOSimpleLockAssert
+ *  @abstract   Assert that spinlock is either held or not held by current thread.
+ *  @discussion Call with either kIOSimpleLockAssertOwned or kIOSimpleLockAssertNotOwned.
+ *  Panics the kernel if the lock is not owned if called with
+ *  kIOSimpleLockAssertOwned and vice-versa.
+ */
+void   IOSimpleLockAssert(IOSimpleLock *lock, IOSimpleLockAssertState type);
+#endif /* !IOLOCKS_INLINE */
+#endif /* !XNU_KERNEL_PRIVATE */
+

 #if __LP64__
 typedef boolean_t IOInterruptState;
 #else
 #if __LP64__
 typedef boolean_t IOInterruptState;
 #else

diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h
index f66f8da1f70f15d978685a2fdf2bee86738ce412..63029bb1d85d5feebed58030559af285ccb10e9d 100644 (file)
--- a/iokit/IOKit/IOPlatformExpert.h
+++ b/iokit/IOKit/IOPlatformExpert.h
@@ -47,6 +47,13 @@ extern "C" {
 
 #include <libkern/OSTypes.h>
 
 
 #include <libkern/OSTypes.h>
 

+typedef enum {
+    kCoprocessorVersionNone    =   0x00000000,
+    kCoprocessorVersion1       =   0x00010000,
+    kCoprocessorVersion2       =   0x00020000,
+} coprocessor_type_t;
+
+

 extern boolean_t PEGetMachineName( char * name, int maxLength );
 extern boolean_t PEGetModelName( char * name, int maxLength );
 extern int PEGetPlatformEpoch( void );
 extern boolean_t PEGetMachineName( char * name, int maxLength );
 extern boolean_t PEGetModelName( char * name, int maxLength );
 extern int PEGetPlatformEpoch( void );


@@ -68,7 +75,13 @@ extern int PEHaltRestart(unsigned int type);
 
 // Save the Panic Info.  Returns the number of bytes saved.
 extern UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length);
 
 // Save the Panic Info.  Returns the number of bytes saved.
 extern UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length);

-extern void PESavePanicInfoAction(void *buffer, size_t length);
+extern void PESavePanicInfoAction(void *buffer, UInt32 offset, UInt32 length);
+
+/* 
+ * SMC requires that all data is flushed in multiples of 16 bytes at 16 byte
+ * boundaries.
+ */
+#define PANIC_FLUSH_BOUNDARY 16

 
 extern long PEGetGMTTimeOfDay( void );
 extern void PESetGMTTimeOfDay( long secs );
 
 extern long PEGetGMTTimeOfDay( void );
 extern void PESetGMTTimeOfDay( long secs );


@@ -84,12 +97,16 @@ extern boolean_t PEReadNVRAMProperty(const char *symbol, void *value, unsigned i
 
 extern boolean_t PERemoveNVRAMProperty(const char *symbol);
 
 
 extern boolean_t PERemoveNVRAMProperty(const char *symbol);
 

+extern coprocessor_type_t PEGetCoprocessorVersion( void );
+

 #ifdef __cplusplus
 } /* extern "C" */
 
 #define kIOPlatformMapperPresentKey "IOPlatformMapperPresent"
 
 
 #ifdef __cplusplus
 } /* extern "C" */
 
 #define kIOPlatformMapperPresentKey "IOPlatformMapperPresent"
 
 

+
+

 extern OSSymbol *               gPlatformInterruptControllerName;
 
 extern const OSSymbol *                gIOPlatformSleepActionKey;
 extern OSSymbol *               gPlatformInterruptControllerName;
 
 extern const OSSymbol *                gIOPlatformSleepActionKey;

diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h
index 0480207842b9f1ffdf5f60b1247701f32b1854fe..504d6f2219315de9d07fa34a97be3e1b4c6c4173 100644 (file)
--- a/iokit/IOKit/IOReturn.h
+++ b/iokit/IOKit/IOReturn.h
@@ -74,12 +74,12 @@ typedef     kern_return_t           IOReturn;
 #endif
 #define sub_iokit_platform                err_sub(0x2A)
 #define sub_iokit_audio_video             err_sub(0x45)
 #endif
 #define sub_iokit_platform                err_sub(0x2A)
 #define sub_iokit_audio_video             err_sub(0x45)

+#define sub_iokit_cec                     err_sub(0x46)

 #define sub_iokit_baseband                err_sub(0x80)
 #define sub_iokit_baseband                err_sub(0x80)

-#define sub_iokit_HDA                     err_sub(254)
+#define sub_iokit_HDA                     err_sub(0xFE)

 #define sub_iokit_hsic                    err_sub(0x147)
 #define sub_iokit_sdio                    err_sub(0x174)
 #define sub_iokit_wlan                    err_sub(0x208)
 #define sub_iokit_hsic                    err_sub(0x147)
 #define sub_iokit_sdio                    err_sub(0x174)
 #define sub_iokit_wlan                    err_sub(0x208)

-

 #define sub_iokit_appleembeddedsleepwakehandler  err_sub(0x209)
 
 #define sub_iokit_vendor_specific         err_sub(-2)
 #define sub_iokit_appleembeddedsleepwakehandler  err_sub(0x209)
 
 #define sub_iokit_vendor_specific         err_sub(-2)

diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h
index b45162f87d2b0f1f061bc7efa3eea35595e111f3..89ce8909d48800ef499e9eab00b4c75d112e4e80 100644 (file)
--- a/iokit/IOKit/IOService.h
+++ b/iokit/IOKit/IOService.h
@@ -1399,7 +1399,9 @@ private:
     bool terminatePhase1( IOOptionBits options = 0 );
     void scheduleTerminatePhase2( IOOptionBits options = 0 );
     void scheduleStop( IOService * provider );
     bool terminatePhase1( IOOptionBits options = 0 );
     void scheduleTerminatePhase2( IOOptionBits options = 0 );
     void scheduleStop( IOService * provider );

-    static void terminateThread( void * arg, wait_result_t unused );
+
+    static void waitToBecomeTerminateThread( void );
+    static void __attribute__((__noreturn__)) terminateThread( void * arg, wait_result_t unused );

     static void terminateWorker( IOOptionBits options );
     static void actionWillTerminate( IOService * victim, IOOptionBits options, 
                                      OSArray * doPhase2List, void*, void * );
     static void terminateWorker( IOOptionBits options );
     static void actionWillTerminate( IOService * victim, IOOptionBits options, 
                                      OSArray * doPhase2List, void*, void * );

diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h
index f9ca6ed8a179ca7cd060b510a01dae8c50a97361..3fc00ebae61aa3c95b4c97c2b413dd73c7734133 100644 (file)
--- a/iokit/IOKit/pwr_mgt/IOPM.h
+++ b/iokit/IOKit/pwr_mgt/IOPM.h
@@ -613,6 +613,8 @@ enum {
 #define kIOPMPSAdapterDetailsDescriptionKey        "Description"
 #define kIOPMPSAdapterDetailsPMUConfigurationKey    "PMUConfiguration"
 #define kIOPMPSAdapterDetailsVoltage            "AdapterVoltage"
 #define kIOPMPSAdapterDetailsDescriptionKey        "Description"
 #define kIOPMPSAdapterDetailsPMUConfigurationKey    "PMUConfiguration"
 #define kIOPMPSAdapterDetailsVoltage            "AdapterVoltage"

+#define kIOPMPSAdapterDetailsSourceIDKey                   "SourceID"
+#define kIOPMPSAdapterDetailsSharedSourceKey           "SharedSource"

 
 // values for kIOPSPowerAdapterFamilyKey
 enum {
 
 // values for kIOPSPowerAdapterFamilyKey
 enum {

diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h
index 5a649dfdfb0ae7b32f479744e92001f099331d31..c5d2c17bc79ca126aa8a86bb3c6adfc9afe6b347 100644 (file)
--- a/iokit/IOKit/pwr_mgt/RootDomain.h
+++ b/iokit/IOKit/pwr_mgt/RootDomain.h
@@ -557,6 +557,8 @@ public:
     void        sleepWakeDebugSaveSpinDumpFile();
     void        swdDebugSetup();
     void        swdDebugTeardown();
     void        sleepWakeDebugSaveSpinDumpFile();
     void        swdDebugSetup();
     void        swdDebugTeardown();

+    bool        checkShutdownTimeout();
+    void        panicWithShutdownLog(uint32_t timeoutInMs);

 
 private:
     friend class PMSettingObject;
 
 private:
     friend class PMSettingObject;

diff --git a/iokit/IOKit/rtc/IORTCController.h b/iokit/IOKit/rtc/IORTCController.h
index eaf681d12d3f4ab0481965ff2d312895c63e885b..2757c8a13de4bc01dd40c7b81b06658260fb843f 100644 (file)
--- a/iokit/IOKit/rtc/IORTCController.h
+++ b/iokit/IOKit/rtc/IORTCController.h
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2017 Apple Computer, Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -28,6 +28,8 @@
 /*
  * 24 Nov  1998 suurballe  Created.
  */
 /*
  * 24 Nov  1998 suurballe  Created.
  */

+#ifndef _IORTCCONTROLLER_H
+#define _IORTCCONTROLLER_H

 
 #include <IOKit/IOService.h>
 
 
 #include <IOKit/IOService.h>
 


@@ -44,3 +46,40 @@ virtual IOReturn getRealTimeClock ( UInt8 * currentTime, IOByteCount * length )
 virtual IOReturn setRealTimeClock ( UInt8 * newTime ) = 0;
 };
 
 virtual IOReturn setRealTimeClock ( UInt8 * newTime ) = 0;
 };
 

+class IORTC: public IOService
+{
+OSDeclareAbstractStructors(IORTC);
+
+protected:
+
+    /*! @var reserved
+        Reserved for future use.  (Internal use only)  */
+    struct ExpansionData { };
+    ExpansionData *reserved;
+
+public:
+
+    virtual long            getGMTTimeOfDay( void ) = 0;
+    virtual void            setGMTTimeOfDay( long secs ) = 0;
+
+       virtual void                    getUTCTimeOfDay( clock_sec_t * secs, clock_nsec_t * nsecs );
+       virtual void                    setUTCTimeOfDay( clock_sec_t secs, clock_nsec_t nsecs );
+
+    virtual void            setAlarmEnable( IOOptionBits message ) = 0;
+
+    virtual IOReturn        getMonotonicClockOffset( int64_t * usecs );
+    virtual IOReturn        setMonotonicClockOffset( int64_t usecs );
+    virtual IOReturn        getMonotonicClockAndTimestamp( uint64_t * usecs, uint64_t *mach_absolute_time );
+
+
+    OSMetaClassDeclareReservedUnused(IORTC, 0);
+    OSMetaClassDeclareReservedUnused(IORTC, 1);
+    OSMetaClassDeclareReservedUnused(IORTC, 2);
+    OSMetaClassDeclareReservedUnused(IORTC, 3);
+    OSMetaClassDeclareReservedUnused(IORTC, 4);
+    OSMetaClassDeclareReservedUnused(IORTC, 5);
+    OSMetaClassDeclareReservedUnused(IORTC, 6);
+    OSMetaClassDeclareReservedUnused(IORTC, 7);
+};
+
+#endif /* !_IORTCCONTROLLER_H */

diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp
index afbad66663d2f1722483e922a1792723860b3394..e9173c6558bf85e48eab0764e60b4e8c210b9009 100644 (file)
--- a/iokit/Kernel/IOCPU.cpp
+++ b/iokit/Kernel/IOCPU.cpp
@@ -183,12 +183,18 @@ IOCPURunPlatformPanicActions(uint32_t message)
 
 
 extern "C" kern_return_t
 
 
 extern "C" kern_return_t

-IOCPURunPlatformPanicSyncAction(void *addr, size_t len)
+IOCPURunPlatformPanicSyncAction(void *addr, uint32_t offset, uint32_t len)

 {
 {

+    PE_panic_save_context_t context = {
+        .psc_buffer = addr,
+        .psc_offset = offset,
+        .psc_length = len
+    };
+

     // Don't allow nested calls of panic actions
     if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady);
     return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1,
     // Don't allow nested calls of panic actions
     if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady);
     return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1,

-                                   (void *)(uintptr_t)(kPEPanicSync), addr, (void *)(uintptr_t)len, FALSE));
+                                   (void *)(uintptr_t)(kPEPanicSync), &context, NULL, FALSE));

 
 }
 
 
 }
 

diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index 66b566ca9a8492b9dac0bcfa06b93167c55e6726..4ceb8cc0f8d2d3bceaa05713a198106ad4770732 100644 (file)
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -437,6 +437,7 @@ IOHibernateSystemSleep(void)
         gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
     }
     IOHibernateSystemPostWakeTrim(NULL, NULL);
         gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
     }
     IOHibernateSystemPostWakeTrim(NULL, NULL);

+    thread_call_cancel(gIOHibernateTrimCalloutEntry);

     if (kFSIdle != gFSState)
     {
        HIBLOG("hibernate file busy\n");
     if (kFSIdle != gFSState)
     {
        HIBLOG("hibernate file busy\n");


@@ -677,14 +678,16 @@ IOHibernateSystemSleep(void)
                    IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
                data->release();
            }
                    IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
                data->release();
            }

-            if (gIOChosenEntry)
+            if (gIOChosenEntry && gIOOptionsEntry)

             {
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
                // set BootNext
                if (!gIOHibernateBoot0082Data)
                {
             {
                 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
                 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
                // set BootNext
                if (!gIOHibernateBoot0082Data)
                {

+                   OSData * fileData = 0;

                    data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
                    data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));

+                   if (data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));

                    if (data)
                    {
                        // AppleNVRAM_EFI_LOAD_OPTION
                    if (data)
                    {
                        // AppleNVRAM_EFI_LOAD_OPTION


@@ -696,11 +699,21 @@ IOHibernateSystemSleep(void)
                        loadOptionHeader.Attributes     = 1;
                        loadOptionHeader.FilePathLength = data->getLength();
                        loadOptionHeader.Desc           = 0;
                        loadOptionHeader.Attributes     = 1;
                        loadOptionHeader.FilePathLength = data->getLength();
                        loadOptionHeader.Desc           = 0;

+                       if (fileData)
+                       {
+                           loadOptionHeader.FilePathLength -= 4;
+                           loadOptionHeader.FilePathLength += fileData->getLength();
+                       }

                        gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
                        if (gIOHibernateBoot0082Data)
                        {
                            gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
                        gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
                        if (gIOHibernateBoot0082Data)
                        {
                            gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));

-                           gIOHibernateBoot0082Data->appendBytes(data);
+                           if (fileData)
+                           {
+                               gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
+                               gIOHibernateBoot0082Data->appendBytes(fileData);
+                           }
+                           else gIOHibernateBoot0082Data->appendBytes(data);

                        }
                    }
                }
                        }
                    }
                }


@@ -1316,12 +1329,20 @@ IOHibernateSystemPostWake(bool now)
 {
     gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
     IOLockLock(gFSLock);
 {
     gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
     IOLockLock(gFSLock);

-    if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL);
+    if (kFSTrimDelay == gFSState)
+    {
+        thread_call_cancel(gIOHibernateTrimCalloutEntry);
+        IOHibernateSystemPostWakeTrim(NULL, NULL);
+    }

     else if (kFSOpened != gFSState) gFSState = kFSIdle;
     else
     {
         gFSState = kFSTrimDelay;
     else if (kFSOpened != gFSState) gFSState = kFSIdle;
     else
     {
         gFSState = kFSTrimDelay;

-       if (now) IOHibernateSystemPostWakeTrim(NULL, NULL);
+       if (now)
+    {
+        thread_call_cancel(gIOHibernateTrimCalloutEntry);
+        IOHibernateSystemPostWakeTrim(NULL, NULL);
+    }

        else
        {
            AbsoluteTime deadline;
        else
        {
            AbsoluteTime deadline;


@@ -1331,8 +1352,6 @@ IOHibernateSystemPostWake(bool now)
     }
     IOLockUnlock(gFSLock);
 
     }
     IOLockUnlock(gFSLock);
 

-    // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
-    IOCloseDebugDataFile( );

     return (kIOReturnSuccess);
 }
 
     return (kIOReturnSuccess);
 }
 

diff --git a/iokit/Kernel/IOLocks.cpp b/iokit/Kernel/IOLocks.cpp
index c2ece6b9c79a9d41ea97c0f6a16a52cccb9a61a9..2bbb712bb52b8c68d2da8d97b82e84a5608b62b2 100644 (file)
--- a/iokit/Kernel/IOLocks.cpp
+++ b/iokit/Kernel/IOLocks.cpp
@@ -80,6 +80,7 @@ void  IOLockWakeup(IOLock * lock, void *event, bool oneThread)
        thread_wakeup_prim((event_t) event, oneThread, THREAD_AWAKENED);
 }   
 
        thread_wakeup_prim((event_t) event, oneThread, THREAD_AWAKENED);
 }   
 

+

 #if defined(__x86_64__)
 /*
  * For backwards compatibility, kexts built against pre-Darwin 14 headers will bind at runtime to this function,
 #if defined(__x86_64__)
 /*
  * For backwards compatibility, kexts built against pre-Darwin 14 headers will bind at runtime to this function,


@@ -308,6 +309,30 @@ lck_spin_t * IOSimpleLockGetMachLock( IOSimpleLock * lock)
     return( (lck_spin_t *)lock);
 }
 
     return( (lck_spin_t *)lock);
 }
 

+#ifndef IOLOCKS_INLINE
+/*
+ * Lock assertions
+ */
+
+void
+IOLockAssert(IOLock * lock, IOLockAssertState type)
+{
+    LCK_MTX_ASSERT(lock, type);
+}
+
+void
+IORWLockAssert(IORWLock * lock, IORWLockAssertState type)
+{
+    LCK_RW_ASSERT(lock, type);
+}
+
+void
+IOSimpleLockAssert(IOSimpleLock *lock, IOSimpleLockAssertState type)
+{
+    LCK_SPIN_ASSERT(l, type);
+}
+#endif /* !IOLOCKS_INLINE */
+

 } /* extern "C" */
 
 
 } /* extern "C" */
 
 

diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp
index 3b778df403ad1329ed7fe1c9fdab5f65ee8c07e1..bf8ecc94f90b70adb79594ed4cb8c4e7c60f4d60 100644 (file)
--- a/iokit/Kernel/IOPMrootDomain.cpp
+++ b/iokit/Kernel/IOPMrootDomain.cpp
@@ -611,7 +611,7 @@ extern "C" IOReturn rootDomainShutdown ( void )
 
 static void halt_log_putc(char c)
 {
 
 static void halt_log_putc(char c)
 {

-    if (gHaltLogPos >= (kHaltLogSize - 1)) return;
+    if (gHaltLogPos >= (kHaltLogSize - 2)) return;

     gHaltLog[gHaltLogPos++] = c;
 }
 
     gHaltLog[gHaltLogPos++] = c;
 }
 


@@ -645,12 +645,15 @@ halt_log_enter(const char * what, const void * pc, uint64_t time)
 
     IOLockLock(gHaltLogLock);
     if (pc) {
 
     IOLockLock(gHaltLogLock);
     if (pc) {

-       halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc));
-       OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log,
-            OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse);
-    } else {
-       halt_log("%s: %qd ms\n", what, millis, VM_KERNEL_UNSLIDE(pc));
+        halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc));
+        OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log,
+                OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse);
+    }
+    else {
+        halt_log("%s: %qd ms\n", what, millis);

     }
     }

+
+    gHaltLog[gHaltLogPos] = 0;

     IOLockUnlock(gHaltLogLock);
 }
 
     IOLockUnlock(gHaltLogLock);
 }
 


@@ -675,6 +678,7 @@ extern "C" void IOSystemShutdownNotification(void)
 #if HIBERNATION
     startTime = mach_absolute_time();
     IOHibernateSystemPostWake(true);
 #if HIBERNATION
     startTime = mach_absolute_time();
     IOHibernateSystemPostWake(true);

+    gRootDomain->swdDebugTeardown();

     halt_log_enter("IOHibernateSystemPostWake", 0, mach_absolute_time() - startTime);
 #endif
     if (OSCompareAndSwap(0, 1, &gPagingOff))
     halt_log_enter("IOHibernateSystemPostWake", 0, mach_absolute_time() - startTime);
 #endif
     if (OSCompareAndSwap(0, 1, &gPagingOff))


@@ -782,13 +786,15 @@ static void swdDebugSetupCallout( thread_call_param_t p0, thread_call_param_t p1
 void IOPMrootDomain::swdDebugSetup( )
 {
 #if    HIBERNATION
 void IOPMrootDomain::swdDebugSetup( )
 {
 #if    HIBERNATION

-    static int32_t mem_only = -1;
-    if ((mem_only == -1) &&
-        (PE_parse_boot_argn("swd_mem_only", &mem_only, sizeof(mem_only)) == false)) {
-        mem_only = 0;
+    static int32_t noDebugFile = -1;
+    if (noDebugFile == -1) {
+        if (PEGetCoprocessorVersion() >= kCoprocessorVersion2)
+            noDebugFile = 1;
+        else if (PE_parse_boot_argn("swd_mem_only", &noDebugFile, sizeof(noDebugFile)) == false)
+            noDebugFile = 0;

     }
 
     }
 

-   if ((mem_only == 1) || (gRootDomain->sleepWakeDebugIsWdogEnabled() == false)) {
+   if ((noDebugFile == 1) || (gRootDomain->sleepWakeDebugIsWdogEnabled() == false)) {

        return;
    }
     DLOG("swdDebugSetup state:%d\n", swd_DebugImageSetup);
        return;
    }
     DLOG("swdDebugSetup state:%d\n", swd_DebugImageSetup);


@@ -797,6 +803,7 @@ void IOPMrootDomain::swdDebugSetup( )
         if (CAP_GAIN(kIOPMSystemCapabilityGraphics) ||
                 (CAP_LOSS(kIOPMSystemCapabilityGraphics))) {
             IOHibernateSystemPostWake(true);
         if (CAP_GAIN(kIOPMSystemCapabilityGraphics) ||
                 (CAP_LOSS(kIOPMSystemCapabilityGraphics))) {
             IOHibernateSystemPostWake(true);

+            IOCloseDebugDataFile();

         }
         IOOpenDebugDataFile(kSleepWakeStackBinFilename, SWD_BUF_SIZE);
     }
         }
         IOOpenDebugDataFile(kSleepWakeStackBinFilename, SWD_BUF_SIZE);
     }


@@ -844,6 +851,11 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 )
     if (ON_STATE == powerState)
     {
         sync_internal();
     if (ON_STATE == powerState)
     {
         sync_internal();

+
+#if HIBERNATION
+        // Block sleep until trim issued on previous wake path is completed.
+        IOHibernateSystemPostWake(true);
+#endif

         swdDebugSetupCallout(p0, NULL);
     }
 #if HIBERNATION
         swdDebugSetupCallout(p0, NULL);
     }
 #if HIBERNATION


@@ -3083,9 +3095,6 @@ IOReturn IOPMrootDomain::sysPowerDownHandler(
             // We will ack within 20 seconds
             params->maxWaitForReply = 20 * 1000 * 1000;
 
             // We will ack within 20 seconds
             params->maxWaitForReply = 20 * 1000 * 1000;
 

-            // Remove EFI/BootRom's previous wake's failure data
-            PERemoveNVRAMProperty(kIOEFIBootRomFailureKey);
-

 #if HIBERNATION
             gRootDomain->evaluateSystemSleepPolicyEarly();
 
 #if HIBERNATION
             gRootDomain->evaluateSystemSleepPolicyEarly();
 


@@ -4622,6 +4631,16 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType, uint32_t * st
 //
 //******************************************************************************
 
 //
 //******************************************************************************
 

+// Phases while performing shutdown/restart
+typedef enum {
+    kNotifyDone                 = 0x00,
+    kNotifyPriorityClients      = 0x10,
+    kNotifyPowerPlaneDrivers    = 0x20,
+    kNotifyHaltRestartAction    = 0x30,
+    kQuiescePM                  = 0x40,
+} shutdownPhase_t;
+
+

 struct HaltRestartApplierContext {
     IOPMrootDomain *    RootDomain;
     unsigned long       PowerState;
 struct HaltRestartApplierContext {
     IOPMrootDomain *    RootDomain;
     unsigned long       PowerState;


@@ -4629,7 +4648,29 @@ struct HaltRestartApplierContext {
     UInt32              MessageType;
     UInt32              Counter;
     const char *        LogString;
     UInt32              MessageType;
     UInt32              Counter;
     const char *        LogString;

-};
+    shutdownPhase_t     phase;
+
+    IOServiceInterestHandler   handler;
+} gHaltRestartCtx;
+
+const char *shutdownPhase2String(shutdownPhase_t phase)
+{
+    switch(phase) {
+        case kNotifyDone:
+            return "Notifications completed";
+        case kNotifyPriorityClients:
+            return "Notifying priority clients";
+        case kNotifyPowerPlaneDrivers:
+            return  "Notifying power plane drivers";
+        case kNotifyHaltRestartAction:
+            return "Notifying HaltRestart action handlers";
+        case kQuiescePM:
+            return "Quiescing PM";
+        default:
+            return "Unknown";
+    }
+
+}

 
 static void
 platformHaltRestartApplier( OSObject * object, void * context )
 
 static void
 platformHaltRestartApplier( OSObject * object, void * context )


@@ -4641,33 +4682,30 @@ platformHaltRestartApplier( OSObject * object, void * context )
 
     ctx = (HaltRestartApplierContext *) context;
 
 
     ctx = (HaltRestartApplierContext *) context;
 

+    _IOServiceInterestNotifier * notifier;
+    notifier = OSDynamicCast(_IOServiceInterestNotifier, object);

     memset(&notify, 0, sizeof(notify));
     notify.powerRef    = (void *)(uintptr_t)ctx->Counter;
     notify.returnValue = 0;
     notify.stateNumber = ctx->PowerState;
     notify.stateFlags  = ctx->PowerFlags;
 
     memset(&notify, 0, sizeof(notify));
     notify.powerRef    = (void *)(uintptr_t)ctx->Counter;
     notify.returnValue = 0;
     notify.stateNumber = ctx->PowerState;
     notify.stateFlags  = ctx->PowerFlags;
 

+    if (notifier) {
+        ctx->handler = notifier->handler;
+    }
+

     clock_get_uptime(&startTime);
     ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)&notify );
     deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);
 
     clock_get_uptime(&startTime);
     ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)&notify );
     deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);
 

-    if ((deltaTime > kPMHaltTimeoutMS) ||
-        (gIOKitDebug & kIOLogPMRootDomain))
-    {
-        _IOServiceInterestNotifier * notifier;
-        notifier = OSDynamicCast(_IOServiceInterestNotifier, object);
-
-        // IOService children of IOPMrootDomain are not instrumented.
-        // Only IORootParent currently falls under that group.
+    if ((deltaTime > kPMHaltTimeoutMS) && notifier) {

 
 

-        if (notifier)
-        {
-            LOG("%s handler %p took %u ms\n",
+        LOG("%s handler %p took %u ms\n",

                 ctx->LogString, OBFUSCATE(notifier->handler), deltaTime);
                 ctx->LogString, OBFUSCATE(notifier->handler), deltaTime);

-            halt_log_enter(ctx->LogString, (const void *) notifier->handler, elapsedTime);
-        }
+        halt_log_enter("PowerOff/Restart message to priority client", (const void *) notifier->handler, elapsedTime);

     }
 
     }
 

+    ctx->handler = 0;

     ctx->Counter++;
 }
 
     ctx->Counter++;
 }
 


@@ -4681,33 +4719,32 @@ static void quiescePowerTreeCallback( void * target, void * param )
 
 void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
 {
 
 void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
 {

-    HaltRestartApplierContext   ctx;

     AbsoluteTime                startTime, elapsedTime;
     uint32_t                    deltaTime;
 
     AbsoluteTime                startTime, elapsedTime;
     uint32_t                    deltaTime;
 

-    memset(&ctx, 0, sizeof(ctx));
-    ctx.RootDomain = this;
+    memset(&gHaltRestartCtx, 0, sizeof(gHaltRestartCtx));
+    gHaltRestartCtx.RootDomain = this;

 
     clock_get_uptime(&startTime);
     switch (pe_type)
     {
         case kPEHaltCPU:
         case kPEUPSDelayHaltCPU:
 
     clock_get_uptime(&startTime);
     switch (pe_type)
     {
         case kPEHaltCPU:
         case kPEUPSDelayHaltCPU:

-            ctx.PowerState  = OFF_STATE;
-            ctx.MessageType = kIOMessageSystemWillPowerOff;
-            ctx.LogString   = "PowerOff";
+            gHaltRestartCtx.PowerState  = OFF_STATE;
+            gHaltRestartCtx.MessageType = kIOMessageSystemWillPowerOff;
+            gHaltRestartCtx.LogString   = "PowerOff";

             break;
 
         case kPERestartCPU:
             break;
 
         case kPERestartCPU:

-            ctx.PowerState  = RESTART_STATE;
-            ctx.MessageType = kIOMessageSystemWillRestart;
-            ctx.LogString   = "Restart";
+            gHaltRestartCtx.PowerState  = RESTART_STATE;
+            gHaltRestartCtx.MessageType = kIOMessageSystemWillRestart;
+            gHaltRestartCtx.LogString   = "Restart";

             break;
 
         case kPEPagingOff:
             break;
 
         case kPEPagingOff:

-            ctx.PowerState  = ON_STATE;
-            ctx.MessageType = kIOMessageSystemPagingOff;
-            ctx.LogString   = "PagingOff";
+            gHaltRestartCtx.PowerState  = ON_STATE;
+            gHaltRestartCtx.MessageType = kIOMessageSystemPagingOff;
+            gHaltRestartCtx.LogString   = "PagingOff";

             IOService::updateConsoleUsers(NULL, kIOMessageSystemPagingOff);
 #if HIBERNATION
             IOHibernateSystemRestart();
             IOService::updateConsoleUsers(NULL, kIOMessageSystemPagingOff);
 #if HIBERNATION
             IOHibernateSystemRestart();


@@ -4718,8 +4755,9 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
             return;
     }
 
             return;
     }
 

+    gHaltRestartCtx.phase = kNotifyPriorityClients;

     // Notify legacy clients
     // Notify legacy clients

-    applyToInterested(gIOPriorityPowerStateInterest, platformHaltRestartApplier, &ctx);
+    applyToInterested(gIOPriorityPowerStateInterest, platformHaltRestartApplier, &gHaltRestartCtx);

 
     // For normal shutdown, turn off File Server Mode.
     if (kPEHaltCPU == pe_type)
 
     // For normal shutdown, turn off File Server Mode.
     if (kPEHaltCPU == pe_type)


@@ -4734,17 +4772,21 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
         }
     }
 
         }
     }
 

+

     if (kPEPagingOff != pe_type)
     {
     if (kPEPagingOff != pe_type)
     {

+        gHaltRestartCtx.phase = kNotifyPowerPlaneDrivers;

         // Notify in power tree order
         // Notify in power tree order

-        notifySystemShutdown(this, ctx.MessageType);
+        notifySystemShutdown(this, gHaltRestartCtx.MessageType);

     }
 
     }
 

+    gHaltRestartCtx.phase = kNotifyHaltRestartAction;

     IOCPURunPlatformHaltRestartActions(pe_type);
 
     // Wait for PM to quiesce
     if ((kPEPagingOff != pe_type) && gPMHaltLock)
     {
     IOCPURunPlatformHaltRestartActions(pe_type);
 
     // Wait for PM to quiesce
     if ((kPEPagingOff != pe_type) && gPMHaltLock)
     {

+        gHaltRestartCtx.phase = kQuiescePM;

         AbsoluteTime quiesceTime = mach_absolute_time();
 
         IOLockLock(gPMHaltLock);
         AbsoluteTime quiesceTime = mach_absolute_time();
 
         IOLockLock(gPMHaltLock);


@@ -4762,23 +4804,47 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
         DLOG("PM quiesce took %u ms\n", deltaTime);
         halt_log_enter("Quiesce", NULL, elapsedTime);
     }
         DLOG("PM quiesce took %u ms\n", deltaTime);
         halt_log_enter("Quiesce", NULL, elapsedTime);
     }

+    gHaltRestartCtx.phase = kNotifyDone;

 
     deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);
 
     deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);

-    LOG("%s all drivers took %u ms\n", ctx.LogString, deltaTime);
+    LOG("%s all drivers took %u ms\n", gHaltRestartCtx.LogString, deltaTime);

 
 

-    halt_log_enter(ctx.LogString, NULL, elapsedTime);
-    if (gHaltLog) gHaltLog[gHaltLogPos] = 0;
+    halt_log_enter(gHaltRestartCtx.LogString, NULL, elapsedTime);

 
     deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime);
 
     deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime);

-    LOG("%s total %u ms\n", ctx.LogString, deltaTime);
+    LOG("%s total %u ms\n", gHaltRestartCtx.LogString, deltaTime);

 
     if (gHaltLog && gHaltTimeMaxLog && (deltaTime >= gHaltTimeMaxLog))
     {
 
     if (gHaltLog && gHaltTimeMaxLog && (deltaTime >= gHaltTimeMaxLog))
     {

-         printf("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog);
+         printf("%s total %d ms:%s\n", gHaltRestartCtx.LogString, deltaTime, gHaltLog);

     }
     }

-    if (gHaltLog && gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic))
-    {
-        panic("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog);
+
+    checkShutdownTimeout();
+}
+
+bool IOPMrootDomain::checkShutdownTimeout()
+{
+    AbsoluteTime   elapsedTime;
+    uint32_t deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime);
+
+    if (gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic)) {
+        return true;
+    }
+    return false;
+}
+
+void IOPMrootDomain::panicWithShutdownLog(uint32_t timeoutInMs)
+{
+    if (gHaltLog) {
+        if ((gHaltRestartCtx.phase == kNotifyPriorityClients) && gHaltRestartCtx.handler) {
+            halt_log_enter("Blocked on priority client", (void *)gHaltRestartCtx.handler, mach_absolute_time() - gHaltStartTime);
+        }
+        panic("%s timed out in phase '%s'. Total %d ms:%s",
+                gHaltRestartCtx.LogString, shutdownPhase2String(gHaltRestartCtx.phase), timeoutInMs, gHaltLog);
+    }
+    else {
+        panic("%s timed out in phase \'%s\'. Total %d ms",
+                gHaltRestartCtx.LogString, shutdownPhase2String(gHaltRestartCtx.phase), timeoutInMs);

     }
 }
 
     }
 }
 


@@ -7801,7 +7867,6 @@ IOReturn IOPMrootDomain::callPlatformFunction(
     void * param3, void * param4 )
 {
     uint32_t bootFailureCode = 0xffffffff;
     void * param3, void * param4 )
 {
     uint32_t bootFailureCode = 0xffffffff;

-    unsigned int len = sizeof(bootFailureCode);

     if (pmTracer && functionName &&
         functionName->isEqualTo(kIOPMRegisterNVRAMTracePointHandlerKey) &&
         !pmTracer->tracePointHandler && !pmTracer->tracePointTarget)
     if (pmTracer && functionName &&
         functionName->isEqualTo(kIOPMRegisterNVRAMTracePointHandlerKey) &&
         !pmTracer->tracePointHandler && !pmTracer->tracePointTarget)


@@ -7814,11 +7879,17 @@ IOReturn IOPMrootDomain::callPlatformFunction(
         tracePointPCI               = (uint32_t)(uintptr_t) param3;
         tracePointPhases            = (uint32_t)(uintptr_t) param4;
         if ((tracePointPhases & 0xff) == kIOPMTracePointSystemSleep) {
         tracePointPCI               = (uint32_t)(uintptr_t) param3;
         tracePointPhases            = (uint32_t)(uintptr_t) param4;
         if ((tracePointPhases & 0xff) == kIOPMTracePointSystemSleep) {

-           if (!PEReadNVRAMProperty(kIOEFIBootRomFailureKey, &bootFailureCode, &len)) {
-              MSG("Failed to read failure code from NVRam\n");
-           }
-           // Failure code from EFI/BootRom is a four byte structure
-           tracePointPCI = OSSwapBigToHostInt32(bootFailureCode);
+
+            IORegistryEntry *node = IORegistryEntry::fromPath( "/chosen", gIODTPlane );
+            if ( node ) {
+                OSData *data = OSDynamicCast( OSData, node->getProperty(kIOEFIBootRomFailureKey) );
+                if ( data && data->getLength() == sizeof(bootFailureCode) ) {
+                    memcpy(&bootFailureCode, data->getBytesNoCopy(), sizeof(bootFailureCode));
+                }
+                node->release();
+            }
+            // Failure code from EFI/BootRom is a four byte structure
+            tracePointPCI = OSSwapBigToHostInt32(bootFailureCode);

         }
         statusCode = (((uint64_t)tracePointPCI) << 32) | tracePointPhases;
         if ((tracePointPhases & 0xff) != kIOPMTracePointSystemUp) {
         }
         statusCode = (((uint64_t)tracePointPCI) << 32) | tracePointPhases;
         if ((tracePointPhases & 0xff) != kIOPMTracePointSystemUp) {


@@ -8454,16 +8525,14 @@ void PMHaltWorker::work( PMHaltWorker * me )
         }
 
         deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);
         }
 
         deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime);

-        if ((deltaTime > kPMHaltTimeoutMS) || timeout ||
-            (gIOKitDebug & kIOLogPMRootDomain))
+        if ((deltaTime > kPMHaltTimeoutMS) || timeout)

         {
             LOG("%s driver %s (0x%llx) took %u ms\n",
                 (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ?
                     "PowerOff" : "Restart",
                 service->getName(), service->getRegistryEntryID(),
                 (uint32_t) deltaTime );
         {
             LOG("%s driver %s (0x%llx) took %u ms\n",
                 (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ?
                     "PowerOff" : "Restart",
                 service->getName(), service->getRegistryEntryID(),
                 (uint32_t) deltaTime );

-            halt_log_enter(
-                (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart",
+            halt_log_enter("PowerOff/Restart handler completed",

                 OSMemberFunctionCast(const void *, service, &IOService::systemWillShutdown),
                 elapsedTime);
         }
                 OSMemberFunctionCast(const void *, service, &IOService::systemWillShutdown),
                 elapsedTime);
         }


@@ -8494,9 +8563,12 @@ void PMHaltWorker::checkTimeout( PMHaltWorker * me, AbsoluteTime * now )
         if (nano > 3000000000ULL)
         {
             me->timeout = true;
         if (nano > 3000000000ULL)
         {
             me->timeout = true;

+
+            halt_log_enter("PowerOff/Restart still waiting on handler",
+                OSMemberFunctionCast(const void *, me->service, &IOService::systemWillShutdown),
+                endTime);

             MSG("%s still waiting on %s\n",
             MSG("%s still waiting on %s\n",

-                (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ?
-                    "PowerOff" : "Restart",
+                (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ?  "PowerOff" : "Restart",

                 me->service->getName());
         }
     }
                 me->service->getName());
         }
     }


@@ -9700,7 +9772,7 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is
    if (wdogTrigger) {
        PE_parse_boot_argn("swd_panic", &wdog_panic, sizeof(wdog_panic));
        PE_parse_boot_argn("stress-rack", &stress_rack, sizeof(stress_rack));
    if (wdogTrigger) {
        PE_parse_boot_argn("swd_panic", &wdog_panic, sizeof(wdog_panic));
        PE_parse_boot_argn("stress-rack", &stress_rack, sizeof(stress_rack));

-       if ((wdog_panic == 1) || (stress_rack == 1)) {
+       if ((wdog_panic == 1) || (stress_rack == 1) || (PEGetCoprocessorVersion() >= kCoprocessorVersion2)) {

            // If boot-arg specifies to panic then panic.
            panic("Sleep/Wake hang detected");
            return;
            // If boot-arg specifies to panic then panic.
            panic("Sleep/Wake hang detected");
            return;


@@ -9886,6 +9958,9 @@ void IOPMrootDomain::sleepWakeDebugMemAlloc( )
     if ( kIOSleepWakeWdogOff & gIOKitDebug )
       return;
 
     if ( kIOSleepWakeWdogOff & gIOKitDebug )
       return;
 

+    if (PEGetCoprocessorVersion() >= kCoprocessorVersion2)
+        return;
+

     if (!OSCompareAndSwap(0, 1, &gRootDomain->swd_lock))
        return;
 
     if (!OSCompareAndSwap(0, 1, &gRootDomain->swd_lock))
        return;
 

diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp
index 274e6904342758ec612bca8dd73ff8a2574af831..59c36062ebd65824669684b6267710c31639d433 100644 (file)
--- a/iokit/Kernel/IOPlatformExpert.cpp
+++ b/iokit/Kernel/IOPlatformExpert.cpp
@@ -54,15 +54,9 @@ extern "C" {
 #include <uuid/uuid.h>
 }
 
 #include <uuid/uuid.h>
 }
 

-#if !CONFIG_EMBEDDED
+#define kShutdownTimeout    30 //in secs

 
 

-/*
- * This will eventually be properly exported in
- * <rdar://problem/31181482> ER: Expose coprocessor version (T208/T290) in a kernel/kext header
- * although we'll always need to hardcode this here since we won't be able to include whatever
- * header this ends up in.
- */
-#define kCoprocessorMinVersion 0x00020000
+#if !CONFIG_EMBEDDED

 
 boolean_t coprocessor_cross_panic_enabled = TRUE;
 #endif /* !CONFIG_EMBEDDED */
 
 boolean_t coprocessor_cross_panic_enabled = TRUE;
 #endif /* !CONFIG_EMBEDDED */


@@ -114,11 +108,6 @@ bool IOPlatformExpert::start( IOService * provider )
     OSData *           busFrequency;
     uint32_t           debugFlags;
 
     OSData *           busFrequency;
     uint32_t           debugFlags;
 

-#if !CONFIG_EMBEDDED
-    IORegistryEntry    *platform_entry = NULL;
-    OSData             *coprocessor_version_obj = NULL;
-    uint64_t           coprocessor_version = 0;
-#endif

     
     if (!super::start(provider))
       return false;
     
     if (!super::start(provider))
       return false;


@@ -171,19 +160,11 @@ bool IOPlatformExpert::start( IOService * provider )
     }
 
 #if !CONFIG_EMBEDDED
     }
 
 #if !CONFIG_EMBEDDED

-    platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform");
-    if (platform_entry != NULL) {
-        coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version"));
-        if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(coprocessor_version))) {
-            memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength());
-            if (coprocessor_version >= kCoprocessorMinVersion) {
-                coprocessor_paniclog_flush = TRUE;
-                extended_debug_log_init();
-            }
-        }
-        platform_entry->release();
+    if (PEGetCoprocessorVersion() >= kCoprocessorVersion2) {
+        coprocessor_paniclog_flush = TRUE;
+        extended_debug_log_init();

     }
     }

-#endif /* !CONFIG_EMBEDDED */
+#endif

 
     return( configure(provider) );
 }
 
     return( configure(provider) );
 }


@@ -788,6 +769,14 @@ static void IOShutdownNotificationsTimedOut(
 
 #else /* ! CONFIG_EMBEDDED */
     int type = (int)(long)p0;
 
 #else /* ! CONFIG_EMBEDDED */
     int type = (int)(long)p0;

+    uint32_t timeout = (uint32_t)(uintptr_t)p1;
+
+    IOPMrootDomain *pmRootDomain = IOService::getPMRootDomain();
+    if (pmRootDomain) {
+        if ((PEGetCoprocessorVersion() >= kCoprocessorVersion2) || pmRootDomain->checkShutdownTimeout()) {
+        pmRootDomain->panicWithShutdownLog(timeout * 1000);
+        }
+    }

 
     /* 30 seconds has elapsed - resume shutdown */
     if(gIOPlatform) gIOPlatform->haltRestart(type);
 
     /* 30 seconds has elapsed - resume shutdown */
     if(gIOPlatform) gIOPlatform->haltRestart(type);


@@ -832,7 +821,7 @@ int PEHaltRestart(unsigned int type)
   thread_call_t     shutdown_hang;
   IORegistryEntry   *node;
   OSData            *data;
   thread_call_t     shutdown_hang;
   IORegistryEntry   *node;
   OSData            *data;

-  uint32_t          timeout = 30;
+  uint32_t          timeout = kShutdownTimeout;

   static boolean_t  panic_begin_called = FALSE;
   
   if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU)
   static boolean_t  panic_begin_called = FALSE;
   
   if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU)


@@ -863,7 +852,7 @@ int PEHaltRestart(unsigned int type)
     shutdown_hang = thread_call_allocate( &IOShutdownNotificationsTimedOut, 
                         (thread_call_param_t)(uintptr_t) type);
     clock_interval_to_deadline( timeout, kSecondScale, &deadline );
     shutdown_hang = thread_call_allocate( &IOShutdownNotificationsTimedOut, 
                         (thread_call_param_t)(uintptr_t) type);
     clock_interval_to_deadline( timeout, kSecondScale, &deadline );

-    thread_call_enter1_delayed( shutdown_hang, 0, deadline );
+    thread_call_enter1_delayed( shutdown_hang, (thread_call_param_t)(uintptr_t)timeout, deadline );

 
     pmRootDomain->handlePlatformHaltRestart(type); 
     /* This notification should have few clients who all do 
 
     pmRootDomain->handlePlatformHaltRestart(type); 
     /* This notification should have few clients who all do 


@@ -925,9 +914,9 @@ UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length)
   else return 0;
 }
 
   else return 0;
 }
 

-void PESavePanicInfoAction(void *buffer, size_t length)
+void PESavePanicInfoAction(void *buffer, UInt32 offset, UInt32 length)

 {
 {

-       IOCPURunPlatformPanicSyncAction(buffer, length);
+       IOCPURunPlatformPanicSyncAction(buffer, offset, length);

        return;
 }
 
        return;
 }
 


@@ -1130,6 +1119,25 @@ void PESetUTCTimeOfDay(clock_sec_t secs, clock_usec_t usecs)
         gIOPlatform->setUTCTimeOfDay(secs, usecs * NSEC_PER_USEC);
 }
 
         gIOPlatform->setUTCTimeOfDay(secs, usecs * NSEC_PER_USEC);
 }
 

+coprocessor_type_t  PEGetCoprocessorVersion( void )
+{
+    coprocessor_type_t coprocessor_version = kCoprocessorVersionNone;
+#if !CONFIG_EMBEDDED
+    IORegistryEntry    *platform_entry = NULL;
+    OSData             *coprocessor_version_obj = NULL;
+
+    platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform");
+    if (platform_entry != NULL) {
+        coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version"));
+        if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(uint64_t))) {
+            memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength());
+        }
+        platform_entry->release();
+    }
+#endif
+    return coprocessor_version;
+}
+

 } /* extern "C" */
 
 void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller)
 } /* extern "C" */
 
 void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller)


@@ -1187,7 +1195,9 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller)
             data = OSDynamicCast( OSData, entry->getProperty( "EffectiveProductionStatus" ) );
             if ( data  && ( data->getLength( ) == sizeof( UInt8 ) ) ) {
                     UInt8 *isProdFused = (UInt8 *) data->getBytesNoCopy( );
             data = OSDynamicCast( OSData, entry->getProperty( "EffectiveProductionStatus" ) );
             if ( data  && ( data->getLength( ) == sizeof( UInt8 ) ) ) {
                     UInt8 *isProdFused = (UInt8 *) data->getBytesNoCopy( );

-                    if ( *isProdFused ) {
+                    UInt32 debug_flags = 0;
+                    if ( *isProdFused || ( PE_i_can_has_debugger(&debug_flags) &&
+                               ( debug_flags & DB_DISABLE_CROSS_PANIC ) ) ) {

                         coprocessor_cross_panic_enabled = FALSE;
                     }
             }
                         coprocessor_cross_panic_enabled = FALSE;
                     }
             }

diff --git a/iokit/Kernel/IOPolledInterface.cpp b/iokit/Kernel/IOPolledInterface.cpp
index 2c9dfba6825b9aebf144b9a743c62963d32d9b79..821bbc81bb820ddf4fd84bb3968d27668fe1adee 100644 (file)
--- a/iokit/Kernel/IOPolledInterface.cpp
+++ b/iokit/Kernel/IOPolledInterface.cpp
@@ -490,9 +490,11 @@ IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
         uuid_t volUuid = {0};
         size_t sizeOut = 0;
         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut);
         uuid_t volUuid = {0};
         size_t sizeOut = 0;
         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut);

-        if (err == kIOReturnSuccess) {
+        if (err == kIOReturnSuccess)
+        {

             // No need to create uuid string if it's not requested
             // No need to create uuid string if it's not requested

-            if (pKeyUUID) {
+            if (pKeyUUID)
+            {

                 uuid_string_t volUuidStr;
                 uuid_unparse(volUuid, volUuidStr);
                 *pKeyUUID = OSString::withCString(volUuidStr);
                 uuid_string_t volUuidStr;
                 uuid_unparse(volUuid, volUuidStr);
                 *pKeyUUID = OSString::withCString(volUuidStr);


@@ -526,11 +528,15 @@ IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
         }
         bzero(&vek, sizeof(vek));
             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
         }
         bzero(&vek, sizeof(vek));

+
+        if (pKeyUUID)
+        {
+            // Create a copy because the caller would release it
+            *pKeyUUID = OSString::withString(keyUUID);
+        }

     }
 
     part->release();
     }
 
     part->release();

-    if (pKeyUUID) *pKeyUUID = keyUUID;
-

     return (err);
 }
 
     return (err);
 }
 


@@ -538,11 +544,11 @@ IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 
 IOReturn
 IOPolledFileOpen(const char * filename,
 
 IOReturn
 IOPolledFileOpen(const char * filename,

-                uint64_t setFileSize, uint64_t fsFreeSize,
-                void * write_file_addr, size_t write_file_len,
-                IOPolledFileIOVars ** fileVars,
-                OSData ** imagePath,
-                uint8_t * volumeCryptKey, size_t keySize)
+                 uint64_t setFileSize, uint64_t fsFreeSize,
+                 void * write_file_addr, size_t write_file_len,
+                 IOPolledFileIOVars ** fileVars,
+                 OSData ** imagePath,
+                 uint8_t * volumeCryptKey, size_t keySize)

 {
     IOReturn             err = kIOReturnSuccess;
     IOPolledFileIOVars * vars;
 {
     IOReturn             err = kIOReturnSuccess;
     IOPolledFileIOVars * vars;


@@ -562,66 +568,66 @@ IOPolledFileOpen(const char * filename,
 
     do
     {
 
     do
     {

-       extentsData = OSData::withCapacity(32);
-       ctx.extents = extentsData;
-       ctx.size    = 0;
-       clock_get_uptime(&startTime);
+        extentsData = OSData::withCapacity(32);
+        ctx.extents = extentsData;
+        ctx.size    = 0;
+        clock_get_uptime(&startTime);

 
 

-       vars->fileRef = kern_open_file_for_direct_io(filename, 
+        vars->fileRef = kern_open_file_for_direct_io(filename,

                                                      (write_file_addr != NULL) || (0 != setFileSize),
                                                      (write_file_addr != NULL) || (0 != setFileSize),

-                                                    &file_extent_callback, &ctx, 
-                                                    setFileSize,
-                                                    fsFreeSize,
-                                                    // write file:
+                                                     &file_extent_callback, &ctx,
+                                                     setFileSize,
+                                                     fsFreeSize,
+                                                     // write file:

                                                      0, write_file_addr, write_file_len,
                                                      // results
                                                      0, write_file_addr, write_file_len,
                                                      // results

-                                                    &block_dev,
-                                                    &image_dev,
+                                                     &block_dev,
+                                                     &image_dev,

                                                      &vars->block0,
                                                      &vars->maxiobytes,
                                                      &vars->flags);
 #if 0
                                                      &vars->block0,
                                                      &vars->maxiobytes,
                                                      &vars->flags);
 #if 0

-       uint32_t msDelay = (131071 & random());
-       HIBLOG("sleep %d\n", msDelay);
-       IOSleep(msDelay);
+        uint32_t msDelay = (131071 & random());
+        HIBLOG("sleep %d\n", msDelay);
+        IOSleep(msDelay);

 #endif
         clock_get_uptime(&endTime);
         SUB_ABSOLUTETIME(&endTime, &startTime);
         absolutetime_to_nanoseconds(endTime, &nsec);
 
 #endif
         clock_get_uptime(&endTime);
         SUB_ABSOLUTETIME(&endTime, &startTime);
         absolutetime_to_nanoseconds(endTime, &nsec);
 

-       if (!vars->fileRef) err = kIOReturnNoSpace;
+        if (!vars->fileRef) err = kIOReturnNoSpace;

 
         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 
         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);

-       if (kIOReturnSuccess != err) break;
+        if (kIOReturnSuccess != err) break;

 
 

-       HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size, 
-                    (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
-                    vars->maxiobytes, kIOPolledFileSSD & vars->flags);
-       assert(!vars->block0);
-       if (extentsData->getLength() < sizeof(IOPolledFileExtent))
-       {
-           err = kIOReturnNoSpace;
-           break;
-       }
+        HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
+               (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
+               vars->maxiobytes, kIOPolledFileSSD & vars->flags);
+        assert(!vars->block0);
+        if (extentsData->getLength() < sizeof(IOPolledFileExtent))
+        {
+            err = kIOReturnNoSpace;
+            break;
+        }

 
 

-       vars->fileSize = ctx.size;
-       vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
+        vars->fileSize = ctx.size;
+        vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();

 
         part = IOCopyMediaForDev(image_dev);
         if (!part)
         {
             err = kIOReturnNotFound;
             break;
 
         part = IOCopyMediaForDev(image_dev);
         if (!part)
         {
             err = kIOReturnNotFound;
             break;

-       }
+        }

 
 

-       if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
+        if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;

 
 

-       if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
-               vars->blockSize = num->unsigned32BitValue();
-       if (vars->blockSize < 4096) vars->blockSize = 4096;
+        if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
+            vars->blockSize = num->unsigned32BitValue();
+        if (vars->blockSize < 4096) vars->blockSize = 4096;

 
         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 
         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",

-               major(image_dev), minor(image_dev), (long)vars->blockSize, 
+               major(image_dev), minor(image_dev), (long)vars->blockSize,

                vars->pollers->pollers->getCount());
 
         OSString * keyUUID = NULL;
                vars->pollers->pollers->getCount());
 
         OSString * keyUUID = NULL;


@@ -630,45 +636,52 @@ IOPolledFileOpen(const char * filename,
             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
         }
 
             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
         }
 

-       *fileVars    = vars;
-       vars->fileExtents = extentsData;
-    
-       // make imagePath
-       OSData * data;
-       if (imagePath)
-       {
+        *fileVars    = vars;
+        vars->fileExtents = extentsData;
+
+        // make imagePath
+        OSData * data;
+        if (imagePath)
+        {

 #if defined(__i386__) || defined(__x86_64__)
 #if defined(__i386__) || defined(__x86_64__)

-           char str2[24 + sizeof(uuid_string_t) + 2];
+            char str2[24 + sizeof(uuid_string_t) + 2];

 
             if (keyUUID)
 
             if (keyUUID)

-                snprintf(str2, sizeof(str2), "%qx:%s", 
-                                vars->extentMap[0].start, keyUUID->getCStringNoCopy());
+                snprintf(str2, sizeof(str2), "%qx:%s",
+                         vars->extentMap[0].start, keyUUID->getCStringNoCopy());

             else
                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 
             else
                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 

-           err = IOService::getPlatform()->callPlatformFunction(
-                                               gIOCreateEFIDevicePathSymbol, false,
-                                               (void *) part, (void *) str2,
-                                               (void *) (uintptr_t) true, (void *) &data);
+            err = IOService::getPlatform()->callPlatformFunction(
+                                                                 gIOCreateEFIDevicePathSymbol, false,
+                                                                 (void *) part, (void *) str2,
+                                                                 (void *) (uintptr_t) true, (void *) &data);

 #else
 #else

-           data = 0;
-           err = kIOReturnSuccess;
+            data = 0;
+            err = kIOReturnSuccess;

 #endif
 #endif

-           if (kIOReturnSuccess != err)
-           {
-               HIBLOG("error 0x%x getting path\n", err);
-               break;
-           }
-           *imagePath = data;
-       }
+            if (kIOReturnSuccess != err)
+            {
+                HIBLOG("error 0x%x getting path\n", err);
+                break;
+            }
+            *imagePath = data;
+        }
+
+        // Release key UUID if we have one
+        if (keyUUID)
+        {
+            keyUUID->release();
+            keyUUID = NULL; // Just in case
+        }

     }
     while (false);
 
     if (kIOReturnSuccess != err)
     {
         HIBLOG("error 0x%x opening polled file\n", err);
     }
     while (false);
 
     if (kIOReturnSuccess != err)
     {
         HIBLOG("error 0x%x opening polled file\n", err);

-       IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
-       if (extentsData) extentsData->release();
+        IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
+        if (extentsData) extentsData->release();

     }
 
     if (part) part->release();
     }
 
     if (part) part->release();


@@ -816,7 +829,15 @@ IOPolledFileWrite(IOPolledFileIOVars * vars,
 
        if (bytes)
        {
 
        if (bytes)
        {

+#if KASAN
+         /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
+          * avoid triggering global redzone sanitizer violations when accessing
+          * interstices between 'C' structures
+          */
+           __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
+#else

            bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
            bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);

+#endif

            bytes += copy;
        }
         else
            bytes += copy;
        }
         else


@@ -955,7 +976,11 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
 
        if (bytes)
        {
 
        if (bytes)
        {

-           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
+#if KASAN
+         __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
+#else
+         bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
+#endif

            bytes += copy;
        }
        size -= copy;
            bytes += copy;
        }
        size -= copy;

diff --git a/iokit/Kernel/IORTC.cpp b/iokit/Kernel/IORTC.cpp
new file mode 100644 (file)
index 0000000..335bfc3
--- /dev/null
+++ b/iokit/Kernel/IORTC.cpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2017 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/rtc/IORTCController.h>
+
+#define super IOService
+
+OSDefineMetaClassAndAbstractStructors(IORTC, IOService);
+
+void IORTC::getUTCTimeOfDay( clock_sec_t * secs, clock_nsec_t * nsecs )
+{
+    *nsecs = 0;
+    *secs = getGMTTimeOfDay();
+}
+
+void IORTC::setUTCTimeOfDay( clock_sec_t secs, clock_nsec_t nsecs )
+{
+    setGMTTimeOfDay(secs);
+}
+
+IOReturn IORTC::getMonotonicClockOffset( int64_t * usecs )
+{
+    return kIOReturnUnsupported;
+}
+
+IOReturn IORTC::setMonotonicClockOffset( int64_t usecs )
+{
+    return kIOReturnUnsupported;
+}
+
+IOReturn IORTC::getMonotonicClockAndTimestamp( uint64_t * usecs, uint64_t *mach_absolute_time )
+{
+    return kIOReturnUnsupported;
+}
+
+OSMetaClassDefineReservedUnused(IORTC, 0);
+OSMetaClassDefineReservedUnused(IORTC, 1);
+OSMetaClassDefineReservedUnused(IORTC, 2);
+OSMetaClassDefineReservedUnused(IORTC, 3);
+OSMetaClassDefineReservedUnused(IORTC, 4);
+OSMetaClassDefineReservedUnused(IORTC, 5);
+OSMetaClassDefineReservedUnused(IORTC, 6);
+OSMetaClassDefineReservedUnused(IORTC, 7);

diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp
index 40055c5c9c5b43ce3963ca5c03838abe79273172..207c8dc29fa316f452a0ca3aed6e07662d89331f 100644 (file)
--- a/iokit/Kernel/IOService.cpp
+++ b/iokit/Kernel/IOService.cpp
@@ -190,6 +190,7 @@ static IOLock *                     gIOServiceBusyLock;
 bool                           gCPUsRunning;
 
 static thread_t                        gIOTerminateThread;
 bool                           gCPUsRunning;
 
 static thread_t                        gIOTerminateThread;

+static thread_t                        gIOTerminateWorkerThread;

 static UInt32                  gIOTerminateWork;
 static OSArray *               gIOTerminatePhase2List;
 static OSArray *               gIOStopList;
 static UInt32                  gIOTerminateWork;
 static OSArray *               gIOTerminatePhase2List;
 static OSArray *               gIOStopList;


@@ -448,6 +449,11 @@ void IOService::initialize( void )
     gIOStopProviderList    = OSArray::withCapacity( 16 );
     gIOFinalizeList       = OSArray::withCapacity( 16 );
     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
     gIOStopProviderList    = OSArray::withCapacity( 16 );
     gIOFinalizeList       = OSArray::withCapacity( 16 );
     assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );

+
+    // worker thread that is responsible for terminating / cleaning up threads
+    kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread);
+    assert(gIOTerminateWorkerThread);
+    thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread");

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */


@@ -2221,13 +2227,27 @@ void IOService::setTerminateDefer(IOService * provider, bool defer)
     }
 }
 
     }
 }
 

+// Must call this while holding gJobsLock
+void IOService::waitToBecomeTerminateThread(void)
+{
+    IOLockAssert(gJobsLock, kIOLockAssertOwned);
+    bool wait;
+    do {
+        wait = (gIOTerminateThread != THREAD_NULL);
+        if (wait) {
+            IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT);
+        }
+    } while (wait);
+    gIOTerminateThread = current_thread();
+}
+

 // call with lockForArbitration
 void IOService::scheduleTerminatePhase2( IOOptionBits options )
 {
     AbsoluteTime       deadline;
     uint64_t           regID1;
     int                        waitResult = THREAD_AWAKENED;
 // call with lockForArbitration
 void IOService::scheduleTerminatePhase2( IOOptionBits options )
 {
     AbsoluteTime       deadline;
     uint64_t           regID1;
     int                        waitResult = THREAD_AWAKENED;

-    bool               wait, haveDeadline = false;
+    bool               wait = false, haveDeadline = false;

 
     if (!(__state[0] & kIOServiceInactiveState)) return;
 
 
     if (!(__state[0] & kIOServiceInactiveState)) return;
 


@@ -2249,15 +2269,7 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options )
     if( (options & kIOServiceSynchronous)
         && (current_thread() != gIOTerminateThread)) {
 
     if( (options & kIOServiceSynchronous)
         && (current_thread() != gIOTerminateThread)) {
 

-        do {
-            wait = (gIOTerminateThread != 0);
-            if( wait) {
-                // wait to become the terminate thread
-                IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT);
-            }
-        } while( wait );
-
-        gIOTerminateThread = current_thread();
+        waitToBecomeTerminateThread();

         gIOTerminatePhase2List->setObject( this );
         gIOTerminateWork++;
 
         gIOTerminatePhase2List->setObject( this );
         gIOTerminateWork++;
 


@@ -2273,8 +2285,8 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options )
                 }
                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
                                                   deadline, THREAD_UNINT );
                 }
                 waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork,
                                                   deadline, THREAD_UNINT );

-                if( waitResult == THREAD_TIMED_OUT) {
-                    IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());
+                if(__improbable(waitResult == THREAD_TIMED_OUT)) {
+                    panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID());

                }
             }
         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
                }
             }
         } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));


@@ -2287,11 +2299,9 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options )
 
         gIOTerminatePhase2List->setObject( this );
         if( 0 == gIOTerminateWork++) {
 
         gIOTerminatePhase2List->setObject( this );
         if( 0 == gIOTerminateWork++) {

-           if( !gIOTerminateThread)
-               kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread);
-           else
-               IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
-       }
+            assert(gIOTerminateWorkerThread);
+            IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );
+        }

     }
 
     IOLockUnlock( gJobsLock );
     }
 
     IOLockUnlock( gJobsLock );


@@ -2299,18 +2309,23 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options )
     release();
 }
 
     release();
 }
 

+__attribute__((__noreturn__))

 void IOService::terminateThread( void * arg, wait_result_t waitResult )
 {
 void IOService::terminateThread( void * arg, wait_result_t waitResult )
 {

-    IOLockLock( gJobsLock );
-
-    while (gIOTerminateWork)
-       terminateWorker( (uintptr_t) arg );
+    // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
+    IOLockLock(gJobsLock);
+    while (true) {
+        if (gIOTerminateThread != gIOTerminateWorkerThread) {
+            waitToBecomeTerminateThread();
+        }

 
 

-    thread_deallocate(gIOTerminateThread);
-    gIOTerminateThread = 0;
-    IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
+        while (gIOTerminateWork)
+            terminateWorker( (uintptr_t)arg );

 
 

-    IOLockUnlock( gJobsLock );
+        gIOTerminateThread = 0;
+        IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false);
+        IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT);
+    }

 }
 
 void IOService::scheduleStop( IOService * provider )
 }
 
 void IOService::scheduleStop( IOService * provider )


@@ -2331,10 +2346,8 @@ void IOService::scheduleStop( IOService * provider )
     gIOStopProviderList->tailQ( provider );
 
     if( 0 == gIOTerminateWork++) {
     gIOStopProviderList->tailQ( provider );
 
     if( 0 == gIOTerminateWork++) {

-        if( !gIOTerminateThread)
-           kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
-        else
-            IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
+        assert(gIOTerminateWorkerThread);
+        IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );

     }
 
     IOLockUnlock( gJobsLock );
     }
 
     IOLockUnlock( gJobsLock );


@@ -2355,12 +2368,9 @@ void IOService::scheduleFinalize(bool now)
     {
        IOLockLock( gJobsLock );
        gIOFinalizeList->tailQ(this);
     {
        IOLockLock( gJobsLock );
        gIOFinalizeList->tailQ(this);

-       if( 0 == gIOTerminateWork++)
-       {
-           if( !gIOTerminateThread)
-               kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread);
-           else
-               IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false );
+       if( 0 == gIOTerminateWork++) {
+           assert(gIOTerminateWorkerThread);
+           IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false );

        }
        IOLockUnlock( gJobsLock );
     }
        }
        IOLockUnlock( gJobsLock );
     }


@@ -5086,6 +5096,7 @@ bool IOResources::matchPropertyTable( OSDictionary * table )
     OSString *         str;
     OSSet *            set;
     OSIterator *       iter;
     OSString *         str;
     OSSet *            set;
     OSIterator *       iter;

+    OSObject *          obj;

     OSArray *           keys;
     bool               ok = true;
 
     OSArray *           keys;
     bool               ok = true;
 


@@ -5106,14 +5117,15 @@ bool IOResources::matchPropertyTable( OSDictionary * table )
     }
     else if ((prop = table->getObject(gIOResourceMatchedKey)))
     {
     }
     else if ((prop = table->getObject(gIOResourceMatchedKey)))
     {

-        keys = (OSArray *) copyProperty(gIOResourceMatchedKey);
+        obj = copyProperty(gIOResourceMatchedKey);
+        keys = OSDynamicCast(OSArray, obj);

         ok = false;
         if (keys)
         {
             // assuming OSSymbol
             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));
         ok = false;
         if (keys)
         {
             // assuming OSSymbol
             ok = ((-1U) != keys->getNextIndexOfObject(prop, 0));

-            keys->release();

         }
         }

+        OSSafeReleaseNULL(obj);

     }
 
     return( ok );
     }
 
     return( ok );

diff --git a/iokit/Tests/Tests.cpp b/iokit/Tests/Tests.cpp
index 6336b67ac777db1810ba1ae3080da6b92476bffd..173cad67188d6e0f9f789e1677f8110f7901489d 100644 (file)
--- a/iokit/Tests/Tests.cpp
+++ b/iokit/Tests/Tests.cpp
@@ -236,6 +236,7 @@ sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused
        data->release();
     }
 
        data->release();
     }
 

+

     if (changed && newValue)
     {
        error = IOWorkLoopTest(newValue);
     if (changed && newValue)
     {
        error = IOWorkLoopTest(newValue);

diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp
index f68eebd1e7e951ea1be46616b3238fd191620090..890b89946a6b782048362e9110331c63a0a4dedf 100644 (file)
--- a/iokit/bsddev/IOKitBSDInit.cpp
+++ b/iokit/bsddev/IOKitBSDInit.cpp
@@ -60,10 +60,19 @@ extern void di_root_ramfile(IORegistryEntry * entry);
 
 #if CONFIG_EMBEDDED
 #define IOPOLLED_COREFILE      (CONFIG_KDP_INTERACTIVE_DEBUGGING)
 
 #if CONFIG_EMBEDDED
 #define IOPOLLED_COREFILE      (CONFIG_KDP_INTERACTIVE_DEBUGGING)

-#define kIOCoreDumpPath        "/private/var/vm/kernelcore"
-#define kIOCoreDumpSize                350ULL*1024ULL*1024ULL
+
+#if defined(XNU_TARGET_OS_BRIDGE)
+#define kIOCoreDumpSize         150ULL*1024ULL*1024ULL
+// leave free space on volume:
+#define kIOCoreDumpFreeSize     150ULL*1024ULL*1024ULL
+#define kIOCoreDumpPath         "/private/var/internal/kernelcore"
+#else
+#define kIOCoreDumpSize         350ULL*1024ULL*1024ULL

 // leave free space on volume:
 // leave free space on volume:

-#define kIOCoreDumpFreeSize    350ULL*1024ULL*1024ULL
+#define kIOCoreDumpFreeSize     350ULL*1024ULL*1024ULL
+#define kIOCoreDumpPath         "/private/var/vm/kernelcore"
+#endif
+

 #elif DEVELOPMENT
 #define IOPOLLED_COREFILE      1
 // no sizing
 #elif DEVELOPMENT
 #define IOPOLLED_COREFILE      1
 // no sizing


@@ -909,7 +918,13 @@ IOBSDMountChange(struct mount * mp, uint32_t op)
            vnode_put(vn);
            if (0 != result) break;
            if (!pathLen) break;
            vnode_put(vn);
            if (0 != result) break;
            if (!pathLen) break;

-           if (0 != bcmp(path, kIOCoreDumpPath, pathLen - 1)) break;
+#if defined(XNU_TARGET_OS_BRIDGE)
+           // on bridgeOS systems we put the core in /private/var/internal. We don't
+           // want to match with /private/var because /private/var/internal is often mounted
+           // over /private/var
+           if ((pathLen - 1) < (int) strlen("/private/var/internal")) break;
+#endif
+           if (0 != strncmp(path, kIOCoreDumpPath, pathLen - 1)) break;

            IOOpenPolledCoreFile(kIOCoreDumpPath);
            break;
 
            IOOpenPolledCoreFile(kIOCoreDumpPath);
            break;
 

diff --git a/iokit/conf/files b/iokit/conf/files
index 8dd87dd659d0f10bbfa40c4c248f2ede9c09bf8f..bc356a362345e5fdc75a4dc764f930a5057b7a7d 100644 (file)
--- a/iokit/conf/files
+++ b/iokit/conf/files
@@ -85,6 +85,7 @@ iokit/Kernel/IOStateReporter.cpp                      optional iokitcpp
 iokit/Kernel/IOHistogramReporter.cpp                   optional iokitcpp
 iokit/Kernel/IOReportLegend.cpp                                optional iokitcpp
 
 iokit/Kernel/IOHistogramReporter.cpp                   optional iokitcpp
 iokit/Kernel/IOReportLegend.cpp                                optional iokitcpp
 

+iokit/Kernel/IORTC.cpp                         optional iokitcpp

 
 iokit/Kernel/IOStringFuncs.c                           standard
 
 
 iokit/Kernel/IOStringFuncs.c                           standard
 

diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h
index 741c7f86460270b9548a0f7a4b9cf2e957ea9185..e358b4ec8f2dc4e8eeaacc8f7e70a05928e7a1e3 100644 (file)
--- a/libkdd/kcdata.h
+++ b/libkdd/kcdata.h
@@ -469,7 +469,7 @@ struct kcdata_type_definition {
 #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */
 #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO  0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */
 #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du       /* struct stackshot_thread_waitinfo */
 #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */
 #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO  0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */
 #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du       /* struct stackshot_thread_waitinfo */

-#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */
+#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot or thread_group_snapshot_v2 */

 #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu          /* uint64_t */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u      /* uint64_t */
 #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu          /* uint64_t */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u      /* uint64_t */


@@ -659,6 +659,17 @@ struct thread_group_snapshot {
        char tgs_name[16];
 } __attribute__((packed));
 
        char tgs_name[16];
 } __attribute__((packed));
 

+enum thread_group_flags {
+       kThreadGroupEfficient = 0x1,
+       kThreadGroupUIApp = 0x2
+};
+
+struct thread_group_snapshot_v2 {
+       uint64_t tgs_id;
+       char tgs_name[16];
+       uint64_t tgs_flags;
+} __attribute__((packed));
+

 enum coalition_flags {
        kCoalitionTermRequested = 0x1,
        kCoalitionTerminated    = 0x2,
 enum coalition_flags {
        kCoalitionTermRequested = 0x1,
        kCoalitionTerminated    = 0x2,

diff --git a/libkdd/kcdtypes.c b/libkdd/kcdtypes.c
index 8fe7aee238ce05d094c361623a528bec00669d0d..73cbeb91896232541bbf814fe1edc5067f958b99 100644 (file)
--- a/libkdd/kcdtypes.c
+++ b/libkdd/kcdtypes.c
@@ -574,8 +574,9 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s
 
        case STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT: {
                i = 0;
 
        case STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT: {
                i = 0;

-               _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot, tgs_id);
-               _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot, tgs_name, 16);
+               _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot_v2, tgs_id);
+               _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot_v2, tgs_name, 16);
+               _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot_v2, tgs_flags);

                setup_type_definition(retval, type_id, i, "thread_group_snapshot");
                break;
        }
                setup_type_definition(retval, type_id, i, "thread_group_snapshot");
                break;
        }

diff --git a/libkdd/kdd.xcodeproj/project.pbxproj b/libkdd/kdd.xcodeproj/project.pbxproj
index 5a2b63714e8425bad7297d98c7e422c2f5340e79..e36cc086ceba5d8773c3b8fe8e60f9e02e2e0c79 100644 (file)
--- a/libkdd/kdd.xcodeproj/project.pbxproj
+++ b/libkdd/kdd.xcodeproj/project.pbxproj
@@ -62,6 +62,8 @@
                13F3DA9E1C7C1C6600ACFFCC /* corpse-twr-sample-v2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */; };
                1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */; };
                1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */; };
                13F3DA9E1C7C1C6600ACFFCC /* corpse-twr-sample-v2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */; };
                1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */; };
                1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */; };

+               18C577C31F96DB5200C67EB3 /* stackshot-sample-thread-groups-flags in Resources */ = {isa = PBXBuildFile; fileRef = 18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */; };
+               18C577C61F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */; };

                18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */ = {isa = PBXBuildFile; fileRef = 18E592961E9451A20018612A /* stackshot-sample-coalitions */; };
                18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */; };
                C91C93CB1ACB58B700119B60 /* kdd.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93CA1ACB58B700119B60 /* kdd.h */; settings = {ATTRIBUTES = (Private, ); }; };
                18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */ = {isa = PBXBuildFile; fileRef = 18E592961E9451A20018612A /* stackshot-sample-coalitions */; };
                18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */; };
                C91C93CB1ACB58B700119B60 /* kdd.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93CA1ACB58B700119B60 /* kdd.h */; settings = {ATTRIBUTES = (Private, ); }; };


@@ -165,6 +167,8 @@
                13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "corpse-twr-sample-v2.plist.gz"; path = "tests/corpse-twr-sample-v2.plist.gz"; sourceTree = SOURCE_ROOT; };
                1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups"; sourceTree = "<group>"; };
                1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups.plist.gz"; sourceTree = "<group>"; };
                13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "corpse-twr-sample-v2.plist.gz"; path = "tests/corpse-twr-sample-v2.plist.gz"; sourceTree = SOURCE_ROOT; };
                1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups"; sourceTree = "<group>"; };
                1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups.plist.gz"; sourceTree = "<group>"; };

+               18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups-flags"; sourceTree = "<group>"; };
+               18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups-flags.plist.gz"; sourceTree = "<group>"; };

                18E592961E9451A20018612A /* stackshot-sample-coalitions */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-coalitions"; sourceTree = "<group>"; };
                18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-coalitions.plist.gz"; sourceTree = "<group>"; };
                C91C93C71ACB58B700119B60 /* libkdd.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libkdd.a; sourceTree = BUILT_PRODUCTS_DIR; };
                18E592961E9451A20018612A /* stackshot-sample-coalitions */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-coalitions"; sourceTree = "<group>"; };
                18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-coalitions.plist.gz"; sourceTree = "<group>"; };
                C91C93C71ACB58B700119B60 /* libkdd.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libkdd.a; sourceTree = BUILT_PRODUCTS_DIR; };


@@ -214,6 +218,8 @@
                08603F351BF69EDE007D3784 /* tests */ = {
                        isa = PBXGroup;
                        children = (
                08603F351BF69EDE007D3784 /* tests */ = {
                        isa = PBXGroup;
                        children = (

+                               18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */,
+                               18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */,

                                C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */,
                                C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */,
                                088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */,
                                C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */,
                                C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */,
                                088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */,


@@ -437,6 +443,8 @@
                        isa = PBXResourcesBuildPhase;
                        buildActionMask = 2147483647;
                        files = (
                        isa = PBXResourcesBuildPhase;
                        buildActionMask = 2147483647;
                        files = (

+                               18C577C61F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz in Resources */,
+                               18C577C31F96DB5200C67EB3 /* stackshot-sample-thread-groups-flags in Resources */,

                                C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */,
                                C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */,
                                088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */,
                                C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */,
                                C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */,
                                088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */,

diff --git a/libkdd/tests/Tests.swift b/libkdd/tests/Tests.swift
index cd7f46ea060194439d7bcb3edfc1ce9f34c3a95b..0886e025190619d47d88021b6c9a41703000f47d 100644 (file)
--- a/libkdd/tests/Tests.swift
+++ b/libkdd/tests/Tests.swift
@@ -1338,6 +1338,10 @@ class Tests: XCTestCase {
         self.testSampleStackshot("stackshot-sample-thread-groups")
     }
 
         self.testSampleStackshot("stackshot-sample-thread-groups")
     }
 

+    func testSampleThreadGroupsFlags() {
+        self.testSampleStackshot("stackshot-sample-thread-groups-flags")
+    }
+

     func testSampleCoalitions() {
         self.testSampleStackshot("stackshot-sample-coalitions")
     }
     func testSampleCoalitions() {
         self.testSampleStackshot("stackshot-sample-coalitions")
     }

diff --git a/libkdd/tests/stackshot-sample-thread-groups-flags b/libkdd/tests/stackshot-sample-thread-groups-flags
new file mode 100644 (file)
index 0000000..7cd85ba
Binary files /dev/null and b/libkdd/tests/stackshot-sample-thread-groups-flags differ

diff --git a/libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz b/libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz
new file mode 100644 (file)
index 0000000..7ca2eb5
Binary files /dev/null and b/libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz differ

diff --git a/libsyscall/Libsyscall.xcodeproj/project.pbxproj b/libsyscall/Libsyscall.xcodeproj/project.pbxproj
index 8175500a054a9a5cda881a628de9d5acebddc061..d0053b2daf82b67d8d7db497e7b86773581f061b 100644 (file)
--- a/libsyscall/Libsyscall.xcodeproj/project.pbxproj
+++ b/libsyscall/Libsyscall.xcodeproj/project.pbxproj
@@ -107,6 +107,7 @@
                29A59AE2183B0DE000E8B896 /* renameat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE1183B0DE000E8B896 /* renameat.c */; };
                29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE5183B110C00E8B896 /* unlinkat.c */; };
                2BA88DCC1810A3CE00EB63F6 /* coalition.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BA88DCB1810A3CE00EB63F6 /* coalition.c */; };
                29A59AE2183B0DE000E8B896 /* renameat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE1183B0DE000E8B896 /* renameat.c */; };
                29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE5183B110C00E8B896 /* unlinkat.c */; };
                2BA88DCC1810A3CE00EB63F6 /* coalition.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BA88DCB1810A3CE00EB63F6 /* coalition.c */; };

+               3695DCC91F3D2C5F0072C0B3 /* reboot.c in Sources */ = {isa = PBXBuildFile; fileRef = 3695DCC81F3D2C5A0072C0B3 /* reboot.c */; };

                374A36E314748F1300AAF39D /* varargs_wrappers.s in Sources */ = {isa = PBXBuildFile; fileRef = 374A36E214748EE400AAF39D /* varargs_wrappers.s */; };
                3F538F891A659C5600B37EFD /* persona.c in Sources */ = {isa = PBXBuildFile; fileRef = 3F538F881A659C5600B37EFD /* persona.c */; };
                401BB71A1BCAE57B005080D3 /* os_channel.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7161BCAE539005080D3 /* os_channel.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; };
                374A36E314748F1300AAF39D /* varargs_wrappers.s in Sources */ = {isa = PBXBuildFile; fileRef = 374A36E214748EE400AAF39D /* varargs_wrappers.s */; };
                3F538F891A659C5600B37EFD /* persona.c in Sources */ = {isa = PBXBuildFile; fileRef = 3F538F881A659C5600B37EFD /* persona.c */; };
                401BB71A1BCAE57B005080D3 /* os_channel.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7161BCAE539005080D3 /* os_channel.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; };


@@ -491,6 +492,7 @@
                29A59AE1183B0DE000E8B896 /* renameat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = renameat.c; sourceTree = "<group>"; };
                29A59AE5183B110C00E8B896 /* unlinkat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = unlinkat.c; sourceTree = "<group>"; };
                2BA88DCB1810A3CE00EB63F6 /* coalition.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = coalition.c; sourceTree = "<group>"; };
                29A59AE1183B0DE000E8B896 /* renameat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = renameat.c; sourceTree = "<group>"; };
                29A59AE5183B110C00E8B896 /* unlinkat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = unlinkat.c; sourceTree = "<group>"; };
                2BA88DCB1810A3CE00EB63F6 /* coalition.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = coalition.c; sourceTree = "<group>"; };

+               3695DCC81F3D2C5A0072C0B3 /* reboot.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = reboot.c; sourceTree = "<group>"; };

                374A36E214748EE400AAF39D /* varargs_wrappers.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = varargs_wrappers.s; sourceTree = "<group>"; };
                37DDFB7614748713009D3355 /* syscall.map */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = syscall.map; sourceTree = "<group>"; };
                3F538F881A659C5600B37EFD /* persona.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = persona.c; sourceTree = "<group>"; };
                374A36E214748EE400AAF39D /* varargs_wrappers.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = varargs_wrappers.s; sourceTree = "<group>"; };
                37DDFB7614748713009D3355 /* syscall.map */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = syscall.map; sourceTree = "<group>"; };
                3F538F881A659C5600B37EFD /* persona.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = persona.c; sourceTree = "<group>"; };


@@ -805,6 +807,7 @@
                                BA9973461C3B4C8A00B14D8C /* quota_obsolete.c */,
                                24B8C2611237F53900D36CC3 /* remove-counter.c */,
                                248AA966122C7CDA0085F5B1 /* rename.c */,
                                BA9973461C3B4C8A00B14D8C /* quota_obsolete.c */,
                                24B8C2611237F53900D36CC3 /* remove-counter.c */,
                                248AA966122C7CDA0085F5B1 /* rename.c */,

+                               3695DCC81F3D2C5A0072C0B3 /* reboot.c */,

                                29A59AE1183B0DE000E8B896 /* renameat.c */,
                                906AA2D018F74CD1001C681A /* renamex.c */,
                                248AA964122C7C330085F5B1 /* rmdir.c */,
                                29A59AE1183B0DE000E8B896 /* renameat.c */,
                                906AA2D018F74CD1001C681A /* renamex.c */,
                                248AA964122C7C330085F5B1 /* rmdir.c */,


@@ -1319,6 +1322,7 @@
                                C9D9BD4D114B00600000D8B9 /* processor.defs in Sources */,
                                C9D9BD53114B00600000D8B9 /* netname.defs in Sources */,
                                C9D9BD57114B00600000D8B9 /* task.defs in Sources */,
                                C9D9BD4D114B00600000D8B9 /* processor.defs in Sources */,
                                C9D9BD53114B00600000D8B9 /* netname.defs in Sources */,
                                C9D9BD57114B00600000D8B9 /* task.defs in Sources */,

+                               3695DCC91F3D2C5F0072C0B3 /* reboot.c in Sources */,

                                C9D9BD58114B00600000D8B9 /* thread_act.defs in Sources */,
                                72E09E941B444B19006F11A4 /* mach_continuous_time.c in Sources */,
                                29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */,
                                C9D9BD58114B00600000D8B9 /* thread_act.defs in Sources */,
                                72E09E941B444B19006F11A4 /* mach_continuous_time.c in Sources */,
                                29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */,

diff --git a/libsyscall/wrappers/libproc/libproc.h b/libsyscall/wrappers/libproc/libproc.h
index 3cfcfdc70af9a4a1cef449b025893957e3580999..ccf413fc3b913b62953f0c9eab4152608405ad28 100644 (file)
--- a/libsyscall/wrappers/libproc/libproc.h
+++ b/libsyscall/wrappers/libproc/libproc.h
@@ -127,6 +127,7 @@ int proc_clear_dirty(pid_t pid, uint32_t flags);
 int proc_terminate(pid_t pid, int *sig);
 
 #ifdef PRIVATE
 int proc_terminate(pid_t pid, int *sig);
 
 #ifdef PRIVATE

+#include <sys/event.h>

 /*
  * Enumerate potential userspace pointers embedded in kernel data structures.
  * Currently inspects kqueues only.
 /*
  * Enumerate potential userspace pointers embedded in kernel data structures.
  * Currently inspects kqueues only.

diff --git a/libsyscall/wrappers/mach_absolute_time.s b/libsyscall/wrappers/mach_absolute_time.s
index cf93161b9d8305c9ef9138aeff6f0dd1da0601fa..2c637bfc933740fb4174eaf6dbe650de6741a5bd 100644 (file)
--- a/libsyscall/wrappers/mach_absolute_time.s
+++ b/libsyscall/wrappers/mach_absolute_time.s
@@ -232,16 +232,13 @@ _mach_continuous_time_kernel:
  * read the register.  If the offset changes, we have gone to sleep in the midst of
  * doing a read.  This case should be exceedingly rare, but could result in a terribly
  * inaccurate result, so we need to get a fresh timebase value.
  * read the register.  If the offset changes, we have gone to sleep in the midst of
  * doing a read.  This case should be exceedingly rare, but could result in a terribly
  * inaccurate result, so we need to get a fresh timebase value.

- *
- * Note that the commpage address construction expects our top 2 bytes to be 0xFFFF.
- * If this changes (i.e, we significantly relocate the commpage), this logic will need
- * to change as well (use 4 movk instructions rather than cheating with the movn).

  */
        .text
        .align 2
        .globl _mach_absolute_time
 _mach_absolute_time:
  */
        .text
        .align 2
        .globl _mach_absolute_time
 _mach_absolute_time:

-       movn    x3, #(~((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32
+       movk    x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 48) & 0x000000000000FFFF), lsl #48
+       movk    x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32

        movk    x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x000000000000FFFF), lsl #16
        movk    x3, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x000000000000FFFF)
        ldrb    w2, [x3, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))]
        movk    x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x000000000000FFFF), lsl #16
        movk    x3, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x000000000000FFFF)
        ldrb    w2, [x3, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))]

diff --git a/libsyscall/wrappers/reboot.c b/libsyscall/wrappers/reboot.c
new file mode 100644 (file)
index 0000000..c703ed4
--- /dev/null
+++ b/libsyscall/wrappers/reboot.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2017 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#include <stdlib.h>
+
+extern int __reboot(int, const char *);
+
+int
+reboot(int howto)
+{
+       int res = __reboot(howto, NULL);
+       return res;
+}
+
+int
+reboot_np(int howto, const char *msg)
+{
+       int res = __reboot(howto, msg);
+       return res;
+}

diff --git a/libsyscall/wrappers/spawn/posix_spawn.c b/libsyscall/wrappers/spawn/posix_spawn.c
index 69002dc0b1c79166d93953666db49c0fbdc7fed0..1334f4c3d91f828c2a29df6bd3eed0f1e7f7ffa9 100644 (file)
--- a/libsyscall/wrappers/spawn/posix_spawn.c
+++ b/libsyscall/wrappers/spawn/posix_spawn.c
@@ -713,7 +713,6 @@ posix_spawn_growportactions_np(posix_spawnattr_t *attr)
 {
        _posix_spawnattr_t psattr;
        _posix_spawn_port_actions_t acts; 
 {
        _posix_spawnattr_t psattr;
        _posix_spawn_port_actions_t acts; 

-       int newnum;

 
        if (attr == NULL || *attr == NULL)
                return EINVAL;
 
        if (attr == NULL || *attr == NULL)
                return EINVAL;


@@ -724,8 +723,14 @@ posix_spawn_growportactions_np(posix_spawnattr_t *attr)
                return EINVAL;
        
        /* Double number of port actions allocated for */
                return EINVAL;
        
        /* Double number of port actions allocated for */

-       newnum = 2 * acts->pspa_alloc;
-       acts = realloc(acts, PS_PORT_ACTIONS_SIZE(newnum));
+       int newnum = 0;
+       if (os_mul_overflow(acts->pspa_alloc, 2, &newnum))
+               return ENOMEM;
+       size_t newsize = PS_PORT_ACTIONS_SIZE(newnum);
+       if (newsize == 0)
+               return ENOMEM;
+
+       acts = realloc(acts, newsize);

        if (acts == NULL)
                return ENOMEM;
        
        if (acts == NULL)
                return ENOMEM;
        


@@ -1031,8 +1036,13 @@ posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *file_actions)
 static int
 _posix_spawn_file_actions_grow(_posix_spawn_file_actions_t *psactsp)
 {
 static int
 _posix_spawn_file_actions_grow(_posix_spawn_file_actions_t *psactsp)
 {

-       int     new_alloc = (*psactsp)->psfa_act_alloc * 2;
-       _posix_spawn_file_actions_t new_psacts;
+       int newnum = 0;
+       if (os_mul_overflow((*psactsp)->psfa_act_alloc, 2, &newnum))
+               return ENOMEM;
+
+       size_t newsize = PSF_ACTIONS_SIZE(newnum);
+       if (newsize == 0)
+               return ENOMEM;

 
        /*
         * XXX may want to impose an administrative limit here; POSIX does
 
        /*
         * XXX may want to impose an administrative limit here; POSIX does


@@ -1040,13 +1050,14 @@ _posix_spawn_file_actions_grow(_posix_spawn_file_actions_t *psactsp)
         * XXX so it's probably acceptable to just fail catastrophically
         * XXX instead of implementing one.
         */
         * XXX so it's probably acceptable to just fail catastrophically
         * XXX instead of implementing one.
         */

-       if ((new_psacts = (_posix_spawn_file_actions_t)realloc((*psactsp), PSF_ACTIONS_SIZE(new_alloc))) == NULL) {
-               return (ENOMEM);
+       _posix_spawn_file_actions_t new_psacts;
+       if ((new_psacts = (_posix_spawn_file_actions_t)realloc((*psactsp), newsize)) == NULL) {
+               return ENOMEM;

        }
        }

-       new_psacts->psfa_act_alloc = new_alloc;
+       new_psacts->psfa_act_alloc = newnum;

        *psactsp = new_psacts;
 
        *psactsp = new_psacts;
 

-       return (0);
+       return 0;

 }
 
 
 }
 
 


@@ -1515,10 +1526,16 @@ posix_spawnattr_setmacpolicyinfo_np(posix_spawnattr_t * __restrict attr,
                psmx->psmx_count = 0;
        }
        else if (psmx->psmx_count == psmx->psmx_alloc) {
                psmx->psmx_count = 0;
        }
        else if (psmx->psmx_count == psmx->psmx_alloc) {

-               psmx = psattr->psa_mac_extensions = reallocf(psmx, PS_MAC_EXTENSIONS_SIZE(psmx->psmx_alloc * 2));
+               int newnum = 0;
+               if (os_mul_overflow(psmx->psmx_alloc, 2, &newnum))
+                       return ENOMEM;
+               size_t extsize = PS_MAC_EXTENSIONS_SIZE(newnum);
+               if (extsize == 0)
+                       return ENOMEM;
+               psmx = psattr->psa_mac_extensions = reallocf(psmx, extsize);

                if (psmx == NULL)
                        return ENOMEM;
                if (psmx == NULL)
                        return ENOMEM;

-               psmx->psmx_alloc *= 2;
+               psmx->psmx_alloc = newnum;

        }
        extension = &psmx->psmx_extensions[psmx->psmx_count];
        strlcpy(extension->policyname, policyname, sizeof(extension->policyname));
        }
        extension = &psmx->psmx_extensions[psmx->psmx_count];
        strlcpy(extension->policyname, policyname, sizeof(extension->policyname));


@@ -1801,14 +1818,24 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path,
                        ad.attrp = psattr;
 
                        if (psattr->psa_ports != NULL) {
                        ad.attrp = psattr;
 
                        if (psattr->psa_ports != NULL) {

+                               size_t psact_size = PS_PORT_ACTIONS_SIZE(psattr->psa_ports->pspa_count);
+                               if (psact_size == 0 && psattr->psa_ports->pspa_count != 0) {
+                                       errno = EINVAL;
+                                       ret = -1;
+                                       goto out;
+                               }

                                ad.port_actions = psattr->psa_ports;
                                ad.port_actions = psattr->psa_ports;

-                               ad.port_actions_size = PS_PORT_ACTIONS_SIZE(
-                                               ad.port_actions->pspa_count);
+                               ad.port_actions_size = psact_size;

                        }
                        if (psattr->psa_mac_extensions != NULL) {
                        }
                        if (psattr->psa_mac_extensions != NULL) {

+                               size_t macext_size = PS_MAC_EXTENSIONS_SIZE(psattr->psa_mac_extensions->psmx_count);
+                               if (macext_size == 0 && psattr->psa_mac_extensions->psmx_count != 0) {
+                                       errno = EINVAL;
+                                       ret = -1;
+                                       goto out;
+                               }

                                ad.mac_extensions = psattr->psa_mac_extensions;
                                ad.mac_extensions = psattr->psa_mac_extensions;

-                               ad.mac_extensions_size = PS_MAC_EXTENSIONS_SIZE(
-                                               ad.mac_extensions->psmx_count);
+                               ad.mac_extensions_size = macext_size;

                        }
                        if (psattr->psa_coalition_info != NULL) {
                                ad.coal_info_size = sizeof(struct _posix_spawn_coalition_info);
                        }
                        if (psattr->psa_coalition_info != NULL) {
                                ad.coal_info_size = sizeof(struct _posix_spawn_coalition_info);


@@ -1824,7 +1851,13 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path,
                                *(_posix_spawn_file_actions_t *)file_actions;
 
                        if (psactsp->psfa_act_count > 0) {
                                *(_posix_spawn_file_actions_t *)file_actions;
 
                        if (psactsp->psfa_act_count > 0) {

-                               ad.file_actions_size = PSF_ACTIONS_SIZE(psactsp->psfa_act_count);
+                               size_t fa_size = PSF_ACTIONS_SIZE(psactsp->psfa_act_count);
+                               if (fa_size == 0 && psactsp->psfa_act_count != 0) {
+                                       errno = EINVAL;
+                                       ret = -1;
+                                       goto out;
+                               }
+                               ad.file_actions_size = fa_size;

                                ad.file_actions = psactsp;
                        }
                }
                                ad.file_actions = psactsp;
                        }
                }


@@ -1833,6 +1866,7 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path,
        } else
                ret = __posix_spawn(pid, path, NULL, argv, envp);
 
        } else
                ret = __posix_spawn(pid, path, NULL, argv, envp);
 

+out:

        if (ret < 0)
                ret = errno;
        errno = saveerrno;
        if (ret < 0)
                ret = errno;
        errno = saveerrno;

diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def
index 4eca357c9657db9339d60fe5c9b39a6df6085347..8c49bd74b01b3c9d58a4c6b1771c4dfe1d6501c9 100644 (file)
--- a/makedefs/MakeInc.def
+++ b/makedefs/MakeInc.def
@@ -60,7 +60,7 @@ else ifeq ($(PLATFORM),tvOS)
 else ifeq ($(PLATFORM),AppleTVOS)
     DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION)
 else ifeq ($(PLATFORM),BridgeOS)
 else ifeq ($(PLATFORM),AppleTVOS)
     DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION)
 else ifeq ($(PLATFORM),BridgeOS)

-    DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION)
+    DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) -DXNU_TARGET_OS_BRIDGE

 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),)
     DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION)
 else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),)
 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),)
     DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION)
 else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),)


@@ -694,14 +694,6 @@ INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR)
 INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR)
 INSTALL_KERNEL_SYM_TO_KDK = 1
 INSTALL_XNU_DEBUG_FILES = 1
 INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR)
 INSTALL_KERNEL_SYM_TO_KDK = 1
 INSTALL_XNU_DEBUG_FILES = 1

-else ifeq ($(RC_ProjectName),xnu_kasan)
-ifeq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),)
-# MacOS
-INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR)
-INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR)
-endif
-INSTALL_KERNEL_SYM_TO_KDK = 1
-INSTALL_KASAN_ONLY = 1

 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),)
 INSTALL_KERNEL_SYM_TO_KDK = 1
 USE_BINARY_PLIST = 1
 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),)
 INSTALL_KERNEL_SYM_TO_KDK = 1
 USE_BINARY_PLIST = 1


@@ -713,4 +705,8 @@ INSTALL_KERNEL_SYM_DIR := $(SYSTEM_LIBRARY_KERNELS_DIR)
 INSTALL_KERNEL_SYM_TO_KDK = $(if $(filter YES,$(DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT)),1,0)
 endif
 
 INSTALL_KERNEL_SYM_TO_KDK = $(if $(filter YES,$(DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT)),1,0)
 endif
 

+ifeq ($(RC_ProjectName),xnu_kasan)
+INSTALL_KASAN_ONLY = 1
+endif
+

 # vim: set ft=make:
 # vim: set ft=make:

diff --git a/osfmk/arm/commpage/commpage.c b/osfmk/arm/commpage/commpage.c
index 520a140d01bea3438450677f6ed442ca0c95dd2d..430a149b71583411acb2497b47987a4527e04852 100644 (file)
--- a/osfmk/arm/commpage/commpage.c
+++ b/osfmk/arm/commpage/commpage.c
@@ -263,22 +263,14 @@ commpage_init_cpu_capabilities( void )
        bits |= (cpus << kNumCPUsShift);
 
        bits |= kFastThreadLocalStorage;        // TPIDRURO for TLS
        bits |= (cpus << kNumCPUsShift);
 
        bits |= kFastThreadLocalStorage;        // TPIDRURO for TLS

+

 #if    __ARM_VFP__
        bits |= kHasVfp;
 #if    __ARM_VFP__
        bits |= kHasVfp;

-#if    (__ARM_VFP__ >= 3)
-       bits |= kHasNeon;
-
-#if defined(__arm64__)
-       bits |= kHasNeonHPFP;
-#else
-       boolean_t intr = ml_set_interrupts_enabled(FALSE);
-       unsigned int mvfr1 = get_mvfr1();
-       
-       if (mvfr1 & MVFR_ASIMD_HPFP)
+       arm_mvfp_info_t *mvfp_info = arm_mvfp_info();
+       if (mvfp_info->neon)
+               bits |= kHasNeon;
+       if (mvfp_info->neon_hpfp)

                bits |= kHasNeonHPFP;
                bits |= kHasNeonHPFP;

-       (void) ml_set_interrupts_enabled(intr);
-#endif
-#endif

 #endif
 #if defined(__arm64__)
        bits |= kHasFMA;
 #endif
 #if defined(__arm64__)
        bits |= kHasFMA;

diff --git a/osfmk/arm/cpu_capabilities.h b/osfmk/arm/cpu_capabilities.h
index f1d16dde684f093c432311d4023225254b393eac..71050291c175e351dba420539dc8a8a719d1031d 100644 (file)
--- a/osfmk/arm/cpu_capabilities.h
+++ b/osfmk/arm/cpu_capabilities.h
@@ -100,8 +100,9 @@ typedef struct {
 
 #if defined(__arm64__)
 
 
 #if defined(__arm64__)
 

-#define _COMM_PAGE64_BASE_ADDRESS              (0xfffffff0001fc000ULL) /* Just below the kernel, safely in TTBR1 */
-#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_PAGE64_BASE_ADDRESS - (PAGE_SIZE))               /* Privileged RO in kernel mode */
+#define _COMM_PAGE64_BASE_ADDRESS              (0x0000000FFFFFC000ULL) /* In TTBR0 */
+#define _COMM_HIGH_PAGE64_BASE_ADDRESS (0xFFFFFFF0001FC000ULL) /* Just below the kernel, safely in TTBR1; only used for testing */
+#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_HIGH_PAGE64_BASE_ADDRESS - (PAGE_SIZE))          /* Privileged RO in kernel mode */

 
 #define _COMM_PAGE64_AREA_LENGTH               (_COMM_PAGE32_AREA_LENGTH)
 #define _COMM_PAGE64_AREA_USED                 (-1)
 
 #define _COMM_PAGE64_AREA_LENGTH               (_COMM_PAGE32_AREA_LENGTH)
 #define _COMM_PAGE64_AREA_USED                 (-1)


@@ -117,12 +118,12 @@ extern vm_address_t                                               sharedpage_rw_addr;
 
 #define        _COMM_PAGE_BASE_ADDRESS                 (sharedpage_rw_addr)
 #define _COMM_PAGE_START_ADDRESS               (sharedpage_rw_addr)
 
 #define        _COMM_PAGE_BASE_ADDRESS                 (sharedpage_rw_addr)
 #define _COMM_PAGE_START_ADDRESS               (sharedpage_rw_addr)

-#else
+#else /* KERNEL_PRIVATE */

 #define        _COMM_PAGE_AREA_LENGTH                  (4096)
 
 #define        _COMM_PAGE_BASE_ADDRESS                 _COMM_PAGE64_BASE_ADDRESS
 #define _COMM_PAGE_START_ADDRESS               _COMM_PAGE64_BASE_ADDRESS
 #define        _COMM_PAGE_AREA_LENGTH                  (4096)
 
 #define        _COMM_PAGE_BASE_ADDRESS                 _COMM_PAGE64_BASE_ADDRESS
 #define _COMM_PAGE_START_ADDRESS               _COMM_PAGE64_BASE_ADDRESS

-#endif
+#endif /* KERNEL_PRIVATE */

 
 #elif defined(__arm__)
 
 
 #elif defined(__arm__)
 

diff --git a/osfmk/arm/cpu_data_internal.h b/osfmk/arm/cpu_data_internal.h
index 264e7ed96065892975e8f16202984d4dfd52c7c5..ecef9f9390109391fb829fd136d32b8b3a20d168 100644 (file)
--- a/osfmk/arm/cpu_data_internal.h
+++ b/osfmk/arm/cpu_data_internal.h
@@ -188,9 +188,9 @@ typedef struct cpu_data
        uint64_t                                cpu_idle_latency;
        uint64_t                                cpu_idle_pop;
 
        uint64_t                                cpu_idle_latency;
        uint64_t                                cpu_idle_pop;
 

-#if    __arm__
+#if    __arm__ || __ARM_KERNEL_PROTECT__

        vm_offset_t                             cpu_exc_vectors;
        vm_offset_t                             cpu_exc_vectors;

-#endif
+#endif /* __ARM_KERNEL_PROTECT__ */

        vm_offset_t                             cpu_reset_handler;
        uint32_t                                cpu_reset_type;
        uintptr_t                               cpu_reset_assist;
        vm_offset_t                             cpu_reset_handler;
        uint32_t                                cpu_reset_type;
        uintptr_t                               cpu_reset_assist;

diff --git a/osfmk/arm/machine_cpuid.c b/osfmk/arm/machine_cpuid.c
index 232c6e64a39d7a2f96858150c439ab7a779a02e4..3943f47d4c862bdcdeec3560db0041fee5865b67 100644 (file)
--- a/osfmk/arm/machine_cpuid.c
+++ b/osfmk/arm/machine_cpuid.c
@@ -135,24 +135,20 @@ machine_arm_debug_info(void)
 void
 machine_do_mvfpid()
 {
 void
 machine_do_mvfpid()
 {

+#if __arm__

        arm_mvfr0_info_t        arm_mvfr0_info;
        arm_mvfr1_info_t        arm_mvfr1_info;
        arm_mvfr0_info_t        arm_mvfr0_info;
        arm_mvfr1_info_t        arm_mvfr1_info;

-       uint64_t                tmp;

 
 

-#if __arm__
-       (void)tmp;

        __asm__ volatile("vmrs  %0, mvfr0":"=r"(arm_mvfr0_info.value));
        __asm__ volatile("vmrs  %0, mvfr1":"=r"(arm_mvfr1_info.value));
        __asm__ volatile("vmrs  %0, mvfr0":"=r"(arm_mvfr0_info.value));
        __asm__ volatile("vmrs  %0, mvfr1":"=r"(arm_mvfr1_info.value));

-#else
-       __asm__ volatile("mrs   %0, MVFR0_EL1":"=r"(tmp));
-       arm_mvfr0_info.value = (uint32_t)tmp;
-
-       __asm__ volatile("mrs   %0, MVFR1_EL1":"=r"(tmp));
-       arm_mvfr1_info.value = (uint32_t)tmp;
-#endif

 
        cpuid_mvfp_info.neon = arm_mvfr1_info.bits.SP;
        cpuid_mvfp_info.neon_hpfp = arm_mvfr1_info.bits.HPFP;
 
        cpuid_mvfp_info.neon = arm_mvfr1_info.bits.SP;
        cpuid_mvfp_info.neon_hpfp = arm_mvfr1_info.bits.HPFP;

+#else
+       cpuid_mvfp_info.neon = 1;
+       cpuid_mvfp_info.neon_hpfp = 1;
+#endif
+

 }
 
 arm_mvfp_info_t *
 }
 
 arm_mvfp_info_t *

diff --git a/osfmk/arm/machine_routines.h b/osfmk/arm/machine_routines.h
index 3510649b4012eb3f4bbf5a262c19c256725ec77e..3e46c61e187fd3c466bf69c49273481c600658f4 100644 (file)
--- a/osfmk/arm/machine_routines.h
+++ b/osfmk/arm/machine_routines.h
@@ -529,10 +529,6 @@ void               set_fpscr(uint32_t);
 
 extern void            init_vfp(void);
 extern boolean_t       get_vfp_enabled(void);
 
 extern void            init_vfp(void);
 extern boolean_t       get_vfp_enabled(void);

-#if     (__ARM_VFP__ >= 3)
-extern unsigned int    get_mvfr0(void);
-extern unsigned int    get_mvfr1(void);
-#endif

 extern void            arm_debug_set_cp14(arm_debug_state_t *debug_state);
 extern void            fiq_context_init(boolean_t enable_fiq);
 
 extern void            arm_debug_set_cp14(arm_debug_state_t *debug_state);
 extern void            fiq_context_init(boolean_t enable_fiq);
 


@@ -551,6 +547,9 @@ void rorgn_stash_range(void);
 void rorgn_lockdown(void);
 #endif /* defined(KERNEL_INTEGRITY_KTRR)*/
 
 void rorgn_lockdown(void);
 #endif /* defined(KERNEL_INTEGRITY_KTRR)*/
 

+#if __ARM_KERNEL_PROTECT__
+extern void set_vbar_el1(uint64_t);
+#endif /* __ARM_KERNEL_PROTECT__ */

 #endif /* MACH_KERNEL_PRIVATE */
 
 extern uint32_t        arm_debug_read_dscr(void);
 #endif /* MACH_KERNEL_PRIVATE */
 
 extern uint32_t        arm_debug_read_dscr(void);

diff --git a/osfmk/arm/machine_routines_asm.s b/osfmk/arm/machine_routines_asm.s
index b64d28373056f96a964e0e91cb4b28a3cb73be10..d0b0a6b96c82e2440e90e7ab4549615799c7d57c 100644 (file)
--- a/osfmk/arm/machine_routines_asm.s
+++ b/osfmk/arm/machine_routines_asm.s
@@ -177,18 +177,6 @@ LEXT(set_fpscr)
 #endif
        bx      lr
 
 #endif
        bx      lr
 

-#if    (__ARM_VFP__ >= 3)
-       .align  2
-       .globl  EXT(get_mvfr0)
-LEXT(get_mvfr0)
-       vmrs    r0, mvfr0
-       bx              lr
-       .globl  EXT(get_mvfr1)
-LEXT(get_mvfr1)
-       vmrs    r0, mvfr1
-       bx              lr
-#endif
-

 /*
  *     void OSSynchronizeIO(void)
  */
 /*
  *     void OSSynchronizeIO(void)
  */

diff --git a/osfmk/arm/model_dep.c b/osfmk/arm/model_dep.c
index ab930d549eabf449726ef891c5a679bd308df194..ad9fd23ea9686892bedee1776341bef98115012c 100644 (file)
--- a/osfmk/arm/model_dep.c
+++ b/osfmk/arm/model_dep.c
@@ -92,7 +92,7 @@ extern int            kdp_stack_snapshot_bytes_traced(void);
  * Increment the PANICLOG_VERSION if you change the format of the panic
  * log in any way.
  */
  * Increment the PANICLOG_VERSION if you change the format of the panic
  * log in any way.
  */

-#define PANICLOG_VERSION 8
+#define PANICLOG_VERSION 9

 static struct kcdata_descriptor kc_panic_data;
 
 extern char                 firmware_version[];
 static struct kcdata_descriptor kc_panic_data;
 
 extern char                 firmware_version[];


@@ -147,6 +147,14 @@ unsigned int          debug_ack_timeout_count = 0;
 volatile unsigned int debugger_sync = 0;
 volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */
 unsigned int          DebugContextCount = 0;
 volatile unsigned int debugger_sync = 0;
 volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */
 unsigned int          DebugContextCount = 0;

+
+#if defined(__arm64__)
+uint8_t PE_smc_stashed_x86_system_state = 0xFF;
+uint8_t PE_smc_stashed_x86_power_state = 0xFF;
+uint8_t PE_smc_stashed_x86_efi_boot_state = 0xFF;
+uint32_t PE_pcie_stashed_link_state = UINT32_MAX;
+#endif
+

  
 // Convenient macros to easily validate one or more pointers if 
 // they have defined types
  
 // Convenient macros to easily validate one or more pointers if 
 // they have defined types


@@ -319,6 +327,14 @@ do_print_all_backtraces(
                if (last_hwaccess_thread) {
                        paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread);
                }
                if (last_hwaccess_thread) {
                        paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread);
                }

+#if defined(XNU_TARGET_OS_BRIDGE)
+               paniclog_append_noflush("PCIeUp link state: ");
+               if (PE_pcie_stashed_link_state != UINT32_MAX) {
+                       paniclog_append_noflush("0x%x", PE_pcie_stashed_link_state);
+               } else {
+                       paniclog_append_noflush("not available\n");
+               }
+#endif

        }
        paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID);
        paniclog_append_noflush("OS version: %.256s\n",
        }
        paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID);
        paniclog_append_noflush("OS version: %.256s\n",


@@ -332,6 +348,26 @@ do_print_all_backtraces(
 
        paniclog_append_noflush("iBoot version: %.128s\n", firmware_version);
        paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES");
 
        paniclog_append_noflush("iBoot version: %.128s\n", firmware_version);
        paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES");

+#if defined(XNU_TARGET_OS_BRIDGE)
+       paniclog_append_noflush("x86 EFI Boot State: ");
+       if (PE_smc_stashed_x86_efi_boot_state != 0xFF) {
+               paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_efi_boot_state);
+       } else {
+               paniclog_append_noflush("not available\n");
+       }
+       paniclog_append_noflush("x86 System State: ");
+       if (PE_smc_stashed_x86_system_state != 0xFF) {
+               paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_system_state);
+       } else {
+               paniclog_append_noflush("not available\n");
+       }
+       paniclog_append_noflush("x86 Power State: ");
+       if (PE_smc_stashed_x86_power_state != 0xFF) {
+               paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_power_state);
+       } else {
+               paniclog_append_noflush("not available\n");
+       }
+#endif

        paniclog_append_noflush("Paniclog version: %d\n", logversion);
 
        panic_display_kernel_aslr();
        paniclog_append_noflush("Paniclog version: %d\n", logversion);
 
        panic_display_kernel_aslr();


@@ -574,6 +610,12 @@ SavePanicInfo(
                panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC;
        }
 
                panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC;
        }
 

+#if defined(XNU_TARGET_OS_BRIDGE)
+       panic_info->eph_x86_power_state = PE_smc_stashed_x86_power_state;
+       panic_info->eph_x86_efi_boot_state = PE_smc_stashed_x86_efi_boot_state;
+       panic_info->eph_x86_system_state = PE_smc_stashed_x86_system_state;
+#endif
+

        /*
         * On newer targets, panic data is stored directly into the iBoot panic region.
         * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the
        /*
         * On newer targets, panic data is stored directly into the iBoot panic region.
         * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the

diff --git a/osfmk/arm/pmap.c b/osfmk/arm/pmap.c
index 327ac9b9f139d3d507b0afacffa2e4c7e0ba2089..990ab667d14fe5c595a0b8ad17d5e153b7da8b1f 100644 (file)
--- a/osfmk/arm/pmap.c
+++ b/osfmk/arm/pmap.c
@@ -88,6 +88,17 @@
 
 #include <san/kasan.h>
 
 
 #include <san/kasan.h>
 

+#if MACH_ASSERT
+int pmap_stats_assert = 1;
+#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...)                   \
+       MACRO_BEGIN                                         \
+       if (pmap_stats_assert && (pmap)->pmap_stats_assert) \
+               assertf(cond, fmt, ##__VA_ARGS__);                  \
+       MACRO_END
+#else /* MACH_ASSERT */
+#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...)
+#endif /* MACH_ASSERT */
+

 #if DEVELOPMENT || DEBUG
 #define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended)
 #else /* DEVELOPMENT || DEBUG */
 #if DEVELOPMENT || DEBUG
 #define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended)
 #else /* DEVELOPMENT || DEBUG */


@@ -366,7 +377,7 @@ SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0;
 static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA;
 
 #if    (__ARM_VMSA__ > 7)
 static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA;
 
 #if    (__ARM_VMSA__ > 7)

-SECURITY_READ_ONLY_LATE(pmap_t) u32_sharedpage_pmap;
+SECURITY_READ_ONLY_LATE(pmap_t) sharedpage_pmap;

 #endif
 
 
 #endif
 
 


@@ -975,7 +986,7 @@ static inline tt_entry_t *pmap_tt2e(
 static inline pt_entry_t *pmap_tt3e(
                                pmap_t, vm_map_address_t);
 
 static inline pt_entry_t *pmap_tt3e(
                                pmap_t, vm_map_address_t);
 

-static void pmap_unmap_sharedpage32(
+static void pmap_unmap_sharedpage(

                                pmap_t pmap);
 
 static void pmap_sharedpage_flush_32_to_64(
                                pmap_t pmap);
 
 static void pmap_sharedpage_flush_32_to_64(


@@ -1605,11 +1616,33 @@ alloc_asid(
                if (temp > 0) {
                        temp -= 1;
                        asid_bitmap[asid_bitmap_index] &= ~(1 << temp);
                if (temp > 0) {
                        temp -= 1;
                        asid_bitmap[asid_bitmap_index] &= ~(1 << temp);

+#if __ARM_KERNEL_PROTECT__
+                       /*
+                        * We need two ASIDs: n and (n | 1).  n is used for EL0,
+                        * (n | 1) for EL1.
+                        */
+                       unsigned int temp2 = temp | 1;
+                       assert(temp2 < MAX_ASID);
+                       assert(temp2 < 32);
+                       assert(temp2 != temp);
+                       assert(asid_bitmap[asid_bitmap_index] & (1 << temp2));
+
+                       /* Grab the second ASID. */
+                       asid_bitmap[asid_bitmap_index] &= ~(1 << temp2);
+#endif /* __ARM_KERNEL_PROTECT__ */

                        simple_unlock(&pmaps_lock);
 
                        simple_unlock(&pmaps_lock);
 

-                       /* We should never vend out physical ASID 0 through this method. */
+                       /*
+                        * We should never vend out physical ASID 0 through this
+                        * method, as it belongs to the kernel.
+                        */

                        assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0);
 
                        assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0);
 

+#if __ARM_KERNEL_PROTECT__
+                       /* Or the kernel EL1 ASID. */
+                       assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 1);
+#endif /* __ARM_KERNEL_PROTECT__ */
+

                        return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp);
                }
        }
                        return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp);
                }
        }


@@ -1631,6 +1664,13 @@ free_asid(
 
        simple_lock(&pmaps_lock);
        setbit(asid, (int *) asid_bitmap);
 
        simple_lock(&pmaps_lock);
        setbit(asid, (int *) asid_bitmap);

+
+#if __ARM_KERNEL_PROTECT__
+       assert((asid | 1) < MAX_ASID);
+       assert((asid | 1) != asid);
+       setbit(asid | 1, (int *) asid_bitmap);
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        simple_unlock(&pmaps_lock);
 }
 
        simple_unlock(&pmaps_lock);
 }
 


@@ -2510,6 +2550,9 @@ pmap_map_bd_with_options(
 
        tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) |
                  mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF;
 
        tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) |
                  mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF;

+#if __ARM_KERNEL_PROTECT__
+       tmplate |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

 
        vaddr = virt;
        paddr = start;
 
        vaddr = virt;
        paddr = start;


@@ -2554,8 +2597,11 @@ pmap_map_bd(
        /* not cacheable and not buffered */
        tmplate = pa_to_pte(start)
                  | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX
        /* not cacheable and not buffered */
        tmplate = pa_to_pte(start)
                  | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX

-                     | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)
+                 | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA)

                  | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);
                  | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE);

+#if __ARM_KERNEL_PROTECT__
+       tmplate |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

 
        vaddr = virt;
        paddr = start;
 
        vaddr = virt;
        paddr = start;


@@ -2644,6 +2690,9 @@ scan:
 #else
                pte |= ARM_PTE_SH;
 #endif
 #else
                pte |= ARM_PTE_SH;
 #endif

+#if __ARM_KERNEL_PROTECT__
+               pte |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

                WRITE_PTE(ptep, pte);
        }
        PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len);
                WRITE_PTE(ptep, pte);
        }
        PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len);


@@ -2898,6 +2947,11 @@ pmap_bootstrap(
         */
        for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) {
                asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY)));
         */
        for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) {
                asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY)));

+
+#if __ARM_KERNEL_PROTECT__
+               assert((i + 1) < MAX_ASID);
+               asid_bitmap[(i + 1) / (sizeof(uint32_t) * NBBY)] &= ~(1 << ((i + 1) % (sizeof(uint32_t) * NBBY)));
+#endif /* __ARM_KERNEL_PROTECT__ */

        }
 
        kernel_pmap->asid = 0;
        }
 
        kernel_pmap->asid = 0;


@@ -2928,6 +2982,12 @@ pmap_bootstrap(
        pmap_nesting_size_max = ARM_NESTING_SIZE_MAX;
 
        simple_lock_init(&phys_backup_lock, 0);
        pmap_nesting_size_max = ARM_NESTING_SIZE_MAX;
 
        simple_lock_init(&phys_backup_lock, 0);

+
+#if MACH_ASSERT
+       PE_parse_boot_argn("pmap_stats_assert",
+                          &pmap_stats_assert,
+                          sizeof (pmap_stats_assert));
+#endif /* MACH_ASSERT */

 }
 
 
 }
 
 


@@ -3231,6 +3291,7 @@ pmap_create_internal(
        p->nested_region_asid_bitmap_size = 0x0UL;
 
 #if MACH_ASSERT
        p->nested_region_asid_bitmap_size = 0x0UL;
 
 #if MACH_ASSERT

+       p->pmap_stats_assert = TRUE;

        p->pmap_pid = 0;
        strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
 #endif /* MACH_ASSERT */
        p->pmap_pid = 0;
        strlcpy(p->pmap_procname, "<nil>", sizeof (p->pmap_procname));
 #endif /* MACH_ASSERT */


@@ -3278,9 +3339,31 @@ pmap_set_process_internal(
 
        pmap->pmap_pid = pid;
        strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));
 
        pmap->pmap_pid = pid;
        strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname));

-#endif
+       if (!strncmp(procname, "corecaptured", sizeof (pmap->pmap_procname))) {
+               /*
+                * XXX FBDP
+                * "corecaptured" somehow triggers some issues that make
+                * the pmap stats and ledgers to go off track, causing
+                * some assertion failures and ledger panics.
+                * Turn that off if the terminating process is "corecaptured".
+                */
+               pmap->pmap_stats_assert = FALSE;
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.phys_footprint);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.internal);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.internal_compressed);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.iokit_mapped);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.alternate_accounting);
+               ledger_disable_panic_on_negative(pmap->ledger,
+                                                task_ledgers.alternate_accounting_compressed);
+       }
+#endif /* MACH_ASSERT */

 }
 }

-#endif
+#endif /* MACH_ASSERT*/

 
 #if MACH_ASSERT
 void
 
 #if MACH_ASSERT
 void


@@ -3513,8 +3596,7 @@ pmap_destroy_internal(
                return;
        }
 
                return;
        }
 

-       if (!pmap->is_64bit)
-               pmap_unmap_sharedpage32(pmap);
+       pmap_unmap_sharedpage(pmap);

 
        if (hw_atomic_sub(&pmap->ref_count, 1) == 0) {
 
 
        if (hw_atomic_sub(&pmap->ref_count, 1) == 0) {
 


@@ -3560,7 +3642,6 @@ pmap_destroy_internal(
 
 
                assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);
 
 
                assert((tt_free_entry_t*)pmap->tt_entry_free == NULL);

-

                flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT);
                free_asid(pmap->vasid);
 
                flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT);
                free_asid(pmap->vasid);
 


@@ -4238,7 +4319,8 @@ pmap_remove_range_options(
                /* sanity checks... */
 #if MACH_ASSERT
                if (pmap->stats.internal < num_internal) {
                /* sanity checks... */
 #if MACH_ASSERT
                if (pmap->stats.internal < num_internal) {

-                       if ((pmap->stats.internal + pmap->stats.reusable) ==
+                       if ((! pmap_stats_assert) ||
+                           (pmap->stats.internal + pmap->stats.reusable) ==

                            (num_internal + num_reusable)) {
                                num_reusable_mismatch++;
                                printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n",
                            (num_internal + num_reusable)) {
                                num_reusable_mismatch++;
                                printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n",


@@ -4282,23 +4364,27 @@ pmap_remove_range_options(
                        }
                }
 #endif /* MACH_ASSERT */
                        }
                }
 #endif /* MACH_ASSERT */

-               assertf(pmap->stats.external >= num_external,
-                       "pmap=%p num_external=%d stats.external=%d",
-                       pmap, num_external, pmap->stats.external);
-               assertf(pmap->stats.internal >= num_internal,
-                       "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
-                       pmap,
-                       num_internal, pmap->stats.internal,
-                       num_reusable, pmap->stats.reusable);
-               assertf(pmap->stats.reusable >= num_reusable,
-                       "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
-                       pmap,
-                       num_internal, pmap->stats.internal,
-                       num_reusable, pmap->stats.reusable);
-               assertf(pmap->stats.compressed >= num_compressed,
-                       "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld",
-                       pmap, num_compressed, num_alt_compressed,
-                       pmap->stats.compressed);
+               PMAP_STATS_ASSERTF(pmap->stats.external >= num_external,
+                                  pmap,
+                                  "pmap=%p num_external=%d stats.external=%d",
+                                  pmap, num_external, pmap->stats.external);
+               PMAP_STATS_ASSERTF(pmap->stats.internal >= num_internal,
+                                  pmap,
+                                  "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
+                                  pmap,
+                                  num_internal, pmap->stats.internal,
+                                  num_reusable, pmap->stats.reusable);
+               PMAP_STATS_ASSERTF(pmap->stats.reusable >= num_reusable,
+                                  pmap,
+                                  "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d",
+                                  pmap,
+                                  num_internal, pmap->stats.internal,
+                                  num_reusable, pmap->stats.reusable);
+               PMAP_STATS_ASSERTF(pmap->stats.compressed >= num_compressed,
+                                  pmap,
+                                  "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld",
+                                  pmap, num_compressed, num_alt_compressed,
+                                  pmap->stats.compressed);

 
                /* update pmap stats... */
                OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);
 
                /* update pmap stats... */
                OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);


@@ -4493,6 +4579,9 @@ pmap_flush_core_tlb_asid(pmap_t pmap)
        flush_core_tlb_asid(pmap->asid);
 #else
        flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT);
        flush_core_tlb_asid(pmap->asid);
 #else
        flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT);

+#if __ARM_KERNEL_PROTECT__
+       flush_core_tlb_asid(((uint64_t) pmap->asid + 1) << TLBI_ASID_SHIFT);
+#endif /* __ARM_KERNEL_PROTECT__ */

 #endif
 }
 
 #endif
 }
 


@@ -4758,13 +4847,13 @@ pmap_page_protect_options_internal(
                                if (IS_REUSABLE_PAGE(pai) &&
                                    IS_INTERNAL_PAGE(pai) &&
                                    !is_altacct) {
                                if (IS_REUSABLE_PAGE(pai) &&
                                    IS_INTERNAL_PAGE(pai) &&
                                    !is_altacct) {

-                                       assert(pmap->stats.reusable > 0);
+                                       PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);

                                        OSAddAtomic(-1, &pmap->stats.reusable);
                                } else if (IS_INTERNAL_PAGE(pai)) {
                                        OSAddAtomic(-1, &pmap->stats.reusable);
                                } else if (IS_INTERNAL_PAGE(pai)) {

-                                       assert(pmap->stats.internal > 0);
+                                       PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);

                                        OSAddAtomic(-1, &pmap->stats.internal);
                                } else {
                                        OSAddAtomic(-1, &pmap->stats.internal);
                                } else {

-                                       assert(pmap->stats.external > 0);
+                                       PMAP_STATS_ASSERTF(pmap->stats.external > 0, pmap, "stats.external %d", pmap->stats.external);

                                        OSAddAtomic(-1, &pmap->stats.external);
                                }
                                if ((options & PMAP_OPTIONS_COMPRESSOR) &&
                                        OSAddAtomic(-1, &pmap->stats.external);
                                }
                                if ((options & PMAP_OPTIONS_COMPRESSOR) &&


@@ -5579,6 +5668,9 @@ Pmap_enter_retry:
        pte |= wimg_to_pte(wimg_bits);
 
        if (pmap == kernel_pmap) {
        pte |= wimg_to_pte(wimg_bits);
 
        if (pmap == kernel_pmap) {

+#if __ARM_KERNEL_PROTECT__
+               pte |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

                if (prot & VM_PROT_WRITE) {
                        pte |= ARM_PTE_AP(AP_RWNA);
                        pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED);
                if (prot & VM_PROT_WRITE) {
                        pte |= ARM_PTE_AP(AP_RWNA);
                        pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED);


@@ -5960,7 +6052,7 @@ pmap_change_wiring_internal(
                OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
                pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
        } else if (!wired && pte_is_wired(*pte_p)) {
                OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
                pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
        } else if (!wired && pte_is_wired(*pte_p)) {

-                assert(pmap->stats.wired_count >= 1);
+                PMAP_STATS_ASSERTF(pmap->stats.wired_count >= 1, pmap, "stats.wired_count %d", pmap->stats.wired_count);

                pte_set_wired(pte_p, wired);
                OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
                pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);
                pte_set_wired(pte_p, wired);
                OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
                pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE);


@@ -7294,12 +7386,12 @@ arm_force_fast_fault_internal(
                           is_internal &&
                           pmap != kernel_pmap) {
                        /* one less "reusable" */
                           is_internal &&
                           pmap != kernel_pmap) {
                        /* one less "reusable" */

-                       assert(pmap->stats.reusable > 0);
+                       PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);

                        OSAddAtomic(-1, &pmap->stats.reusable);
                        /* one more "internal" */
                        OSAddAtomic(+1, &pmap->stats.internal);
                        PMAP_STATS_PEAK(pmap->stats.internal);
                        OSAddAtomic(-1, &pmap->stats.reusable);
                        /* one more "internal" */
                        OSAddAtomic(+1, &pmap->stats.internal);
                        PMAP_STATS_PEAK(pmap->stats.internal);

-                       assert(pmap->stats.internal > 0);
+                       PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);

                        pmap_ledger_credit(pmap,
                                           task_ledgers.internal,
                                           machine_ptob(1));
                        pmap_ledger_credit(pmap,
                                           task_ledgers.internal,
                                           machine_ptob(1));


@@ -7325,9 +7417,9 @@ arm_force_fast_fault_internal(
                        /* one more "reusable" */
                        OSAddAtomic(+1, &pmap->stats.reusable);
                        PMAP_STATS_PEAK(pmap->stats.reusable);
                        /* one more "reusable" */
                        OSAddAtomic(+1, &pmap->stats.reusable);
                        PMAP_STATS_PEAK(pmap->stats.reusable);

-                       assert(pmap->stats.reusable > 0);
+                       PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable);

                        /* one less "internal" */
                        /* one less "internal" */

-                       assert(pmap->stats.internal > 0);
+                       PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal);

                        OSAddAtomic(-1, &pmap->stats.internal);
                        pmap_ledger_debit(pmap,
                                          task_ledgers.internal,
                        OSAddAtomic(-1, &pmap->stats.internal);
                        pmap_ledger_debit(pmap,
                                          task_ledgers.internal,


@@ -7696,6 +7788,9 @@ pmap_map_globals(
        assert(*ptep == ARM_PTE_EMPTY);
 
        pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE;
        assert(*ptep == ARM_PTE_EMPTY);
 
        pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE;

+#if __ARM_KERNEL_PROTECT__
+       pte |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

        pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
 #if    (__ARM_VMSA__ > 7)
        pte |= ARM_PTE_SH(SH_OUTER_MEMORY);
        pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
 #if    (__ARM_VMSA__ > 7)
        pte |= ARM_PTE_SH(SH_OUTER_MEMORY);


@@ -7738,6 +7833,9 @@ pmap_map_cpu_windows_copy_internal(
        }
 
        pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX;
        }
 
        pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX;

+#if __ARM_KERNEL_PROTECT__
+       pte |= ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

 
        pte |= wimg_to_pte(wimg_bits);
 
 
        pte |= wimg_to_pte(wimg_bits);
 


@@ -8606,7 +8704,7 @@ pmap_update_cache_attributes_locked(
                va = ptep_get_va(pte_p);
 
                tmplate = *pte_p;
                va = ptep_get_va(pte_p);
 
                tmplate = *pte_p;

-               tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_SHMASK);
+               tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK);

                tmplate |= wimg_to_pte(attributes);
 
                WRITE_PTE(pte_p, tmplate);
                tmplate |= wimg_to_pte(attributes);
 
                WRITE_PTE(pte_p, tmplate);


@@ -8670,20 +8768,24 @@ pmap_create_sharedpage(
        pmap_paddr_t    pa = 0;
 
 
        pmap_paddr_t    pa = 0;
 
 

-       kr = pmap_expand(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
-       assert(kr == KERN_SUCCESS);
-

        (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
 
        memset((char *) phystokv(pa), 0, PAGE_SIZE);
 
        /*
        (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0);
 
        memset((char *) phystokv(pa), 0, PAGE_SIZE);
 
        /*

-        * This is the mapping which U64 will refer to.
-        * Create with common path, update to be non-global and user-readable.
+        * The kernel pmap maintains a user accessible mapping of the commpage
+        * to test PAN.

         */
         */

-       kr = pmap_enter(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
+       kr = pmap_expand(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL);
+       assert(kr == KERN_SUCCESS);
+       kr = pmap_enter(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);

        assert(kr == KERN_SUCCESS);
        assert(kr == KERN_SUCCESS);

-       pmap_update_tt3e(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
+
+       /*
+        * This mapping should not be global (as we only expect to reference it
+        * during testing).
+        */
+       pmap_update_tt3e(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE | ARM_PTE_NG);

 
        /*
         * With PAN enabled kernel drivers can no longer use the previous mapping which is user readable
 
        /*
         * With PAN enabled kernel drivers can no longer use the previous mapping which is user readable


@@ -8695,31 +8797,50 @@ pmap_create_sharedpage(
        assert(kr == KERN_SUCCESS);
 
        /*
        assert(kr == KERN_SUCCESS);
 
        /*

-        * Preferrable to just use a single entry,but we consume
-        * the full entry 0 of the L1 page table for U32 (i.e.
-        * for the 1GB of user address space), so we have no choice
-        * but to burn another L1 entry for the shared page... unless
-        * something clever comes to us.  For the short term (i.e.
-        * bringup) do a kind of forced nesting (we don't have a
-        * notion of nesting more than one pmap, and the shared cache
-        * wins).  In effect, just allocate a pmap, fill it out
-        * to include the one entry we care about, and then
-        * use its L1 pointer in U32 TTBR0 page tables.
+        * In order to avoid burning extra pages on mapping the shared page, we
+        * create a dedicated pmap for the shared page.  We forcibly nest the
+        * translation tables from this pmap into other pmaps.  The level we
+        * will nest at depends on the MMU configuration (page size, TTBR range,
+        * etc).
+        *
+        * Note that this is NOT "the nested pmap" (which is used to nest the
+        * shared cache).

         *
         *

-        * Note that we update parameters of entry for our unique
-        * needs (NG entry, etc.).
+        * Note that we update parameters of the entry for our unique needs (NG
+        * entry, etc.).

         */
         */

-       u32_sharedpage_pmap = pmap_create(NULL, 0x0, FALSE);
-       assert(u32_sharedpage_pmap != NULL);
-       kr = pmap_enter(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
+       sharedpage_pmap = pmap_create(NULL, 0x0, FALSE);
+       assert(sharedpage_pmap != NULL);
+
+       /* The user 64-bit mapping... */
+       kr = pmap_enter(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);
+       assert(kr == KERN_SUCCESS);
+       pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
+
+       /* ...and the user 32-bit mapping. */
+       kr = pmap_enter(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE);

        assert(kr == KERN_SUCCESS);
        assert(kr == KERN_SUCCESS);

-       pmap_update_tt3e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);
+       pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE);

 
        /* For manipulation in kernel, go straight to physical page */
        sharedpage_rw_addr = phystokv(pa);
        return((vm_map_address_t)sharedpage_rw_addr);
 }
 
 
        /* For manipulation in kernel, go straight to physical page */
        sharedpage_rw_addr = phystokv(pa);
        return((vm_map_address_t)sharedpage_rw_addr);
 }
 

+/*
+ * Asserts to ensure that the TTEs we nest to map the shared page do not overlap
+ * with user controlled TTEs.
+ */
+#if (ARM_PGSHIFT == 14) || __ARM64_TWO_LEVEL_PMAP__
+static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= MACH_VM_MAX_ADDRESS);
+static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= VM_MAX_ADDRESS);
+#elif (ARM_PGSHIFT == 12)
+static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= MACH_VM_MAX_ADDRESS);
+static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= VM_MAX_ADDRESS);
+#else
+#error Nested shared page mapping is unsupported on this config
+#endif
+

 static void
 pmap_insert_sharedpage_internal(
        pmap_t pmap)
 static void
 pmap_insert_sharedpage_internal(
        pmap_t pmap)


@@ -8727,14 +8848,16 @@ pmap_insert_sharedpage_internal(
 #if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__
        kern_return_t kr;
 #endif
 #if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__
        kern_return_t kr;
 #endif

+       vm_offset_t sharedpage_vaddr;

        pt_entry_t *ttep, *src_ttep;
 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
 #error We assume a single page.
 #endif
 
        if (pmap_is_64bit(pmap)) {
        pt_entry_t *ttep, *src_ttep;
 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
 #error We assume a single page.
 #endif
 
        if (pmap_is_64bit(pmap)) {

-               /* Already in kernel pmap */
-               return;
+               sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
+       } else {
+               sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;

        }
 
        PMAP_LOCK(pmap);
        }
 
        PMAP_LOCK(pmap);


@@ -8751,12 +8874,13 @@ pmap_insert_sharedpage_internal(
 #error A two level page table with a page shift of 12 is not currently supported
 #endif
        /* Just slam in the L1 entry.  */
 #error A two level page table with a page shift of 12 is not currently supported
 #endif
        /* Just slam in the L1 entry.  */

-       ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS);
+       ttep = pmap_tt1e(pmap, sharedpage_vaddr);
+

        if (*ttep != ARM_PTE_EMPTY) {
                panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
        }
 
        if (*ttep != ARM_PTE_EMPTY) {
                panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
        }
 

-       src_ttep = pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS);
+       src_ttep = pmap_tt1e(sharedpage_pmap, sharedpage_vaddr);

 #elif (ARM_PGSHIFT == 14)
 #if !__ARM64_TWO_LEVEL_PMAP__
        /* Allocate for the L2 entry if necessary, and slam it into place. */
 #elif (ARM_PGSHIFT == 14)
 #if !__ARM64_TWO_LEVEL_PMAP__
        /* Allocate for the L2 entry if necessary, and slam it into place. */


@@ -8764,7 +8888,7 @@ pmap_insert_sharedpage_internal(
         * As long as we are use a three level page table, the first level
         * should always exist, so we don't need to check for it.
         */
         * As long as we are use a three level page table, the first level
         * should always exist, so we don't need to check for it.
         */

-       while (*pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS) == ARM_PTE_EMPTY) {
+       while (*pmap_tt1e(pmap, sharedpage_vaddr) == ARM_PTE_EMPTY) {

                PMAP_UNLOCK(pmap);
 
                kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL);
                PMAP_UNLOCK(pmap);
 
                kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL);


@@ -8777,23 +8901,26 @@ pmap_insert_sharedpage_internal(
        }
 #endif
 
        }
 #endif
 

-       ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS);
+       ttep = pmap_tt2e(pmap, sharedpage_vaddr);
+

        if (*ttep != ARM_PTE_EMPTY) {
                panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
        }
 
        if (*ttep != ARM_PTE_EMPTY) {
                panic("%s: Found something mapped at the commpage address?!", __FUNCTION__);
        }
 

-       src_ttep = pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS);
+       src_ttep = pmap_tt2e(sharedpage_pmap, sharedpage_vaddr);

 #endif
 
        *ttep =  *src_ttep;
 #ifndef __ARM_L1_PTW__
        CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
 #endif
 #endif
 
        *ttep =  *src_ttep;
 #ifndef __ARM_L1_PTW__
        CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t));
 #endif

-       flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
+       /* TODO: Should we flush in the 64-bit case? */
+       flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE);
+

 #if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__
 #if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__

-       flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
+       flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));

 #elif (ARM_PGSHIFT == 14)
 #elif (ARM_PGSHIFT == 14)

-       flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
+       flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));

 #endif
 
        PMAP_UNLOCK(pmap);
 #endif
 
        PMAP_UNLOCK(pmap);


@@ -8807,50 +8934,59 @@ pmap_sharedpage_flush_32_to_64(
 }
 
 static void
 }
 
 static void

-pmap_unmap_sharedpage32(
+pmap_unmap_sharedpage(

        pmap_t pmap)
 {
        pt_entry_t *ttep;
        pmap_t pmap)
 {
        pt_entry_t *ttep;

+       vm_offset_t sharedpage_vaddr;

 
 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
 #error We assume a single page.
 #endif
 
 
 #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE
 #error We assume a single page.
 #endif
 

+       if (pmap_is_64bit(pmap)) {
+               sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS;
+       } else {
+               sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS;
+       }
+

 #if (ARM_PGSHIFT == 12)
 #if __ARM64_TWO_LEVEL_PMAP__
 #error A two level page table with a page shift of 12 is not currently supported
 #endif
 #if (ARM_PGSHIFT == 12)
 #if __ARM64_TWO_LEVEL_PMAP__
 #error A two level page table with a page shift of 12 is not currently supported
 #endif

-       ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS);
+       ttep = pmap_tt1e(pmap, sharedpage_vaddr);
+

        if (ttep == NULL) {
                return;
        }
 
        /* It had better be mapped to the shared page */
        if (ttep == NULL) {
                return;
        }
 
        /* It had better be mapped to the shared page */

-       if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) {
+       if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(sharedpage_pmap, sharedpage_vaddr)) {

                panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
        }
 #elif (ARM_PGSHIFT == 14)
                panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
        }
 #elif (ARM_PGSHIFT == 14)

-       ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS);
+       ttep = pmap_tt2e(pmap, sharedpage_vaddr);
+

        if (ttep == NULL) {
                return;
        }
 
        /* It had better be mapped to the shared page */
        if (ttep == NULL) {
                return;
        }
 
        /* It had better be mapped to the shared page */

-       if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) {
+       if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(sharedpage_pmap, sharedpage_vaddr)) {

                panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
        }
 #endif
 
        *ttep = ARM_TTE_EMPTY;
                panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__);
        }
 #endif
 
        *ttep = ARM_TTE_EMPTY;

-       flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE);
+       flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE);

 
 #if (ARM_PGSHIFT == 12)
 #if __ARM64_TWO_LEVEL_PMAP__
 #error A two level page table with a page shift of 12 is not currently supported
 #endif
 
 #if (ARM_PGSHIFT == 12)
 #if __ARM64_TWO_LEVEL_PMAP__
 #error A two level page table with a page shift of 12 is not currently supported
 #endif

-       flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));
+       flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid));

 #elif (ARM_PGSHIFT == 14)
 #elif (ARM_PGSHIFT == 14)

-       flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));
+       flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid));

 #endif
 }
 
 #endif
 }
 


@@ -9478,15 +9614,15 @@ pmap_check_ledgers(
                }
        }
 
                }
        }
 

-       assert(pmap->stats.resident_count == 0);
+       PMAP_STATS_ASSERTF(pmap->stats.resident_count == 0, pmap, "stats.resident_count %d", pmap->stats.resident_count);

 #if 00
 #if 00

-       assert(pmap->stats.wired_count == 0);
+       PMAP_STATS_ASSERTF(pmap->stats.wired_count == 0, pmap, "stats.wired_count %d", pmap->stats.wired_count);

 #endif
 #endif

-       assert(pmap->stats.device == 0);
-       assert(pmap->stats.internal == 0);
-       assert(pmap->stats.external == 0);
-       assert(pmap->stats.reusable == 0);
-       assert(pmap->stats.compressed == 0);
+       PMAP_STATS_ASSERTF(pmap->stats.device == 0, pmap, "stats.device %d", pmap->stats.device);
+       PMAP_STATS_ASSERTF(pmap->stats.internal == 0, pmap, "stats.internal %d", pmap->stats.internal);
+       PMAP_STATS_ASSERTF(pmap->stats.external == 0, pmap, "stats.external %d", pmap->stats.external);
+       PMAP_STATS_ASSERTF(pmap->stats.reusable == 0, pmap, "stats.reusable %d", pmap->stats.reusable);
+       PMAP_STATS_ASSERTF(pmap->stats.compressed == 0, pmap, "stats.compressed %lld", pmap->stats.compressed);

 }
 #endif /* MACH_ASSERT */
 
 }
 #endif /* MACH_ASSERT */
 

diff --git a/osfmk/arm/pmap.h b/osfmk/arm/pmap.h
index 75ed29d26d4a8e12fd94776fa58ad57c095869a0..7653401cf7e68e4fb8cdb6583a985adc67990e03 100644 (file)
--- a/osfmk/arm/pmap.h
+++ b/osfmk/arm/pmap.h
@@ -44,6 +44,42 @@
 
 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>
 
 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>

+
+#if __ARM_KERNEL_PROTECT__
+/*
+ * For __ARM_KERNEL_PROTECT__, we need twice as many ASIDs to support having
+ * unique EL0 and EL1 ASIDs for each pmap.
+ */
+#define ASID_SHIFT                     (12)                            /* Shift for the maximum virtual ASID value (2048)*/
+#else /* __ARM_KERNEL_PROTECT__ */
+#define ASID_SHIFT                     (11)                            /* Shift for the maximum virtual ASID value (2048) */
+#endif /* __ARM_KERNEL_PROTECT__ */
+#define MAX_ASID                       (1 << ASID_SHIFT)               /* Max supported ASIDs (can be virtual) */
+#define ARM_ASID_SHIFT                 (8)                             /* Shift for the maximum ARM ASID value (256) */
+#define ARM_MAX_ASID                   (1 << ARM_ASID_SHIFT)           /* Max ASIDs supported by the hardware */
+#define ASID_VIRT_BITS                 (ASID_SHIFT - ARM_ASID_SHIFT)   /* The number of virtual bits in a virtaul ASID */
+#define NBBY                           8
+
+struct pmap_cpu_data {
+       pmap_t cpu_user_pmap;
+       unsigned int cpu_number;
+       unsigned int cpu_user_pmap_stamp;
+
+       /*
+        * This supports overloading of ARM ASIDs by the pmap.  The field needs
+        * to be wide enough to cover all the virtual bits in a virtual ASID.
+        * With 256 physical ASIDs, 8-bit fields let us support up to 65536
+        * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global
+        * ASID).
+        *
+        * If we were to use bitfield shenanigans here, we could save a bit of
+        * memory by only having enough bits to support MAX_ASID.  However, such
+        * an implementation would be more error prone.
+        */
+       uint8_t cpu_asid_high_bits[ARM_MAX_ASID];
+};
+typedef struct pmap_cpu_data pmap_cpu_data_t;
+

 #include <mach/vm_prot.h>
 #include <mach/vm_statistics.h>
 #include <mach/machine/vm_param.h>
 #include <mach/vm_prot.h>
 #include <mach/vm_statistics.h>
 #include <mach/machine/vm_param.h>


@@ -136,18 +172,8 @@ extern void flush_mmu_tlb_entry(uint64_t);
 extern void flush_mmu_tlb_entries(uint64_t, uint64_t);
 extern void flush_mmu_tlb_asid(uint64_t);
 extern void flush_core_tlb_asid(uint64_t);
 extern void flush_mmu_tlb_entries(uint64_t, uint64_t);
 extern void flush_mmu_tlb_asid(uint64_t);
 extern void flush_core_tlb_asid(uint64_t);

-/*
- * TLBI appers to only deal in 4KB page addresses, so give
- * it an explicit shift of 12.
- */
-#define TLBI_ADDR_SIZE 44
-#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1)
-#define TLBI_ADDR_SHIFT (12)
-#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK)

 
 

-#define        TLBI_ASID_SHIFT 48
-#define TLBI_ASID_SIZE 16
-#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT)
+#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK)

 #define tlbi_asid(x) (((uint64_t)x << TLBI_ASID_SHIFT) & TLBI_ASID_MASK)
 #else
 extern void flush_mmu_tlb_entry(uint32_t);
 #define tlbi_asid(x) (((uint64_t)x << TLBI_ASID_SHIFT) & TLBI_ASID_MASK)
 #else
 extern void flush_mmu_tlb_entry(uint32_t);


@@ -268,8 +294,9 @@ struct pmap {
        decl_simple_lock_data(,tt1_lock)        /* lock on tt1 */
 #endif
 #if MACH_ASSERT
        decl_simple_lock_data(,tt1_lock)        /* lock on tt1 */
 #endif
 #if MACH_ASSERT

-       int                                     pmap_pid;
-       char                            pmap_procname[17];
+       boolean_t               pmap_stats_assert;
+       int                     pmap_pid;
+       char                    pmap_procname[17];

 #endif /* MACH_ASSERT */
 #if DEVELOPMENT || DEBUG
        boolean_t               footprint_suspended;
 #endif /* MACH_ASSERT */
 #if DEVELOPMENT || DEBUG
        boolean_t               footprint_suspended;


@@ -423,12 +450,6 @@ extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset
 #define ARM_PMAP_MAX_OFFSET_DEVICE     0x08
 #define ARM_PMAP_MAX_OFFSET_JUMBO      0x10
 
 #define ARM_PMAP_MAX_OFFSET_DEVICE     0x08
 #define ARM_PMAP_MAX_OFFSET_JUMBO      0x10
 

-#define ASID_SHIFT                     (11)                            /* Shift for the maximum virtual ASID value (2048) */
-#define MAX_ASID                       (1 << ASID_SHIFT)               /* Max supported ASIDs (can be virtual) */
-#define ARM_ASID_SHIFT                 (8)                             /* Shift for the maximum ARM ASID value (256) */
-#define ARM_MAX_ASID                   (1 << ARM_ASID_SHIFT)           /* Max ASIDs supported by the hardware */
-#define ASID_VIRT_BITS                 (ASID_SHIFT - ARM_ASID_SHIFT)   /* The number of virtual bits in a virtaul ASID */
-#define NBBY                           8

 
 extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option);
 
 
 extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option);
 


@@ -478,27 +499,6 @@ boolean_t pmap_enforces_execute_only(pmap_t pmap);
 
 #define PMAP_INVALID_CPU_NUM (~0U)
 
 
 #define PMAP_INVALID_CPU_NUM (~0U)
 

-struct pmap_cpu_data {
-       pmap_t cpu_user_pmap;
-       unsigned int cpu_number;
-       unsigned int cpu_user_pmap_stamp;
-
-       /*
-        * This supports overloading of ARM ASIDs by the pmap.  The field needs
-        * to be wide enough to cover all the virtual bits in a virtual ASID.
-        * With 256 physical ASIDs, 8-bit fields let us support up to 65536
-        * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global
-        * ASID).
-        *
-        * If we were to use bitfield shenanigans here, we could save a bit of
-        * memory by only having enough bits to support MAX_ASID.  However, such
-        * an implementation would be more error prone.
-        */
-       uint8_t cpu_asid_high_bits[ARM_MAX_ASID];
-};
-
-typedef struct pmap_cpu_data pmap_cpu_data_t;
-

 /* Initialize the pmap per-CPU data for the current CPU. */
 extern void pmap_cpu_data_init(void);
 
 /* Initialize the pmap per-CPU data for the current CPU. */
 extern void pmap_cpu_data_init(void);
 


@@ -512,4 +512,12 @@ extern kern_return_t pmap_return(boolean_t do_panic, boolean_t do_recurse);
 
 #endif /* #ifndef ASSEMBLER */
 
 
 #endif /* #ifndef ASSEMBLER */
 

+#if __ARM_KERNEL_PROTECT__
+/*
+ * The exception vector mappings start at the middle of the kernel page table
+ * range  (so that the EL0 mapping can be located at the base of the range).
+ */
+#define ARM_KERNEL_PROTECT_EXCEPTION_START ((~((ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK) / 2ULL)) + 1ULL)
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #endif /* #ifndef _ARM_PMAP_H_ */
 #endif /* #ifndef _ARM_PMAP_H_ */

diff --git a/osfmk/arm/proc_reg.h b/osfmk/arm/proc_reg.h
index ca58e18ae76ed429d826a7a4e523cf3005763f8d..9dc5e2ec8c81b9c14dc1767a38709e2b948858d4 100644 (file)
--- a/osfmk/arm/proc_reg.h
+++ b/osfmk/arm/proc_reg.h
@@ -64,6 +64,17 @@
 #ifndef        _ARM_PROC_REG_H_
 #define        _ARM_PROC_REG_H_
 
 #ifndef        _ARM_PROC_REG_H_
 #define        _ARM_PROC_REG_H_
 

+#if __ARM_KERNEL_PROTECT__
+/*
+ * This feature is not currently implemented for 32-bit ARM CPU architectures.
+ * A discussion of this feature for 64-bit ARM CPU architectures can be found
+ * in the ARM64 version of this file.
+ */
+#if __arm__
+#error __ARM_KERNEL_PROTECT__ is not supported on ARM32
+#endif
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #if defined (__arm64__)
 #include <pexpert/arm64/board_config.h>
 #elif defined (__arm__)
 #if defined (__arm64__)
 #include <pexpert/arm64/board_config.h>
 #elif defined (__arm__)


@@ -110,6 +121,7 @@
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1

+#define __ARM_KERNEL_PROTECT__ 1

 
 #elif defined (APPLETYPHOON)
 #define        __ARM_ARCH__    8
 
 #elif defined (APPLETYPHOON)
 #define        __ARM_ARCH__    8


@@ -124,6 +136,7 @@
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1

+#define __ARM_KERNEL_PROTECT__ 1

 
 #elif defined (APPLETWISTER)
 #define        __ARM_ARCH__    8
 
 #elif defined (APPLETWISTER)
 #define        __ARM_ARCH__    8


@@ -138,7 +151,8 @@
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define        __ARM_16K_PG__  1
 #define __ARM_ENABLE_SWAP__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define        __ARM_16K_PG__  1

-#define __ARM64_TWO_LEVEL_PMAP__ 1
+#define __ARM64_PMAP_SUBPAGE_L1__ 1
+#define __ARM_KERNEL_PROTECT__ 1

 
 #elif defined (APPLEHURRICANE)
 #define        __ARM_ARCH__    8
 
 #elif defined (APPLEHURRICANE)
 #define        __ARM_ARCH__    8


@@ -154,6 +168,7 @@
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define        __ARM_16K_PG__  1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1
 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1
 #define        __ARM_16K_PG__  1
 #define __ARM64_PMAP_SUBPAGE_L1__ 1

+#define __ARM_KERNEL_PROTECT__ 1

 #define __ARM_GLOBAL_SLEEP_BIT__ 1
 #define __ARM_PAN_AVAILABLE__ 1
 
 #define __ARM_GLOBAL_SLEEP_BIT__ 1
 #define __ARM_PAN_AVAILABLE__ 1
 


@@ -678,6 +693,7 @@
  * Convenience definitions for:
  *   ARM_TT_LEAF: The last level of the configured page table format.
  *   ARM_TT_TWIG: The second to last level of the configured page table format.
  * Convenience definitions for:
  *   ARM_TT_LEAF: The last level of the configured page table format.
  *   ARM_TT_TWIG: The second to last level of the configured page table format.

+ *   ARM_TT_ROOT: The first level of the configured page table format.

  *
  *   My apologies to any botanists who may be reading this.
  */
  *
  *   My apologies to any botanists who may be reading this.
  */


@@ -691,6 +707,11 @@
 #define ARM_TT_TWIG_SHIFT                              ARM_TT_L1_SHIFT
 #define ARM_TT_TWIG_INDEX_MASK                 ARM_TT_L1_INDEX_MASK
 
 #define ARM_TT_TWIG_SHIFT                              ARM_TT_L1_SHIFT
 #define ARM_TT_TWIG_INDEX_MASK                 ARM_TT_L1_INDEX_MASK
 

+#define ARM_TT_ROOT_SIZE                               ARM_TT_L1_SIZE
+#define ARM_TT_ROOT_OFFMASK                            ARM_TT_L1_OFFMASK
+#define ARM_TT_ROOT_SHIFT                              ARM_TT_L1_SHIFT
+#define ARM_TT_ROOT_INDEX_MASK                 ARM_TT_L1_INDEX_MASK
+

 /*
  *     Level 1 Translation Table Entry
  *
 /*
  *     Level 1 Translation Table Entry
  *

diff --git a/osfmk/arm64/arm_vm_init.c b/osfmk/arm64/arm_vm_init.c
index dc0cb74304e450a47371e583ba69d41c5bf7b199..9b06f950478d0dace888789179068a0f9af23ef2 100644 (file)
--- a/osfmk/arm64/arm_vm_init.c
+++ b/osfmk/arm64/arm_vm_init.c
@@ -47,6 +47,28 @@
 #include <libkern/kernel_mach_header.h>
 #include <libkern/section_keywords.h>
 
 #include <libkern/kernel_mach_header.h>
 #include <libkern/section_keywords.h>
 

+#if __ARM_KERNEL_PROTECT__
+#include <arm/atomic.h>
+#endif /* __ARM_KERNEL_PROTECT__ */
+
+#if __ARM_KERNEL_PROTECT__
+/*
+ * If we want to support __ARM_KERNEL_PROTECT__, we need a sufficient amount of
+ * mappable space preceeding the kernel (as we unmap the kernel by cutting the
+ * range covered by TTBR1 in half).  This must also cover the exception vectors.
+ */
+static_assert(KERNEL_PMAP_HEAP_RANGE_START > ARM_KERNEL_PROTECT_EXCEPTION_START);
+
+/* The exception vectors and the kernel cannot share root TTEs. */
+static_assert((KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_ROOT_OFFMASK) > ARM_KERNEL_PROTECT_EXCEPTION_START);
+
+/*
+ * We must have enough space in the TTBR1_EL1 range to create the EL0 mapping of
+ * the exception vectors.
+ */
+static_assert((((~ARM_KERNEL_PROTECT_EXCEPTION_START) + 1) * 2ULL) <= (ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK));
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #if KASAN
 extern vm_offset_t shadow_pbase;
 extern vm_offset_t shadow_ptop;
 #if KASAN
 extern vm_offset_t shadow_pbase;
 extern vm_offset_t shadow_ptop;


@@ -178,6 +200,11 @@ SECURITY_READ_ONLY_LATE(vm_offset_t)     static_memory_end;
 SECURITY_READ_ONLY_LATE(pmap_paddr_t)    avail_start;
 SECURITY_READ_ONLY_LATE(pmap_paddr_t)    avail_end;
 
 SECURITY_READ_ONLY_LATE(pmap_paddr_t)    avail_start;
 SECURITY_READ_ONLY_LATE(pmap_paddr_t)    avail_end;
 

+#if __ARM_KERNEL_PROTECT__
+extern void ExceptionVectorsBase;
+extern void ExceptionVectorsEnd;
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #define        MEM_SIZE_MAX            0x100000000ULL
 
 #if defined(KERNEL_INTEGRITY_KTRR)
 #define        MEM_SIZE_MAX            0x100000000ULL
 
 #if defined(KERNEL_INTEGRITY_KTRR)


@@ -344,6 +371,240 @@ void dump_kva_space() {
 
 #endif /* DEBUG */
 
 
 #endif /* DEBUG */
 

+#if __ARM_KERNEL_PROTECT__
+/*
+ * arm_vm_map:
+ *   root_ttp: The kernel virtual address for the root of the target page tables
+ *   vaddr: The target virtual address
+ *   pte: A page table entry value (may be ARM_PTE_EMPTY)
+ *
+ * This function installs pte at vaddr in root_ttp.  Any page table pages needed
+ * to install pte will be allocated by this function.
+ */
+static void
+arm_vm_map(tt_entry_t * root_ttp, vm_offset_t vaddr, pt_entry_t pte)
+{
+       vm_offset_t ptpage = 0;
+       tt_entry_t * ttp = root_ttp;
+
+#if !__ARM64_TWO_LEVEL_PMAP__
+       tt_entry_t * l1_ttep = NULL;
+       tt_entry_t l1_tte = 0;
+#endif
+
+       tt_entry_t * l2_ttep = NULL;
+       tt_entry_t l2_tte = 0;
+       pt_entry_t * ptep = NULL;
+       pt_entry_t cpte = 0;
+
+       /*
+        * Walk the target page table to find the PTE for the given virtual
+        * address.  Allocate any page table pages needed to do this.
+        */
+#if !__ARM64_TWO_LEVEL_PMAP__
+       l1_ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
+       l1_tte = *l1_ttep;
+
+       if (l1_tte == ARM_TTE_EMPTY) {
+               ptpage = alloc_ptpage(TRUE);
+               bzero((void *)ptpage, ARM_PGBYTES);
+               l1_tte = kvtophys(ptpage);
+               l1_tte &= ARM_TTE_TABLE_MASK;
+               l1_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE;
+               *l1_ttep = l1_tte;
+               ptpage = 0;
+       }
+
+       ttp = (tt_entry_t *)phystokv(l1_tte & ARM_TTE_TABLE_MASK);
+#endif
+
+       l2_ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
+       l2_tte = *l2_ttep;
+
+       if (l2_tte == ARM_TTE_EMPTY) {
+               ptpage = alloc_ptpage(TRUE);
+               bzero((void *)ptpage, ARM_PGBYTES);
+               l2_tte = kvtophys(ptpage);
+               l2_tte &= ARM_TTE_TABLE_MASK;
+               l2_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE;
+               *l2_ttep = l2_tte;
+               ptpage = 0;
+       }
+
+       ttp = (tt_entry_t *)phystokv(l2_tte & ARM_TTE_TABLE_MASK);
+
+       ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
+       cpte = *ptep;
+
+       /*
+        * If the existing PTE is not empty, then we are replacing a valid
+        * mapping.
+        */
+       if (cpte != ARM_PTE_EMPTY) {
+               panic("%s: cpte=%#llx is not empty, "
+                     "vaddr=%#lx, pte=%#llx",
+                     __FUNCTION__, cpte,
+                     vaddr, pte);
+       }
+
+       *ptep = pte;
+}
+
+/*
+ * arm_vm_kernel_el0_map:
+ *   vaddr: The target virtual address
+ *   pte: A page table entry value (may be ARM_PTE_EMPTY)
+ *
+ * This function installs pte at vaddr for the EL0 kernel mappings.
+ */
+static void
+arm_vm_kernel_el0_map(vm_offset_t vaddr, pt_entry_t pte)
+{
+       /* Calculate where vaddr will be in the EL1 kernel page tables. */
+       vm_offset_t kernel_pmap_vaddr = vaddr - ((ARM_TT_ROOT_INDEX_MASK + ARM_TT_ROOT_SIZE) / 2ULL);
+       arm_vm_map(cpu_tte, kernel_pmap_vaddr, pte);
+}
+
+/*
+ * arm_vm_kernel_el1_map:
+ *   vaddr: The target virtual address
+ *   pte: A page table entry value (may be ARM_PTE_EMPTY)
+ *
+ * This function installs pte at vaddr for the EL1 kernel mappings.
+ */
+static void
+arm_vm_kernel_el1_map(vm_offset_t vaddr, pt_entry_t pte) {
+       arm_vm_map(cpu_tte, vaddr, pte);
+}
+
+/*
+ * arm_vm_kernel_pte:
+ *   vaddr: The target virtual address
+ *
+ * This function returns the PTE value for the given vaddr from the kernel page
+ * tables.  If the region has been been block mapped, we return what an
+ * equivalent PTE value would be (as regards permissions and flags).  We also
+ * remove the HINT bit (as we are not necessarily creating contiguous mappings.
+ */
+static pt_entry_t
+arm_vm_kernel_pte(vm_offset_t vaddr)
+{
+       tt_entry_t * ttp = cpu_tte;
+       tt_entry_t * ttep = NULL;
+       tt_entry_t tte = 0;
+       pt_entry_t * ptep = NULL;
+       pt_entry_t pte = 0;
+
+#if !__ARM64_TWO_LEVEL_PMAP__
+       ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
+       tte = *ttep;
+
+       assert(tte & ARM_TTE_VALID);
+
+       if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
+               /* This is a block mapping; return the equivalent PTE value. */
+               pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK);
+               pte |= ARM_PTE_TYPE_VALID;
+               pte |= vaddr & ((ARM_TT_L1_SIZE - 1) & ARM_PTE_PAGE_MASK);
+               pte &= ~ARM_PTE_HINT_MASK;
+               return pte;
+       }
+
+       ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK);
+#endif
+       ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT);
+       tte = *ttep;
+
+       assert(tte & ARM_TTE_VALID);
+
+       if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) {
+               /* This is a block mapping; return the equivalent PTE value. */
+               pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK);
+               pte |= ARM_PTE_TYPE_VALID;
+               pte |= vaddr & ((ARM_TT_L2_SIZE - 1) & ARM_PTE_PAGE_MASK);
+               pte &= ~ARM_PTE_HINT_MASK;
+               return pte;
+       }
+
+       ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK);
+
+       ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT);
+       pte = *ptep;
+       pte &= ~ARM_PTE_HINT_MASK;
+       return pte;
+}
+
+/*
+ * arm_vm_prepare_kernel_el0_mappings:
+ *   alloc_only: Indicates if PTE values should be copied from the EL1 kernel
+ *     mappings.
+ *
+ * This function expands the kernel page tables to support the EL0 kernel
+ * mappings, and conditionally installs the PTE values for the EL0 kernel
+ * mappings (if alloc_only is false).
+ */
+static void
+arm_vm_prepare_kernel_el0_mappings(bool alloc_only)
+{
+       pt_entry_t pte = 0;
+       vm_offset_t start = ((vm_offset_t)&ExceptionVectorsBase) & ~PAGE_MASK;
+       vm_offset_t end = (((vm_offset_t)&ExceptionVectorsEnd) + PAGE_MASK) & ~PAGE_MASK;
+       vm_offset_t cur = 0;
+       vm_offset_t cur_fixed = 0;
+
+       /* Expand for/map the exceptions vectors in the EL0 kernel mappings. */
+       for (cur = start, cur_fixed = ARM_KERNEL_PROTECT_EXCEPTION_START; cur < end; cur += ARM_PGBYTES, cur_fixed += ARM_PGBYTES) {
+               /*
+                * We map the exception vectors at a different address than that
+                * of the kernelcache to avoid sharing page table pages with the
+                * kernelcache (as this may cause issues with TLB caching of
+                * page table pages.
+                */
+               if (!alloc_only) {
+                       pte = arm_vm_kernel_pte(cur);
+               }
+
+               arm_vm_kernel_el1_map(cur_fixed, pte);
+               arm_vm_kernel_el0_map(cur_fixed, pte);
+       }
+
+       __builtin_arm_dmb(DMB_ISH);
+       __builtin_arm_isb(ISB_SY);
+
+       if (!alloc_only) {
+               /*
+                * If we have created the alternate exception vector mappings,
+                * the boot CPU may now switch over to them.
+                */
+               set_vbar_el1(ARM_KERNEL_PROTECT_EXCEPTION_START);
+               __builtin_arm_isb(ISB_SY);
+       }
+}
+
+/*
+ * arm_vm_populate_kernel_el0_mappings:
+ *
+ * This function adds all required mappings to the EL0 kernel mappings.
+ */
+static void
+arm_vm_populate_kernel_el0_mappings(void)
+{
+       arm_vm_prepare_kernel_el0_mappings(FALSE);
+}
+
+/*
+ * arm_vm_expand_kernel_el0_mappings:
+ *
+ * This function expands the kernel page tables to accomodate the EL0 kernel
+ * mappings.
+ */
+static void
+arm_vm_expand_kernel_el0_mappings(void)
+{
+       arm_vm_prepare_kernel_el0_mappings(TRUE);
+}
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #if defined(KERNEL_INTEGRITY_KTRR)
 extern void bootstrap_instructions;
 
 #if defined(KERNEL_INTEGRITY_KTRR)
 extern void bootstrap_instructions;
 


@@ -474,6 +735,9 @@ arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va,
                                ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
                                ptmp = ptmp | ARM_PTE_AP(pte_prot_APX);
                                ptmp = ptmp | ARM_PTE_NX;
                                ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT);
                                ptmp = ptmp | ARM_PTE_AP(pte_prot_APX);
                                ptmp = ptmp | ARM_PTE_NX;

+#if __ARM_KERNEL_PROTECT__
+                               ptmp = ptmp | ARM_PTE_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

 
                                if (pte_prot_XN) {
                                        ptmp = ptmp | ARM_PTE_PNX;
 
                                if (pte_prot_XN) {
                                        ptmp = ptmp | ARM_PTE_PNX;


@@ -566,6 +830,9 @@ arm_vm_page_granular_prot(vm_offset_t start, unsigned long size,
 
                        tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX);
                        tmplate = tmplate | ARM_TTE_BLOCK_NX;
 
                        tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX);
                        tmplate = tmplate | ARM_TTE_BLOCK_NX;

+#if __ARM_KERNEL_PROTECT__
+                       tmplate = tmplate | ARM_TTE_BLOCK_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

                        if (tte_prot_XN)
                                tmplate = tmplate | ARM_TTE_BLOCK_PNX;
 
                        if (tte_prot_XN)
                                tmplate = tmplate | ARM_TTE_BLOCK_PNX;
 


@@ -742,6 +1009,10 @@ arm_vm_prot_finalize(boot_args * args)
                arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE);
        }
 
                arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE);
        }
 

+#if __ARM_KERNEL_PROTECT__
+       arm_vm_populate_kernel_el0_mappings();
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 #if defined(KERNEL_INTEGRITY_KTRR)
        /*
         * __LAST,__pinst should no longer be executable.
 #if defined(KERNEL_INTEGRITY_KTRR)
        /*
         * __LAST,__pinst should no longer be executable.


@@ -754,6 +1025,7 @@ arm_vm_prot_finalize(boot_args * args)
         * and will become immutable.
         */
 #endif
         * and will become immutable.
         */
 #endif

+

        arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, FALSE);
 
 #ifndef __ARM_L1_PTW__
        arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, FALSE);
 
 #ifndef __ARM_L1_PTW__


@@ -776,6 +1048,10 @@ boolean_t user_tbi = TRUE;
 static void
 set_tbi(void)
 {
 static void
 set_tbi(void)
 {

+#if !__ARM_KERNEL_PROTECT__
+       /* If we are not built with __ARM_KERNEL_PROTECT__, TBI can be turned
+        * off with a boot-arg.
+        */

        uint64_t old_tcr, new_tcr;
        int tbi = 0;
 
        uint64_t old_tcr, new_tcr;
        int tbi = 0;
 


@@ -789,6 +1065,7 @@ set_tbi(void)
                set_tcr(new_tcr);
                sysreg_restore.tcr_el1 = new_tcr;
        }
                set_tcr(new_tcr);
                sysreg_restore.tcr_el1 = new_tcr;
        }

+#endif /* !__ARM_KERNEL_PROTECT__ */

 }
 
 void
 }
 
 void


@@ -863,7 +1140,6 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
        cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE);
        cpu_ttep = kvtophys((vm_offset_t)cpu_tte);
        bzero(cpu_tte, ARM_PGBYTES);
        cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE);
        cpu_ttep = kvtophys((vm_offset_t)cpu_tte);
        bzero(cpu_tte, ARM_PGBYTES);

-

        avail_end = gPhysBase + mem_size;
 
        /*
        avail_end = gPhysBase + mem_size;
 
        /*


@@ -873,7 +1149,6 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
         *
         *   the so called physical aperture should be statically mapped
         */
         *
         *   the so called physical aperture should be statically mapped
         */

-

 #if !__ARM64_TWO_LEVEL_PMAP__
        pa_l1 = gPhysBase;
        va_l1 = gVirtBase;
 #if !__ARM64_TWO_LEVEL_PMAP__
        pa_l1 = gPhysBase;
        va_l1 = gVirtBase;


@@ -920,6 +1195,9 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
                                      | ARM_TTE_VALID | ARM_TTE_BLOCK_AF
                                      | ARM_TTE_BLOCK_AP(AP_RWNA) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)
                                      | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);
                                      | ARM_TTE_VALID | ARM_TTE_BLOCK_AF
                                      | ARM_TTE_BLOCK_AP(AP_RWNA) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)
                                      | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK);

+#if __ARM_KERNEL_PROTECT__
+                       *cpu_l2_tte |= ARM_TTE_BLOCK_NG;
+#endif /* __ARM_KERNEL_PROTECT__ */

                        va_l2 += ARM_TT_L2_SIZE;
                        pa_l2 += ARM_TT_L2_SIZE;
                        cpu_l2_tte++;
                        va_l2 += ARM_TT_L2_SIZE;
                        pa_l2 += ARM_TT_L2_SIZE;
                        cpu_l2_tte++;


@@ -931,6 +1209,11 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
        }
 #endif
 
        }
 #endif
 

+#if __ARM_KERNEL_PROTECT__
+       /* Expand the page tables to prepare for the EL0 mappings. */
+       arm_vm_expand_kernel_el0_mappings();
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        /*
         * Now retrieve addresses for end, edata, and etext from MACH-O headers
         */
        /*
         * Now retrieve addresses for end, edata, and etext from MACH-O headers
         */


@@ -1010,7 +1293,7 @@ arm_vm_init(uint64_t memory_size, boot_args * args)
         */
 #if !__ARM64_TWO_LEVEL_PMAP__
        va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL;
         */
 #if !__ARM64_TWO_LEVEL_PMAP__
        va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL;

-       va_l1_end = VM_MAX_KERNEL_ADDRESS; 
+       va_l1_end = VM_MAX_KERNEL_ADDRESS;

        cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
 
        while (va_l1 < va_l1_end) {
        cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT);
 
        while (va_l1 < va_l1_end) {

diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c
index 6e0d5ed52fa0da1bc4465d71ad1c131eaf192866..d4712a612c328038ee5120ab8407b178d4087b31 100644 (file)
--- a/osfmk/arm64/cpu.c
+++ b/osfmk/arm64/cpu.c
@@ -78,6 +78,10 @@ void sleep_token_buffer_init(void);
 extern uintptr_t resume_idle_cpu;
 extern uintptr_t start_cpu;
 
 extern uintptr_t resume_idle_cpu;
 extern uintptr_t start_cpu;
 

+#if __ARM_KERNEL_PROTECT__
+extern void exc_vectors_table;
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void);
 extern void arm64_force_wfi_clock_gate(void);
 #if (defined(APPLECYCLONE) || defined(APPLETYPHOON))
 extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void);
 extern void arm64_force_wfi_clock_gate(void);
 #if (defined(APPLECYCLONE) || defined(APPLETYPHOON))


@@ -537,6 +541,9 @@ cpu_data_init(cpu_data_t *cpu_data_ptr)
                pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0;
        }
        cpu_data_ptr->halt_status = CPU_NOT_HALTED;
                pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0;
        }
        cpu_data_ptr->halt_status = CPU_NOT_HALTED;

+#if __ARM_KERNEL_PROTECT__
+       cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table;
+#endif /* __ARM_KERNEL_PROTECT__ */

 }
 
 kern_return_t
 }
 
 kern_return_t

diff --git a/osfmk/arm64/genassym.c b/osfmk/arm64/genassym.c
index 5c06aabce0675053b8fab8d66a610763af73760f..6e47758b1ae0bfa14d9960832cc1e70510a8bbe8 100644 (file)
--- a/osfmk/arm64/genassym.c
+++ b/osfmk/arm64/genassym.c
@@ -296,6 +296,10 @@ main(
                offsetof(cpu_data_t, fiqstackptr));
         DECLARE("CPU_FIQSTACK_TOP",
                offsetof(cpu_data_t, fiqstack_top));
                offsetof(cpu_data_t, fiqstackptr));
         DECLARE("CPU_FIQSTACK_TOP",
                offsetof(cpu_data_t, fiqstack_top));

+#if __ARM_KERNEL_PROTECT__
+       DECLARE("CPU_EXC_VECTORS",
+               offsetof(cpu_data_t, cpu_exc_vectors));
+#endif /* __ARM_KERNEL_PROTECT__ */

         DECLARE("CPU_NUMBER_GS",
                offsetof(cpu_data_t,cpu_number));
         DECLARE("CPU_IDENT",
         DECLARE("CPU_NUMBER_GS",
                offsetof(cpu_data_t,cpu_number));
         DECLARE("CPU_IDENT",

diff --git a/osfmk/arm64/locore.s b/osfmk/arm64/locore.s
index 4f8b9e9ad285848014eaa591db4ea111ca6d7788..ea33d6b42c6a4d64e2cf9b282702dbfa1370bc81 100644 (file)
--- a/osfmk/arm64/locore.s
+++ b/osfmk/arm64/locore.s
@@ -34,6 +34,10 @@
 #include <config_dtrace.h>
 #include "assym.s"
 
 #include <config_dtrace.h>
 #include "assym.s"
 

+#if __ARM_KERNEL_PROTECT__
+#include <arm/pmap.h>
+#endif
+

 
 /*
  * INIT_SAVED_STATE_FLAVORS
 
 /*
  * INIT_SAVED_STATE_FLAVORS


@@ -53,16 +57,6 @@
        str             $1, [$0, NS_COUNT]
 .endmacro
 
        str             $1, [$0, NS_COUNT]
 .endmacro
 

-.macro EL1_SP0_VECTOR
-       msr             SPSel, #0                                                       // Switch to SP0
-       sub             sp, sp, ARM_CONTEXT_SIZE                        // Create exception frame
-       stp             x0, x1, [sp, SS64_X0]                           // Save x0, x1 to exception frame
-       add             x0, sp, ARM_CONTEXT_SIZE                        // Calculate the original stack pointer
-       str             x0, [sp, SS64_SP]                                       // Save stack pointer to exception frame
-       stp             fp, lr, [sp, SS64_FP]                           // Save fp and lr to exception frame
-       INIT_SAVED_STATE_FLAVORS sp, w0, w1
-       mov             x0, sp                                                          // Copy saved state pointer to x0
-.endmacro

 
 /*
  * SPILL_REGISTERS
 
 /*
  * SPILL_REGISTERS


@@ -142,11 +136,183 @@
 #endif
 .endmacro
 
 #endif
 .endmacro
 

+/*
+ * MAP_KERNEL
+ *
+ * Restores the kernel EL1 mappings, if necessary.
+ *
+ * This may mutate x18.
+ */
+.macro MAP_KERNEL
+#if __ARM_KERNEL_PROTECT__
+       /* Switch to the kernel ASID (low bit set) for the task. */
+       mrs             x18, TTBR0_EL1
+       orr             x18, x18, #(1 << TTBR_ASID_SHIFT)
+       msr             TTBR0_EL1, x18
+
+       /*
+        * We eschew some barriers on Apple CPUs, as relative ordering of writes
+        * to the TTBRs and writes to the TCR should be ensured by the
+        * microarchitecture.
+        */
+#if !defined(APPLE_ARM64_ARCH_FAMILY)
+       isb             sy
+#endif
+
+       /*
+        * Update the TCR to map the kernel now that we are using the kernel
+        * ASID.
+        */
+       MOV64           x18, TCR_EL1_BOOT
+       msr             TCR_EL1, x18
+       isb             sy
+#endif /* __ARM_KERNEL_PROTECT__ */
+.endmacro
+
+/*
+ * BRANCH_TO_KVA_VECTOR
+ *
+ * Branches to the requested long exception vector in the kernelcache.
+ *   arg0 - The label to branch to
+ *   arg1 - The index of the label in exc_vectors_tables
+ *
+ * This may mutate x18.
+ */
+.macro BRANCH_TO_KVA_VECTOR
+#if __ARM_KERNEL_PROTECT__
+       /*
+        * Find the kernelcache table for the exception vectors by accessing
+        * the per-CPU data.
+        */
+       mrs             x18, TPIDR_EL1
+       ldr             x18, [x18, ACT_CPUDATAP]
+       ldr             x18, [x18, CPU_EXC_VECTORS]
+
+       /*
+        * Get the handler for this exception and jump to it.
+        */
+       ldr             x18, [x18, #($1 << 3)]
+       br              x18
+#else
+       b               $0
+#endif /* __ARM_KERNEL_PROTECT__ */
+.endmacro
+
+#if __ARM_KERNEL_PROTECT__
+       .data
+       .align 3
+       .globl EXT(exc_vectors_table)
+LEXT(exc_vectors_table)
+       /* Table of exception handlers. */
+       .quad Lel1_sp0_synchronous_vector_long
+       .quad Lel1_sp0_irq_vector_long
+       .quad Lel1_sp0_fiq_vector_long
+       .quad Lel1_sp0_serror_vector_long
+       .quad Lel1_sp1_synchronous_vector_long
+       .quad Lel1_sp1_irq_vector_long
+       .quad Lel1_sp1_fiq_vector_long
+       .quad Lel1_sp1_serror_vector_long
+       .quad Lel0_synchronous_vector_64_long
+       .quad Lel0_irq_vector_64_long
+       .quad Lel0_fiq_vector_64_long
+       .quad Lel0_serror_vector_64_long
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        .text
        .text

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * We need this to be on a page boundary so that we may avoiding mapping
+        * other text along with it.  As this must be on the VM page boundary
+        * (due to how the coredumping code currently works), this will be a
+        * 16KB page boundary.
+        */
+       .align 14
+#else

        .align 12
        .align 12

+#endif /* __ARM_KERNEL_PROTECT__ */

        .globl EXT(ExceptionVectorsBase)
 LEXT(ExceptionVectorsBase)
 Lel1_sp0_synchronous_vector:
        .globl EXT(ExceptionVectorsBase)
 LEXT(ExceptionVectorsBase)
 Lel1_sp0_synchronous_vector:

+       BRANCH_TO_KVA_VECTOR Lel1_sp0_synchronous_vector_long, 0
+
+       .text
+       .align 7
+Lel1_sp0_irq_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp0_irq_vector_long, 1
+
+       .text
+       .align 7
+Lel1_sp0_fiq_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp0_fiq_vector_long, 2
+
+       .text
+       .align 7
+Lel1_sp0_serror_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp0_serror_vector_long, 3
+
+       .text
+       .align 7
+Lel1_sp1_synchronous_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp1_synchronous_vector_long, 4
+
+       .text
+       .align 7
+Lel1_sp1_irq_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp1_irq_vector_long, 5
+
+       .text
+       .align 7
+Lel1_sp1_fiq_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp1_fiq_vector_long, 6
+
+       .text
+       .align 7
+Lel1_sp1_serror_vector:
+       BRANCH_TO_KVA_VECTOR Lel1_sp1_serror_vector, 7
+
+       .text
+       .align 7
+Lel0_synchronous_vector_64:
+       MAP_KERNEL
+       BRANCH_TO_KVA_VECTOR Lel0_synchronous_vector_64_long, 8
+
+       .text
+       .align 7
+Lel0_irq_vector_64:
+       MAP_KERNEL
+       BRANCH_TO_KVA_VECTOR Lel0_irq_vector_64_long, 9
+
+       .text
+       .align 7
+Lel0_fiq_vector_64:
+       MAP_KERNEL
+       BRANCH_TO_KVA_VECTOR Lel0_fiq_vector_64_long, 10
+
+       .text
+       .align 7
+Lel0_serror_vector_64:
+       MAP_KERNEL
+       BRANCH_TO_KVA_VECTOR Lel0_serror_vector_64_long, 11
+
+       /* Fill out the rest of the page */
+       .align 12
+
+/*********************************
+ * END OF EXCEPTION VECTORS PAGE *
+ *********************************/
+
+.macro EL1_SP0_VECTOR
+       msr             SPSel, #0                                                       // Switch to SP0
+       sub             sp, sp, ARM_CONTEXT_SIZE                        // Create exception frame
+       stp             x0, x1, [sp, SS64_X0]                           // Save x0, x1 to exception frame
+       add             x0, sp, ARM_CONTEXT_SIZE                        // Calculate the original stack pointer
+       str             x0, [sp, SS64_SP]                                       // Save stack pointer to exception frame
+       stp             fp, lr, [sp, SS64_FP]                           // Save fp and lr to exception frame
+       INIT_SAVED_STATE_FLAVORS sp, w0, w1
+       mov             x0, sp                                                          // Copy saved state pointer to x0
+.endmacro
+
+Lel1_sp0_synchronous_vector_long:

        sub             sp, sp, ARM_CONTEXT_SIZE                        // Make space on the exception stack
        stp             x0, x1, [sp, SS64_X0]                           // Save x0, x1 to the stack
        mrs             x1, ESR_EL1                                                     // Get the exception syndrome
        sub             sp, sp, ARM_CONTEXT_SIZE                        // Make space on the exception stack
        stp             x0, x1, [sp, SS64_X0]                           // Save x0, x1 to the stack
        mrs             x1, ESR_EL1                                                     // Get the exception syndrome


@@ -168,9 +334,7 @@ Lkernel_stack_valid:
        add             x1, x1, fleh_synchronous@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_synchronous@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp0_irq_vector:
+Lel1_sp0_irq_vector_long:

        EL1_SP0_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]
        EL1_SP0_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]


@@ -180,9 +344,7 @@ Lel1_sp0_irq_vector:
        add             x1, x1, fleh_irq@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_irq@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp0_fiq_vector:
+Lel1_sp0_fiq_vector_long:

        // ARM64_TODO write optimized decrementer
        EL1_SP0_VECTOR
        mrs             x1, TPIDR_EL1
        // ARM64_TODO write optimized decrementer
        EL1_SP0_VECTOR
        mrs             x1, TPIDR_EL1


@@ -193,9 +355,7 @@ Lel1_sp0_fiq_vector:
        add             x1, x1, fleh_fiq@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_fiq@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp0_serror_vector:
+Lel1_sp0_serror_vector_long:

        EL1_SP0_VECTOR
        adrp    x1, fleh_serror@page                            // Load address for fleh
        add             x1, x1, fleh_serror@pageoff
        EL1_SP0_VECTOR
        adrp    x1, fleh_serror@page                            // Load address for fleh
        add             x1, x1, fleh_serror@pageoff


@@ -211,9 +371,7 @@ Lel1_sp0_serror_vector:
        mov             x0, sp                                                          // Copy saved state pointer to x0
 .endmacro
 
        mov             x0, sp                                                          // Copy saved state pointer to x0
 .endmacro
 

-       .text
-       .align 7
-Lel1_sp1_synchronous_vector:
+Lel1_sp1_synchronous_vector_long:

 #if defined(KERNEL_INTEGRITY_KTRR)
        b               check_ktrr_sctlr_trap
 Lel1_sp1_synchronous_vector_continue:
 #if defined(KERNEL_INTEGRITY_KTRR)
        b               check_ktrr_sctlr_trap
 Lel1_sp1_synchronous_vector_continue:


@@ -223,25 +381,19 @@ Lel1_sp1_synchronous_vector_continue:
        add             x1, x1, fleh_synchronous_sp1@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_synchronous_sp1@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp1_irq_vector:
+Lel1_sp1_irq_vector_long:

        EL1_SP1_VECTOR
        adrp    x1, fleh_irq_sp1@page
        add             x1, x1, fleh_irq_sp1@pageoff
        b               fleh_dispatch64
 
        EL1_SP1_VECTOR
        adrp    x1, fleh_irq_sp1@page
        add             x1, x1, fleh_irq_sp1@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp1_fiq_vector:
+Lel1_sp1_fiq_vector_long:

        EL1_SP1_VECTOR
        adrp    x1, fleh_fiq_sp1@page
        add             x1, x1, fleh_fiq_sp1@pageoff
        b               fleh_dispatch64
 
        EL1_SP1_VECTOR
        adrp    x1, fleh_fiq_sp1@page
        add             x1, x1, fleh_fiq_sp1@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel1_sp1_serror_vector:
+Lel1_sp1_serror_vector_long:

        EL1_SP1_VECTOR
        adrp    x1, fleh_serror_sp1@page
        add             x1, x1, fleh_serror_sp1@pageoff
        EL1_SP1_VECTOR
        adrp    x1, fleh_serror_sp1@page
        add             x1, x1, fleh_serror_sp1@pageoff


@@ -264,9 +416,8 @@ Lel1_sp1_serror_vector:
        mov             x0, sp                                                          // Copy the user PCB pointer to x0
 .endmacro
 
        mov             x0, sp                                                          // Copy the user PCB pointer to x0
 .endmacro
 

-       .text
-       .align 7
-Lel0_synchronous_vector_64:
+
+Lel0_synchronous_vector_64_long:

        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1                                           // Load the thread register
        ldr             x1, [x1, TH_KSTACKPTR]                          // Load the top of the kernel stack to x1
        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1                                           // Load the thread register
        ldr             x1, [x1, TH_KSTACKPTR]                          // Load the top of the kernel stack to x1


@@ -275,9 +426,7 @@ Lel0_synchronous_vector_64:
        add             x1, x1, fleh_synchronous@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_synchronous@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel0_irq_vector_64:
+Lel0_irq_vector_64_long:

        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]
        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]


@@ -287,9 +436,7 @@ Lel0_irq_vector_64:
        add             x1, x1, fleh_irq@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_irq@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel0_fiq_vector_64:
+Lel0_fiq_vector_64_long:

        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]
        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1
        ldr             x1, [x1, ACT_CPUDATAP]


@@ -299,9 +446,7 @@ Lel0_fiq_vector_64:
        add             x1, x1, fleh_fiq@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_fiq@pageoff
        b               fleh_dispatch64
 

-       .text
-       .align 7
-Lel0_serror_vector_64:
+Lel0_serror_vector_64_long:

        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1                                           // Load the thread register
        ldr             x1, [x1, TH_KSTACKPTR]                          // Load the top of the kernel stack to x1
        EL0_64_VECTOR
        mrs             x1, TPIDR_EL1                                           // Load the thread register
        ldr             x1, [x1, TH_KSTACKPTR]                          // Load the top of the kernel stack to x1


@@ -310,13 +455,6 @@ Lel0_serror_vector_64:
        add             x1, x1, fleh_serror@pageoff
        b               fleh_dispatch64
 
        add             x1, x1, fleh_serror@pageoff
        b               fleh_dispatch64
 

-       /* Fill out the rest of the page */
-       .align 12
-
-/*********************************
- * END OF EXCEPTION VECTORS PAGE *
- *********************************/
-

 
 /*
  * check_kernel_stack
 
 /*
  * check_kernel_stack


@@ -755,6 +893,26 @@ exception_return:
        ldr             x0, [x3, TH_CTH_DATA]                           // Load cthread data pointer
        str             x0, [sp, SS64_X18]                                      // and use it to trash x18
 
        ldr             x0, [x3, TH_CTH_DATA]                           // Load cthread data pointer
        str             x0, [sp, SS64_X18]                                      // and use it to trash x18
 

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * If we are going to eret to userspace, we must return through the EL0
+        * eret mapping.
+        */
+       ldr             w1, [sp, SS64_CPSR]                                                                     // Load CPSR
+       tbnz            w1, PSR64_MODE_EL_SHIFT, Lskip_el0_eret_mapping // Skip if returning to EL1
+
+       /* We need to switch to the EL0 mapping of this code to eret to EL0. */
+       adrp            x0, EXT(ExceptionVectorsBase)@page                              // Load vector base
+       adrp            x1, Lexception_return_restore_registers@page    // Load target PC
+       add             x1, x1, Lexception_return_restore_registers@pageoff
+       MOV64           x2, ARM_KERNEL_PROTECT_EXCEPTION_START                  // Load EL0 vector address
+       sub             x1, x1, x0                                                                                      // Calculate delta
+       add             x0, x2, x1                                                                                      // Convert KVA to EL0 vector address
+       br              x0
+
+Lskip_el0_eret_mapping:
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 Lexception_return_restore_registers:
        /* Restore special register state */
        ldr             x0, [sp, SS64_PC]                                       // Get the return address
 Lexception_return_restore_registers:
        /* Restore special register state */
        ldr             x0, [sp, SS64_PC]                                       // Get the return address


@@ -809,8 +967,39 @@ Lexception_return_restore_registers:
        // Restore stack pointer and our last two GPRs
        ldr             x1, [x0, SS64_SP]
        mov             sp, x1
        // Restore stack pointer and our last two GPRs
        ldr             x1, [x0, SS64_SP]
        mov             sp, x1

+
+#if __ARM_KERNEL_PROTECT__
+       ldr             w18, [x0, SS64_CPSR]                            // Stash CPSR
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        ldp             x0, x1, [x0, SS64_X0]                           // Restore the GPRs
 
        ldp             x0, x1, [x0, SS64_X0]                           // Restore the GPRs
 

+#if __ARM_KERNEL_PROTECT__
+       /* If we are going to eret to userspace, we must unmap the kernel. */
+       tbnz            w18, PSR64_MODE_EL_SHIFT, Lskip_ttbr1_switch
+
+       /* Update TCR to unmap the kernel. */
+       MOV64           x18, TCR_EL1_USER
+       msr             TCR_EL1, x18
+
+       /*
+        * On Apple CPUs, TCR writes and TTBR writes should be ordered relative to
+        * each other due to the microarchitecture.
+        */
+#if !defined(APPLE_ARM64_ARCH_FAMILY)
+       isb             sy
+#endif
+
+       /* Switch to the user ASID (low bit clear) for the task. */
+       mrs             x18, TTBR0_EL1
+       bic             x18, x18, #(1 << TTBR_ASID_SHIFT)
+       msr             TTBR0_EL1, x18
+       mov             x18, xzr
+
+       /* We don't need an ISB here, as the eret is synchronizing. */
+Lskip_ttbr1_switch:
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        eret
 
 user_take_ast:
        eret
 
 user_take_ast:


@@ -856,6 +1045,18 @@ L_rwlock_count_notzero_str:
        .asciz "RW lock count not 0 on thread %p (%u)"
 .align 2
 
        .asciz "RW lock count not 0 on thread %p (%u)"
 .align 2
 

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * This symbol denotes the end of the exception vector/eret range; we page
+        * align it so that we can avoid mapping other text in the EL0 exception
+        * vector mapping.
+        */
+       .text
+       .align 14
+       .globl EXT(ExceptionVectorsEnd)
+LEXT(ExceptionVectorsEnd)
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        .text
        .align 2
        .globl EXT(ml_panic_trap_to_debugger)
        .text
        .align 2
        .globl EXT(ml_panic_trap_to_debugger)

diff --git a/osfmk/arm64/machine_routines.c b/osfmk/arm64/machine_routines.c
index fefbaa51be31e8353487a28bb8086674f9056db2..defb3f8376a5fcbd1c8d5c535f35b0c5ee9ad563 100644 (file)
--- a/osfmk/arm64/machine_routines.c
+++ b/osfmk/arm64/machine_routines.c
@@ -186,6 +186,11 @@ pmap_paddr_t get_mmu_ttb(void)
 MARK_AS_PMAP_TEXT
 void set_mmu_ttb(pmap_paddr_t value)
 {
 MARK_AS_PMAP_TEXT
 void set_mmu_ttb(pmap_paddr_t value)
 {

+#if __ARM_KERNEL_PROTECT__
+       /* All EL1-mode ASIDs are odd. */
+       value |= (1ULL << TTBR_ASID_SHIFT);
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        __builtin_arm_dsb(DSB_ISH);
        MSR("TTBR0_EL1", value);
        __builtin_arm_isb(ISB_SY);
        __builtin_arm_dsb(DSB_ISH);
        MSR("TTBR0_EL1", value);
        __builtin_arm_isb(ISB_SY);

diff --git a/osfmk/arm64/machine_routines_asm.s b/osfmk/arm64/machine_routines_asm.s
index c23ec81da36b989388c180442b7b44e060aa2ca0..04224600684bd51327de71ffff2c2ad35ef37db7 100644 (file)
--- a/osfmk/arm64/machine_routines_asm.s
+++ b/osfmk/arm64/machine_routines_asm.s
@@ -82,19 +82,19 @@ LEXT(set_fpscr)
 #endif
        ret
 
 #endif
        ret
 

-#if    (__ARM_VFP__ >= 3)
-       .align  2
-       .globl  EXT(get_mvfr0)
-LEXT(get_mvfr0)
-       mrs x0, MVFR0_EL1
-       ret
-
-       .globl  EXT(get_mvfr1)
-LEXT(get_mvfr1)
-       mrs x0, MVFR1_EL1
-       ret
-
-#endif
+#if __ARM_KERNEL_PROTECT__
+/*
+ * __ARM_KERNEL_PROTECT__ adds two complications to TLB management:
+ *
+ * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must
+ *   target both ASIDs for the pmap that owns the target ASID.
+ *
+ * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all
+ *   ASIDs (as kernel_pmap mappings may be referenced while using an ASID that
+ *   belongs to another pmap).  We expect these routines to be called with the
+ *   EL0 ASID for the target; not the EL1 ASID.
+ */
+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  *     void flush_mmu_tlb(void)
 
 /*
  *     void flush_mmu_tlb(void)


@@ -154,12 +154,11 @@ LEXT(flush_mmu_tlb_allentries)
        add             x1, x1, #0x3
        and             x1, x1, #~0x3
 #endif
        add             x1, x1, #0x3
        and             x1, x1, #~0x3
 #endif

-
-1:
+Lflush_mmu_tlb_allentries_loop:

        tlbi    vaae1is, x0
        add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are 4KB pages, as defined by the ISA
        cmp             x0, x1
        tlbi    vaae1is, x0
        add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are 4KB pages, as defined by the ISA
        cmp             x0, x1

-       b.lt    1b
+       b.lt    Lflush_mmu_tlb_allentries_loop

        dsb             ish
        isb             sy
        ret
        dsb             ish
        isb             sy
        ret


@@ -173,10 +172,30 @@ LEXT(flush_mmu_tlb_allentries)
        .align 2
        .globl EXT(flush_mmu_tlb_entry)
 LEXT(flush_mmu_tlb_entry)
        .align 2
        .globl EXT(flush_mmu_tlb_entry)
 LEXT(flush_mmu_tlb_entry)

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * If we are flushing ASID 0, this is a kernel operation.  With this
+        * ASID scheme, this means we should flush all ASIDs.
+        */
+       lsr             x2, x0, #TLBI_ASID_SHIFT
+       cmp             x2, #0
+       b.eq            Lflush_mmu_tlb_entry_globally
+
+       bic             x0, x0, #(1 << TLBI_ASID_SHIFT)

        tlbi    vae1is, x0
        tlbi    vae1is, x0

+       orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */
+       tlbi    vae1is, x0
+       dsb             ish
+       isb             sy
+       ret
+#if __ARM_KERNEL_PROTECT__
+Lflush_mmu_tlb_entry_globally:
+       tlbi    vaae1is, x0

        dsb             ish
        isb             sy
        ret
        dsb             ish
        isb             sy
        ret

+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  *     void flush_mmu_tlb_entries(uint64_t, uint64_t)
 
 /*
  *     void flush_mmu_tlb_entries(uint64_t, uint64_t)


@@ -207,16 +226,41 @@ LEXT(flush_mmu_tlb_entries)
         */
        add             x1, x1, #0x3
        and             x1, x1, #~0x3
         */
        add             x1, x1, #0x3
        and             x1, x1, #~0x3

-#endif
+#endif /* __ARM_KERNEL_PROTECT__ */
+#if __ARM_KERNEL_PROTECT__
+       /*
+        * If we are flushing ASID 0, this is a kernel operation.  With this
+        * ASID scheme, this means we should flush all ASIDs.
+        */
+       lsr             x2, x0, #TLBI_ASID_SHIFT
+       cmp             x2, #0
+       b.eq            Lflush_mmu_tlb_entries_globally_loop

 
 

-1:
+       bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */
+Lflush_mmu_tlb_entries_loop:
+       tlbi    vae1is, x0
+#if __ARM_KERNEL_PROTECT__
+       orr             x0, x0, #(1 << TLBI_ASID_SHIFT)

        tlbi    vae1is, x0
        tlbi    vae1is, x0

+       bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */
+       add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are pages
+       cmp             x0, x1
+       b.lt    Lflush_mmu_tlb_entries_loop
+       dsb             ish
+       isb             sy
+       ret
+#if __ARM_KERNEL_PROTECT__
+Lflush_mmu_tlb_entries_globally_loop:
+       tlbi    vaae1is, x0

        add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are pages
        cmp             x0, x1
        add             x0, x0, #(ARM_PGBYTES / 4096)   // Units are pages
        cmp             x0, x1

-       b.lt    1b
+       b.lt    Lflush_mmu_tlb_entries_globally_loop

        dsb             ish
        isb             sy
        ret
        dsb             ish
        isb             sy
        ret

+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  *     void flush_mmu_tlb_asid(uint64_t)
 
 /*
  *     void flush_mmu_tlb_asid(uint64_t)


@@ -227,10 +271,30 @@ LEXT(flush_mmu_tlb_entries)
        .align 2
        .globl EXT(flush_mmu_tlb_asid)
 LEXT(flush_mmu_tlb_asid)
        .align 2
        .globl EXT(flush_mmu_tlb_asid)
 LEXT(flush_mmu_tlb_asid)

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * If we are flushing ASID 0, this is a kernel operation.  With this
+        * ASID scheme, this means we should flush all ASIDs.
+        */
+       lsr             x1, x0, #TLBI_ASID_SHIFT
+       cmp             x1, #0
+       b.eq            Lflush_mmu_tlb_globally
+
+       bic             x0, x0, #(1 << TLBI_ASID_SHIFT)

        tlbi    aside1is, x0
        tlbi    aside1is, x0

+       orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */
+       tlbi    aside1is, x0
+       dsb             ish
+       isb             sy
+       ret
+#if __ARM_KERNEL_PROTECT__
+Lflush_mmu_tlb_globally:
+       tlbi    vmalle1is

        dsb             ish
        isb             sy
        ret
        dsb             ish
        isb             sy
        ret

+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  *     void flush_core_tlb_asid(uint64_t)
 
 /*
  *     void flush_core_tlb_asid(uint64_t)


@@ -241,10 +305,30 @@ LEXT(flush_mmu_tlb_asid)
        .align 2
        .globl EXT(flush_core_tlb_asid)
 LEXT(flush_core_tlb_asid)
        .align 2
        .globl EXT(flush_core_tlb_asid)
 LEXT(flush_core_tlb_asid)

+#if __ARM_KERNEL_PROTECT__
+       /*
+        * If we are flushing ASID 0, this is a kernel operation.  With this
+        * ASID scheme, this means we should flush all ASIDs.
+        */
+       lsr             x1, x0, #TLBI_ASID_SHIFT
+       cmp             x1, #0
+       b.eq            Lflush_core_tlb_asid_globally
+
+       bic             x0, x0, #(1 << TLBI_ASID_SHIFT)
+       tlbi    aside1, x0
+       orr             x0, x0, #(1 << TLBI_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */

        tlbi    aside1, x0
        dsb             ish
        isb             sy
        ret
        tlbi    aside1, x0
        dsb             ish
        isb             sy
        ret

+#if __ARM_KERNEL_PROTECT__
+Lflush_core_tlb_asid_globally:
+       tlbi    vmalle1
+       dsb             ish
+       isb             sy
+       ret
+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  *     Set MMU Translation Table Base Alternate
 
 /*
  *     Set MMU Translation Table Base Alternate


@@ -277,6 +361,19 @@ LEXT(set_aux_control)
        isb             sy
        ret
 
        isb             sy
        ret
 

+#if __ARM_KERNEL_PROTECT__
+       .text
+       .align 2
+       .globl EXT(set_vbar_el1)
+LEXT(set_vbar_el1)
+#if defined(KERNEL_INTEGRITY_KTRR)
+       b               EXT(pinst_set_vbar)
+#else
+       msr             VBAR_EL1, x0
+       ret
+#endif
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 
 /*
  *     set translation control register
 
 /*
  *     set translation control register

diff --git a/osfmk/arm64/platform_tests.c b/osfmk/arm64/platform_tests.c
index 689250acfd82e525fab29f9fce6bcad1e77cfba0..d5391b3279a757fb2ab62d123c45c87ec1151a93 100644 (file)
--- a/osfmk/arm64/platform_tests.c
+++ b/osfmk/arm64/platform_tests.c
@@ -1049,10 +1049,10 @@ arm64_pan_test()
        pan_exception_level = 0;
        pan_fault_value = 0xDE;
        // convert priv_addr to one that is accessible from user mode
        pan_exception_level = 0;
        pan_fault_value = 0xDE;
        // convert priv_addr to one that is accessible from user mode

-       pan_test_addr = priv_addr + _COMM_PAGE64_BASE_ADDRESS - 
+       pan_test_addr = priv_addr + _COMM_HIGH_PAGE64_BASE_ADDRESS -

                _COMM_PAGE_START_ADDRESS;
 
                _COMM_PAGE_START_ADDRESS;
 

-       // Below should trigger a PAN exception as pan_test_addr is accessible 
+       // Below should trigger a PAN exception as pan_test_addr is accessible

        // in user mode
        // The exception handler, upon recognizing the fault address is pan_test_addr,
        // will disable PAN and rerun this instruction successfully
        // in user mode
        // The exception handler, upon recognizing the fault address is pan_test_addr,
        // will disable PAN and rerun this instruction successfully

diff --git a/osfmk/arm64/proc_reg.h b/osfmk/arm64/proc_reg.h
index 55c370635e19f1408cb4b0cd04e7739b46262c9f..ce62e62bd616deda253e40d84c5549df13f339bc 100644 (file)
--- a/osfmk/arm64/proc_reg.h
+++ b/osfmk/arm64/proc_reg.h
@@ -33,6 +33,61 @@
 
 #include <arm/proc_reg.h>
 
 
 #include <arm/proc_reg.h>
 

+#if __ARM_KERNEL_PROTECT__
+/*
+ * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
+ * architectural or microarchitectural vulnerabilities that could allow cores to
+ * read/access EL1-only mappings while in EL0 mode.  This is achieved by
+ * removing as many mappings as possible when the core transitions to EL0 mode
+ * from EL1 mode, and restoring those mappings when the core transitions to EL1
+ * mode from EL0 mode.
+ *
+ * At the moment, this is achieved through use of ASIDs and TCR_EL1.  TCR_EL1 is
+ * used to map and unmap the ordinary kernel mappings, by contracting and
+ * expanding translation zone size for TTBR1 when exiting and entering EL1,
+ * respectively:
+ *
+ * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
+ *   is in EL0.
+ * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
+ *   EL1.
+ *
+ * T1SZ_USER: T1SZ_BOOT + 1
+ * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
+ * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
+ * TTBR1_EL1_MAX: (2^64) - 1
+ *
+ * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
+ * the following TTBR1 layout:
+ *
+ *  TTBR1_EL1_BASE_BOOT   TTBR1_EL1_BASE_USER   TTBR1_EL1_MAX
+ * +---------------------------------------------------------+
+ * | Kernel EL0 Mappings |        Kernel EL1 Mappings        |
+ * +---------------------------------------------------------+
+ *
+ * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
+ *
+ *  TTBR1_EL1_BASE_USER                         TTBR1_EL1_MAX
+ * +---------------------------------------------------------+
+ * |                   Kernel EL0 Mappings                   |
+ * +---------------------------------------------------------+
+ *
+ * With the current implementation, both the EL0 and EL1 mappings for the kernel
+ * use otherwise empty translation tables for mapping the exception vectors (so
+ * that we do not need to TLB flush the exception vector address when switching
+ * between EL0 and EL1).  The rationale here is that the TLBI would require a
+ * DSB, and DSBs can be extremely expensive.
+ *
+ * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
+ * ASID.  The core switches between ASIDs on EL transitions, so that the TLB
+ * does not need to be fully invalidated on an EL transition.
+ *
+ * Most kernel mappings will be marked non-global in this configuration, as
+ * global mappings would be visible to userspace unless we invalidate them on
+ * eret.
+ */
+#endif /* __ARM_KERNEL_PROTECT */
+

 /*
  * 64-bit Program Status Register (PSR64)
  *
 /*
  * 64-bit Program Status Register (PSR64)
  *


@@ -484,22 +539,30 @@
  * we support the following:
  *
  * 4KB pages, full page L1: 39 bit range.
  * we support the following:
  *
  * 4KB pages, full page L1: 39 bit range.

- * 4KB pages, sub-page L1: 36 bit range.
+ * 4KB pages, sub-page L1: 38 bit range.

  * 16KB pages, full page L1: 47 bit range.
  * 16KB pages, full page L1: 47 bit range.

- * 16KB pages, sub-page L1: 37 bit range.
+ * 16KB pages, sub-page L1: 39 bit range.

  * 16KB pages, two level page tables: 36 bit range.
  */
  * 16KB pages, two level page tables: 36 bit range.
  */

+#if __ARM_KERNEL_PROTECT__
+/*
+ * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
+ * address space is used for the mappings that will remain in place when in EL0.
+ * As a result, 1 bit less of address space is available to the rest of the
+ * the kernel.
+ */
+#endif /* __ARM_KERNEL_PROTECT__ */

 #ifdef __ARM_16K_PG__
 #if __ARM64_TWO_LEVEL_PMAP__
 #define T0SZ_BOOT                                              28ULL
 #elif __ARM64_PMAP_SUBPAGE_L1__
 #ifdef __ARM_16K_PG__
 #if __ARM64_TWO_LEVEL_PMAP__
 #define T0SZ_BOOT                                              28ULL
 #elif __ARM64_PMAP_SUBPAGE_L1__

-#define T0SZ_BOOT                                              27ULL
+#define T0SZ_BOOT                                              25ULL

 #else /* __ARM64_TWO_LEVEL_PMAP__ */
 #define T0SZ_BOOT                                              17ULL
 #endif /* __ARM64_TWO_LEVEL_PMAP__ */
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__
 #else /* __ARM64_TWO_LEVEL_PMAP__ */
 #define T0SZ_BOOT                                              17ULL
 #endif /* __ARM64_TWO_LEVEL_PMAP__ */
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__

-#define T0SZ_BOOT                                              28ULL
+#define T0SZ_BOOT                                              26ULL

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #define T0SZ_BOOT                                              25ULL
 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #define T0SZ_BOOT                                              25ULL
 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */


@@ -513,22 +576,32 @@
 #if __ARM64_TWO_LEVEL_PMAP__
 #define T1SZ_BOOT                                              28ULL
 #elif __ARM64_PMAP_SUBPAGE_L1__
 #if __ARM64_TWO_LEVEL_PMAP__
 #define T1SZ_BOOT                                              28ULL
 #elif __ARM64_PMAP_SUBPAGE_L1__

-#define T1SZ_BOOT                                              27ULL
+#define T1SZ_BOOT                                              25ULL

 #else /* __ARM64_TWO_LEVEL_PMAP__ */
 #define T1SZ_BOOT                                              17ULL
 #endif /* __ARM64_TWO_LEVEL_PMAP__ */
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__
 #else /* __ARM64_TWO_LEVEL_PMAP__ */
 #define T1SZ_BOOT                                              17ULL
 #endif /* __ARM64_TWO_LEVEL_PMAP__ */
 #else /* __ARM_16K_PG__ */
 #if __ARM64_PMAP_SUBPAGE_L1__

-#define T1SZ_BOOT                                              28ULL
+#define T1SZ_BOOT                                              26ULL

 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #define T1SZ_BOOT                                              25ULL
 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 #endif /* __ARM_16K_PG__ */
 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 
 #else /* __ARM64_PMAP_SUBPAGE_L1__ */
 #define T1SZ_BOOT                                              25ULL
 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/
 #endif /* __ARM_16K_PG__ */
 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
 

-#define TCR_EL1_BOOT   (TCR_IPS_40BITS | \
+#define TCR_EL1_BASE   (TCR_IPS_40BITS | \

                                                 TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |  TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\
                                                 TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK |  TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\

-                                                TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK |  TCR_IRGN1_WRITEBACK | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG1_GRANULE_SIZE))
+                                                TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK |  TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE))
+
+#if __ARM_KERNEL_PROTECT__
+#define TCR_EL1_BOOT   (TCR_EL1_BASE | \
+                                                (T1SZ_BOOT << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
+#define T1SZ_USER      (T1SZ_BOOT + 1)
+#define TCR_EL1_USER   (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
+#else
+#define TCR_EL1_BOOT   (TCR_EL1_BASE | \
+                                                (T1SZ_BOOT << TCR_T1SZ_SHIFT))
+#endif /* __ARM_KERNEL_PROTECT__ */

 
 /*
  * Translation Table Base Register (TTBR)
 
 /*
  * Translation Table Base Register (TTBR)


@@ -686,21 +759,21 @@
 #define ARM_TT_L1_OFFMASK                              0x0000000fffffffffULL           /* offset within an L1 entry */
 #define ARM_TT_L1_SHIFT                                        36                                                      /* page descriptor shift */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 #define ARM_TT_L1_OFFMASK                              0x0000000fffffffffULL           /* offset within an L1 entry */
 #define ARM_TT_L1_SHIFT                                        36                                                      /* page descriptor shift */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__

-/* This config supports 128GB per TTBR. */
-#define ARM_TT_L1_INDEX_MASK                   0x0000001000000000ULL           /* mask for getting index into L1 table from virtual address */
-#else
+/* This config supports 512GB per TTBR. */
+#define ARM_TT_L1_INDEX_MASK                   0x0000007000000000ULL           /* mask for getting index into L1 table from virtual address */
+#else /* __ARM64_PMAP_SUBPAGE_L1__ */

 #define ARM_TT_L1_INDEX_MASK                   0x00007ff000000000ULL           /* mask for getting index into L1 table from virtual address */
 #define ARM_TT_L1_INDEX_MASK                   0x00007ff000000000ULL           /* mask for getting index into L1 table from virtual address */

-#endif
-#else
+#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
+#else /* __ARM_16K_PG__ */

 #define ARM_TT_L1_SIZE                                 0x0000000040000000ULL           /* size of area covered by a tte */
 #define ARM_TT_L1_OFFMASK                              0x000000003fffffffULL           /* offset within an L1 entry */
 #define ARM_TT_L1_SHIFT                                        30                                                      /* page descriptor shift */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__
 #define ARM_TT_L1_SIZE                                 0x0000000040000000ULL           /* size of area covered by a tte */
 #define ARM_TT_L1_OFFMASK                              0x000000003fffffffULL           /* offset within an L1 entry */
 #define ARM_TT_L1_SHIFT                                        30                                                      /* page descriptor shift */
 #ifdef __ARM64_PMAP_SUBPAGE_L1__

-/* This config supports 64GB per TTBR. */
-#define ARM_TT_L1_INDEX_MASK                   0x0000000fc0000000ULL           /* mask for getting index into L1 table from virtual address */
-#else
+/* This config supports 256GB per TTBR. */
+#define ARM_TT_L1_INDEX_MASK                   0x0000003fc0000000ULL           /* mask for getting index into L1 table from virtual address */
+#else /* __ARM64_PMAP_SUBPAGE_L1__ */

 #define ARM_TT_L1_INDEX_MASK                   0x0000007fc0000000ULL           /* mask for getting index into L1 table from virtual address */
 #define ARM_TT_L1_INDEX_MASK                   0x0000007fc0000000ULL           /* mask for getting index into L1 table from virtual address */

-#endif
+#endif /* __ARM64_PMAP_SUBPAGE_L1__ */

 #endif
 
 /*
 #endif
 
 /*


@@ -759,6 +832,7 @@
  * Convenience definitions for:
  *   ARM_TT_LEAF: The last level of the configured page table format.
  *   ARM_TT_TWIG: The second to last level of the configured page table format.
  * Convenience definitions for:
  *   ARM_TT_LEAF: The last level of the configured page table format.
  *   ARM_TT_TWIG: The second to last level of the configured page table format.

+ *   ARM_TT_ROOT: The first level of the configured page table format.

  *
  *   My apologies to any botanists who may be reading this.
  */
  *
  *   My apologies to any botanists who may be reading this.
  */


@@ -772,6 +846,18 @@
 #define ARM_TT_TWIG_SHIFT                              ARM_TT_L2_SHIFT
 #define ARM_TT_TWIG_INDEX_MASK                 ARM_TT_L2_INDEX_MASK
 
 #define ARM_TT_TWIG_SHIFT                              ARM_TT_L2_SHIFT
 #define ARM_TT_TWIG_INDEX_MASK                 ARM_TT_L2_INDEX_MASK
 

+#if __ARM64_TWO_LEVEL_PMAP__
+#define ARM_TT_ROOT_SIZE                               ARM_TT_L2_SIZE
+#define ARM_TT_ROOT_OFFMASK                            ARM_TT_L2_OFFMASK
+#define ARM_TT_ROOT_SHIFT                              ARM_TT_L2_SHIFT
+#define ARM_TT_ROOT_INDEX_MASK                 ARM_TT_L2_INDEX_MASK
+#else
+#define ARM_TT_ROOT_SIZE                               ARM_TT_L1_SIZE
+#define ARM_TT_ROOT_OFFMASK                            ARM_TT_L1_OFFMASK
+#define ARM_TT_ROOT_SHIFT                              ARM_TT_L1_SHIFT
+#define ARM_TT_ROOT_INDEX_MASK                 ARM_TT_L1_INDEX_MASK
+#endif
+

 /*
  * 4KB granule size:
  *
 /*
  * 4KB granule size:
  *


@@ -941,8 +1027,14 @@
 #define ARM_TTE_TABLE_PXN                      0x0800000000000000ULL   /* value for privilege no execute bit */
 #define ARM_TTE_TABLE_PXNMASK          0x0800000000000000ULL   /* privilege execute mask */
 
 #define ARM_TTE_TABLE_PXN                      0x0800000000000000ULL   /* value for privilege no execute bit */
 #define ARM_TTE_TABLE_PXNMASK          0x0800000000000000ULL   /* privilege execute mask */
 

+#if __ARM_KERNEL_PROTECT__
+#define ARM_TTE_BOOT_BLOCK                     (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID |  ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)        \
+                                                                        | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF \
+                                                                        | ARM_TTE_BLOCK_NG)
+#else /* __ARM_KERNEL_PROTECT__ */

 #define ARM_TTE_BOOT_BLOCK                     (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID |  ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)        \
                                                                         | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
 #define ARM_TTE_BOOT_BLOCK                     (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID |  ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY)        \
                                                                         | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)

+#endif /* __ARM_KERNEL_PROTECT__ */

 
 #define ARM_TTE_BOOT_TABLE                     (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
 /*
 
 #define ARM_TTE_BOOT_TABLE                     (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
 /*


@@ -1083,9 +1175,26 @@
 #define ARM_PTE_PGTRACE_MASK        0x0200000000000000ULL   /* software trace mask */
 #endif
 
 #define ARM_PTE_PGTRACE_MASK        0x0200000000000000ULL   /* software trace mask */
 #endif
 

-#define ARM_PTE_BOOT_PAGE                      (ARM_PTE_TYPE_VALID |  ARM_PTE_SH(SH_OUTER_MEMORY)      \
+#define ARM_PTE_BOOT_PAGE_BASE                 (ARM_PTE_TYPE_VALID |  ARM_PTE_SH(SH_OUTER_MEMORY) \

                                                                         | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
 
                                                                         | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
 

+#if __ARM_KERNEL_PROTECT__
+#define ARM_PTE_BOOT_PAGE                      (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
+#else /* __ARM_KERNEL_PROTECT__ */
+#define ARM_PTE_BOOT_PAGE                      (ARM_PTE_BOOT_PAGE_BASE)
+#endif /* __ARM_KERNEL_PROTECT__ */
+
+/*
+ * TLBI appers to only deal in 4KB page addresses, so give
+ * it an explicit shift of 12.
+ */
+#define TLBI_ADDR_SIZE  (44)
+#define TLBI_ADDR_MASK  ((1ULL << TLBI_ADDR_SIZE) - 1)
+#define TLBI_ADDR_SHIFT (12)
+#define TLBI_ASID_SHIFT (48)
+#define TLBI_ASID_SIZE  (16)
+#define TLBI_ASID_MASK  (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT)
+

 /*
  * Exception Syndrome Register
  *
 /*
  * Exception Syndrome Register
  *

diff --git a/osfmk/arm64/sleh.c b/osfmk/arm64/sleh.c
index f77de678b6ab061308e061f6056cd832545e4ba8..6e65c759849552df208944cae939f587f1761891 100644 (file)
--- a/osfmk/arm64/sleh.c
+++ b/osfmk/arm64/sleh.c
@@ -942,7 +942,7 @@ handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr
        thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
 
        if (is_vm_fault(fault_code)) {
        thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */
 
        if (is_vm_fault(fault_code)) {

-               kern_return_t   result;
+               kern_return_t   result = KERN_FAILURE;

                vm_map_t                map = thread->map;
                vm_offset_t             vm_fault_addr = fault_addr;
 
                vm_map_t                map = thread->map;
                vm_offset_t             vm_fault_addr = fault_addr;
 


@@ -982,7 +982,10 @@ handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr
 #endif
 
                /* check to see if it is just a pmap ref/modify fault */
 #endif
 
                /* check to see if it is just a pmap ref/modify fault */

-               result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE);
+
+               if (result != KERN_SUCCESS) {
+                       result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE);
+               }

                if (result != KERN_SUCCESS) {
 
                        {
                if (result != KERN_SUCCESS) {
 
                        {

diff --git a/osfmk/arm64/start.s b/osfmk/arm64/start.s
index 2776c8f4628a48fccab7ca492c0242bedcec415c..ad181f7056bb549016c1c5a67330251cd3a1f9a5 100644 (file)
--- a/osfmk/arm64/start.s
+++ b/osfmk/arm64/start.s
@@ -35,6 +35,10 @@
 #include <machine/asm.h>
 #include "assym.s"
 
 #include <machine/asm.h>
 #include "assym.s"
 

+#if __ARM_KERNEL_PROTECT__
+#include <arm/pmap.h>
+#endif /* __ARM_KERNEL_PROTECT__ */
+

 
 .macro MSR_VBAR_EL1_X0
 #if defined(KERNEL_INTEGRITY_KTRR)
 
 .macro MSR_VBAR_EL1_X0
 #if defined(KERNEL_INTEGRITY_KTRR)


@@ -180,6 +184,15 @@ Lfound_cpu_data_entry:
 
 
 
 
 
 

+#if __ARM_KERNEL_PROTECT__ && defined(KERNEL_INTEGRITY_KTRR)
+       /*
+        * Populate TPIDR_EL1 (in case the CPU takes an exception while
+        * turning on the MMU).
+        */
+       ldr             x13, [x21, CPU_ACTIVE_THREAD]
+       msr             TPIDR_EL1, x13
+#endif /* __ARM_KERNEL_PROTECT__ */
+

        blr             x0
 Lskip_cpu_reset_handler:
        b               .                                                                       // Hang if the handler is NULL or returns
        blr             x0
 Lskip_cpu_reset_handler:
        b               .                                                                       // Hang if the handler is NULL or returns


@@ -586,6 +599,10 @@ common_start:
         */
 #endif
        and             x0, x25, #(TTBR_BADDR_MASK)
         */
 #endif
        and             x0, x25, #(TTBR_BADDR_MASK)

+#if __ARM_KERNEL_PROTECT__
+       /* We start out with a kernel ASID. */
+       orr             x0, x0, #(1 << TTBR_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */

        msr             TTBR0_EL1, x0
 #if __ARM64_TWO_LEVEL_PMAP__
        /*
        msr             TTBR0_EL1, x0
 #if __ARM64_TWO_LEVEL_PMAP__
        /*


@@ -656,10 +673,20 @@ common_start:
 1:
 
        // Set up the exception vectors
 1:
 
        // Set up the exception vectors

+#if __ARM_KERNEL_PROTECT__
+       /* If this is not the first reset of the boot CPU, the alternate mapping
+        * for the exception vectors will be set up, so use it.  Otherwise, we
+        * should use the mapping located in the kernelcache mapping.
+        */
+       MOV64   x0, ARM_KERNEL_PROTECT_EXCEPTION_START
+
+       cbnz            x21, 1f
+#endif /* __ARM_KERNEL_PROTECT__ */

        adrp    x0, EXT(ExceptionVectorsBase)@page                      // Load exception vectors base address
        add             x0, x0, EXT(ExceptionVectorsBase)@pageoff
        add             x0, x0, x22                                                                     // Convert exception vector address to KVA
        sub             x0, x0, x23
        adrp    x0, EXT(ExceptionVectorsBase)@page                      // Load exception vectors base address
        add             x0, x0, EXT(ExceptionVectorsBase)@pageoff
        add             x0, x0, x22                                                                     // Convert exception vector address to KVA
        sub             x0, x0, x23

+1:

        MSR_VBAR_EL1_X0
 
 
        MSR_VBAR_EL1_X0
 
 


@@ -874,6 +901,10 @@ arm_init_tramp:
        adrp    x0, EXT(invalid_ttep)@page
        add             x0, x0, EXT(invalid_ttep)@pageoff
        ldr             x0, [x0]
        adrp    x0, EXT(invalid_ttep)@page
        add             x0, x0, EXT(invalid_ttep)@pageoff
        ldr             x0, [x0]

+#if __ARM_KERNEL_PROTECT__
+       /* We start out with a kernel ASID. */
+       orr             x0, x0, #(1 << TTBR_ASID_SHIFT)
+#endif /* __ARM_KERNEL_PROTECT__ */

 
        msr             TTBR0_EL1, x0
 
 
        msr             TTBR0_EL1, x0
 

diff --git a/osfmk/bank/bank.c b/osfmk/bank/bank.c
index 66a4798f5e19ccb8f9f4cb49f8ad442f5eee0f81..b7d5d11d0fd3a6e55f533b729bf95256d9a8cdf5 100644 (file)
--- a/osfmk/bank/bank.c
+++ b/osfmk/bank/bank.c
@@ -44,6 +44,7 @@
 #include <sys/kdebug.h>
 #include <IOKit/IOBSD.h>
 #include <mach/mach_voucher_attr_control.h>
 #include <sys/kdebug.h>
 #include <IOKit/IOBSD.h>
 #include <mach/mach_voucher_attr_control.h>

+#include <kern/policy_internal.h>

 
 static zone_t bank_task_zone, bank_account_zone;
 #define MAX_BANK_TASK     (CONFIG_TASK_MAX)
 
 static zone_t bank_task_zone, bank_account_zone;
 #define MAX_BANK_TASK     (CONFIG_TASK_MAX)


@@ -69,13 +70,15 @@ struct _bank_ledger_indices bank_ledgers = { -1, -1 };
 
 static bank_task_t bank_task_alloc_init(task_t task);
 static bank_account_t bank_account_alloc_init(bank_task_t bank_holder, bank_task_t bank_merchant,
 
 static bank_task_t bank_task_alloc_init(task_t task);
 static bank_account_t bank_account_alloc_init(bank_task_t bank_holder, bank_task_t bank_merchant,

-               bank_task_t bank_secureoriginator, bank_task_t bank_proximateprocess);
+               bank_task_t bank_secureoriginator, bank_task_t bank_proximateprocess, struct thread_group* banktg);

 static bank_task_t get_bank_task_context(task_t task, boolean_t initialize);
 static void bank_task_dealloc(bank_task_t bank_task, mach_voucher_attr_value_reference_t sync);
 static kern_return_t bank_account_dealloc_with_sync(bank_account_t bank_account, mach_voucher_attr_value_reference_t sync);
 static void bank_rollup_chit_to_tasks(ledger_t bill, bank_task_t bank_holder, bank_task_t bank_merchant);
 static void init_bank_ledgers(void);
 static boolean_t bank_task_is_propagate_entitled(task_t t);
 static bank_task_t get_bank_task_context(task_t task, boolean_t initialize);
 static void bank_task_dealloc(bank_task_t bank_task, mach_voucher_attr_value_reference_t sync);
 static kern_return_t bank_account_dealloc_with_sync(bank_account_t bank_account, mach_voucher_attr_value_reference_t sync);
 static void bank_rollup_chit_to_tasks(ledger_t bill, bank_task_t bank_holder, bank_task_t bank_merchant);
 static void init_bank_ledgers(void);
 static boolean_t bank_task_is_propagate_entitled(task_t t);

+static struct thread_group *bank_get_bank_task_thread_group(bank_task_t bank_task __unused);
+static struct thread_group *bank_get_bank_account_thread_group(bank_account_t bank_account __unused);

 
 static lck_spin_t g_bank_task_lock_data;    /* lock to protect task->bank_context transition */
 
 
 static lck_spin_t g_bank_task_lock_data;    /* lock to protect task->bank_context transition */
 


@@ -322,6 +325,8 @@ bank_get_value(
        task_t task;
        kern_return_t kr = KERN_SUCCESS;
        mach_msg_type_number_t i;
        task_t task;
        kern_return_t kr = KERN_SUCCESS;
        mach_msg_type_number_t i;

+       struct thread_group *thread_group = NULL;
+       struct thread_group *cur_thread_group = NULL;

 
        assert(MACH_VOUCHER_ATTR_KEY_BANK == key);
        assert(manager == &bank_manager);
 
        assert(MACH_VOUCHER_ATTR_KEY_BANK == key);
        assert(manager == &bank_manager);


@@ -354,11 +359,14 @@ bank_get_value(
                                bank_holder = CAST_TO_BANK_TASK(bank_element);
                                bank_secureoriginator = bank_holder;
                                bank_proximateprocess = bank_holder;
                                bank_holder = CAST_TO_BANK_TASK(bank_element);
                                bank_secureoriginator = bank_holder;
                                bank_proximateprocess = bank_holder;

+                               thread_group = bank_get_bank_task_thread_group(bank_holder);

                        } else if (bank_element->be_type == BANK_ACCOUNT) {
                                old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element);
                                bank_holder = old_bank_account->ba_holder;
                                bank_secureoriginator = old_bank_account->ba_secureoriginator;
                                bank_proximateprocess = old_bank_account->ba_proximateprocess;
                        } else if (bank_element->be_type == BANK_ACCOUNT) {
                                old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element);
                                bank_holder = old_bank_account->ba_holder;
                                bank_secureoriginator = old_bank_account->ba_secureoriginator;
                                bank_proximateprocess = old_bank_account->ba_proximateprocess;

+                               thread_group = bank_get_bank_account_thread_group(old_bank_account);
+

                        } else {
                                panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type);
                        }
                        } else {
                                panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type);
                        }


@@ -367,17 +375,26 @@ bank_get_value(
                        if (bank_merchant == BANK_TASK_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 
                        if (bank_merchant == BANK_TASK_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 

+                       cur_thread_group = bank_get_bank_task_thread_group(bank_merchant);
+
+                       /* Change voucher thread group to current thread group for Apps */
+                       if (task_is_app(task)) {
+                               thread_group = cur_thread_group;
+                       }
+

                        /* Check if trying to redeem for self task, return the default bank task */
                        if (bank_holder == bank_merchant && 
                                bank_holder == bank_secureoriginator &&
                        /* Check if trying to redeem for self task, return the default bank task */
                        if (bank_holder == bank_merchant && 
                                bank_holder == bank_secureoriginator &&

-                               bank_holder == bank_proximateprocess) {
+                               bank_holder == bank_proximateprocess &&
+                               thread_group == cur_thread_group) {

                                *out_value = BANK_ELEMENT_TO_HANDLE(BANK_DEFAULT_TASK_VALUE);
                                *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_PERSIST;
                                return kr;
                        }
 
                        bank_account = bank_account_alloc_init(bank_holder, bank_merchant,
                                *out_value = BANK_ELEMENT_TO_HANDLE(BANK_DEFAULT_TASK_VALUE);
                                *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_PERSIST;
                                return kr;
                        }
 
                        bank_account = bank_account_alloc_init(bank_holder, bank_merchant,

-                                               bank_secureoriginator, bank_proximateprocess);
+                                               bank_secureoriginator, bank_proximateprocess,
+                                               thread_group);

                        if (bank_account == BANK_ACCOUNT_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 
                        if (bank_account == BANK_ACCOUNT_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 


@@ -405,10 +422,12 @@ bank_get_value(
                        if (bank_element->be_type == BANK_TASK) {
                                bank_holder = CAST_TO_BANK_TASK(bank_element);
                                bank_secureoriginator = bank_holder;
                        if (bank_element->be_type == BANK_TASK) {
                                bank_holder = CAST_TO_BANK_TASK(bank_element);
                                bank_secureoriginator = bank_holder;

+                               thread_group = bank_get_bank_task_thread_group(bank_holder);

                        } else if (bank_element->be_type == BANK_ACCOUNT) {
                                old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element);
                                bank_holder = old_bank_account->ba_holder;
                                bank_secureoriginator = old_bank_account->ba_secureoriginator;
                        } else if (bank_element->be_type == BANK_ACCOUNT) {
                                old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element);
                                bank_holder = old_bank_account->ba_holder;
                                bank_secureoriginator = old_bank_account->ba_secureoriginator;

+                               thread_group = bank_get_bank_account_thread_group(old_bank_account);

                        } else {
                                panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type);
                        }
                        } else {
                                panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type);
                        }


@@ -417,6 +436,8 @@ bank_get_value(
                        if (bank_merchant == BANK_TASK_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 
                        if (bank_merchant == BANK_TASK_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 

+                       cur_thread_group = bank_get_bank_task_thread_group(bank_merchant);
+

                        /*
                         * If the process doesn't have secure persona entitlement,
                         * then replace the secure originator to current task.
                        /*
                         * If the process doesn't have secure persona entitlement,
                         * then replace the secure originator to current task.


@@ -433,7 +454,8 @@ bank_get_value(
                        /* Check if trying to redeem for self task, return the bank task */
                        if (bank_holder == bank_merchant && 
                                bank_holder == bank_secureoriginator &&
                        /* Check if trying to redeem for self task, return the bank task */
                        if (bank_holder == bank_merchant && 
                                bank_holder == bank_secureoriginator &&

-                               bank_holder == bank_proximateprocess) {
+                               bank_holder == bank_proximateprocess &&
+                               thread_group == cur_thread_group) {

 
                                lck_mtx_lock(&bank_holder->bt_acc_to_pay_lock);
                                bank_task_made_reference(bank_holder);
 
                                lck_mtx_lock(&bank_holder->bt_acc_to_pay_lock);
                                bank_task_made_reference(bank_holder);


@@ -448,7 +470,8 @@ bank_get_value(
                                return kr;
                        }
                        bank_account = bank_account_alloc_init(bank_holder, bank_merchant,
                                return kr;
                        }
                        bank_account = bank_account_alloc_init(bank_holder, bank_merchant,

-                                               bank_secureoriginator, bank_proximateprocess);
+                                               bank_secureoriginator, bank_proximateprocess,
+                                               thread_group);

                        if (bank_account == BANK_ACCOUNT_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 
                        if (bank_account == BANK_ACCOUNT_NULL)
                                return KERN_RESOURCE_SHORTAGE;
 


@@ -782,7 +805,8 @@ bank_account_alloc_init(
        bank_task_t bank_holder,
        bank_task_t bank_merchant,
        bank_task_t bank_secureoriginator,
        bank_task_t bank_holder,
        bank_task_t bank_merchant,
        bank_task_t bank_secureoriginator,

-       bank_task_t bank_proximateprocess)
+       bank_task_t bank_proximateprocess,
+       struct thread_group *thread_group)

 {
        bank_account_t new_bank_account;
        bank_account_t bank_account;
 {
        bank_account_t new_bank_account;
        bank_account_t bank_account;


@@ -815,7 +839,8 @@ bank_account_alloc_init(
        queue_iterate(&bank_holder->bt_accounts_to_pay, bank_account, bank_account_t, ba_next_acc_to_pay) {
                if (bank_account->ba_merchant != bank_merchant ||
                    bank_account->ba_secureoriginator != bank_secureoriginator ||
        queue_iterate(&bank_holder->bt_accounts_to_pay, bank_account, bank_account_t, ba_next_acc_to_pay) {
                if (bank_account->ba_merchant != bank_merchant ||
                    bank_account->ba_secureoriginator != bank_secureoriginator ||

-                   bank_account->ba_proximateprocess != bank_proximateprocess)
+                   bank_account->ba_proximateprocess != bank_proximateprocess ||
+                   bank_get_bank_account_thread_group(bank_account) != thread_group)

                        continue;
 
                entry_found = TRUE;
                        continue;
 
                entry_found = TRUE;


@@ -1322,7 +1347,8 @@ bank_serviced_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energ
  * Routine: bank_get_voucher_bank_account
  * Purpose: Get the bank account from the voucher.
  * Returns: bank_account if bank_account attribute present in voucher.
  * Routine: bank_get_voucher_bank_account
  * Purpose: Get the bank account from the voucher.
  * Returns: bank_account if bank_account attribute present in voucher.

- *          NULL on no attribute, no bank_element, or if holder and merchant bank accounts are the same.
+ *          NULL on no attribute, no bank_element, or if holder and merchant bank accounts
+ *          and voucher thread group and current thread group are the same.

  */
 static bank_account_t
 bank_get_voucher_bank_account(ipc_voucher_t voucher)
  */
 static bank_account_t
 bank_get_voucher_bank_account(ipc_voucher_t voucher)


@@ -1352,12 +1378,20 @@ bank_get_voucher_bank_account(ipc_voucher_t voucher)
                return BANK_ACCOUNT_NULL;
        } else if (bank_element->be_type == BANK_ACCOUNT) {
                bank_account = CAST_TO_BANK_ACCOUNT(bank_element);
                return BANK_ACCOUNT_NULL;
        } else if (bank_element->be_type == BANK_ACCOUNT) {
                bank_account = CAST_TO_BANK_ACCOUNT(bank_element);

-               if (bank_account->ba_holder != bank_account->ba_merchant) {
-                       bank_task_t bank_merchant = get_bank_task_context(current_task(), FALSE);
-                       if (bank_account->ba_merchant == bank_merchant)
-                               return bank_account;
-                       else
-                               return BANK_ACCOUNT_NULL;
+               /*
+                * Return BANK_ACCOUNT_NULL if the ba_holder is same as ba_merchant
+                * and bank account thread group is same as current thread group
+                * i.e. ba_merchant's thread group.
+                *
+                * The bank account might have ba_holder same as ba_merchant but different
+                * thread group if daemon sends a voucher to an App and then App sends the
+                * same voucher back to the daemon (IPC code will replace thread group in the
+                * voucher to App's thread group when it gets auto redeemed by the App).
+                */
+               if (bank_account->ba_holder != bank_account->ba_merchant ||
+                       bank_get_bank_account_thread_group(bank_account) !=
+                               bank_get_bank_task_thread_group(bank_account->ba_merchant)) {
+                       return bank_account;

                } else {
                        return BANK_ACCOUNT_NULL;
                }
                } else {
                        return BANK_ACCOUNT_NULL;
                }


@@ -1369,19 +1403,45 @@ bank_get_voucher_bank_account(ipc_voucher_t voucher)
 
 /*
  * Routine: bank_get_bank_account_ledger
 
 /*
  * Routine: bank_get_bank_account_ledger

- * Purpose: Get the bankledger from the bank account
+ * Purpose: Get the bankledger from the bank account if ba_merchant different than ba_holder

  */
 static ledger_t
 bank_get_bank_account_ledger(bank_account_t bank_account)
 {
        ledger_t bankledger = NULL;
 
  */
 static ledger_t
 bank_get_bank_account_ledger(bank_account_t bank_account)
 {
        ledger_t bankledger = NULL;
 

-       if (bank_account != BANK_ACCOUNT_NULL)
+       if (bank_account != BANK_ACCOUNT_NULL &&
+               bank_account->ba_holder != bank_account->ba_merchant)

                bankledger = bank_account->ba_bill;
 
        return (bankledger);
 }
 
                bankledger = bank_account->ba_bill;
 
        return (bankledger);
 }
 

+/*
+ * Routine: bank_get_bank_task_thread_group
+ * Purpose: Get the bank task's thread group from the bank task
+ */
+static struct thread_group *
+bank_get_bank_task_thread_group(bank_task_t bank_task __unused)
+{
+       struct thread_group *banktg = NULL;
+
+
+       return (banktg);
+}
+
+/*
+ * Routine: bank_get_bank_account_thread_group
+ * Purpose: Get the bank account's thread group from the bank account
+ */
+static struct thread_group *
+bank_get_bank_account_thread_group(bank_account_t bank_account __unused)
+{
+       thread_group_t banktg = NULL;
+
+
+       return (banktg);
+}

 
 /*
  * Routine: bank_get_bank_ledger_and_thread_group
 
 /*
  * Routine: bank_get_bank_ledger_and_thread_group


@@ -1393,12 +1453,21 @@ kern_return_t
 bank_get_bank_ledger_and_thread_group(
        ipc_voucher_t     voucher,
        ledger_t          *bankledger,
 bank_get_bank_ledger_and_thread_group(
        ipc_voucher_t     voucher,
        ledger_t          *bankledger,

-       thread_group_t    *banktg __unused)
+       thread_group_t    *banktg)

 {
        bank_account_t bank_account;
 {
        bank_account_t bank_account;

+       struct thread_group *thread_group = NULL;

 
        bank_account = bank_get_voucher_bank_account(voucher);
        *bankledger = bank_get_bank_account_ledger(bank_account);
 
        bank_account = bank_get_voucher_bank_account(voucher);
        *bankledger = bank_get_bank_account_ledger(bank_account);

+       thread_group = bank_get_bank_account_thread_group(bank_account);
+
+       /* Return NULL thread group if voucher has current task's thread group */
+       if (thread_group == bank_get_bank_task_thread_group(
+                       get_bank_task_context(current_task(), FALSE))) {
+               thread_group = NULL;
+       }
+       *banktg = thread_group;

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

diff --git a/osfmk/i386/AT386/model_dep.c b/osfmk/i386/AT386/model_dep.c
index 2d29bf9685c84fa59049c0d45c7fcae52886c025..10b4c4c3ff085657d18ac8b8fb8a39cf9fb072cd 100644 (file)
--- a/osfmk/i386/AT386/model_dep.c
+++ b/osfmk/i386/AT386/model_dep.c
@@ -134,10 +134,26 @@
 #define DPRINTF(x...)
 #endif
 
 #define DPRINTF(x...)
 #endif
 

+#ifndef ROUNDUP
+#define ROUNDUP(a, b) (((a) + ((b) - 1)) & (~((b) - 1)))
+#endif
+
+#ifndef ROUNDDOWN
+#define ROUNDDOWN(x,y) (((x)/(y))*(y))
+#endif
+

 static void machine_conf(void);
 void panic_print_symbol_name(vm_address_t search);
 void RecordPanicStackshot(void);
 
 static void machine_conf(void);
 void panic_print_symbol_name(vm_address_t search);
 void RecordPanicStackshot(void);
 

+typedef enum paniclog_flush_type {
+    kPaniclogFlushBase         = 1, /* Flush the initial log and paniclog header */
+    kPaniclogFlushStackshot    = 2, /* Flush only the stackshot data, then flush the header */
+    kPaniclogFlushOtherLog     = 3  /* Flush the other log, then flush the header */
+} paniclog_flush_type_t;
+
+void paniclog_flush_internal(paniclog_flush_type_t variant);
+

 extern const char      version[];
 extern char    osversion[];
 extern int             max_unsafe_quanta;
 extern const char      version[];
 extern char    osversion[];
 extern int             max_unsafe_quanta;


@@ -919,8 +935,6 @@ RecordPanicStackshot()
                                }
                        }
                }
                                }
                        }
                }

-
-               paniclog_flush();

 #if DEVELOPMENT || DEBUG
                if (panic_stackshot_buf != 0) {
                        // We're going to try to take another stackshot, reset the state.
 #if DEVELOPMENT || DEBUG
                if (panic_stackshot_buf != 0) {
                        // We're going to try to take another stackshot, reset the state.


@@ -1008,7 +1022,7 @@ SavePanicInfo(
        }
 
        /* Flush the panic log */
        }
 
        /* Flush the panic log */

-       paniclog_flush();
+       paniclog_flush_internal(kPaniclogFlushBase);

 
        /* Try to take a panic stackshot */
        RecordPanicStackshot();
 
        /* Try to take a panic stackshot */
        RecordPanicStackshot();


@@ -1018,17 +1032,12 @@ SavePanicInfo(
         * from when we tried to capture it.
         */
        if (extended_debug_log_enabled) {
         * from when we tried to capture it.
         */
        if (extended_debug_log_enabled) {

-               paniclog_flush();
+               paniclog_flush_internal(kPaniclogFlushStackshot);

        }
 }
 
        }
 }
 

-void
-paniclog_flush()
+void paniclog_flush_internal(paniclog_flush_type_t variant)

 {
 {

-       unsigned long pi_size = 0;
-
-       assert(panic_info != NULL);
-

        /* Update the other log offset if we've opened the other log */
        if (panic_info->mph_other_log_offset != 0) {
                panic_info->mph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->mph_other_log_offset;
        /* Update the other log offset if we've opened the other log */
        if (panic_info->mph_other_log_offset != 0) {
                panic_info->mph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->mph_other_log_offset;


@@ -1039,7 +1048,8 @@ paniclog_flush()
         * panic callbacks, otherwise we flush via nvram (unless that has been disabled).
         */
        if (coprocessor_paniclog_flush) {
         * panic callbacks, otherwise we flush via nvram (unless that has been disabled).
         */
        if (coprocessor_paniclog_flush) {

-               unsigned int size_to_flush = debug_buf_size;
+               uint32_t overall_buffer_size = debug_buf_size;
+               uint32_t size_to_flush = 0, offset_to_flush = 0;

                if (extended_debug_log_enabled) {
                        /*
                         * debug_buf_size for the extended log does not include the length of the header.
                if (extended_debug_log_enabled) {
                        /*
                         * debug_buf_size for the extended log does not include the length of the header.


@@ -1047,16 +1057,47 @@ paniclog_flush()
                         * for the non-extended case (this is a concession we make to not shrink the paniclog data
                         * for non-coprocessor systems that only use the basic log).
                         */
                         * for the non-extended case (this is a concession we make to not shrink the paniclog data
                         * for non-coprocessor systems that only use the basic log).
                         */

-                       size_to_flush = debug_buf_size + sizeof(struct macos_panic_header);
+                       overall_buffer_size = debug_buf_size + sizeof(struct macos_panic_header);

                }
 
                }
 

-               /* We need to calculate the CRC for co-processor platforms */
-               panic_info->mph_crc = crc32(0L, &panic_info->mph_version, (size_to_flush - offsetof(struct macos_panic_header, mph_version)));
+               /* Update the CRC */
+               panic_info->mph_crc = crc32(0L, &panic_info->mph_version, (overall_buffer_size - offsetof(struct macos_panic_header, mph_version)));
+
+               if (variant == kPaniclogFlushBase) {
+                       /* Flush the header and base panic log. */
+                       kprintf("Flushing base panic log\n");
+                       size_to_flush = ROUNDUP((panic_info->mph_panic_log_offset + panic_info->mph_panic_log_len), PANIC_FLUSH_BOUNDARY);
+                       offset_to_flush = 0;
+                       PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush);
+               } else if ((variant == kPaniclogFlushStackshot) || (variant == kPaniclogFlushOtherLog)) {
+                       if (variant == kPaniclogFlushStackshot) {
+                               /*
+                                * We flush the stackshot before flushing the updated header because the stackshot
+                                * can take a while to flush. We want the paniclog header to be as consistent as possible even
+                                * if the stackshot isn't flushed completely. Flush starting from the end of the panic log.
+                                */
+                               kprintf("Flushing panic log stackshot\n");
+                               offset_to_flush = ROUNDDOWN((panic_info->mph_panic_log_offset + panic_info->mph_panic_log_len), PANIC_FLUSH_BOUNDARY);
+                               size_to_flush = ROUNDUP((panic_info->mph_stackshot_len + (panic_info->mph_stackshot_offset - offset_to_flush)), PANIC_FLUSH_BOUNDARY);
+                               PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush);
+                       }

 
 

-               PESavePanicInfoAction(panic_info, size_to_flush);
+                       /* Flush the other log -- everything after the stackshot */
+                       kprintf("Flushing panic 'other' log\n");
+                       offset_to_flush = ROUNDDOWN((panic_info->mph_stackshot_offset + panic_info->mph_stackshot_len), PANIC_FLUSH_BOUNDARY);
+                       size_to_flush = ROUNDUP((panic_info->mph_other_log_len + (panic_info->mph_other_log_offset - offset_to_flush)), PANIC_FLUSH_BOUNDARY);
+                       PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush);
+
+                       /* Flush the header -- everything before the paniclog */
+                       kprintf("Flushing panic log header\n");
+                       size_to_flush = ROUNDUP(panic_info->mph_panic_log_offset, PANIC_FLUSH_BOUNDARY);
+                       offset_to_flush = 0;
+                       PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush);
+               }

        } else if (commit_paniclog_to_nvram) {
                assert(debug_buf_size != 0);
                unsigned int bufpos;
        } else if (commit_paniclog_to_nvram) {
                assert(debug_buf_size != 0);
                unsigned int bufpos;

+               unsigned long pi_size = 0;

                uintptr_t cr0;
 
                debug_putc(0);
                uintptr_t cr0;
 
                debug_putc(0);


@@ -1109,6 +1150,14 @@ paniclog_flush()
        }
 }
 
        }
 }
 

+void
+paniclog_flush()
+{
+       /* Called outside of this file to update logging appended to the "other" log */
+       paniclog_flush_internal(kPaniclogFlushOtherLog);
+       return;
+}
+

 char *
 machine_boot_info(char *buf, __unused vm_size_t size)
 {
 char *
 machine_boot_info(char *buf, __unused vm_size_t size)
 {

diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h
index 27c493c248a6ae7c767b58ee64776ef62779da7b..618389fd9d3bd8c9bbb6d3bd985e31623b3f242c 100644 (file)
--- a/osfmk/i386/cpu_data.h
+++ b/osfmk/i386/cpu_data.h
@@ -78,11 +78,17 @@ typedef struct rtclock_timer {
 } rtclock_timer_t;
 
 typedef struct {
 } rtclock_timer_t;
 
 typedef struct {

-       struct x86_64_tss       *cdi_ktss;
-       x86_64_desc_register_t  cdi_gdt;
-       x86_64_desc_register_t  cdi_idt;
-       struct fake_descriptor  *cdi_ldt;
-       vm_offset_t             cdi_sstk;
+       /* The 'u' suffixed fields store the double-mapped descriptor addresses */
+       struct x86_64_tss       *cdi_ktssu;
+       struct x86_64_tss       *cdi_ktssb;
+       x86_64_desc_register_t  cdi_gdtu;
+       x86_64_desc_register_t  cdi_gdtb;
+       x86_64_desc_register_t  cdi_idtu;
+       x86_64_desc_register_t  cdi_idtb;
+       struct fake_descriptor  *cdi_ldtu;
+       struct fake_descriptor  *cdi_ldtb;
+       vm_offset_t             cdi_sstku;
+       vm_offset_t             cdi_sstkb;

 } cpu_desc_index_t;
 
 typedef enum {
 } cpu_desc_index_t;
 
 typedef enum {


@@ -109,7 +115,7 @@ typedef     uint8_t         pcid_ref_t;
 #define CPU_RTIME_BINS (12)
 #define CPU_ITIME_BINS (CPU_RTIME_BINS)
 
 #define CPU_RTIME_BINS (12)
 #define CPU_ITIME_BINS (CPU_RTIME_BINS)
 

-#define MAXPLFRAMES (32)
+#define MAXPLFRAMES (16)

 typedef struct {
        boolean_t pltype;
        int plevel;
 typedef struct {
        boolean_t pltype;
        int plevel;


@@ -129,6 +135,13 @@ typedef struct {
  * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
  * pointers.
  */
  * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
  * pointers.
  */

+typedef struct {
+       pcid_t                  cpu_pcid_free_hint;
+#define        PMAP_PCID_MAX_PCID      (0x800)
+       pcid_ref_t              cpu_pcid_refcounts[PMAP_PCID_MAX_PCID];
+       pmap_t                  cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID];
+} pcid_cdata_t;
+

 typedef struct cpu_data
 {
        struct pal_cpu_data     cpu_pal_data;           /* PAL-specific data */
 typedef struct cpu_data
 {
        struct pal_cpu_data     cpu_pal_data;           /* PAL-specific data */


@@ -143,8 +156,6 @@ typedef struct cpu_data
        vm_offset_t             cpu_kernel_stack;       /* kernel stack top */
        vm_offset_t             cpu_int_stack_top;
        int                     cpu_interrupt_level;
        vm_offset_t             cpu_kernel_stack;       /* kernel stack top */
        vm_offset_t             cpu_int_stack_top;
        int                     cpu_interrupt_level;

-       int                     cpu_phys_number;        /* Physical CPU */
-       cpu_id_t                cpu_id;                 /* Platform Expert */

        volatile int            cpu_signals;            /* IPI events */
        volatile int            cpu_prior_signals;      /* Last set of events,
                                                         * debugging
        volatile int            cpu_signals;            /* IPI events */
        volatile int            cpu_prior_signals;      /* Last set of events,
                                                         * debugging


@@ -166,18 +177,22 @@ typedef struct cpu_data
        };
        volatile task_map_t     cpu_task_map;
        volatile addr64_t       cpu_task_cr3;
        };
        volatile task_map_t     cpu_task_map;
        volatile addr64_t       cpu_task_cr3;

+       volatile addr64_t       cpu_ucr3;

        addr64_t                cpu_kernel_cr3;
        boolean_t               cpu_pagezero_mapped;
        cpu_uber_t              cpu_uber;
        addr64_t                cpu_kernel_cr3;
        boolean_t               cpu_pagezero_mapped;
        cpu_uber_t              cpu_uber;

-       void                    *cpu_chud;
-       void                    *cpu_console_buf;
-       struct x86_lcpu         lcpu;
+       /* Double-mapped per-CPU exception stack address */
+       uintptr_t               cd_estack;
+       /* Address of shadowed, partially mirrored CPU data structures located
+        * in the double mapped PML4
+        */
+       void                    *cd_shadow;

        struct processor        *cpu_processor;
 #if NCOPY_WINDOWS > 0
        struct cpu_pmap         *cpu_pmap;
 #endif
        struct cpu_desc_table   *cpu_desc_tablep;
        struct processor        *cpu_processor;
 #if NCOPY_WINDOWS > 0
        struct cpu_pmap         *cpu_pmap;
 #endif
        struct cpu_desc_table   *cpu_desc_tablep;

-       struct fake_descriptor  *cpu_ldtp;
+       struct real_descriptor  *cpu_ldtp;

        cpu_desc_index_t        cpu_desc_index;
        int                     cpu_ldt;
 #if NCOPY_WINDOWS > 0
        cpu_desc_index_t        cpu_desc_index;
        int                     cpu_ldt;
 #if NCOPY_WINDOWS > 0


@@ -191,7 +206,6 @@ typedef struct cpu_data
 #define HWINTCNT_SIZE 256
        uint32_t                cpu_hwIntCnt[HWINTCNT_SIZE];    /* Interrupt counts */
        uint64_t                cpu_hwIntpexits[HWINTCNT_SIZE];
 #define HWINTCNT_SIZE 256
        uint32_t                cpu_hwIntCnt[HWINTCNT_SIZE];    /* Interrupt counts */
        uint64_t                cpu_hwIntpexits[HWINTCNT_SIZE];

-       uint64_t                cpu_hwIntcexits[HWINTCNT_SIZE];

        uint64_t                cpu_dr7; /* debug control register */
        uint64_t                cpu_int_event_time;     /* intr entry/exit time */
        pal_rtc_nanotime_t      *cpu_nanotime;          /* Nanotime info */
        uint64_t                cpu_dr7; /* debug control register */
        uint64_t                cpu_int_event_time;     /* intr entry/exit time */
        pal_rtc_nanotime_t      *cpu_nanotime;          /* Nanotime info */


@@ -211,10 +225,7 @@ typedef struct cpu_data
        pcid_t                  cpu_kernel_pcid;
        volatile pcid_ref_t     *cpu_pmap_pcid_coherentp;
        volatile pcid_ref_t     *cpu_pmap_pcid_coherentp_kernel;
        pcid_t                  cpu_kernel_pcid;
        volatile pcid_ref_t     *cpu_pmap_pcid_coherentp;
        volatile pcid_ref_t     *cpu_pmap_pcid_coherentp_kernel;

-#define        PMAP_PCID_MAX_PCID      (0x1000)
-       pcid_t                  cpu_pcid_free_hint;
-       pcid_ref_t              cpu_pcid_refcounts[PMAP_PCID_MAX_PCID];
-       pmap_t                  cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID];
+       pcid_cdata_t            *cpu_pcid_data;

 #ifdef PCID_STATS
        uint64_t                cpu_pmap_pcid_flushes;
        uint64_t                cpu_pmap_pcid_preserves;
 #ifdef PCID_STATS
        uint64_t                cpu_pmap_pcid_flushes;
        uint64_t                cpu_pmap_pcid_preserves;


@@ -263,10 +274,20 @@ typedef struct cpu_data
        boolean_t               cpu_iflag;
        boolean_t               cpu_boot_complete;
        int                     cpu_hibernate;
        boolean_t               cpu_iflag;
        boolean_t               cpu_boot_complete;
        int                     cpu_hibernate;

-#define MAX_PREEMPTION_RECORDS (128)
+#define MAX_PREEMPTION_RECORDS (8)

 #if    DEVELOPMENT || DEBUG
        int                     cpu_plri;
        plrecord_t              plrecords[MAX_PREEMPTION_RECORDS];
 #if    DEVELOPMENT || DEBUG
        int                     cpu_plri;
        plrecord_t              plrecords[MAX_PREEMPTION_RECORDS];

+#endif
+       void                    *cpu_chud;
+       void                    *cpu_console_buf;
+       struct x86_lcpu         lcpu;
+       int                     cpu_phys_number;        /* Physical CPU */
+       cpu_id_t                cpu_id;                 /* Platform Expert */
+#if DEBUG
+       uint64_t                cpu_entry_cr3;
+       uint64_t                cpu_exit_cr3;
+       uint64_t                cpu_pcid_last_cr3;

 #endif
 } cpu_data_t;
 
 #endif
 } cpu_data_t;
 


@@ -585,12 +606,18 @@ _mp_enable_preemption_no_check(void) {
 #define MACHINE_PREEMPTION_MACROS (1)
 #endif
 
 #define MACHINE_PREEMPTION_MACROS (1)
 #endif
 

-

 static inline cpu_data_t *
 cpu_datap(int cpu) {
        return cpu_data_ptr[cpu];
 }
 
 static inline cpu_data_t *
 cpu_datap(int cpu) {
        return cpu_data_ptr[cpu];
 }
 

+#ifdef MACH_KERNEL_PRIVATE
+static inline cpu_data_t *
+cpu_shadowp(int cpu) {
+       return cpu_data_ptr[cpu]->cd_shadow;
+}
+
+#endif

 extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
 extern void cpu_data_realloc(void);
 
 extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
 extern void cpu_data_realloc(void);
 

diff --git a/osfmk/i386/cpu_topology.c b/osfmk/i386/cpu_topology.c
index a8517d148d92ff9ba4d5274e93d48a3d3bb12904..e079e6e38ff011912ec22a44908b28eabd777367 100644 (file)
--- a/osfmk/i386/cpu_topology.c
+++ b/osfmk/i386/cpu_topology.c
@@ -50,6 +50,19 @@ static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
 x86_affinity_set_t     *x86_affinities = NULL;
 static int             x86_affinity_count = 0;
 
 x86_affinity_set_t     *x86_affinities = NULL;
 static int             x86_affinity_count = 0;
 

+extern cpu_data_t cpshadows[];
+/* Re-sort double-mapped CPU data shadows after topology discovery sorts the
+ * primary CPU data structures by physical/APIC CPU ID.
+ */
+static void cpu_shadow_sort(int ncpus) {
+       for (int i = 0; i < ncpus; i++) {
+               cpu_data_t      *cpup = cpu_datap(i);
+               ptrdiff_t       coff = cpup - cpu_datap(0);
+
+               cpup->cd_shadow = &cpshadows[coff];
+       }
+}
+

 /*
  * cpu_topology_sort() is called after all processors have been registered
  * but before any non-boot processor id started.
 /*
  * cpu_topology_sort() is called after all processors have been registered
  * but before any non-boot processor id started.


@@ -122,6 +135,7 @@ cpu_topology_sort(int ncpus)
                x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
        }
 
                x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i);
        }
 

+       cpu_shadow_sort(ncpus);

        x86_validate_topology();
 
        ml_set_interrupts_enabled(istate);
        x86_validate_topology();
 
        ml_set_interrupts_enabled(istate);

diff --git a/osfmk/i386/genassym.c b/osfmk/i386/genassym.c
index 510bdf89ffa3e0ba70f34a4bab6ad2489829f404..f6812c6f0cca2877d62f230d53d351193a34ccb9 100644 (file)
--- a/osfmk/i386/genassym.c
+++ b/osfmk/i386/genassym.c
@@ -311,11 +311,11 @@ main(
        DECLARE("KERNEL_PML4_INDEX", KERNEL_PML4_INDEX);
        DECLARE("IDTSZ",        IDTSZ);
        DECLARE("GDTSZ",        GDTSZ);
        DECLARE("KERNEL_PML4_INDEX", KERNEL_PML4_INDEX);
        DECLARE("IDTSZ",        IDTSZ);
        DECLARE("GDTSZ",        GDTSZ);

-       DECLARE("LDTSZ",        LDTSZ);

 
        DECLARE("KERNEL_DS",    KERNEL_DS);
        DECLARE("USER_CS",      USER_CS);
        DECLARE("USER_DS",      USER_DS);
 
        DECLARE("KERNEL_DS",    KERNEL_DS);
        DECLARE("USER_CS",      USER_CS);
        DECLARE("USER_DS",      USER_DS);

+       DECLARE("USER_CTHREAD", USER_CTHREAD);

        DECLARE("KERNEL32_CS",  KERNEL32_CS);
        DECLARE("KERNEL64_CS",  KERNEL64_CS);
        DECLARE("USER64_CS",    USER64_CS);
        DECLARE("KERNEL32_CS",  KERNEL32_CS);
        DECLARE("KERNEL64_CS",  KERNEL64_CS);
        DECLARE("USER64_CS",    USER64_CS);


@@ -350,14 +350,6 @@ main(
                offsetof(cpu_data_t,cpu_running));
        DECLARE("CPU_PENDING_AST",
                offsetof(cpu_data_t,cpu_pending_ast));
                offsetof(cpu_data_t,cpu_running));
        DECLARE("CPU_PENDING_AST",
                offsetof(cpu_data_t,cpu_pending_ast));

-       DECLARE("CPU_DESC_TABLEP",
-               offsetof(cpu_data_t,cpu_desc_tablep));
-       DECLARE("CPU_DESC_INDEX",
-               offsetof(cpu_data_t,cpu_desc_index));
-       DECLARE("CDI_GDT",
-               offsetof(cpu_desc_index_t,cdi_gdt));
-       DECLARE("CDI_IDT",
-               offsetof(cpu_desc_index_t,cdi_idt));

        DECLARE("CPU_PROCESSOR",
                offsetof(cpu_data_t,cpu_processor));
         DECLARE("CPU_INT_STATE",
        DECLARE("CPU_PROCESSOR",
                offsetof(cpu_data_t,cpu_processor));
         DECLARE("CPU_INT_STATE",


@@ -371,6 +363,14 @@ main(
                offsetof(cpu_data_t, cpu_active_cr3));
         DECLARE("CPU_KERNEL_CR3",
                offsetof(cpu_data_t, cpu_kernel_cr3));
                offsetof(cpu_data_t, cpu_active_cr3));
         DECLARE("CPU_KERNEL_CR3",
                offsetof(cpu_data_t, cpu_kernel_cr3));

+       DECLARE("CPU_UCR3",
+               offsetof(cpu_data_t, cpu_ucr3));
+#if    DEBUG
+       DECLARE("CPU_ENTRY_CR3",
+               offsetof(cpu_data_t, cpu_entry_cr3));
+       DECLARE("CPU_EXIT_CR3",
+               offsetof(cpu_data_t, cpu_exit_cr3));
+#endif

        DECLARE("CPU_TLB_INVALID",
                offsetof(cpu_data_t, cpu_tlb_invalid));
        DECLARE("CPU_PAGEZERO_MAPPED",
        DECLARE("CPU_TLB_INVALID",
                offsetof(cpu_data_t, cpu_tlb_invalid));
        DECLARE("CPU_PAGEZERO_MAPPED",


@@ -418,6 +418,11 @@ main(
            offsetof(cpu_data_t, cpu_tlb_invalid_local));
        DECLARE("CPU_TLB_INVALID_GLOBAL",
                offsetof(cpu_data_t, cpu_tlb_invalid_global));
            offsetof(cpu_data_t, cpu_tlb_invalid_local));
        DECLARE("CPU_TLB_INVALID_GLOBAL",
                offsetof(cpu_data_t, cpu_tlb_invalid_global));

+       DECLARE("CPU_ESTACK",
+               offsetof(cpu_data_t, cd_estack));
+       DECLARE("CPU_DSHADOW",
+               offsetof(cpu_data_t, cd_shadow));
+

        DECLARE("enaExpTrace",  enaExpTrace);
        DECLARE("enaUsrFCall",  enaUsrFCall);
        DECLARE("enaUsrPhyMp",  enaUsrPhyMp);
        DECLARE("enaExpTrace",  enaExpTrace);
        DECLARE("enaUsrFCall",  enaUsrFCall);
        DECLARE("enaUsrPhyMp",  enaUsrPhyMp);

diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c
index 73a30a22b93b387fb33ccc94a0bdef13df3316db..5a7655de9fbb7679518e0e157f9cd6697366a799 100644 (file)
--- a/osfmk/i386/i386_init.c
+++ b/osfmk/i386/i386_init.c
@@ -137,6 +137,7 @@ pdpt_entry_t                *IdlePDPT;
 pml4_entry_t           *IdlePML4;
 
 char *physfree;
 pml4_entry_t           *IdlePML4;
 
 char *physfree;

+void idt64_remap(void);

 
 /*
  * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()
 
 /*
  * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init()


@@ -249,32 +250,7 @@ physmap_init(void)
                KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
 }
 
                KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]);
 }
 

-static void
-descriptor_alias_init()
-{
-       vm_offset_t     master_gdt_phys;
-       vm_offset_t     master_gdt_alias_phys;
-       vm_offset_t     master_idt_phys;
-       vm_offset_t     master_idt_alias_phys;
-
-       assert(((vm_offset_t)master_gdt & PAGE_MASK) == 0);
-       assert(((vm_offset_t)master_idt64 & PAGE_MASK) == 0);
-
-       master_gdt_phys       = (vm_offset_t) ID_MAP_VTOP(master_gdt);
-       master_idt_phys       = (vm_offset_t) ID_MAP_VTOP(master_idt64);
-       master_gdt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_GDT_ALIAS);
-       master_idt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_IDT_ALIAS);
-       
-       DBG("master_gdt_phys:       %p\n", (void *) master_gdt_phys);
-       DBG("master_idt_phys:       %p\n", (void *) master_idt_phys);
-       DBG("master_gdt_alias_phys: %p\n", (void *) master_gdt_alias_phys);
-       DBG("master_idt_alias_phys: %p\n", (void *) master_idt_alias_phys);
-
-       KPTphys[atop_kernel(master_gdt_alias_phys)] = master_gdt_phys |
-               INTEL_PTE_VALID | INTEL_PTE_NX | INTEL_PTE_WRITE;
-       KPTphys[atop_kernel(master_idt_alias_phys)] = master_idt_phys |
-               INTEL_PTE_VALID | INTEL_PTE_NX; /* read-only */
-}
+void doublemap_init(void);

 
 static void
 Idle_PTs_init(void)
 
 static void
 Idle_PTs_init(void)


@@ -304,10 +280,8 @@ Idle_PTs_init(void)
        postcode(VSTART_PHYSMAP_INIT);
 
        physmap_init();
        postcode(VSTART_PHYSMAP_INIT);
 
        physmap_init();

-
-       postcode(VSTART_DESC_ALIAS_INIT);
-
-       descriptor_alias_init();
+       doublemap_init();
+       idt64_remap();

 
        postcode(VSTART_SET_CR3);
 
 
        postcode(VSTART_SET_CR3);
 


@@ -685,4 +659,115 @@ i386_init_slave_fast(void)
        do_init_slave(TRUE);
 }
 
        do_init_slave(TRUE);
 }
 

+#include <libkern/kernel_mach_header.h>
+
+/* TODO: Evaluate global PTEs for the double-mapped translations */
+
+uint64_t dblmap_base, dblmap_max;
+kernel_segment_command_t *hdescseg;

 
 

+pt_entry_t *dblmapL3;
+unsigned int dblallocs;
+uint64_t dblmap_dist;
+extern uint64_t idt64_hndl_table0[];
+
+
+void doublemap_init(void) {
+       dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries
+       dblallocs++;
+
+       struct {
+               pt_entry_t entries[PTE_PER_PAGE];
+       } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries
+       dblallocs++;
+
+       dblmapL3[0] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0]))
+           | INTEL_PTE_VALID
+           | INTEL_PTE_WRITE;
+
+       hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB");
+
+       vm_offset_t hdescb = hdescseg->vmaddr;
+       unsigned long hdescsz = hdescseg->vmsize;
+       unsigned long hdescszr = round_page_64(hdescsz);
+       vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr;
+
+       kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text");
+       vm_offset_t thdescb = thdescsect->addr;
+       unsigned long thdescsz = thdescsect->size;
+       unsigned long thdescszr = round_page_64(thdescsz);
+       vm_offset_t thdesce = thdescb + thdescszr;
+
+       assert((hdescb & 0xFFF) == 0);
+       /* Mirror HIB translations into the double-mapped pagetable subtree*/
+       for(int i = 0; hdescc < hdesce; i++) {
+               struct {
+                       pt_entry_t entries[PTE_PER_PAGE];
+               } * dblmapL1 = ALLOCPAGES(1);
+               dblallocs++;
+               dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
+               int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE);
+               for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) {
+                       uint64_t template = INTEL_PTE_VALID;
+                       if ((hdescc >= thdescb) && (hdescc < thdesce)) {
+                               /* executable */
+                       } else {
+                               template |= INTEL_PTE_WRITE | INTEL_PTE_NX ; /* Writeable, NX */
+                       }
+                       dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template;
+                       hdescc += PAGE_SIZE;
+               }
+       }
+
+       IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF;
+
+       dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, dblmapL3, 0, 0);
+       dblmap_max = dblmap_base + hdescszr;
+       /* Calculate the double-map distance, which accounts for the current
+        * KASLR slide
+        */
+
+       dblmap_dist = dblmap_base - hdescb;
+       idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]);
+
+       extern cpu_data_t cpshadows[], scdatas[];
+       uintptr_t cd1 = (uintptr_t) &cpshadows[0];
+       uintptr_t cd2 = (uintptr_t) &scdatas[0];
+/* Record the displacement from the kernel's per-CPU data pointer, eventually
+ * programmed into GSBASE, to the "shadows" in the doublemapped
+ * region. These are not aliases, but separate physical allocations
+ * containing data required in the doublemapped trampolines.
+*/
+       idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2;
+
+       DBG("Double map base: 0x%qx\n", dblmap_base);
+       DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]);
+       assert(LDTSZ > LDTSZ_MIN);
+}
+
+vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
+
+#include <i386/pmap_internal.h>
+
+/* Use of this routine is expected to be synchronized by callers
+ * Creates non-executable aliases.
+ */
+vm_offset_t dyn_dblmap(vm_offset_t cva, vm_offset_t sz) {
+       vm_offset_t ava = dblmap_max;
+
+       assert((sz & PAGE_MASK) == 0);
+       assert(cva != 0);
+
+       pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP);
+       dblmap_max += sz;
+       return (ava - cva);
+}
+/* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect
+ * control to the double-mapped interrupt vectors. The IDTR proper will be
+ * programmed via cpu_desc_load()
+ */
+void idt64_remap(void) {
+       for (int i = 0; i < IDTSZ; i++) {
+               master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64);
+       }
+}

diff --git a/osfmk/i386/ktss.c b/osfmk/i386/ktss.c
index f5d3118f7c447b431b0294172fad19f813d8dd6e..9d49d25713221d94f372c4c8d070b8441bf24cbe 100644 (file)
--- a/osfmk/i386/ktss.c
+++ b/osfmk/i386/ktss.c
@@ -73,10 +73,10 @@
  * due to a single-step trace trap at system call entry.
  */
 struct sysenter_stack master_sstk
  * due to a single-step trace trap at system call entry.
  */
 struct sysenter_stack master_sstk

-       __attribute__ ((section ("__DESC, master_sstk")))
+       __attribute__ ((section ("__HIB, __desc")))

        __attribute__ ((aligned (16)))  = { {0}, 0 };
 
        __attribute__ ((aligned (16)))  = { {0}, 0 };
 

-struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) = {
+struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) __attribute__ ((section ("__HIB, __desc"))) = {

        .io_bit_map_offset = 0x0FFF,
 };
 
        .io_bit_map_offset = 0x0FFF,
 };
 

diff --git a/osfmk/i386/ldt.c b/osfmk/i386/ldt.c
index 4027c05f2cbe424852e01dafe79d0cb217035d65..6e1a1e1150ef9a2c8b1360f733b51050394b4016 100644 (file)
--- a/osfmk/i386/ldt.c
+++ b/osfmk/i386/ldt.c
@@ -62,8 +62,9 @@
  */
 #include <i386/seg.h>
 
  */
 #include <i386/seg.h>
 

-
-struct real_descriptor master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = {
+struct real_descriptor master_ldt[LDTSZ] __attribute__ ((aligned (4096)))
+       __attribute__ ((section ("__HIB, __desc")))
+       = {

        [SEL_TO_INDEX(USER_CTHREAD)] = MAKE_REAL_DESCRIPTOR(    /* user cthread segment */
                0,
                0xfffff,
        [SEL_TO_INDEX(USER_CTHREAD)] = MAKE_REAL_DESCRIPTOR(    /* user cthread segment */
                0,
                0xfffff,


@@ -71,3 +72,5 @@ struct real_descriptor        master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = {
                ACC_P|ACC_PL_U|ACC_DATA_W
        ),
 };
                ACC_P|ACC_PL_U|ACC_DATA_W
        ),
 };

+
+unsigned mldtsz = sizeof(master_ldt);

diff --git a/osfmk/i386/mp.c b/osfmk/i386/mp.c
index 0c971b1e00d35f0a774692c6175d4741d41f1343..1c11d96015dacf27f7ff3c98e5821966ec188f8f 100644 (file)
--- a/osfmk/i386/mp.c
+++ b/osfmk/i386/mp.c
@@ -189,7 +189,7 @@ boolean_t i386_smp_init(int nmi_vector, i386_intr_func_t nmi_handler,
                int ipi_vector, i386_intr_func_t ipi_handler);
 void i386_start_cpu(int lapic_id, int cpu_num);
 void i386_send_NMI(int cpu);
                int ipi_vector, i386_intr_func_t ipi_handler);
 void i386_start_cpu(int lapic_id, int cpu_num);
 void i386_send_NMI(int cpu);

-
+void NMIPI_enable(boolean_t);

 #if GPROF
 /*
  * Initialize dummy structs for profiling. These aren't used but
 #if GPROF
 /*
  * Initialize dummy structs for profiling. These aren't used but


@@ -1034,6 +1034,7 @@ NMIPI_panic(cpumask_t cpu_mask, NMI_reason_t why) {
        unsigned int cpu, cpu_bit;
        uint64_t deadline;
 
        unsigned int cpu, cpu_bit;
        uint64_t deadline;
 

+       NMIPI_enable(TRUE);

        NMI_panic_reason = why;
 
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
        NMI_panic_reason = why;
 
        for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {


@@ -1647,8 +1648,12 @@ mp_kdp_enter(boolean_t proceed_on_failure)
                        cpu_pause();
                }
                /* If we've timed out, and some processor(s) are still unresponsive,
                        cpu_pause();
                }
                /* If we've timed out, and some processor(s) are still unresponsive,

-                * interrupt them with an NMI via the local APIC.
+                * interrupt them with an NMI via the local APIC, iff a panic is
+                * in progress.

                 */
                 */

+               if (panic_active()) {
+                       NMIPI_enable(TRUE);
+               }

                if (mp_kdp_ncpus != ncpus) {
                        DBG("mp_kdp_enter() timed-out on cpu %d, NMI-ing\n", my_cpu);
                        for (cpu = 0; cpu < real_ncpus; cpu++) {
                if (mp_kdp_ncpus != ncpus) {
                        DBG("mp_kdp_enter() timed-out on cpu %d, NMI-ing\n", my_cpu);
                        for (cpu = 0; cpu < real_ncpus; cpu++) {

diff --git a/osfmk/i386/mp_desc.c b/osfmk/i386/mp_desc.c
index 5f95beb4da5bc2afebba9cc75f1a3e8b7e929d66..788e71663931347b217f1dd0b3550d4f6c7cb776 100644 (file)
--- a/osfmk/i386/mp_desc.c
+++ b/osfmk/i386/mp_desc.c
@@ -176,15 +176,17 @@ extern uint32_t           low_eintstack[];        /* top */
 
 /*
  * Per-cpu data area pointers.
 
 /*
  * Per-cpu data area pointers.

- * The master cpu (cpu 0) has its data area statically allocated;
- * others are allocated dynamically and this array is updated at runtime.

  */
  */

-static cpu_data_t      cpu_data_master = {
-       .cpu_this = &cpu_data_master,
-       .cpu_nanotime = &pal_rtc_nanotime_info,
-       .cpu_int_stack_top = (vm_offset_t) low_eintstack,
+cpu_data_t cpshadows[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
+cpu_data_t scdatas[MAX_CPUS] __attribute__((aligned(64))) = {
+       [0].cpu_this = &scdatas[0],
+       [0].cpu_nanotime = &pal_rtc_nanotime_info,
+       [0].cpu_int_stack_top = (vm_offset_t) low_eintstack,
+       [0].cd_shadow = &cpshadows[0]

 };
 };

-cpu_data_t     *cpu_data_ptr[MAX_CPUS] = { [0] = &cpu_data_master };
+cpu_data_t *cpu_data_master = &scdatas[0];
+
+cpu_data_t     *cpu_data_ptr[MAX_CPUS] = { [0] = &scdatas[0] };

 
 decl_simple_lock_data(,ncpus_lock);    /* protects real_ncpus */
 unsigned int   real_ncpus = 1;
 
 decl_simple_lock_data(,ncpus_lock);    /* protects real_ncpus */
 unsigned int   real_ncpus = 1;


@@ -193,6 +195,15 @@ unsigned int       max_ncpus = MAX_CPUS;
 extern void hi64_sysenter(void);
 extern void hi64_syscall(void);
 
 extern void hi64_sysenter(void);
 extern void hi64_syscall(void);
 

+typedef struct {
+       struct real_descriptor pcldts[LDTSZ];
+} cldt_t;
+
+cpu_desc_table64_t scdtables[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
+cpu_fault_stack_t scfstks[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc")));
+
+cldt_t *dyn_ldts;
+

 /*
  * Multiprocessor i386/i486 systems use a separate copy of the
  * GDT, IDT, LDT, and kernel TSS per processor.  The first three
 /*
  * Multiprocessor i386/i486 systems use a separate copy of the
  * GDT, IDT, LDT, and kernel TSS per processor.  The first three


@@ -356,51 +367,36 @@ fix_desc64(void *descp, int count)
        }
 }
 
        }
 }
 

-static void
-cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias)
-{
-       pt_entry_t *pte = NULL;
-
-       /* Require page alignment */
-       assert(page_aligned(gdt));
-       assert(page_aligned(alias));
-
-       pte = pmap_pte(kernel_pmap, alias);
-       pmap_store_pte(pte, kvtophys(gdt) | INTEL_PTE_REF
-                                         | INTEL_PTE_MOD
-                                         | INTEL_PTE_WIRED
-                                         | INTEL_PTE_VALID
-                                         | INTEL_PTE_WRITE
-                                         | INTEL_PTE_NX);
-#if KASAN
-       kasan_notify_address(alias, PAGE_SIZE);
-#endif
-}
-
-
+extern unsigned mldtsz;

 void
 cpu_desc_init(cpu_data_t *cdp)
 {
        cpu_desc_index_t        *cdi = &cdp->cpu_desc_index;
 
 void
 cpu_desc_init(cpu_data_t *cdp)
 {
        cpu_desc_index_t        *cdi = &cdp->cpu_desc_index;
 

-       if (cdp == &cpu_data_master) {
+       if (cdp == cpu_data_master) {

                /*
                /*

-                * Master CPU uses the tables built at boot time.
-                * Just set the index pointers to the low memory space.
+                * Populate the double-mapped 'u' and base 'b' fields in the
+                * KTSS with I/G/LDT and sysenter stack data.

                 */
                 */

-               cdi->cdi_ktss = (void *)&master_ktss64;
-               cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
-               cdi->cdi_gdt.ptr  = (void *)MASTER_GDT_ALIAS;
-               cdi->cdi_idt.ptr  = (void *)MASTER_IDT_ALIAS;
-               cdi->cdi_ldt  = (struct fake_descriptor *) master_ldt;
+               cdi->cdi_ktssu = (void *)DBLMAP(&master_ktss64);
+               cdi->cdi_ktssb = (void *)&master_ktss64;
+               cdi->cdi_sstku = (vm_offset_t) DBLMAP(&master_sstk.top);
+               cdi->cdi_sstkb = (vm_offset_t) &master_sstk.top;
+
+               cdi->cdi_gdtu.ptr = (void *)DBLMAP((uintptr_t) &master_gdt);
+               cdi->cdi_gdtb.ptr = (void *)&master_gdt;
+               cdi->cdi_idtu.ptr  = (void *)DBLMAP((uintptr_t) &master_idt64);
+               cdi->cdi_idtb.ptr  = (void *)((uintptr_t) &master_idt64);
+               cdi->cdi_ldtu  = (struct fake_descriptor *) (void *) DBLMAP((uintptr_t)&master_ldt[0]);
+               cdi->cdi_ldtb  = (struct fake_descriptor *) (void *) &master_ldt[0];

 
                /* Replace the expanded LDTs and TSS slots in the GDT */
 
                /* Replace the expanded LDTs and TSS slots in the GDT */

-               kernel_ldt_desc64.offset64 = (uintptr_t) &master_ldt;
+               kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;

                *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
                        kernel_ldt_desc64;
                *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] =
                        kernel_ldt_desc64;
                *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
                        kernel_ldt_desc64;
                *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] =
                        kernel_ldt_desc64;

-               kernel_tss_desc64.offset64 = (uintptr_t) &master_ktss64;
+               kernel_tss_desc64.offset64 = (uintptr_t) DBLMAP(&master_ktss64);

                *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
                        kernel_tss_desc64;
 
                *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
                        kernel_tss_desc64;
 


@@ -412,69 +408,62 @@ cpu_desc_init(cpu_data_t *cdp)
 
                /*
                 * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS
 
                /*
                 * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS

-                * Note: this will be dynamically re-allocated in VM later. 

                 */
                master_ktss64.ist2 = (uintptr_t) low_eintstack;
                 */
                master_ktss64.ist2 = (uintptr_t) low_eintstack;

-               master_ktss64.ist1 = (uintptr_t) low_eintstack
-                                       - sizeof(x86_64_intr_stack_frame_t);
-       } else if (cdi->cdi_ktss == NULL) {     /* Skipping re-init on wake */
+               master_ktss64.ist1 = (uintptr_t) low_eintstack - sizeof(x86_64_intr_stack_frame_t);
+       } else if (cdi->cdi_ktssu == NULL) {    /* Skipping re-init on wake */

                cpu_desc_table64_t      *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
 
                cpu_desc_table64_t      *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
 

-               /*
-                * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel 
-                * heap (cpu_desc_table). 
-                * LDT descriptors are mapped into a separate area.
-                * GDT descriptors are addressed by alias to avoid sgdt leaks to user-space.
-                */
-               cdi->cdi_idt.ptr  = (void *)MASTER_IDT_ALIAS;
-               cdi->cdi_gdt.ptr  = (void *)CPU_GDT_ALIAS(cdp->cpu_number);
-               cdi->cdi_ktss = (void *)&cdt->ktss;
-               cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top;
-               cdi->cdi_ldt  = cdp->cpu_ldtp;
+               cdi->cdi_idtu.ptr  = (void *)DBLMAP((uintptr_t) &master_idt64);

 
 

-               /* Make the virtual alias address for the GDT */
-               cpu_gdt_alias((vm_map_offset_t) &cdt->gdt,
-                             (vm_map_offset_t) cdi->cdi_gdt.ptr);
+               cdi->cdi_ktssu = (void *)DBLMAP(&cdt->ktss);
+               cdi->cdi_ktssb = (void *)(&cdt->ktss);
+               cdi->cdi_sstku = (vm_offset_t)DBLMAP(&cdt->sstk.top);
+               cdi->cdi_sstkb = (vm_offset_t)(&cdt->sstk.top);
+               cdi->cdi_ldtu  = (void *)LDTALIAS(cdp->cpu_ldtp);
+               cdi->cdi_ldtb  = (void *)(cdp->cpu_ldtp);

 
                /*
                 * Copy the tables
                 */
                bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt));
 
                /*
                 * Copy the tables
                 */
                bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt));

-               bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, sizeof(master_ldt));
+               bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, mldtsz);

                bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss));
                bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss));

-
+               cdi->cdi_gdtu.ptr  = (void *)DBLMAP(cdt->gdt);
+               cdi->cdi_gdtb.ptr  = (void *)(cdt->gdt);

                /*
                 * Fix up the entries in the GDT to point to
                 * this LDT and this TSS.
                /*
                 * Fix up the entries in the GDT to point to
                 * this LDT and this TSS.

+                * Note reuse of global 'kernel_ldt_desc64, which is not
+                * concurrency-safe. Higher level synchronization is expected

                 */
                 */

-               kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
+               kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;

                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
                        kernel_ldt_desc64;
                fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
 
                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
                        kernel_ldt_desc64;
                fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
 

-               kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
+               kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu;

                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
                        kernel_ldt_desc64;
                fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
 
                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
                        kernel_ldt_desc64;
                fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
 

-               kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktss;
+               kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktssu;

                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
                        kernel_tss_desc64;
                fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
 
                /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */
                *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
                        kernel_tss_desc64;
                fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
 
                /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */

-               bzero((void *) cdt->fstk, sizeof(cdt->fstk));
-               cdt->ktss.ist2 = (unsigned long)cdt->fstk + sizeof(cdt->fstk);
-               cdt->ktss.ist1 = cdt->ktss.ist2
-                                       - sizeof(x86_64_intr_stack_frame_t);
+               uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0];
+               cdt->fstkp = cfstk;
+               bzero((void *) cfstk, FSTK_SZ);
+               cdt->ktss.ist2 = DBLMAP((uint64_t)cdt->fstkp + FSTK_SZ);
+               cdt->ktss.ist1 = cdt->ktss.ist2 - sizeof(x86_64_intr_stack_frame_t);

        }
 
        /* Require that the top of the sysenter stack is 16-byte aligned */
        }
 
        /* Require that the top of the sysenter stack is 16-byte aligned */

-       if ((cdi->cdi_sstk % 16) != 0)
+       if ((cdi->cdi_sstku % 16) != 0)

                panic("cpu_desc_init() sysenter stack not 16-byte aligned");
 }
                panic("cpu_desc_init() sysenter stack not 16-byte aligned");
 }

-
-

 void
 cpu_desc_load(cpu_data_t *cdp)
 {
 void
 cpu_desc_load(cpu_data_t *cdp)
 {


@@ -496,13 +485,15 @@ cpu_desc_load(cpu_data_t *cdp)
        gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
 
        /* Load the GDT, LDT, IDT and TSS */
        gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
 
        /* Load the GDT, LDT, IDT and TSS */

-       cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1;
-       cdi->cdi_idt.size = 0x1000 + cdp->cpu_number;
-       
+       cdi->cdi_gdtb.size = sizeof(struct real_descriptor)*GDTSZ - 1;
+       cdi->cdi_gdtu.size = cdi->cdi_gdtb.size;
+       cdi->cdi_idtb.size = 0x1000 + cdp->cpu_number;
+       cdi->cdi_idtu.size = cdi->cdi_idtb.size;
+

        postcode(CPU_DESC_LOAD_GDT);
        postcode(CPU_DESC_LOAD_GDT);

-       lgdt((uintptr_t *) &cdi->cdi_gdt);
+       lgdt((uintptr_t *) &cdi->cdi_gdtu);

        postcode(CPU_DESC_LOAD_IDT);
        postcode(CPU_DESC_LOAD_IDT);

-       lidt((uintptr_t *) &cdi->cdi_idt);
+       lidt((uintptr_t *) &cdi->cdi_idtu);

        postcode(CPU_DESC_LOAD_LDT);
        lldt(KERNEL_LDT);
        postcode(CPU_DESC_LOAD_TSS);
        postcode(CPU_DESC_LOAD_LDT);
        lldt(KERNEL_LDT);
        postcode(CPU_DESC_LOAD_TSS);


@@ -514,7 +505,6 @@ cpu_desc_load(cpu_data_t *cdp)
        postcode(CPU_DESC_LOAD_EXIT);
 }
 
        postcode(CPU_DESC_LOAD_EXIT);
 }
 

-

 /*
  * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
  */
 /*
  * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
  */


@@ -527,8 +517,8 @@ cpu_syscall_init(cpu_data_t *cdp)
 #pragma unused(cdp)
 #endif /* !MONOTONIC */
        wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); 
 #pragma unused(cdp)
 #endif /* !MONOTONIC */
        wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); 

-       wrmsr64(MSR_IA32_SYSENTER_EIP, (uintptr_t) hi64_sysenter);
-       wrmsr64(MSR_IA32_SYSENTER_ESP, current_sstk());
+       wrmsr64(MSR_IA32_SYSENTER_EIP, DBLMAP((uintptr_t) hi64_sysenter));
+       wrmsr64(MSR_IA32_SYSENTER_ESP, current_cpu_datap()->cpu_desc_index.cdi_sstku);

        /* Enable syscall/sysret */
        wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
 
        /* Enable syscall/sysret */
        wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
 


@@ -537,9 +527,8 @@ cpu_syscall_init(cpu_data_t *cdp)
         * Note USER_CS because sysret uses this + 16 when returning to
         * 64-bit code.
         */
         * Note USER_CS because sysret uses this + 16 when returning to
         * 64-bit code.
         */

-       wrmsr64(MSR_IA32_LSTAR, (uintptr_t) hi64_syscall);
-       wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) |
-                               (((uint64_t)KERNEL64_CS) << 32));
+       wrmsr64(MSR_IA32_LSTAR, DBLMAP((uintptr_t) hi64_syscall));
+       wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) | (((uint64_t)KERNEL64_CS) << 32));

        /*
         * Emulate eflags cleared by sysenter but note that
         * we also clear the trace trap to avoid the complications
        /*
         * Emulate eflags cleared by sysenter but note that
         * we also clear the trace trap to avoid the complications


@@ -550,7 +539,8 @@ cpu_syscall_init(cpu_data_t *cdp)
        wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
 
 }
        wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
 
 }

-
+extern vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t);
+uint64_t ldt_alias_offset;

 
 cpu_data_t *
 cpu_data_alloc(boolean_t is_boot_cpu)
 
 cpu_data_t *
 cpu_data_alloc(boolean_t is_boot_cpu)


@@ -571,17 +561,24 @@ cpu_data_alloc(boolean_t is_boot_cpu)
                return cdp;
        }
 
                return cdp;
        }
 

+       boolean_t do_ldt_alloc = FALSE;
+       simple_lock(&ncpus_lock);
+       int cnum = real_ncpus;
+       real_ncpus++;
+       if (dyn_ldts == NULL) {
+               do_ldt_alloc = TRUE;
+       }
+       simple_unlock(&ncpus_lock);
+

        /*
         * Allocate per-cpu data:
         */
        /*
         * Allocate per-cpu data:
         */

-       ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
-       if (ret != KERN_SUCCESS) {
-               printf("cpu_data_alloc() failed, ret=%d\n", ret);
-               goto abort;
-       }
+
+       cdp = &scdatas[cnum];

        bzero((void*) cdp, sizeof(cpu_data_t));
        cdp->cpu_this = cdp;
        bzero((void*) cdp, sizeof(cpu_data_t));
        cdp->cpu_this = cdp;

-
+       cdp->cpu_number = cnum;
+       cdp->cd_shadow = &cpshadows[cnum];

        /*
         * Allocate interrupt stack:
         */
        /*
         * Allocate interrupt stack:
         */


@@ -589,8 +586,7 @@ cpu_data_alloc(boolean_t is_boot_cpu)
                         (vm_offset_t *) &cdp->cpu_int_stack_top,
                         INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
        if (ret != KERN_SUCCESS) {
                         (vm_offset_t *) &cdp->cpu_int_stack_top,
                         INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
        if (ret != KERN_SUCCESS) {

-               printf("cpu_data_alloc() int stack failed, ret=%d\n", ret);
-               goto abort;
+               panic("cpu_data_alloc() int stack failed, ret=%d\n", ret);

        }
        bzero((void*) cdp->cpu_int_stack_top, INTSTACK_SIZE);
        cdp->cpu_int_stack_top += INTSTACK_SIZE;
        }
        bzero((void*) cdp->cpu_int_stack_top, INTSTACK_SIZE);
        cdp->cpu_int_stack_top += INTSTACK_SIZE;


@@ -598,39 +594,49 @@ cpu_data_alloc(boolean_t is_boot_cpu)
        /*
         * Allocate descriptor table:
         */
        /*
         * Allocate descriptor table:
         */

-       ret = kmem_alloc(kernel_map, 
-                        (vm_offset_t *) &cdp->cpu_desc_tablep,
-                        sizeof(cpu_desc_table64_t),
-                        VM_KERN_MEMORY_CPU);
-       if (ret != KERN_SUCCESS) {
-               printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret);
-               goto abort;
-       }

 
 

+       cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[cnum];

        /*
         * Allocate LDT
         */
        /*
         * Allocate LDT
         */

-       ret = kmem_alloc(kernel_map, 
-                        (vm_offset_t *) &cdp->cpu_ldtp,
-                        sizeof(struct real_descriptor) * LDTSZ,
-                        VM_KERN_MEMORY_CPU);
-       if (ret != KERN_SUCCESS) {
-               printf("cpu_data_alloc() ldt failed, ret=%d\n", ret);
-               goto abort;
+       if (do_ldt_alloc) {
+               boolean_t do_ldt_free = FALSE;
+               vm_offset_t sldtoffset = 0;
+               /*
+                * Allocate LDT
+                */
+               vm_offset_t ldtalloc = 0, ldtallocsz = round_page_64(MAX_CPUS * sizeof(struct real_descriptor) * LDTSZ);
+               ret = kmem_alloc(kernel_map, (vm_offset_t *) &ldtalloc, ldtallocsz, VM_KERN_MEMORY_CPU);
+               if (ret != KERN_SUCCESS) {
+                       panic("cpu_data_alloc() ldt failed, kmem_alloc=%d\n", ret);
+               }
+
+               simple_lock(&ncpus_lock);
+               if (dyn_ldts == NULL) {
+                       dyn_ldts = (cldt_t *)ldtalloc;
+               } else {
+                       do_ldt_free = TRUE;
+               }
+               simple_unlock(&ncpus_lock);
+
+               if (do_ldt_free) {
+                       kmem_free(kernel_map, ldtalloc, ldtallocsz);
+               } else {
+                       /* CPU registration and startup are expected to execute
+                        * serially, as invoked by the platform driver.
+                        * Create trampoline alias of LDT region.
+                        */
+                       sldtoffset = dyn_dblmap(ldtalloc, ldtallocsz);
+                       ldt_alias_offset = sldtoffset;
+               }

        }
        }

+       cdp->cpu_ldtp = &dyn_ldts[cnum].pcldts[0];

 
 #if CONFIG_MCA
        /* Machine-check shadow register allocation. */
        mca_cpu_alloc(cdp);
 #endif
 
 
 #if CONFIG_MCA
        /* Machine-check shadow register allocation. */
        mca_cpu_alloc(cdp);
 #endif
 

-       simple_lock(&ncpus_lock);
-
-       cpu_data_ptr[real_ncpus] = cdp;
-       cdp->cpu_number = real_ncpus;
-       real_ncpus++;
-       simple_unlock(&ncpus_lock);
-

        /*
         * Before this cpu has been assigned a real thread context,
         * we give it a fake, unique, non-zero thread id which the locking
        /*
         * Before this cpu has been assigned a real thread context,
         * we give it a fake, unique, non-zero thread id which the locking


@@ -648,20 +654,10 @@ cpu_data_alloc(boolean_t is_boot_cpu)
                "int_stack: 0x%lx-0x%lx\n",
                cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
                (long)(cdp->cpu_int_stack_top - INTSTACK_SIZE), (long)(cdp->cpu_int_stack_top));
                "int_stack: 0x%lx-0x%lx\n",
                cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
                (long)(cdp->cpu_int_stack_top - INTSTACK_SIZE), (long)(cdp->cpu_int_stack_top));

+       cpu_data_ptr[cnum] = cdp;

 
        return cdp;
 
 
        return cdp;
 

-abort:
-       if (cdp) {
-               if (cdp->cpu_desc_tablep)
-                       kfree((void *) cdp->cpu_desc_tablep,
-                               sizeof(cpu_desc_table64_t));
-               if (cdp->cpu_int_stack_top)
-                       kfree((void *) (cdp->cpu_int_stack_top - INTSTACK_SIZE),
-                               INTSTACK_SIZE);
-               kfree((void *) cdp, sizeof(*cdp));
-       }
-       return NULL;

 }
 
 boolean_t
 }
 
 boolean_t


@@ -678,7 +674,6 @@ valid_user_data_selector(uint16_t selector)
        if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U)
            return (TRUE);
     }
        if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U)
            return (TRUE);
     }

-               

     return (FALSE);
 }
 
     return (FALSE);
 }
 


@@ -841,7 +836,6 @@ cpu_data_realloc(void)
 {
        int             ret;
        vm_offset_t     istk;
 {
        int             ret;
        vm_offset_t     istk;

-       vm_offset_t     fstk;

        cpu_data_t      *cdp;
        boolean_t       istate;
 
        cpu_data_t      *cdp;
        boolean_t       istate;
 


@@ -852,10 +846,7 @@ cpu_data_realloc(void)
        bzero((void*) istk, INTSTACK_SIZE);
        istk += INTSTACK_SIZE;
 
        bzero((void*) istk, INTSTACK_SIZE);
        istk += INTSTACK_SIZE;
 

-       ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
-       if (ret != KERN_SUCCESS) {
-               panic("cpu_data_realloc() cpu data alloc, ret=%d\n", ret);
-       }
+       cdp = &scdatas[0];

 
        /* Copy old contents into new area and make fix-ups */
        assert(cpu_number() == 0);
 
        /* Copy old contents into new area and make fix-ups */
        assert(cpu_number() == 0);


@@ -863,28 +854,25 @@ cpu_data_realloc(void)
        cdp->cpu_this = cdp;
        cdp->cpu_int_stack_top = istk;
        timer_call_queue_init(&cdp->rtclock_timer.queue);
        cdp->cpu_this = cdp;
        cdp->cpu_int_stack_top = istk;
        timer_call_queue_init(&cdp->rtclock_timer.queue);

+       cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[0];
+       cpu_desc_table64_t      *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;

 
 

-       /* Allocate the separate fault stack */
-       ret = kmem_alloc(kernel_map, &fstk, PAGE_SIZE, VM_KERN_MEMORY_CPU);
-       if (ret != KERN_SUCCESS) {
-               panic("cpu_data_realloc() fault stack alloc, ret=%d\n", ret);
-       }
-       bzero((void*) fstk, PAGE_SIZE);
-       fstk += PAGE_SIZE;
+       uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0];
+       cdt->fstkp = cfstk;
+       cfstk += FSTK_SZ;

 
        /*
         * With interrupts disabled commmit the new areas.
         */
        istate = ml_set_interrupts_enabled(FALSE);
        cpu_data_ptr[0] = cdp;
 
        /*
         * With interrupts disabled commmit the new areas.
         */
        istate = ml_set_interrupts_enabled(FALSE);
        cpu_data_ptr[0] = cdp;

-       master_ktss64.ist2 = (uintptr_t) fstk;
-       master_ktss64.ist1 = (uintptr_t) fstk
-                               - sizeof(x86_64_intr_stack_frame_t);
+       master_ktss64.ist2 = DBLMAP((uintptr_t) cfstk);
+       master_ktss64.ist1 = DBLMAP((uintptr_t) cfstk - sizeof(x86_64_intr_stack_frame_t));

        wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
        wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
        (void) ml_set_interrupts_enabled(istate);
 
        kprintf("Reallocated master cpu data: %p,"
                " interrupt stack: %p, fault stack: %p\n",
        wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
        wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
        (void) ml_set_interrupts_enabled(istate);
 
        kprintf("Reallocated master cpu data: %p,"
                " interrupt stack: %p, fault stack: %p\n",

-               (void *) cdp, (void *) istk, (void *) fstk);
+               (void *) cdp, (void *) istk, (void *) cfstk);

 }
 }

diff --git a/osfmk/i386/mp_desc.h b/osfmk/i386/mp_desc.h
index ebc75cfb0a7a35ebd68042b51ebbf23ba420379a..50f6c62a692ccb8667fac187a4e2e9c2d261a157 100644 (file)
--- a/osfmk/i386/mp_desc.h
+++ b/osfmk/i386/mp_desc.h
@@ -80,18 +80,23 @@ __BEGIN_DECLS
  * The descriptor tables are together in a structure
  * allocated one per processor (except for the boot processor).
  */
  * The descriptor tables are together in a structure
  * allocated one per processor (except for the boot processor).
  */

+#define FSTK_SZ (PAGE_SIZE/2)

 typedef struct cpu_desc_table64 {
        struct fake_descriptor  gdt[GDTSZ]       __attribute__ ((aligned (16)));
        struct x86_64_tss       ktss             __attribute__ ((aligned (16)));
        struct sysenter_stack   sstk             __attribute__ ((aligned (16)));
 typedef struct cpu_desc_table64 {
        struct fake_descriptor  gdt[GDTSZ]       __attribute__ ((aligned (16)));
        struct x86_64_tss       ktss             __attribute__ ((aligned (16)));
        struct sysenter_stack   sstk             __attribute__ ((aligned (16)));

-       uint8_t                 fstk[PAGE_SIZE]  __attribute__ ((aligned (16)));
+       uint8_t                 *fstkp;

 } cpu_desc_table64_t;
 
 } cpu_desc_table64_t;
 

-#define        current_gdt()   (current_cpu_datap()->cpu_desc_index.cdi_gdt.ptr)
-#define        current_idt()   (current_cpu_datap()->cpu_desc_index.cdi_idt.ptr)
-#define        current_ldt()   (current_cpu_datap()->cpu_desc_index.cdi_ldt)
-#define        current_ktss()  (current_cpu_datap()->cpu_desc_index.cdi_ktss)
-#define        current_sstk()  (current_cpu_datap()->cpu_desc_index.cdi_sstk)
+typedef struct {
+       uint8_t                 fstk[FSTK_SZ]  __attribute__ ((aligned (16)));
+} cpu_fault_stack_t;
+
+#define        current_gdt()   (current_cpu_datap()->cpu_desc_index.cdi_gdtb.ptr)
+#define        current_idt()   (current_cpu_datap()->cpu_desc_index.cdi_idtb.ptr)
+#define        current_ldt()   (current_cpu_datap()->cpu_desc_index.cdi_ldtb)
+#define        current_ktss()  (current_cpu_datap()->cpu_desc_index.cdi_ktssb)
+#define        current_sstk()  (current_cpu_datap()->cpu_desc_index.cdi_sstkb)

 
 #define        current_ktss64() ((struct x86_64_tss *) current_ktss())
 #define        current_sstk64() ((addr64_t *) current_sstk())
 
 #define        current_ktss64() ((struct x86_64_tss *) current_ktss())
 #define        current_sstk64() ((addr64_t *) current_sstk())

diff --git a/osfmk/i386/mp_native.c b/osfmk/i386/mp_native.c
index f517f173ae223a61f47ff8b75b84612bc58aa3cc..e64344db7230893bb8f78858649532b8cce76aa0 100644 (file)
--- a/osfmk/i386/mp_native.c
+++ b/osfmk/i386/mp_native.c
@@ -42,6 +42,7 @@ boolean_t i386_smp_init(int nmi_vector, i386_intr_func_t nmi_handler,
 void i386_start_cpu(int lapic_id, int cpu_num);
 void i386_send_NMI(int cpu);
 void handle_pending_TLB_flushes(void);
 void i386_start_cpu(int lapic_id, int cpu_num);
 void i386_send_NMI(int cpu);
 void handle_pending_TLB_flushes(void);

+void NMIPI_enable(boolean_t);

 
 extern void    slave_pstart(void);
 
 
 extern void    slave_pstart(void);
 


@@ -75,16 +76,27 @@ i386_start_cpu(int lapic_id, __unused int cpu_num )
                        LAPIC_ICR_DM_STARTUP|(REAL_MODE_BOOTSTRAP_OFFSET>>12));
 }
 
                        LAPIC_ICR_DM_STARTUP|(REAL_MODE_BOOTSTRAP_OFFSET>>12));
 }
 

+static boolean_t NMIPIs_enabled = FALSE;
+
+void NMIPI_enable(boolean_t enable) {
+       NMIPIs_enabled = enable;
+}
+

 void
 i386_send_NMI(int cpu)
 {
        boolean_t state = ml_set_interrupts_enabled(FALSE);
 void
 i386_send_NMI(int cpu)
 {
        boolean_t state = ml_set_interrupts_enabled(FALSE);

+
+       if (NMIPIs_enabled == FALSE) {
+               i386_cpu_IPI(cpu);
+       } else {

        /* Program the interrupt command register */
        /* The vector is ignored in this case--the target CPU will enter on the
         * NMI vector.
         */
        LAPIC_WRITE_ICR(cpu_to_lapic[cpu],
                        LAPIC_VECTOR(INTERPROCESSOR)|LAPIC_ICR_DM_NMI);
        /* Program the interrupt command register */
        /* The vector is ignored in this case--the target CPU will enter on the
         * NMI vector.
         */
        LAPIC_WRITE_ICR(cpu_to_lapic[cpu],
                        LAPIC_VECTOR(INTERPROCESSOR)|LAPIC_ICR_DM_NMI);

+       }

        (void) ml_set_interrupts_enabled(state);
 }
 
        (void) ml_set_interrupts_enabled(state);
 }
 

diff --git a/osfmk/i386/pcb_native.c b/osfmk/i386/pcb_native.c
index c260ec7cd2ac075ec1388d2067b0b3e123d14995..34df7c11932c2d5fa78822a2c2941602ae98c345 100644 (file)
--- a/osfmk/i386/pcb_native.c
+++ b/osfmk/i386/pcb_native.c
@@ -137,6 +137,7 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new)
        set_ds(NULL_SEG);
        set_es(NULL_SEG);
        set_fs(NULL_SEG);
        set_ds(NULL_SEG);
        set_es(NULL_SEG);
        set_fs(NULL_SEG);

+

        if (get_gs() != NULL_SEG) {
                swapgs();               /* switch to user's GS context */
                set_gs(NULL_SEG);
        if (get_gs() != NULL_SEG) {
                swapgs();               /* switch to user's GS context */
                set_gs(NULL_SEG);


@@ -158,9 +159,7 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new)
        /* require 16-byte alignment */
        assert((pcb_stack_top & 0xF) == 0);
 
        /* require 16-byte alignment */
        assert((pcb_stack_top & 0xF) == 0);
 

-       /* Interrupt stack is pcb */
-       current_ktss64()->rsp0 = pcb_stack_top;
-
+       current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku;

        /*
         * Top of temporary sysenter stack points to pcb stack.
         * Although this is not normally used by 64-bit users,
        /*
         * Top of temporary sysenter stack points to pcb stack.
         * Although this is not normally used by 64-bit users,


@@ -168,6 +167,8 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new)
         */
        *current_sstk64() = pcb_stack_top;
 
         */
        *current_sstk64() = pcb_stack_top;
 

+       cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku;
+

        if (is_saved_state64(pcb->iss)) {
 
                cdp->cpu_task_map = new->map->pmap->pm_task_map; 
        if (is_saved_state64(pcb->iss)) {
 
                cdp->cpu_task_map = new->map->pmap->pm_task_map; 


@@ -195,7 +196,6 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new)
                                wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
                        }
                }
                                wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
                        }
                }

-

        } else {
 
                cdp->cpu_task_map = TASK_MAP_32BIT;
        } else {
 
                cdp->cpu_task_map = TASK_MAP_32BIT;


@@ -403,9 +403,7 @@ machine_thread_create(
         * segment.
         */
        if ((pcb->cthread_desc.access & ACC_P) == 0) {
         * segment.
         */
        if ((pcb->cthread_desc.access & ACC_P) == 0) {

-               struct real_descriptor  *ldtp;
-               ldtp = (struct real_descriptor *)current_ldt();
-               pcb->cthread_desc = ldtp[sel_idx(USER_DS)];
+               pcb->cthread_desc = *gdt_desc_p(USER_DS);

        }
 
        return(KERN_SUCCESS);
        }
 
        return(KERN_SUCCESS);

diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h
index 0c26eee608345e84ed96b643b181f8a70f169ff2..3458ce7cf57806b6f60624d5275fc5944582e3fe 100644 (file)
--- a/osfmk/i386/pmap.h
+++ b/osfmk/i386/pmap.h
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 


@@ -70,7 +70,6 @@
 
 #ifndef        ASSEMBLER
 
 
 #ifndef        ASSEMBLER
 

-

 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>
 #include <mach/vm_prot.h>
 #include <mach/kern_return.h>
 #include <mach/machine/vm_types.h>
 #include <mach/vm_prot.h>


@@ -110,10 +109,7 @@
 
 #define PTESHIFT        12ULL
 
 
 #define PTESHIFT        12ULL
 

-
-#ifdef __x86_64__

 #define LOW_4GB_MASK   ((vm_offset_t)0x00000000FFFFFFFFUL)
 #define LOW_4GB_MASK   ((vm_offset_t)0x00000000FFFFFFFFUL)

-#endif

 
 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */
 
 #define PDESIZE                sizeof(pd_entry_t) /* for assembly files */
 #define PTESIZE                sizeof(pt_entry_t) /* for assembly files */


@@ -194,11 +190,7 @@ extern int pmap_asserts_traced;
 #endif
 
 /* superpages */
 #endif
 
 /* superpages */

-#ifdef __x86_64__

 #define SUPERPAGE_NBASEPAGES 512
 #define SUPERPAGE_NBASEPAGES 512

-#else
-#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */
-#endif

 
 /*
  * Atomic 64-bit store of a page table entry.
 
 /*
  * Atomic 64-bit store of a page table entry.


@@ -220,10 +212,11 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 
 #define KERNEL_PML4_INDEX              511
 #define NPTEPGS         (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t))))
 
 #define KERNEL_PML4_INDEX              511

-#define KERNEL_KEXTS_INDEX     510     /* Home of KEXTs - the basement */
+#define KERNEL_KEXTS_INDEX             510     /* Home of KEXTs - the basement */

 #define KERNEL_PHYSMAP_PML4_INDEX      509     /* virtual to physical map */ 
 #define KERNEL_KASAN_PML4_INDEX0       508
 #define KERNEL_KASAN_PML4_INDEX1       507
 #define KERNEL_PHYSMAP_PML4_INDEX      509     /* virtual to physical map */ 
 #define KERNEL_KASAN_PML4_INDEX0       508
 #define KERNEL_KASAN_PML4_INDEX1       507

+#define KERNEL_DBLMAP_PML4_INDEX       (506)

 #define KERNEL_BASE            (0ULL - NBPML4)
 #define KERNEL_BASEMENT                (KERNEL_BASE - NBPML4)
 
 #define KERNEL_BASE            (0ULL - NBPML4)
 #define KERNEL_BASEMENT                (KERNEL_BASE - NBPML4)
 


@@ -460,9 +453,6 @@ extern uint64_t             pmap_pv_hashlist_cnts;
 extern uint32_t                pmap_pv_hashlist_max;
 extern uint32_t                pmap_kernel_text_ps;
 
 extern uint32_t                pmap_pv_hashlist_max;
 extern uint32_t                pmap_kernel_text_ps;
 

-
-
-#ifdef __x86_64__

 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 
 extern uint64_t physmap_base, physmap_max;
 #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK))
 
 extern uint64_t physmap_base, physmap_max;


@@ -484,6 +474,30 @@ static     inline void * PHYSMAP_PTOV_check(void *paddr) {
 }
 
 #define PHYSMAP_PTOV(x)        (PHYSMAP_PTOV_check((void*) (x)))
 }
 
 #define PHYSMAP_PTOV(x)        (PHYSMAP_PTOV_check((void*) (x)))

+#if MACH_KERNEL_PRIVATE
+extern uint64_t dblmap_base, dblmap_max, dblmap_dist;
+
+static inline uint64_t DBLMAP_CHECK(uintptr_t x) {
+       uint64_t dbladdr = (uint64_t)x + dblmap_dist;
+       if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+               panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+                   (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+       }
+       return dbladdr;
+
+}
+#define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x))
+extern uint64_t ldt_alias_offset;
+static inline uint64_t LDTALIAS_CHECK(uintptr_t x) {
+       uint64_t dbladdr = (uint64_t)x + ldt_alias_offset;
+       if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) {
+               panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx",
+                   (uint64_t)x, dbladdr, dblmap_base, dblmap_max);
+       }
+       return dbladdr;
+}
+#define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x))
+#endif

 
 /*
  * For KASLR, we alias the master processor's IDT and GDT at fixed
 
 /*
  * For KASLR, we alias the master processor's IDT and GDT at fixed


@@ -491,10 +505,7 @@ static     inline void * PHYSMAP_PTOV_check(void *paddr) {
  * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
  * The low global vector page is mapped at a fixed alias also.
  */
  * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS)
  * The low global vector page is mapped at a fixed alias also.
  */

-#define MASTER_IDT_ALIAS       (VM_MIN_KERNEL_ADDRESS + 0x0000)
-#define MASTER_GDT_ALIAS       (VM_MIN_KERNEL_ADDRESS + 0x1000)

 #define LOWGLOBAL_ALIAS                (VM_MIN_KERNEL_ADDRESS + 0x2000)
 #define LOWGLOBAL_ALIAS                (VM_MIN_KERNEL_ADDRESS + 0x2000)

-#define CPU_GDT_ALIAS(_cpu)    (LOWGLOBAL_ALIAS + (0x1000*(_cpu)))

 
 /*
  * This indicates (roughly) where there is free space for the VM
 
 /*
  * This indicates (roughly) where there is free space for the VM


@@ -502,8 +513,6 @@ static      inline void * PHYSMAP_PTOV_check(void *paddr) {
  */
 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
 
  */
 #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS
 

-#endif /*__x86_64__ */
-

 #include <vm/vm_page.h>
 
 /*
 #include <vm/vm_page.h>
 
 /*


@@ -514,7 +523,8 @@ static      inline void * PHYSMAP_PTOV_check(void *paddr) {
 
 struct pmap {
        decl_simple_lock_data(,lock)    /* lock on map */
 
 struct pmap {
        decl_simple_lock_data(,lock)    /* lock on map */

-       pmap_paddr_t    pm_cr3;         /* physical addr */
+       pmap_paddr_t    pm_cr3;         /* Kernel+user shared PML4 physical*/
+       pmap_paddr_t    pm_ucr3;        /* Mirrored user PML4 physical */

         task_map_t      pm_task_map;
        boolean_t       pm_shared;
        boolean_t       pagezero_accessible;
         task_map_t      pm_task_map;
        boolean_t       pm_shared;
        boolean_t       pagezero_accessible;


@@ -524,13 +534,12 @@ struct pmap {
        struct pmap_statistics  stats;  /* map statistics */
        int             ref_count;      /* reference count */
         int            nx_enabled;
        struct pmap_statistics  stats;  /* map statistics */
        int             ref_count;      /* reference count */
         int            nx_enabled;

-       pdpt_entry_t    *pm_pdpt;       /* KVA of 3rd level page */

        pml4_entry_t    *pm_pml4;       /* VKA of top level */
        pml4_entry_t    *pm_pml4;       /* VKA of top level */

+       pml4_entry_t    *pm_upml4;      /* Shadow VKA of top level */

        vm_object_t     pm_obj;         /* object to hold pde's */
        vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
        vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
        pmap_paddr_t    pm_eptp;        /* EPTP */
        vm_object_t     pm_obj;         /* object to hold pde's */
        vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
        vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
        pmap_paddr_t    pm_eptp;        /* EPTP */

-       pd_entry_t      *dirbase;        /* page directory pointer */

        ledger_t        ledger;         /* ledger tracking phys mappings */
 #if MACH_ASSERT
        int             pmap_pid;
        ledger_t        ledger;         /* ledger tracking phys mappings */
 #if MACH_ASSERT
        int             pmap_pid;


@@ -602,7 +611,13 @@ extern pmap_memory_region_t pmap_memory_regions[];
 static inline void
 set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
        int ccpu = my_cpu;
 static inline void
 set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
        int ccpu = my_cpu;

-       cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3;
+       uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3;
+       cpu_datap(ccpu)->cpu_task_cr3 = pcr3;
+       cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3;
+
+       cpu_datap(ccpu)->cpu_ucr3 = ucr3;
+       cpu_shadowp(ccpu)->cpu_ucr3 = ucr3;
+

        cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
 
        assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));
        cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map;
 
        assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE));


@@ -650,10 +665,6 @@ set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) {
 extern void            process_pmap_updates(void);
 extern void            pmap_update_interrupt(void);
 
 extern void            process_pmap_updates(void);
 extern void            pmap_update_interrupt(void);
 

-/*
- *     Machine dependent routines that are used only for i386/i486/i860.
- */
-

 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 
 extern addr64_t                (kvtophys)(
                                vm_offset_t     addr);
 


@@ -661,23 +672,6 @@ extern kern_return_t       pmap_expand(
                                pmap_t          pmap,
                                vm_map_offset_t addr,
                                unsigned int options);
                                pmap_t          pmap,
                                vm_map_offset_t addr,
                                unsigned int options);

-#if    !defined(__x86_64__)
-extern pt_entry_t      *pmap_pte(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pd_entry_t      *pmap_pde(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pd_entry_t      *pmap64_pde(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-
-extern pdpt_entry_t    *pmap64_pdpt(
-                               struct pmap     *pmap,
-                               vm_map_offset_t addr);
-#endif

 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,
 extern vm_offset_t     pmap_map(
                                vm_offset_t     virt,
                                vm_map_offset_t start,


@@ -691,7 +685,6 @@ extern vm_offset_t  pmap_map_bd(
                                vm_map_offset_t end,
                                vm_prot_t       prot,
                                unsigned int    flags);
                                vm_map_offset_t end,
                                vm_prot_t       prot,
                                unsigned int    flags);

-

 extern void            pmap_bootstrap(
                                vm_offset_t     load_start,
                                boolean_t       IA32e);
 extern void            pmap_bootstrap(
                                vm_offset_t     load_start,
                                boolean_t       IA32e);


@@ -870,11 +863,6 @@ extern int pmap_stats_assert;
 #else /* MACH_ASSERT */
 #define PMAP_STATS_ASSERTF(args)
 #endif /* MACH_ASSERT */
 #else /* MACH_ASSERT */
 #define PMAP_STATS_ASSERTF(args)
 #endif /* MACH_ASSERT */

-

 #endif /* ASSEMBLER */
 #endif /* ASSEMBLER */

-
-

 #endif /* _PMAP_MACHINE_ */
 #endif /* _PMAP_MACHINE_ */

-
-

 #endif  /* KERNEL_PRIVATE */
 #endif  /* KERNEL_PRIVATE */

diff --git a/osfmk/i386/pmap_common.c b/osfmk/i386/pmap_common.c
index d948f34a3986e18c4235f674a91ab0254fdebd28..9436719d4b0dfb9d15ab3c133bc96a9f3f372ea7 100644 (file)
--- a/osfmk/i386/pmap_common.c
+++ b/osfmk/i386/pmap_common.c
@@ -61,7 +61,7 @@ event_t       mapping_replenish_event, pmap_user_pv_throttle_event;
 
 uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
 
 
 uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters;
 

-int pmap_asserts_enabled = DEBUG;
+int pmap_asserts_enabled = (DEBUG);

 int pmap_asserts_traced = 0;
 
 unsigned int pmap_cache_attributes(ppnum_t pn) {
 int pmap_asserts_traced = 0;
 
 unsigned int pmap_cache_attributes(ppnum_t pn) {

diff --git a/osfmk/i386/pmap_internal.h b/osfmk/i386/pmap_internal.h
index 3523d8a14716d0a630d534c86a96a5c3eb6cde71..4ddabaa2049b5df0a7eee6adf2c7843357a7d100 100644 (file)
--- a/osfmk/i386/pmap_internal.h
+++ b/osfmk/i386/pmap_internal.h
@@ -407,7 +407,7 @@ extern boolean_t    pmap_disable_kstack_nx;
 #define PMAP_EXPAND_OPTIONS_NONE (0x0)
 #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT)
 #define PMAP_EXPAND_OPTIONS_NOENTER (PMAP_OPTIONS_NOENTER)
 #define PMAP_EXPAND_OPTIONS_NONE (0x0)
 #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT)
 #define PMAP_EXPAND_OPTIONS_NOENTER (PMAP_OPTIONS_NOENTER)

-
+#define PMAP_EXPAND_OPTIONS_ALIASMAP (0x40000000U)

 /*
  *     Amount of virtual memory mapped by one
  *     page-directory entry.
 /*
  *     Amount of virtual memory mapped by one
  *     page-directory entry.


@@ -473,11 +473,14 @@ extern ppnum_t    highest_hi;
 #define MAX_PREEMPTION_LATENCY_NS 20000
 extern uint64_t max_preemption_latency_tsc;
 
 #define MAX_PREEMPTION_LATENCY_NS 20000
 extern uint64_t max_preemption_latency_tsc;
 

-/* #define DEBUGINTERRUPTS 1  uncomment to ensure pmap callers have interrupts enabled */
-#ifdef DEBUGINTERRUPTS
+#if DEBUG
+#define PMAP_INTR_DEBUG (1)
+#endif
+
+#if PMAP_INTR_DEBUG

 #define pmap_intr_assert() {                                                   \
        if (processor_avail_count > 1 && !ml_get_interrupts_enabled())          \
 #define pmap_intr_assert() {                                                   \
        if (processor_avail_count > 1 && !ml_get_interrupts_enabled())          \

-               panic("pmap interrupt assert %s, %d",__FILE__, __LINE__);       \
+               panic("pmap interrupt assert %d %s, %d", processor_avail_count, __FILE__, __LINE__); \

 }
 #else
 #define pmap_intr_assert()
 }
 #else
 #define pmap_intr_assert()


@@ -495,7 +498,6 @@ pvhashidx(pmap_t pmap, vm_map_offset_t va)
            return hashidx;
 }
 
            return hashidx;
 }
 

-

 /*
  * unlinks the pv_hashed_entry_t pvh from the singly linked hash chain.
  * properly deals with the anchor.
 /*
  * unlinks the pv_hashed_entry_t pvh from the singly linked hash chain.
  * properly deals with the anchor.


@@ -964,11 +966,6 @@ PMAP_ZINFO_SFREE(pmap_t pmap, vm_size_t bytes)
 extern boolean_t       pmap_initialized;/* Has pmap_init completed? */
 #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
 
 extern boolean_t       pmap_initialized;/* Has pmap_init completed? */
 #define valid_page(x) (pmap_initialized && pmap_valid_page(x))
 

-// XXX
-#define HIGH_MEM_BASE  ((uint32_t)( -NBPDE) )  /* shared gdt etc seg addr */ /* XXX64 ?? */
-// XXX
-
-

 int            phys_attribute_test(
                        ppnum_t         phys,
                        int             bits);
 int            phys_attribute_test(
                        ppnum_t         phys,
                        int             bits);


@@ -1033,7 +1030,6 @@ static inline void pmap_update_pte(pt_entry_t *mptep, uint64_t pclear_bits, uint
        }       while (!pmap_cmpx_pte(mptep, opte, npte));
 }
 
        }       while (!pmap_cmpx_pte(mptep, opte, npte));
 }
 

-#if    defined(__x86_64__)

 /*
  * The single pml4 page per pmap is allocated at pmap create time and exists
  * for the duration of the pmap. we allocate this page in kernel vm.
 /*
  * The single pml4 page per pmap is allocated at pmap create time and exists
  * for the duration of the pmap. we allocate this page in kernel vm.


@@ -1055,6 +1051,21 @@ pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr)
 #endif
 }
 
 #endif
 }
 

+static inline pml4_entry_t *
+pmap64_user_pml4(pmap_t pmap, vm_map_offset_t vaddr)
+{
+       if (__improbable((vaddr > 0x00007FFFFFFFFFFFULL) &&
+               (vaddr < 0xFFFF800000000000ULL))) {
+               return (NULL);
+       }
+
+#if    DEBUG
+       return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_ucr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]);
+#else
+       return &pmap->pm_upml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)];
+#endif
+}
+

 /*
  * Returns address of requested PDPT entry in the physmap.
  */
 /*
  * Returns address of requested PDPT entry in the physmap.
  */


@@ -1134,7 +1145,13 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
        }
        return (NULL);
 }
        }
        return (NULL);
 }

-#endif
+extern void    pmap_alias(
+                               vm_offset_t     ava,
+                               vm_map_offset_t start,
+                               vm_map_offset_t end,
+                               vm_prot_t       prot,
+                               unsigned int options);
+

 #if    DEBUG
 #define DPRINTF(x...)  kprintf(x)
 #else
 #if    DEBUG
 #define DPRINTF(x...)  kprintf(x)
 #else

diff --git a/osfmk/i386/pmap_pcid.h b/osfmk/i386/pmap_pcid.h
index fc0854d7ee8ee75646721b47cbb993a75494baa2..372f0a4ed67b3d42c153739d886564fdbf47625a 100644 (file)
--- a/osfmk/i386/pmap_pcid.h
+++ b/osfmk/i386/pmap_pcid.h
@@ -61,8 +61,12 @@ tlb_flush_global(void) {
         * count of kernel invalidations, but that scheme
         * has its disadvantages as well.
         */
         * count of kernel invalidations, but that scheme
         * has its disadvantages as well.
         */

-       set_cr4(cr4 & ~CR4_PGE);
-       set_cr4(cr4 | CR4_PGE); 
+       if (cr4 & CR4_PGE) {
+               set_cr4(cr4 & ~CR4_PGE);
+               set_cr4(cr4 | CR4_PGE);
+       } else {
+               set_cr3_raw(get_cr3_raw());
+       }

        return;
 }
 
        return;
 }
 


@@ -85,7 +89,6 @@ static inline void pmap_pcid_validate_current(void) {
        if (cptr) {
                *cptr = 0;
        }
        if (cptr) {
                *cptr = 0;
        }

-

 }
 
 static inline void pmap_pcid_invalidate_cpu(pmap_t tpmap, int ccpu) {
 }
 
 static inline void pmap_pcid_invalidate_cpu(pmap_t tpmap, int ccpu) {

diff --git a/osfmk/i386/pmap_x86_common.c b/osfmk/i386/pmap_x86_common.c
index 72fd27e76c955e9f79af2dffee915526e71e0104..b66630233d686a24a621d04f9c05173c34c10227 100644 (file)
--- a/osfmk/i386/pmap_x86_common.c
+++ b/osfmk/i386/pmap_x86_common.c
@@ -2234,9 +2234,11 @@ pmap_map_bd(
        unsigned int    flags)
 {
        pt_entry_t      template;
        unsigned int    flags)
 {
        pt_entry_t      template;

-       pt_entry_t      *pte;
-       spl_t           spl;
+       pt_entry_t      *ptep;
+

        vm_offset_t     base = virt;
        vm_offset_t     base = virt;

+       boolean_t       doflush = FALSE;
+

        template = pa_to_pte(start_addr)
                | INTEL_PTE_REF
                | INTEL_PTE_MOD
        template = pa_to_pte(start_addr)
                | INTEL_PTE_REF
                | INTEL_PTE_MOD


@@ -2249,31 +2251,80 @@ pmap_map_bd(
                        template |= INTEL_PTE_PTA;
        }
 
                        template |= INTEL_PTE_PTA;
        }
 

-#if    defined(__x86_64__)

        if ((prot & VM_PROT_EXECUTE) == 0)
                template |= INTEL_PTE_NX;
        if ((prot & VM_PROT_EXECUTE) == 0)
                template |= INTEL_PTE_NX;

-#endif

 
        if (prot & VM_PROT_WRITE)
                template |= INTEL_PTE_WRITE;
 
        while (start_addr < end_addr) {
 
        if (prot & VM_PROT_WRITE)
                template |= INTEL_PTE_WRITE;
 
        while (start_addr < end_addr) {

-               spl = splhigh();
-               pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
-               if (pte == PT_ENTRY_NULL) {
-                       panic("pmap_map_bd: Invalid kernel address\n");
+               ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
+               if (ptep == PT_ENTRY_NULL) {
+                       panic("pmap_map_bd: Invalid kernel address");
+               }
+               if (pte_to_pa(*ptep)) {
+                       doflush = TRUE;

                }
                }

-               pmap_store_pte(pte, template);
-               splx(spl);
+               pmap_store_pte(ptep, template);

                pte_increment_pa(template);
                virt += PAGE_SIZE;
                start_addr += PAGE_SIZE;
        }
                pte_increment_pa(template);
                virt += PAGE_SIZE;
                start_addr += PAGE_SIZE;
        }

-       flush_tlb_raw();
-       PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+       if (doflush) {
+               flush_tlb_raw();
+               PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr);
+       }

        return(virt);
 }
 
        return(virt);
 }
 

+/* Create a virtual alias beginning at 'ava' of the specified kernel virtual
+ * range. The aliased pagetable range is expanded if
+ * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization,
+ * assumes caller has stabilized the source and destination ranges. Currently
+ * used to populate sections of the trampoline "doublemap" at CPU startup.
+ */
+
+void
+pmap_alias(
+       vm_offset_t     ava,
+       vm_map_offset_t start_addr,
+       vm_map_offset_t end_addr,
+       vm_prot_t       prot,
+       unsigned int    eoptions)
+{
+       pt_entry_t      prot_template, template;
+       pt_entry_t      *aptep, *sptep;
+
+       prot_template =  INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID;
+       if ((prot & VM_PROT_EXECUTE) == 0)
+               prot_template |= INTEL_PTE_NX;
+
+       if (prot & VM_PROT_WRITE)
+               prot_template |= INTEL_PTE_WRITE;
+       assert(((start_addr | end_addr) & PAGE_MASK) == 0);
+       while (start_addr < end_addr) {
+               aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+               if (aptep == PT_ENTRY_NULL) {
+                       if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) {
+                               pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP);
+                               aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava);
+                       } else {
+                               panic("pmap_alias: Invalid alias address");
+                       }
+               }
+               /* The aliased range should not have any active mappings */
+               assert(pte_to_pa(*aptep) == 0);
+
+               sptep = pmap_pte(kernel_pmap, start_addr);
+               assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0));
+               template = pa_to_pte(pte_to_pa(*sptep)) | prot_template;
+               pmap_store_pte(aptep, template);
+
+               ava += PAGE_SIZE;
+               start_addr += PAGE_SIZE;
+       }
+}
+

 mach_vm_size_t
 pmap_query_resident(
        pmap_t          pmap,
 mach_vm_size_t
 pmap_query_resident(
        pmap_t          pmap,

diff --git a/osfmk/i386/proc_reg.h b/osfmk/i386/proc_reg.h
index 9e1e744d1c4e33691a3fa84f47e826d3586cf328..530f9e61c7fac099d5384eda721e13fcaaeec50d 100644 (file)
--- a/osfmk/i386/proc_reg.h
+++ b/osfmk/i386/proc_reg.h
@@ -306,9 +306,9 @@ static inline uintptr_t get_cr3_base(void)
        return(cr3 & ~(0xFFFULL));
 }
 
        return(cr3 & ~(0xFFFULL));
 }
 

-static inline void set_cr3_composed(uintptr_t base, uint16_t pcid, uint32_t preserve)
+static inline void set_cr3_composed(uintptr_t base, uint16_t pcid, uint64_t preserve)

 {
 {

-       __asm__ volatile("mov %0, %%cr3" : : "r" (base | pcid | ( ( (uint64_t)preserve) << 63) ) );
+       __asm__ volatile("mov %0, %%cr3" : : "r" (base | pcid | ( (preserve) << 63) ) );

 }
 
 static inline uintptr_t get_cr4(void)
 }
 
 static inline uintptr_t get_cr4(void)


@@ -383,7 +383,13 @@ static inline void hlt(void)
 
 static inline void flush_tlb_raw(void)
 {
 
 static inline void flush_tlb_raw(void)
 {

-       set_cr3_raw(get_cr3_raw());
+       uintptr_t cr4 = get_cr4();
+       if (cr4 & CR4_PGE) {
+               set_cr4(cr4 & ~CR4_PGE);
+               set_cr4(cr4 | CR4_PGE);
+       } else {
+               set_cr3_raw(get_cr3_raw());
+       }

 }
 extern int rdmsr64_carefully(uint32_t msr, uint64_t *val);
 extern int wrmsr64_carefully(uint32_t msr, uint64_t val);
 }
 extern int rdmsr64_carefully(uint32_t msr, uint64_t *val);
 extern int wrmsr64_carefully(uint32_t msr, uint64_t val);

diff --git a/osfmk/i386/rtclock_asm.h b/osfmk/i386/rtclock_asm.h
index dba69a6e742ae828f223b408b5c383f5a313afb3..9a380be1edaeffdb74aa8577b01340d3983fd362 100644 (file)
--- a/osfmk/i386/rtclock_asm.h
+++ b/osfmk/i386/rtclock_asm.h
@@ -54,7 +54,7 @@
 #define NANOTIME                                                              \
        movq    %gs:CPU_NANOTIME,%rdi                                        ; \
        PAL_RTC_NANOTIME_READ_FAST()
 #define NANOTIME                                                              \
        movq    %gs:CPU_NANOTIME,%rdi                                        ; \
        PAL_RTC_NANOTIME_READ_FAST()

-
+/* TODO nobarrier */

 /*
  * Add 64-bit delta in register reg to timer pointed to by register treg.
  */
 /*
  * Add 64-bit delta in register reg to timer pointed to by register treg.
  */

diff --git a/osfmk/i386/rtclock_asm_native.h b/osfmk/i386/rtclock_asm_native.h
index ab16668e3a8d8f71cfb814e310be2ae3bc4bad27..50919f1fd8819093f8a009651f9a5d9b79b0a802 100644 (file)
--- a/osfmk/i386/rtclock_asm_native.h
+++ b/osfmk/i386/rtclock_asm_native.h
@@ -69,7 +69,23 @@
         jz        0b                   /* - wait if so */              ; \
        lfence                                                          ; \
        rdtsc                                                           ; \
         jz        0b                   /* - wait if so */              ; \
        lfence                                                          ; \
        rdtsc                                                           ; \

-       lfence                                                          ; \
+       shlq    $32,%rdx                                                ; \
+       movl    RNT_SHIFT(%rdi),%ecx                                    ; \
+       orq     %rdx,%rax                       /* %rax := tsc */       ; \
+       subq    RNT_TSC_BASE(%rdi),%rax         /* tsc - tsc_base */    ; \
+       shlq    %cl,%rax                                                ; \
+       movl    RNT_SCALE(%rdi),%ecx                                    ; \
+       mulq    %rcx                            /* delta * scale */     ; \
+       shrdq   $32,%rdx,%rax                   /* %rdx:%rax >>= 32 */  ; \
+       addq    RNT_NS_BASE(%rdi),%rax          /* add ns_base */       ; \
+       cmpl    RNT_GENERATION(%rdi),%esi       /* repeat if changed */ ; \
+       jne     0b
+
+#define PAL_RTC_NANOTIME_READ_NOBARRIER()                                        \
+0:     movl    RNT_GENERATION(%rdi),%esi                               ; \
+       test        %esi,%esi           /* info updating? */            ; \
+        jz        0b                   /* - wait if so */              ; \
+       rdtsc                                                           ; \

        shlq    $32,%rdx                                                ; \
        movl    RNT_SHIFT(%rdi),%ecx                                    ; \
        orq     %rdx,%rax                       /* %rax := tsc */       ; \
        shlq    $32,%rdx                                                ; \
        movl    RNT_SHIFT(%rdi),%ecx                                    ; \
        orq     %rdx,%rax                       /* %rax := tsc */       ; \

diff --git a/osfmk/i386/seg.h b/osfmk/i386/seg.h
index 75f5182e08b30ef5320ba7c46bca64ecbdec24fa..d5760f7aae1ff38a775a61788a14f15cbc2710a2 100644 (file)
--- a/osfmk/i386/seg.h
+++ b/osfmk/i386/seg.h
@@ -90,7 +90,9 @@ selector_to_sel(uint16_t selector)
     return (tconv.sel);
 }
 
     return (tconv.sel);
 }
 

-#define LDTSZ          8192            /* size of the kernel ldt in entries */
+#define LDTSZ_MAX      8192    /* maximal size of the kernel ldt in entries */
+#define LDTSZ_DFL      (128)
+#define LDTSZ          (LDTSZ_MAX)

 #define        LDTSZ_MIN       SEL_TO_INDEX(USER_SETTABLE)
                                        /* kernel ldt entries */
 
 #define        LDTSZ_MIN       SEL_TO_INDEX(USER_SETTABLE)
                                        /* kernel ldt entries */
 


@@ -182,11 +184,13 @@ typedef struct __attribute__((packed)) {
        void            *ptr;
 } x86_64_desc_register_t;
 
        void            *ptr;
 } x86_64_desc_register_t;
 

+
+

 /*
  * Boot-time data for master (or only) CPU
  */
 extern struct real_descriptor  master_gdt[GDTSZ];
 /*
  * Boot-time data for master (or only) CPU
  */
 extern struct real_descriptor  master_gdt[GDTSZ];

-extern struct real_descriptor  master_ldt[LDTSZ];
+extern struct real_descriptor  master_ldt[];

 extern struct i386_tss         master_ktss;
 extern struct sysenter_stack   master_sstk;
 
 extern struct i386_tss         master_ktss;
 extern struct sysenter_stack   master_sstk;
 


@@ -285,6 +289,7 @@ __END_DECLS
 #define SYSENTER_TF_CS (USER_CS|0x10000)
 #define        SYSENTER_DS     KERNEL64_SS     /* sysenter kernel data segment */
 
 #define SYSENTER_TF_CS (USER_CS|0x10000)
 #define        SYSENTER_DS     KERNEL64_SS     /* sysenter kernel data segment */
 

+#endif /* _I386_SEG_H_ */

 #ifdef __x86_64__
 /*
  * 64-bit kernel LDT descriptors
 #ifdef __x86_64__
 /*
  * 64-bit kernel LDT descriptors


@@ -293,5 +298,3 @@ __END_DECLS
 #define        USER_CTHREAD    0x0f            /* user cthread area */
 #define        USER_SETTABLE   0x1f            /* start of user settable ldt entries */
 #endif
 #define        USER_CTHREAD    0x0f            /* user cthread area */
 #define        USER_SETTABLE   0x1f            /* start of user settable ldt entries */
 #endif

-
-#endif /* _I386_SEG_H_ */

diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c
index ad07c88874d3da1fca1c985719f2fd2dd1e34f7a..bb1f9a17e4eb3ab14290e68c948ec2fdb1406e94 100644 (file)
--- a/osfmk/i386/trap.c
+++ b/osfmk/i386/trap.c
@@ -790,7 +790,6 @@ debugger_entry:
         */
 }
 
         */
 }
 

-

 static void
 set_recovery_ip(x86_saved_state64_t  *saved_state, vm_offset_t ip)
 {
 static void
 set_recovery_ip(x86_saved_state64_t  *saved_state, vm_offset_t ip)
 {


@@ -814,8 +813,8 @@ panic_trap(x86_saved_state64_t *regs, uint32_t pl, kern_return_t fault_result)
         */
        panic_io_port_read();
 
         */
        panic_io_port_read();
 

-       kprintf("panic trap number 0x%x, rip 0x%016llx\n",
-               regs->isf.trapno, regs->isf.rip);
+       kprintf("CPU %d panic trap number 0x%x, rip 0x%016llx\n",
+           cpu_number(), regs->isf.trapno, regs->isf.rip);

        kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
                cr0, cr2, cr3, cr4);
 
        kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n",
                cr0, cr2, cr3, cr4);
 


@@ -940,11 +939,6 @@ user_trap(
        subcode = 0;
        exc = 0;
 
        subcode = 0;
        exc = 0;
 

-#if DEBUG_TRACE
-       kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n",
-               saved_state, type, vaddr);
-#endif
-

        perfASTCallback astfn = perfASTHook;
        if (__improbable(astfn != NULL)) {
                myast = ast_pending();
        perfASTCallback astfn = perfASTHook;
        if (__improbable(astfn != NULL)) {
                myast = ast_pending();


@@ -1245,6 +1239,10 @@ sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
 }
 
 #if DEBUG
 }
 
 #if DEBUG

+#define TERI 1
+#endif
+
+#if TERI

 extern void    thread_exception_return_internal(void) __dead2;
 
 void thread_exception_return(void) {
 extern void    thread_exception_return_internal(void) __dead2;
 
 void thread_exception_return(void) {

diff --git a/osfmk/i386/tss.h b/osfmk/i386/tss.h
index 866d2056bbd0fc611730eecd0f599c30345a4d7a..eab037c242db91d9c13915a794797bf0db328759 100644 (file)
--- a/osfmk/i386/tss.h
+++ b/osfmk/i386/tss.h
@@ -105,8 +105,9 @@ struct i386_tss {
  * Although this is defined as a 64-bit stack, the space is also used in
  * 32-bit legacy mode. For 64-bit the stack is 16-byte aligned.
  */
  * Although this is defined as a 64-bit stack, the space is also used in
  * 32-bit legacy mode. For 64-bit the stack is 16-byte aligned.
  */

+

 struct sysenter_stack {
 struct sysenter_stack {

-       uint64_t        stack[16];      /* Space for a 64-bit frame and some */
+       uint64_t        sysestack[64];  /* Space for a 64-bit frame and some */

        uint64_t        top;            /* Top and pointer to ISS in PCS */
 };
 
        uint64_t        top;            /* Top and pointer to ISS in PCS */
 };
 

diff --git a/osfmk/i386/user_ldt.c b/osfmk/i386/user_ldt.c
index 8b57914558b3924c89ef073b3a9ccf1ff4147c7e..35dd2cef786250a813c63b76ef44612c0a45efd7 100644 (file)
--- a/osfmk/i386/user_ldt.c
+++ b/osfmk/i386/user_ldt.c
@@ -224,7 +224,6 @@ i386_set_ldt(
            } else {
                bzero(&new_ldt->ldt[start_sel - begin_sel], num_sels * sizeof(struct real_descriptor));
            }
            } else {
                bzero(&new_ldt->ldt[start_sel - begin_sel], num_sels * sizeof(struct real_descriptor));
            }

-

            /*
             * Validate descriptors.
             * Only allow descriptors with user privileges.
            /*
             * Validate descriptors.
             * Only allow descriptors with user privileges.


@@ -253,6 +252,12 @@ i386_set_ldt(
                        user_ldt_free(new_ldt);
                        return EACCES;
                }
                        user_ldt_free(new_ldt);
                        return EACCES;
                }

+               /* Reject attempts to create segments with 64-bit granules */
+               if (dp->granularity & SZ_64) {
+                       task_unlock(task);
+                       user_ldt_free(new_ldt);
+                       return EACCES;
+               }

            }
        }
 
            }
        }
 


@@ -292,9 +297,9 @@ i386_get_ldt(
        unsigned int    ldt_count;
        kern_return_t   err;
 
        unsigned int    ldt_count;
        kern_return_t   err;
 

-       if (start_sel >= 8192)
+       if (start_sel >= LDTSZ)

            return EINVAL;
            return EINVAL;

-       if ((uint64_t)start_sel + (uint64_t)num_sels > 8192)
+       if ((uint64_t)start_sel + (uint64_t)num_sels > LDTSZ)

            return EINVAL;
        if (descs == 0)
            return EINVAL;
            return EINVAL;
        if (descs == 0)
            return EINVAL;

diff --git a/osfmk/ipc/ipc_entry.c b/osfmk/ipc/ipc_entry.c
index 677e054eacc9f7a01b3bcc12faae445a6438d876..288ea6a5801857470956296ef1c8c343f6932734 100644 (file)
--- a/osfmk/ipc/ipc_entry.c
+++ b/osfmk/ipc/ipc_entry.c
@@ -103,8 +103,9 @@ ipc_entry_lookup(
        if (index <  space->is_table_size) {
                 entry = &space->is_table[index];
                if (IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) ||
        if (index <  space->is_table_size) {
                 entry = &space->is_table[index];
                if (IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) ||

-                   IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
+                   IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE) {

                        entry = IE_NULL;                
                        entry = IE_NULL;                

+               }

        }
        else {
                entry = IE_NULL;
        }
        else {
                entry = IE_NULL;


@@ -189,10 +190,14 @@ ipc_entry_claim(
        assert(table->ie_next < space->is_table_size);
 
        /*
        assert(table->ie_next < space->is_table_size);
 
        /*

-        *      Initialize the new entry.  We need only
-        *      increment the generation number and clear ie_request.
+        *      Initialize the new entry: increment gencount and reset
+        *      rollover point if it rolled over, and clear ie_request.

         */
         */

-       gen = IE_BITS_NEW_GEN(entry->ie_bits);
+       gen = ipc_entry_new_gen(entry->ie_bits);
+       if (__improbable(ipc_entry_gen_rolled(entry->ie_bits, gen))) {
+               ipc_entry_bits_t roll = ipc_space_get_rollpoint(space);
+               gen = ipc_entry_new_rollpoint(roll);
+       }

        entry->ie_bits = gen;
        entry->ie_request = IE_REQ_NONE;
 
        entry->ie_bits = gen;
        entry->ie_request = IE_REQ_NONE;
 


@@ -434,7 +439,7 @@ ipc_entry_dealloc(
 
        if ((index < size) && (entry == &table[index])) {
                assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name));
 
        if ((index < size) && (entry == &table[index])) {
                assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name));

-               entry->ie_bits &= IE_BITS_GEN_MASK;
+               entry->ie_bits &= (IE_BITS_GEN_MASK | IE_BITS_ROLL_MASK);

                entry->ie_next = table->ie_next;
                table->ie_next = index;
                space->is_table_free++;
                entry->ie_next = table->ie_next;
                table->ie_next = index;
                space->is_table_free++;


@@ -611,14 +616,7 @@ ipc_entry_grow_table(
                return KERN_RESOURCE_SHORTAGE;
        }
 
                return KERN_RESOURCE_SHORTAGE;
        }
 

-       /* initialize new entries (free chain in backwards order) */
-       for (i = osize; i < size; i++) {
-               table[i].ie_object = IO_NULL;
-               table[i].ie_bits = IE_BITS_GEN_MASK;
-               table[i].ie_index = 0;
-               table[i].ie_next = i + 1;
-       }
-       table[size-1].ie_next = 0;
+       ipc_space_rand_freelist(space, table, osize, size);

 
        /* clear out old entries in new table */
        memset((void *)table, 0, osize * sizeof(*table));
 
        /* clear out old entries in new table */
        memset((void *)table, 0, osize * sizeof(*table));


@@ -771,10 +769,10 @@ mach_port_name_t
 ipc_entry_name_mask(mach_port_name_t name)
 {
 #ifndef NO_PORT_GEN
 ipc_entry_name_mask(mach_port_name_t name)
 {
 #ifndef NO_PORT_GEN

-       static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_NEW_GEN(IE_BITS_GEN_MASK));
+       static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_GEN_MASK + IE_BITS_GEN_ONE);

        return name | null_name;
 #else
        return name | null_name;
 #else

-       static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_NEW_GEN(IE_BITS_GEN_MASK)));
+       static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_GEN_MASK + IE_BITS_GEN_ONE));

        return name & ~null_name;
 #endif
 }
        return name & ~null_name;
 #endif
 }

diff --git a/osfmk/ipc/ipc_entry.h b/osfmk/ipc/ipc_entry.h
index 531434edf3c0cefbbd7bcac99bd04a73c5359893..285a6adf258afe58e15cff9a5b96196a8bf95f49 100644 (file)
--- a/osfmk/ipc/ipc_entry.h
+++ b/osfmk/ipc/ipc_entry.h
@@ -119,17 +119,79 @@ struct ipc_entry {
 #define        IE_BITS_GEN_MASK        0xff000000      /* 8 bits for generation */
 #define        IE_BITS_GEN(bits)       ((bits) & IE_BITS_GEN_MASK)
 #define        IE_BITS_GEN_ONE         0x04000000      /* low bit of generation */
 #define        IE_BITS_GEN_MASK        0xff000000      /* 8 bits for generation */
 #define        IE_BITS_GEN(bits)       ((bits) & IE_BITS_GEN_MASK)
 #define        IE_BITS_GEN_ONE         0x04000000      /* low bit of generation */

-#define IE_BITS_NEW_GEN(old)   (((old) + IE_BITS_GEN_ONE) & IE_BITS_GEN_MASK)
+#define IE_BITS_ROLL_POS       22              /* LSB pos of generation rollover */
+#define IE_BITS_ROLL_BITS      2               /* number of generation rollover bits */
+#define IE_BITS_ROLL_MASK      (((1 << IE_BITS_ROLL_BITS) - 1) << IE_BITS_ROLL_POS)
+#define IE_BITS_ROLL(bits)     ((((bits) & IE_BITS_ROLL_MASK) << 8) ^ IE_BITS_GEN_MASK)
+
+/*
+ * Restart a generation counter with the specified bits for the rollover point.
+ * There are 4 different rollover points:
+ * bits    rollover period
+ * 0 0     64
+ * 0 1     48
+ * 1 0     32
+ * 1 1     16
+ */
+static inline ipc_entry_bits_t ipc_entry_new_rollpoint(
+       ipc_entry_bits_t rollbits)
+{
+       rollbits = (rollbits << IE_BITS_ROLL_POS) & IE_BITS_ROLL_MASK;
+       ipc_entry_bits_t newgen = IE_BITS_GEN_MASK + IE_BITS_GEN_ONE;
+       return (newgen | rollbits);
+}
+
+/*
+ * Get the next gencount, modulo the entry's rollover point. If the sum rolls over,
+ * the caller should re-start the generation counter with a different rollpoint.
+ */
+static inline ipc_entry_bits_t ipc_entry_new_gen(
+       ipc_entry_bits_t oldgen)
+{
+       ipc_entry_bits_t sum  = (oldgen + IE_BITS_GEN_ONE) & IE_BITS_GEN_MASK;
+       ipc_entry_bits_t roll = oldgen & IE_BITS_ROLL_MASK;
+       ipc_entry_bits_t newgen = (sum % IE_BITS_ROLL(oldgen)) | roll;
+       return newgen;
+}
+
+/* Determine if a gencount has rolled over or not. */
+static inline boolean_t ipc_entry_gen_rolled(
+       ipc_entry_bits_t oldgen,
+       ipc_entry_bits_t newgen)
+{
+       return (oldgen & IE_BITS_GEN_MASK) > (newgen & IE_BITS_GEN_MASK);
+}
+

 #else
 #define        IE_BITS_GEN_MASK        0
 #define        IE_BITS_GEN(bits)       0
 #define        IE_BITS_GEN_ONE         0
 #else
 #define        IE_BITS_GEN_MASK        0
 #define        IE_BITS_GEN(bits)       0
 #define        IE_BITS_GEN_ONE         0

-#define IE_BITS_NEW_GEN(old)   (old)
-#endif /* !USE_PORT_GEN */
+#define IE_BITS_ROLL_POS       0
+#define IE_BITS_ROLL_MASK      0
+#define IE_BITS_ROLL(bits)     (bits)
+
+static inline ipc_entry_bits_t ipc_entry_new_rollpoint(
+       ipc_entry_bits_t rollbits)
+{
+       return 0;
+}
+
+static inline ipc_entry_bits_t ipc_entry_new_gen(
+       ipc_entry_bits_t oldgen)
+{
+       return 0;
+}
+
+static inline boolean_t ipc_entry_gen_rolled(
+       ipc_entry_bits_t oldgen,
+       ipc_entry_bits_t newgen)
+{
+       return FALSE;
+}

 
 

+#endif /* !USE_PORT_GEN */

 
 #define        IE_BITS_RIGHT_MASK      0x007fffff      /* relevant to the right */
 
 #define        IE_BITS_RIGHT_MASK      0x007fffff      /* relevant to the right */

-

 /*
  * Exported interfaces
  */
 /*
  * Exported interfaces
  */

diff --git a/osfmk/ipc/ipc_init.c b/osfmk/ipc/ipc_init.c
index 49221fa52e5be02cb9aa3cf0a9ef070f01a6ec8e..8d5ea071f771fa8fbb3b287f74b40b32b10a9624 100644 (file)
--- a/osfmk/ipc/ipc_init.c
+++ b/osfmk/ipc/ipc_init.c
@@ -275,7 +275,6 @@ ipc_init(void)
        msg_ool_size_small -= cpy_kdata_hdr_sz;
 
        ipc_host_init();
        msg_ool_size_small -= cpy_kdata_hdr_sz;
 
        ipc_host_init();

-

 }
 
 
 }
 
 

diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c
index 4ec900c7b9ea5000e8b4ee1ce1c118c089ace2f0..e5445a290db53c702c70b7cc776f532674f5f8ca 100644 (file)
--- a/osfmk/ipc/ipc_kmsg.c
+++ b/osfmk/ipc/ipc_kmsg.c
@@ -1160,6 +1160,12 @@ ipc_kmsg_rmqueue(
 
                queue->ikmq_base = IKM_NULL;
        } else {
 
                queue->ikmq_base = IKM_NULL;
        } else {

+               if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) {
+                       panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. "
+                               "(prev->next: %p, next->prev: %p, kmsg: %p)",
+                               prev->ikm_next, next->ikm_prev, kmsg);
+               }
+

                if (queue->ikmq_base == kmsg)
                        queue->ikmq_base = next;
 
                if (queue->ikmq_base == kmsg)
                        queue->ikmq_base = next;
 

diff --git a/osfmk/ipc/ipc_pset.c b/osfmk/ipc/ipc_pset.c
index 2fa71904dab4e769d187209e081a88c84af35ad5..fe989b283d21577f2a0e2401008f2a8d85d922f0 100644 (file)
--- a/osfmk/ipc/ipc_pset.c
+++ b/osfmk/ipc/ipc_pset.c
@@ -192,8 +192,10 @@ ipc_pset_alloc_special(
        reserved_link = waitq_link_reserve(NULL);
 
        __IGNORE_WCASTALIGN(pset = (ipc_pset_t)io_alloc(IOT_PORT_SET));
        reserved_link = waitq_link_reserve(NULL);
 
        __IGNORE_WCASTALIGN(pset = (ipc_pset_t)io_alloc(IOT_PORT_SET));

-       if (pset == IPS_NULL)
+       if (pset == IPS_NULL) {
+               waitq_link_release(reserved_link);

                return IPS_NULL;
                return IPS_NULL;

+       }

 
        bzero((char *)pset, sizeof(*pset));
 
 
        bzero((char *)pset, sizeof(*pset));
 

diff --git a/osfmk/ipc/ipc_space.c b/osfmk/ipc/ipc_space.c
index 4c38dbab7aeae1a23df8dd9f488728e4f7f6b69d..8596ad530f1f38f356bd191ea060d9b86cc877e5 100644 (file)
--- a/osfmk/ipc/ipc_space.c
+++ b/osfmk/ipc/ipc_space.c
@@ -83,8 +83,15 @@
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_right.h>
 #include <ipc/ipc_port.h>
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_right.h>

+#include <prng/random.h>

 #include <string.h>
 
 #include <string.h>
 

+/* Remove this in the future so port names are less predictable. */
+#define CONFIG_SEMI_RANDOM_ENTRIES
+#ifdef CONFIG_SEMI_RANDOM_ENTRIES
+#define NUM_SEQ_ENTRIES 8
+#endif
+

 zone_t ipc_space_zone;
 ipc_space_t ipc_space_kernel;
 ipc_space_t ipc_space_reply;
 zone_t ipc_space_zone;
 ipc_space_t ipc_space_kernel;
 ipc_space_t ipc_space_reply;


@@ -110,6 +117,102 @@ ipc_space_release(
        is_release(space);
 }
 
        is_release(space);
 }
 

+/*     Routine:                ipc_space_get_rollpoint
+ *     Purpose:
+ *             Generate a new gencount rollover point from a space's entropy pool
+ */
+ipc_entry_bits_t
+ipc_space_get_rollpoint(
+       ipc_space_t     space)
+{
+       return random_bool_gen_bits(
+                       &space->bool_gen,
+                       &space->is_entropy[0],
+                       IS_ENTROPY_CNT,
+                       IE_BITS_ROLL_BITS);
+}
+
+/*
+ *     Routine:        ipc_entry_rand_freelist
+ *     Purpose:
+ *             Pseudo-randomly permute the order of entries in an IPC space
+ *     Arguments:
+ *             space:  the ipc space to initialize.
+ *             table:  the corresponding ipc table to initialize.
+ *             bottom: the start of the range to initialize (inclusive).
+ *             top:    the end of the range to initialize (noninclusive).
+ */
+void
+ipc_space_rand_freelist(
+       ipc_space_t             space,
+       ipc_entry_t             table,
+       mach_port_index_t       bottom,
+       mach_port_index_t       top)
+{
+#ifdef CONFIG_SEMI_RANDOM_ENTRIES
+       /*
+        * Only make sequential entries at the start of the table, and not when
+        * we're growing the space.
+        */
+       int at_start = (bottom == 0);
+       ipc_entry_num_t total = 0;
+#endif
+
+       /* First entry in the free list is always free, and is the start of the free list. */
+       mach_port_index_t curr = bottom;
+       bottom++;
+       top--;
+
+       /*
+        *      Initialize the free list in the table.
+        *      Add the entries in pseudo-random order and randomly set the generation
+        *      number, in order to frustrate attacks involving port name reuse.
+        */
+       while (bottom <= top) {
+               ipc_entry_t entry = &table[curr];
+               int which;
+#ifdef CONFIG_SEMI_RANDOM_ENTRIES
+               /*
+                * XXX: This is a horrible hack to make sure that randomizing the port
+                * doesn't break programs that might have (sad) hard-coded values for
+                * certain port names.
+                */
+               if (at_start && total++ < NUM_SEQ_ENTRIES)
+                       which = 0;
+               else
+#endif
+                       which = random_bool_gen_bits(
+                                               &space->bool_gen,
+                                               &space->is_entropy[0],
+                                               IS_ENTROPY_CNT,
+                                               1);
+
+               mach_port_index_t next;
+               if (which) {
+                       next = top;
+                       top--;
+               } else {
+                       next = bottom;
+                       bottom++;
+               }
+
+               /*
+                * The entry's gencount will roll over on its first allocation, at which
+                * point a random rollover will be set for the entry.
+                */
+               entry->ie_bits = IE_BITS_GEN_MASK;
+               entry->ie_next   = next;
+               entry->ie_object = IO_NULL;
+               entry->ie_index  = 0;
+               curr = next;
+       }
+       table[curr].ie_next   = 0;
+       table[curr].ie_object = IO_NULL;
+       table[curr].ie_index  = 0;
+       table[curr].ie_bits   = IE_BITS_GEN_MASK;
+}
+
+

 /*
  *     Routine:        ipc_space_create
  *     Purpose:
 /*
  *     Routine:        ipc_space_create
  *     Purpose:


@@ -132,7 +235,6 @@ ipc_space_create(
        ipc_space_t space;
        ipc_entry_t table;
        ipc_entry_num_t new_size;
        ipc_space_t space;
        ipc_entry_t table;
        ipc_entry_num_t new_size;

-       mach_port_index_t index;

 
        space = is_alloc();
        if (space == IS_NULL)
 
        space = is_alloc();
        if (space == IS_NULL)


@@ -147,19 +249,11 @@ ipc_space_create(
        new_size = initial->its_size;
        memset((void *) table, 0, new_size * sizeof(struct ipc_entry));
 
        new_size = initial->its_size;
        memset((void *) table, 0, new_size * sizeof(struct ipc_entry));
 

-       /*
-        *      Initialize the free list in the table.
-        *      Add the entries in reverse order, and
-        *      set the generation number to -1, so that
-        *      initial allocations produce "natural" names.
-        */
-       for (index = 0; index < new_size; index++) {
-               ipc_entry_t entry = &table[index];
+       /* Set to 0 so entropy pool refills */
+       memset((void *) space->is_entropy, 0, sizeof(space->is_entropy));

 
 

-               entry->ie_bits = IE_BITS_GEN_MASK;
-               entry->ie_next = index+1;
-       }
-       table[new_size-1].ie_next = 0;
+       random_bool_init(&space->bool_gen);
+       ipc_space_rand_freelist(space, table, 0, new_size);

 
        is_lock_init(space);
        space->is_bits = 2; /* 2 refs, active, not growing */
 
        is_lock_init(space);
        space->is_bits = 2; /* 2 refs, active, not growing */

diff --git a/osfmk/ipc/ipc_space.h b/osfmk/ipc/ipc_space.h
index 60662c8ee015a12371e37cfee91cbba33f5b89ca..06fdf3e79a80a61312b3962b17dc730aecbf4db3 100644 (file)
--- a/osfmk/ipc/ipc_space.h
+++ b/osfmk/ipc/ipc_space.h
@@ -73,6 +73,7 @@
 #define _IPC_IPC_SPACE_H_
 
 
 #define _IPC_IPC_SPACE_H_
 
 

+#include <prng/random.h>

 #include <mach/mach_types.h>
 #include <mach/boolean.h>
 #include <mach/kern_return.h>
 #include <mach/mach_types.h>
 #include <mach/boolean.h>
 #include <mach/kern_return.h>


@@ -109,6 +110,7 @@ typedef natural_t ipc_space_refs_t;
 #define IS_REFS_MAX    0x0fffffff
 #define IS_INACTIVE    0x40000000      /* space is inactive */
 #define IS_GROWING     0x20000000      /* space is growing */
 #define IS_REFS_MAX    0x0fffffff
 #define IS_INACTIVE    0x40000000      /* space is inactive */
 #define IS_GROWING     0x20000000      /* space is growing */

+#define IS_ENTROPY_CNT  1              /* per-space entropy pool size */

 
 struct ipc_space {
        lck_spin_t      is_lock_data;
 
 struct ipc_space {
        lck_spin_t      is_lock_data;


@@ -120,6 +122,8 @@ struct ipc_space {
        struct ipc_table_size *is_table_next; /* info for larger table */
        ipc_entry_num_t is_low_mod;     /* lowest modified entry during growth */
        ipc_entry_num_t is_high_mod;    /* highest modified entry during growth */
        struct ipc_table_size *is_table_next; /* info for larger table */
        ipc_entry_num_t is_low_mod;     /* lowest modified entry during growth */
        ipc_entry_num_t is_high_mod;    /* highest modified entry during growth */

+       struct bool_gen bool_gen;       /* state for boolean RNG */
+       unsigned int is_entropy[IS_ENTROPY_CNT]; /* pool of entropy taken from RNG */

        int is_node_id;                 /* HOST_LOCAL_NODE, or remote node if proxy space */
 };
 
        int is_node_id;                 /* HOST_LOCAL_NODE, or remote node if proxy space */
 };
 


@@ -215,7 +219,7 @@ is_release(ipc_space_t is) {
 extern kern_return_t ipc_space_create_special(
        ipc_space_t     *spacep);
 
 extern kern_return_t ipc_space_create_special(
        ipc_space_t     *spacep);
 

-/* Create  new IPC space */
+/* Create a new IPC space */

 extern kern_return_t ipc_space_create(
        ipc_table_size_t        initial,
        ipc_space_t             *spacep);
 extern kern_return_t ipc_space_create(
        ipc_table_size_t        initial,
        ipc_space_t             *spacep);


@@ -228,6 +232,15 @@ extern void ipc_space_terminate(
 extern void ipc_space_clean(
        ipc_space_t     space);
 
 extern void ipc_space_clean(
        ipc_space_t     space);
 

+/* Permute the order of a range within an IPC space */
+extern void ipc_space_rand_freelist(
+       ipc_space_t             space,
+       ipc_entry_t             table,
+       mach_port_index_t       bottom,
+       mach_port_index_t       top);
+
+/* Generate a new gencount rollover point from a space's entropy pool */
+extern ipc_entry_bits_t ipc_space_get_rollpoint(ipc_space_t space);

 #endif /* MACH_KERNEL_PRIVATE */
 #endif /* __APPLE_API_PRIVATE */
 
 #endif /* MACH_KERNEL_PRIVATE */
 #endif /* __APPLE_API_PRIVATE */
 

diff --git a/osfmk/kdp/kdp_core.c b/osfmk/kdp/kdp_core.c
index 5cf505770059970168578d81175b34b79127265e..ac71c806165f94d3dfd6e03c215b1f888a36fec6 100644 (file)
--- a/osfmk/kdp/kdp_core.c
+++ b/osfmk/kdp/kdp_core.c
@@ -709,7 +709,7 @@ kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphy
     }
     else
 #if defined(__arm64__)
     }
     else
 #if defined(__arm64__)

-    if (vaddr == _COMM_PAGE64_BASE_ADDRESS)
+    if (vaddr == _COMM_HIGH_PAGE64_BASE_ADDRESS)

     {
        /* not readable */
        ppn = 0;
     {
        /* not readable */
        ppn = 0;


@@ -1148,8 +1148,10 @@ do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant)
        existing_log_size = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
                                panic_info->eph_panic_log_len + panic_info->eph_other_log_len;
 #else /* CONFIG_EMBEDDED */
        existing_log_size = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) +
                                panic_info->eph_panic_log_len + panic_info->eph_other_log_len;
 #else /* CONFIG_EMBEDDED */

-       existing_log_size = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +
+       if (panic_info->mph_panic_log_offset != 0) {
+               existing_log_size = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) +

                                panic_info->mph_panic_log_len + panic_info->mph_other_log_len;
                                panic_info->mph_panic_log_len + panic_info->mph_other_log_len;

+       }

 #endif /* CONFIG_EMBEDDED */
 
        assert (existing_log_size <= debug_buf_size);
 #endif /* CONFIG_EMBEDDED */
 
        assert (existing_log_size <= debug_buf_size);


@@ -1261,7 +1263,9 @@ do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant)
 #if CONFIG_EMBEDDED
                existing_log_size -= panic_info->eph_other_log_len;
 #else
 #if CONFIG_EMBEDDED
                existing_log_size -= panic_info->eph_other_log_len;
 #else

-               existing_log_size -= panic_info->mph_other_log_len;
+               if (existing_log_size) {
+                       existing_log_size -= panic_info->mph_other_log_len;
+               }

 #endif
 
                /*
 #endif
 
                /*


@@ -1308,15 +1312,18 @@ exit:
                dump_succeeded = FALSE;
        }
 
                dump_succeeded = FALSE;
        }
 

+       /* If applicable, update the panic header and flush it so we update the CRC */

 #if CONFIG_EMBEDDED
        panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
                        EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED);
 #if CONFIG_EMBEDDED
        panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
                        EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED);

+       paniclog_flush();

 #else
 #else

-       panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :
+       if (panic_info->mph_panic_log_offset != 0) {
+               panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE :

                        MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED);
                        MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED);

+               paniclog_flush();
+       }

 #endif
 #endif

-       /* We touched the panic header, flush it so we update the CRC */
-       paniclog_flush();

 
        return (dump_succeeded ? 0 : -1);
 }
 
        return (dump_succeeded ? 0 : -1);
 }

diff --git a/osfmk/kdp/processor_core.c b/osfmk/kdp/processor_core.c
index e6ebc85a21000e016dd4ee25bc01ff10a8823396..f33d9915c57a1115c1db617ba6f193b0bd0d96f6 100644 (file)
--- a/osfmk/kdp/processor_core.c
+++ b/osfmk/kdp/processor_core.c
@@ -678,8 +678,8 @@ kern_do_coredump(void *core_outvars, boolean_t kernel_only, uint64_t first_file_
        current_core = kern_coredump_core_list;
        while (current_core) {
                /* Seek to the beginning of the next file */
        current_core = kern_coredump_core_list;
        while (current_core) {
                /* Seek to the beginning of the next file */

-               ret = kern_dump_seek_to_next_file(core_outvars, *last_file_offset);
-               if (ret != KERN_SUCCESS) {
+               cur_ret = kern_dump_seek_to_next_file(core_outvars, *last_file_offset);
+               if (cur_ret != KERN_SUCCESS) {

                        kern_coredump_log(NULL, "Failed to seek to beginning of next core\n");
                        return KERN_FAILURE;
                }
                        kern_coredump_log(NULL, "Failed to seek to beginning of next core\n");
                        return KERN_FAILURE;
                }

diff --git a/osfmk/kern/debug.c b/osfmk/kern/debug.c
index e330be0d6d799d4a916d341f5807bafcc1815c6a..a84f86d75bcbb44112598c6aff029358e6b5023b 100644 (file)
--- a/osfmk/kern/debug.c
+++ b/osfmk/kern/debug.c
@@ -181,6 +181,7 @@ int mach_assert = 1;
  * EXTENDED_/DEBUG_BUF_SIZE can't grow without updates to SMC and iBoot to store larger panic logs on co-processor systems */
 #define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data))
 #define EXTENDED_DEBUG_BUF_SIZE 0x0013ff80
  * EXTENDED_/DEBUG_BUF_SIZE can't grow without updates to SMC and iBoot to store larger panic logs on co-processor systems */
 #define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data))
 #define EXTENDED_DEBUG_BUF_SIZE 0x0013ff80

+static_assert(((EXTENDED_DEBUG_BUF_SIZE % PANIC_FLUSH_BOUNDARY) == 0), "Extended debug buf size must match SMC alignment requirements");

 #define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace"
 #endif
 
 #define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace"
 #endif
 


@@ -315,7 +316,12 @@ extended_debug_log_init(void)
         * to point at this new buffer.
         */
        char *new_debug_buf = kalloc(EXTENDED_DEBUG_BUF_SIZE);
         * to point at this new buffer.
         */
        char *new_debug_buf = kalloc(EXTENDED_DEBUG_BUF_SIZE);

-       bzero(new_debug_buf, EXTENDED_DEBUG_BUF_SIZE);
+       /*
+        * iBoot pre-initializes the panic region with the NULL character. We set this here
+        * so we can accurately calculate the CRC for the region without needing to flush the
+        * full region over SMC.
+        */
+       memset(new_debug_buf, '\0', EXTENDED_DEBUG_BUF_SIZE);

 
        panic_info = (struct macos_panic_header *)new_debug_buf;
        debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));
 
        panic_info = (struct macos_panic_header *)new_debug_buf;
        debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));


@@ -846,8 +852,6 @@ debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned
        }
 
 #if CONFIG_KDP_INTERACTIVE_DEBUGGING
        }
 
 #if CONFIG_KDP_INTERACTIVE_DEBUGGING

-       PE_i_can_has_debugger(NULL);
-

        /*
         * If reboot on panic is enabled and the caller of panic indicated that we should skip
         * local coredumps, don't try to write these and instead go straight to reboot. This
        /*
         * If reboot on panic is enabled and the caller of panic indicated that we should skip
         * local coredumps, don't try to write these and instead go straight to reboot. This


@@ -890,6 +894,10 @@ debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned
                }
        }
 
                }
        }
 

+       if (debug_boot_arg & DB_REBOOT_ALWAYS) {
+               kdp_machine_reboot_type(kPEPanicRestartCPU);
+       }
+

        /* If KDP is configured, try to trap to the debugger */
        if (current_debugger != NO_CUR_DB) {
                kdp_raise_exception(exception, code, subcode, state);
        /* If KDP is configured, try to trap to the debugger */
        if (current_debugger != NO_CUR_DB) {
                kdp_raise_exception(exception, code, subcode, state);

diff --git a/osfmk/kern/debug.h b/osfmk/kern/debug.h
index ecf9967013adf3371c9674c6cc346a9ea2b5f2fd..6289eeecd3593b538c53dd0b0f17438e8e680379 100644 (file)
--- a/osfmk/kern/debug.h
+++ b/osfmk/kern/debug.h
@@ -276,6 +276,10 @@ struct embedded_panic_header {
        uint32_t eph_stackshot_len;        /* length of the panic stackshot (0 if not valid ) */
        uint32_t eph_other_log_offset;     /* Offset of the other log (any logging subsequent to the stackshot) from the beginning of the header */
        uint32_t eph_other_log_len;        /* length of the other log */
        uint32_t eph_stackshot_len;        /* length of the panic stackshot (0 if not valid ) */
        uint32_t eph_other_log_offset;     /* Offset of the other log (any logging subsequent to the stackshot) from the beginning of the header */
        uint32_t eph_other_log_len;        /* length of the other log */

+       uint64_t eph_x86_power_state:8,
+                eph_x86_efi_boot_state:8,
+                eph_x86_system_state:8,
+                eph_x86_unused_bits:40;

 } __attribute__((packed));
 
 #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE             0x01
 } __attribute__((packed));
 
 #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE             0x01


@@ -289,7 +293,7 @@ struct embedded_panic_header {
 #define EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC        0x100
 #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED               0x200
 
 #define EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC        0x100
 #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED               0x200
 

-#define EMBEDDED_PANIC_HEADER_CURRENT_VERSION 1
+#define EMBEDDED_PANIC_HEADER_CURRENT_VERSION 2

 #define EMBEDDED_PANIC_MAGIC 0x46554E4B /* FUNK */
 
 struct macos_panic_header {
 #define EMBEDDED_PANIC_MAGIC 0x46554E4B /* FUNK */
 
 struct macos_panic_header {


@@ -336,6 +340,12 @@ __END_DECLS
 
 #ifdef KERNEL_PRIVATE
 #if DEBUG
 
 #ifdef KERNEL_PRIVATE
 #if DEBUG

+#ifndef DKPR
+#define DKPR 1
+#endif
+#endif
+
+#if DKPR

 /*
  * For the DEBUG kernel, support the following:
  *     sysctl -w debug.kprint_syscall=<syscall_mask> 
 /*
  * For the DEBUG kernel, support the following:
  *     sysctl -w debug.kprint_syscall=<syscall_mask> 


@@ -403,6 +413,12 @@ enum {
 #define DB_PRT_KDEBUG          0x10000 /* kprintf KDEBUG traces */
 #define DB_DISABLE_LOCAL_CORE   0x20000 /* ignore local kernel core dump support */
 #define DB_DISABLE_GZIP_CORE    0x40000 /* don't gzip kernel core dumps */
 #define DB_PRT_KDEBUG          0x10000 /* kprintf KDEBUG traces */
 #define DB_DISABLE_LOCAL_CORE   0x20000 /* ignore local kernel core dump support */
 #define DB_DISABLE_GZIP_CORE    0x40000 /* don't gzip kernel core dumps */

+#define DB_DISABLE_CROSS_PANIC  0x80000 /* x86 only - don't trigger cross panics. Only
+                                         * necessary to enable x86 kernel debugging on
+                                         * configs with a dev-fused co-processor running
+                                         * release bridgeOS.
+                                         */
+#define DB_REBOOT_ALWAYS        0x100000 /* Don't wait for debugger connection */

 
 /*
  * Values for a 64-bit mask that's passed to the debugger.
 
 /*
  * Values for a 64-bit mask that's passed to the debugger.

diff --git a/osfmk/kern/gzalloc.c b/osfmk/kern/gzalloc.c
index 915b2278276d9ae546d3c51ad7949185f8c10101..9bd11481f4e8df91eaf61936febeff603897ab22 100644 (file)
--- a/osfmk/kern/gzalloc.c
+++ b/osfmk/kern/gzalloc.c
@@ -57,7 +57,7 @@
  *     -gzalloc_noconsistency: disable consistency checks that flag mismatched
  *     frees, corruptions of the header/trailer signatures etc.
  *     -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
  *     -gzalloc_noconsistency: disable consistency checks that flag mismatched
  *     frees, corruptions of the header/trailer signatures etc.
  *     -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel

- *     enables the guard allocator for zones sized 8K-16K (if present) by
+ *     enables the guard allocator for zones sized 1K (if present) by

  *     default, this option can disable that behaviour.
  *     gzname=<name> target a zone by name. Can be coupled with size-based
  *     targeting. Naming conventions match those of the zlog boot-arg, i.e.
  *     default, this option can disable that behaviour.
  *     gzname=<name> target a zone by name. Can be coupled with size-based
  *     targeting. Naming conventions match those of the zlog boot-arg, i.e.

diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c
index 97b04c739d114b8a169bc0b84947ba8a36c01cd8..fa13463fe5573025040e9873d3addcce17a975d1 100644 (file)
--- a/osfmk/kern/kalloc.c
+++ b/osfmk/kern/kalloc.c
@@ -140,48 +140,63 @@ KALLOC_ZINFO_SFREE(vm_size_t bytes)
  *     kalloc_max_prerounded is the smallest allocation size, before
  *     rounding, for which no zone exists.
  *
  *     kalloc_max_prerounded is the smallest allocation size, before
  *     rounding, for which no zone exists.
  *

- *     Also if the allocation size is more than kalloc_kernmap_size 
+ *     Also if the allocation size is more than kalloc_kernmap_size

  *     then allocate from kernel map rather than kalloc_map.
  */
 
 #if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
 
  *     then allocate from kernel map rather than kalloc_map.
  */
 
 #if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
 

-#define K_ZONE_SIZES                   \
-       16,                             \
-       32,                             \
-       48,                             \
-/* 3 */        64,                             \
-       80,                             \
-       96,                             \
-/* 6 */        128,                            \
-       160, 192,                               \
-       256,                            \
-/* 9 */        288,                            \
-       512, 576,                               \
-       1024, 1152,                             \
-/* C */        1280,                           \
-       2048,                           \
-       4096
-
-#define K_ZONE_NAMES                   \
-       "kalloc.16",                    \
-       "kalloc.32",                    \
-       "kalloc.48",                    \
-/* 3 */        "kalloc.64",                    \
-       "kalloc.80",                    \
-       "kalloc.96",                    \
-/* 6 */        "kalloc.128",                   \
-       "kalloc.160",                   \
-       "kalloc.192",                   \
-       "kalloc.256",                   \
-/* 9 */        "kalloc.288",                   \
-       "kalloc.512",                   \
-       "kalloc.576",                   \
-       "kalloc.1024",                  \
-       "kalloc.1152",                  \
-/* C */        "kalloc.1280",                  \
-       "kalloc.2048",                  \
-       "kalloc.4096"
+#define K_ZONE_SIZES \
+       16, \
+       32, \
+       48, \
+       64, /* 2^6 */ \
+       80, \
+       96, \
+       128, /* 2^7 */ \
+       160, \
+       192, \
+       224, \
+       256, /* 2^8 */ \
+       288, \
+       368, \
+       400, \
+       512, /* 2^9 */\
+       576, \
+       768, \
+       1024, /* 2^10 */ \
+       1152, \
+       1280, \
+       1664, \
+       2048, /* 2^11 */ \
+       4096, /* 2^12 */ \
+       6144
+
+#define K_ZONE_NAMES \
+       "kalloc.16", \
+       "kalloc.32", \
+       "kalloc.48", \
+       "kalloc.64", /* 2^6 */ \
+       "kalloc.80", \
+       "kalloc.96", \
+       "kalloc.128", /* 2^7 */ \
+       "kalloc.160", \
+       "kalloc.192", \
+       "kalloc.224", \
+       "kalloc.256", /* 2^8 */\
+       "kalloc.288", \
+       "kalloc.368", \
+       "kalloc.400", \
+       "kalloc.512", /* 2^9 */ \
+       "kalloc.576", \
+       "kalloc.768", \
+       "kalloc.1024", /* 2^10 */ \
+       "kalloc.1152", \
+       "kalloc.1280", \
+       "kalloc.1664", \
+       "kalloc.2048", /* 2^11 */ \
+       "kalloc.4096", /* 2^12 */ \
+       "kalloc.6144"

 
 #elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3
 
 
 #elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3
 


@@ -214,7 +229,7 @@ KALLOC_ZINFO_SFREE(vm_size_t bytes)
        "kalloc.4096",  "kalloc.6144"
 
 #else
        "kalloc.4096",  "kalloc.6144"
 
 #else

-#error missing zone size parameters for kalloc
+#error missing or invalid zone size parameters for kalloc

 #endif
 
 #define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN)
 #endif
 
 #define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN)

diff --git a/osfmk/kern/kcdata.h b/osfmk/kern/kcdata.h
index 741c7f86460270b9548a0f7a4b9cf2e957ea9185..e358b4ec8f2dc4e8eeaacc8f7e70a05928e7a1e3 100644 (file)
--- a/osfmk/kern/kcdata.h
+++ b/osfmk/kern/kcdata.h
@@ -469,7 +469,7 @@ struct kcdata_type_definition {
 #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */
 #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO  0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */
 #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du       /* struct stackshot_thread_waitinfo */
 #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */
 #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO  0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */
 #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du       /* struct stackshot_thread_waitinfo */

-#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */
+#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot or thread_group_snapshot_v2 */

 #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu          /* uint64_t */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u      /* uint64_t */
 #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu          /* uint64_t */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */
 #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u      /* uint64_t */


@@ -659,6 +659,17 @@ struct thread_group_snapshot {
        char tgs_name[16];
 } __attribute__((packed));
 
        char tgs_name[16];
 } __attribute__((packed));
 

+enum thread_group_flags {
+       kThreadGroupEfficient = 0x1,
+       kThreadGroupUIApp = 0x2
+};
+
+struct thread_group_snapshot_v2 {
+       uint64_t tgs_id;
+       char tgs_name[16];
+       uint64_t tgs_flags;
+} __attribute__((packed));
+

 enum coalition_flags {
        kCoalitionTermRequested = 0x1,
        kCoalitionTerminated    = 0x2,
 enum coalition_flags {
        kCoalitionTermRequested = 0x1,
        kCoalitionTerminated    = 0x2,

diff --git a/osfmk/kern/ledger.c b/osfmk/kern/ledger.c
index 4c27ea7fb82c25b407f2d5097c20b6c824a64178..ec35a3a52351b35d29d703bad016b6c94ab89288 100644 (file)
--- a/osfmk/kern/ledger.c
+++ b/osfmk/kern/ledger.c
@@ -178,8 +178,8 @@ static kern_return_t ledger_perform_blocking(ledger_t l);
 static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
 static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
 
 static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
 static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
 

-static void ledger_entry_check_new_balance(ledger_t ledger, int entry,
-                                          struct ledger_entry *le);
+static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
+                                           int entry, struct ledger_entry *le);

 
 #if 0
 static void
 
 #if 0
 static void


@@ -697,7 +697,8 @@ ledger_refill(uint64_t now, ledger_t ledger, int entry)
 #define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
 
 void
 #define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE)
 
 void

-ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *le)
+ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
+                               int entry, struct ledger_entry *le)

 {
        ledger_amount_t credit, debit;
 
 {
        ledger_amount_t credit, debit;
 


@@ -749,7 +750,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *
                if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
                    (!(le->le_flags & LF_CALLED_BACK) &&
                    entry_get_callback(ledger, entry))) {
                if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
                    (!(le->le_flags & LF_CALLED_BACK) &&
                    entry_get_callback(ledger, entry))) {

-                       set_astledger(current_thread());
+                       act_set_astledger_async(thread);

                }
        } else {
                /*
                }
        } else {
                /*


@@ -778,7 +779,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *
                                         * set the AST so it can be done before returning
                                         * to userland.
                                         */
                                         * set the AST so it can be done before returning
                                         * to userland.
                                         */

-                                       set_astledger(current_thread());
+                                       act_set_astledger_async(thread);

                                }
                        } else {
                                /*
                                }
                        } else {
                                /*


@@ -792,7 +793,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *
                                         * know the ledger balance is now back below
                                         * the warning level.
                                         */
                                         * know the ledger balance is now back below
                                         * the warning level.
                                         */

-                                       set_astledger(current_thread());
+                                       act_set_astledger_async(thread);

                                }
                        }
                }
                                }
                        }
                }


@@ -812,20 +813,19 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *
 }
 
 void
 }
 
 void

-ledger_check_new_balance(ledger_t ledger, int entry)
+ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)

 {
        struct ledger_entry *le;
        assert(entry > 0 && entry <= ledger->l_size);
        le = &ledger->l_entries[entry];
 {
        struct ledger_entry *le;
        assert(entry > 0 && entry <= ledger->l_size);
        le = &ledger->l_entries[entry];

-       ledger_entry_check_new_balance(ledger, entry, le);
+       ledger_entry_check_new_balance(thread, ledger, entry, le);

 }
 
 }
 

-

 /*
 /*

- * Add value to an entry in a ledger.
+ * Add value to an entry in a ledger for a specific thread.

  */
 kern_return_t
  */
 kern_return_t

-ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
+ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)

 {
        ledger_amount_t old, new;
        struct ledger_entry *le;
 {
        ledger_amount_t old, new;
        struct ledger_entry *le;


@@ -840,12 +840,22 @@ ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
 
        old = OSAddAtomic64(amount, &le->le_credit);
        new = old + amount;
 
        old = OSAddAtomic64(amount, &le->le_credit);
        new = old + amount;

-       lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new));
-       ledger_entry_check_new_balance(ledger, entry, le);
+       lprintf(("%p Credit %lld->%lld\n", thread, old, new));
+
+       ledger_entry_check_new_balance(thread, ledger, entry, le);

 
        return (KERN_SUCCESS);
 }
 
 
        return (KERN_SUCCESS);
 }
 

+/*
+ * Add value to an entry in a ledger.
+ */
+kern_return_t
+ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+       return ledger_credit_thread(current_thread(), ledger, entry, amount);
+}
+

 /* Add all of one ledger's values into another.
  * They must have been created from the same template.
  * This is not done atomically. Another thread (if not otherwise synchronized)
 /* Add all of one ledger's values into another.
  * They must have been created from the same template.
  * This is not done atomically. Another thread (if not otherwise synchronized)


@@ -1285,7 +1295,7 @@ ledger_set_action(ledger_t ledger, int entry, int action)
 }
 
 kern_return_t
 }
 
 kern_return_t

-ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
+ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)

 {
        struct ledger_entry *le;
        ledger_amount_t old, new;
 {
        struct ledger_entry *le;
        ledger_amount_t old, new;


@@ -1308,9 +1318,15 @@ ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
        }
        lprintf(("%p Debit %lld->%lld\n", thread, old, new));
 
        }
        lprintf(("%p Debit %lld->%lld\n", thread, old, new));
 

-       ledger_entry_check_new_balance(ledger, entry, le);
+       ledger_entry_check_new_balance(thread, ledger, entry, le);
+

        return (KERN_SUCCESS);
        return (KERN_SUCCESS);

+}

 
 

+kern_return_t
+ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
+{
+       return ledger_debit_thread(current_thread(), ledger, entry, amount);

 }
 
 void
 }
 
 void

diff --git a/osfmk/kern/ledger.h b/osfmk/kern/ledger.h
index 8e627bef4048b41e28d0e062ecea430ea42019ec..d6b27ce11276b8320c24cf0a85c7e463bc4f85a0 100644 (file)
--- a/osfmk/kern/ledger.h
+++ b/osfmk/kern/ledger.h
@@ -129,11 +129,15 @@ extern kern_return_t ledger_set_period(ledger_t ledger, int entry,
        uint64_t period);
 extern kern_return_t ledger_disable_refill(ledger_t l, int entry);
 extern kern_return_t ledger_entry_setactive(ledger_t ledger, int entry);
        uint64_t period);
 extern kern_return_t ledger_disable_refill(ledger_t l, int entry);
 extern kern_return_t ledger_entry_setactive(ledger_t ledger, int entry);

-extern void ledger_check_new_balance(ledger_t ledger, int entry);
+extern void ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry);

 extern kern_return_t ledger_credit(ledger_t ledger, int entry,
        ledger_amount_t amount);
 extern kern_return_t ledger_debit(ledger_t ledger, int entry,
        ledger_amount_t amount);
 extern kern_return_t ledger_credit(ledger_t ledger, int entry,
        ledger_amount_t amount);
 extern kern_return_t ledger_debit(ledger_t ledger, int entry,
        ledger_amount_t amount);

+extern kern_return_t ledger_credit_thread(thread_t thread, ledger_t ledger,
+                                          int entry, ledger_amount_t amount);
+extern kern_return_t ledger_debit_thread(thread_t thread, ledger_t ledger,
+                                         int entry, ledger_amount_t amount);

 extern kern_return_t ledger_zero_balance(ledger_t ledger, int entry);
 extern kern_return_t ledger_get_entries(ledger_t ledger, int entry,
        ledger_amount_t *credit, ledger_amount_t *debit);
 extern kern_return_t ledger_zero_balance(ledger_t ledger, int entry);
 extern kern_return_t ledger_get_entries(ledger_t ledger, int entry,
        ledger_amount_t *credit, ledger_amount_t *debit);

diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c
index bd590a0424d9998647324f2d6c9c46de1644bdd4..5f7be7132c5f4f57f654ec6ac17d0a17d628b359 100644 (file)
--- a/osfmk/kern/sched_prim.c
+++ b/osfmk/kern/sched_prim.c
@@ -744,7 +744,6 @@ thread_unblock(
        ml_get_power_state(&aticontext, &pidle);
 
        if (__improbable(aticontext && !(thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT))) {
        ml_get_power_state(&aticontext, &pidle);
 
        if (__improbable(aticontext && !(thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT))) {

-               ledger_credit(thread->t_ledger, task_ledgers.interrupt_wakeups, 1);

                DTRACE_SCHED2(iwakeup, struct thread *, thread, struct proc *, thread->task->bsd_info);
 
                uint64_t ttd = PROCESSOR_DATA(current_processor(), timer_call_ttd);
                DTRACE_SCHED2(iwakeup, struct thread *, thread, struct proc *, thread->task->bsd_info);
 
                uint64_t ttd = PROCESSOR_DATA(current_processor(), timer_call_ttd);


@@ -757,23 +756,30 @@ thread_unblock(
                                        thread->thread_timer_wakeups_bin_2++;
                }
 
                                        thread->thread_timer_wakeups_bin_2++;
                }
 

+               ledger_credit_thread(thread, thread->t_ledger,
+                                    task_ledgers.interrupt_wakeups, 1);

                if (pidle) {
                if (pidle) {

-                       ledger_credit(thread->t_ledger, task_ledgers.platform_idle_wakeups, 1);
+                       ledger_credit_thread(thread, thread->t_ledger,
+                                            task_ledgers.platform_idle_wakeups, 1);

                }
 
        } else if (thread_get_tag_internal(cthread) & THREAD_TAG_CALLOUT) {
                }
 
        } else if (thread_get_tag_internal(cthread) & THREAD_TAG_CALLOUT) {

+               /* TODO: what about an interrupt that does a wake taken on a callout thread? */

                if (cthread->callout_woken_from_icontext) {
                if (cthread->callout_woken_from_icontext) {

-                       ledger_credit(thread->t_ledger, task_ledgers.interrupt_wakeups, 1);
+                       ledger_credit_thread(thread, thread->t_ledger,
+                                            task_ledgers.interrupt_wakeups, 1);

                        thread->thread_callout_interrupt_wakeups++;
                        thread->thread_callout_interrupt_wakeups++;

+

                        if (cthread->callout_woken_from_platform_idle) {
                        if (cthread->callout_woken_from_platform_idle) {

-                               ledger_credit(thread->t_ledger, task_ledgers.platform_idle_wakeups, 1);
+                               ledger_credit_thread(thread, thread->t_ledger,
+                                                    task_ledgers.platform_idle_wakeups, 1);

                                thread->thread_callout_platform_idle_wakeups++;
                        }
                                thread->thread_callout_platform_idle_wakeups++;
                        }

-                       
+

                        cthread->callout_woke_thread = TRUE;
                }
        }
                        cthread->callout_woke_thread = TRUE;
                }
        }

-       
+

        if (thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT) {
                thread->callout_woken_from_icontext = aticontext;
                thread->callout_woken_from_platform_idle = pidle;
        if (thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT) {
                thread->callout_woken_from_icontext = aticontext;
                thread->callout_woken_from_platform_idle = pidle;


@@ -2621,17 +2627,16 @@ thread_dispatch(
                                 * Bill CPU time to both the task and
                                 * the individual thread.
                                 */
                                 * Bill CPU time to both the task and
                                 * the individual thread.
                                 */

-                               ledger_credit(thread->t_ledger,
-                                   task_ledgers.cpu_time, consumed);
-                               ledger_credit(thread->t_threadledger,
-                                   thread_ledgers.cpu_time, consumed);
+                               ledger_credit_thread(thread, thread->t_ledger,
+                                                    task_ledgers.cpu_time, consumed);
+                               ledger_credit_thread(thread, thread->t_threadledger,
+                                                    thread_ledgers.cpu_time, consumed);

                                if (thread->t_bankledger) {
                                if (thread->t_bankledger) {

-                                       ledger_credit(thread->t_bankledger,
-                                               bank_ledgers.cpu_time,
-                                               (consumed - thread->t_deduct_bank_ledger_time));
-
+                                       ledger_credit_thread(thread, thread->t_bankledger,
+                                                            bank_ledgers.cpu_time,
+                                                            (consumed - thread->t_deduct_bank_ledger_time));

                                }
                                }

-                               thread->t_deduct_bank_ledger_time =0;
+                               thread->t_deduct_bank_ledger_time = 0;

                        }
 
                        wake_lock(thread);
                        }
 
                        wake_lock(thread);

diff --git a/osfmk/kern/task.c b/osfmk/kern/task.c
index c2a29291e5c75e1f08b8520f77ce51390ebe6d1b..d58d29cdbc186852040e2a3fdc7f9973f67b9e10 100644 (file)
--- a/osfmk/kern/task.c
+++ b/osfmk/kern/task.c
@@ -2267,21 +2267,6 @@ task_terminate_internal(
        vm_map_disable_hole_optimization(task->map);
        vm_map_unlock(task->map);
 
        vm_map_disable_hole_optimization(task->map);
        vm_map_unlock(task->map);
 

-       vm_map_remove(task->map,
-                     task->map->min_offset,
-                     task->map->max_offset,
-                     /*
-                      * Final cleanup:
-                      * + no unnesting
-                      * + remove immutable mappings
-                      */
-                     (VM_MAP_REMOVE_NO_UNNESTING |
-                      VM_MAP_REMOVE_IMMUTABLE));
-
-       /* release our shared region */
-       vm_shared_region_set(task, NULL);
-
-

 #if MACH_ASSERT
        /*
         * Identify the pmap's process, in case the pmap ledgers drift
 #if MACH_ASSERT
        /*
         * Identify the pmap's process, in case the pmap ledgers drift


@@ -2298,6 +2283,21 @@ task_terminate_internal(
        pmap_set_process(task->map->pmap, pid, procname);
 #endif /* MACH_ASSERT */
 
        pmap_set_process(task->map->pmap, pid, procname);
 #endif /* MACH_ASSERT */
 

+       vm_map_remove(task->map,
+                     task->map->min_offset,
+                     task->map->max_offset,
+                     /*
+                      * Final cleanup:
+                      * + no unnesting
+                      * + remove immutable mappings
+                      */
+                     (VM_MAP_REMOVE_NO_UNNESTING |
+                      VM_MAP_REMOVE_IMMUTABLE));
+
+       /* release our shared region */
+       vm_shared_region_set(task, NULL);
+
+

        lck_mtx_lock(&tasks_threads_lock);
        queue_remove(&tasks, task, task_t, tasks);
        queue_enter(&terminated_tasks, task, task_t, tasks);
        lck_mtx_lock(&tasks_threads_lock);
        queue_remove(&tasks, task, task_t, tasks);
        queue_enter(&terminated_tasks, task, task_t, tasks);


@@ -5071,8 +5071,13 @@ PROC_CROSSED_HIGH_WATERMARK__SEND_EXC_RESOURCE_AND_SUSPEND(int max_footprint_mb,
                        task_resume_internal(task);
                }
        } else {
                        task_resume_internal(task);
                }
        } else {

-               task_enqueue_exception_with_corpse(task, EXC_RESOURCE,
+               if (audio_active) {
+                       printf("process %s[%d] crossed memory high watermark (%d MB); EXC_RESOURCE "
+                       "supressed due to audio playback.\n", procname, pid, max_footprint_mb);
+               } else {
+                       task_enqueue_exception_with_corpse(task, EXC_RESOURCE,

                                code, EXCEPTION_CODE_MAX, NULL);
                                code, EXCEPTION_CODE_MAX, NULL);

+               }

        }
 
        /*
        }
 
        /*


@@ -5248,7 +5253,8 @@ task_set_phys_footprint_limit_internal(
                (ledger_amount_t)new_limit_mb << 20, PHYS_FOOTPRINT_WARNING_LEVEL);
 
         if (task == current_task()) {
                (ledger_amount_t)new_limit_mb << 20, PHYS_FOOTPRINT_WARNING_LEVEL);
 
         if (task == current_task()) {

-                ledger_check_new_balance(task->ledger, task_ledgers.phys_footprint);
+                ledger_check_new_balance(current_thread(), task->ledger,
+                                         task_ledgers.phys_footprint);

         }
 
        task_unlock(task);
         }
 
        task_unlock(task);

diff --git a/osfmk/kern/task_policy.c b/osfmk/kern/task_policy.c
index b1318652355a44df42cf5fc75c25b273b4584a4b..f876182a5f525b60dee2008399c60d5e471ee2db 100644 (file)
--- a/osfmk/kern/task_policy.c
+++ b/osfmk/kern/task_policy.c
@@ -2533,7 +2533,7 @@ task_suspend_cpumon(task_t task)
         */
        task->rusage_cpu_flags &= ~(TASK_RUSECPU_FLAGS_PERTHR_LIMIT | TASK_RUSECPU_FLAGS_FATAL_CPUMON);         
        queue_iterate(&task->threads, thread, thread_t, task_threads) {
         */
        task->rusage_cpu_flags &= ~(TASK_RUSECPU_FLAGS_PERTHR_LIMIT | TASK_RUSECPU_FLAGS_FATAL_CPUMON);         
        queue_iterate(&task->threads, thread, thread_t, task_threads) {

-               set_astledger(thread);
+               act_set_astledger(thread);

        }
 
        return KERN_SUCCESS;
        }
 
        return KERN_SUCCESS;


@@ -2573,7 +2573,7 @@ task_enable_cpumon_locked(task_t task)
 
        task->rusage_cpu_flags |= TASK_RUSECPU_FLAGS_PERTHR_LIMIT;
        queue_iterate(&task->threads, thread, thread_t, task_threads) {
 
        task->rusage_cpu_flags |= TASK_RUSECPU_FLAGS_PERTHR_LIMIT;
        queue_iterate(&task->threads, thread, thread_t, task_threads) {

-               set_astledger(thread);
+               act_set_astledger(thread);

        }
 
        return KERN_SUCCESS;
        }
 
        return KERN_SUCCESS;

diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index 0057c988f8df7506783312120301a131d5ad7da1..ce815db9ef9a20e24c16f330250c1e437584a77c 100644 (file)
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -1253,7 +1253,7 @@ thread_create_internal(
                 * This task has a per-thread CPU limit; make sure this new thread
                 * gets its limit set too, before it gets out of the kernel.
                 */
                 * This task has a per-thread CPU limit; make sure this new thread
                 * gets its limit set too, before it gets out of the kernel.
                 */

-               set_astledger(new_thread);
+               act_set_astledger(new_thread);

        }
 
        /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
        }
 
        /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */

diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h
index 02bdc0607550f1c5ee2cdacaab06b835b918958c..4601a9ebb6638c9f2ce6f0cc52b105bfd6f474ec 100644 (file)
--- a/osfmk/kern/thread.h
+++ b/osfmk/kern/thread.h
@@ -1077,7 +1077,8 @@ extern boolean_t  thread_should_abort(
 extern int is_64signalregset(void);
 
 extern void act_set_kperf(thread_t);
 extern int is_64signalregset(void);
 
 extern void act_set_kperf(thread_t);

-extern void set_astledger(thread_t thread);
+extern void act_set_astledger(thread_t thread);
+extern void act_set_astledger_async(thread_t thread);

 extern void act_set_io_telemetry_ast(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);
 extern void act_set_io_telemetry_ast(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);

diff --git a/osfmk/kern/thread_act.c b/osfmk/kern/thread_act.c
index 6f16bfd1c7ede6c9770bc733b9b5440f75e7e6d7..3bd8b51829f748ea422e7c7040e9e476e984ade1 100644 (file)
--- a/osfmk/kern/thread_act.c
+++ b/osfmk/kern/thread_act.c
@@ -1023,6 +1023,27 @@ act_set_ast(
        splx(s);
 }
 
        splx(s);
 }
 

+/*
+ * set AST on thread without causing an AST check
+ * and without taking the thread lock
+ *
+ * If thread is not the current thread, then it may take
+ * up until the next context switch or quantum expiration
+ * on that thread for it to notice the AST.
+ */
+static void
+act_set_ast_async(thread_t  thread,
+                  ast_t     ast)
+{
+       thread_ast_set(thread, ast);
+
+       if (thread == current_thread()) {
+               spl_t s = splsched();
+               ast_propagate(thread);
+               splx(s);
+       }
+}
+

 void
 act_set_astbsd(
        thread_t        thread)
 void
 act_set_astbsd(
        thread_t        thread)


@@ -1033,22 +1054,10 @@ act_set_astbsd(
 void
 act_set_astkevent(thread_t thread, uint16_t bits)
 {
 void
 act_set_astkevent(thread_t thread, uint16_t bits)
 {

-       spl_t s = splsched();
-
-       /*
-        * Do not send an IPI if the thread is running on
-        * another processor, wait for the next quantum
-        * expiration to load the AST.
-        */
-

        atomic_fetch_or(&thread->kevent_ast_bits, bits);
        atomic_fetch_or(&thread->kevent_ast_bits, bits);

-       thread_ast_set(thread, AST_KEVENT);
-
-       if (thread == current_thread()) {
-               ast_propagate(thread);
-       }

 
 

-       splx(s);
+       /* kevent AST shouldn't send immediate IPIs */
+       act_set_ast_async(thread, AST_KEVENT);

 }
 
 void
 }
 
 void


@@ -1073,11 +1082,25 @@ act_set_astmacf(
 #endif
 
 void
 #endif
 
 void

-set_astledger(thread_t thread)
+act_set_astledger(thread_t thread)

 {
        act_set_ast(thread, AST_LEDGER);
 }
 
 {
        act_set_ast(thread, AST_LEDGER);
 }
 

+/*
+ * The ledger AST may need to be set while already holding
+ * the thread lock.  This routine skips sending the IPI,
+ * allowing us to avoid the lock hold.
+ *
+ * However, it means the targeted thread must context switch
+ * to recognize the ledger AST.
+ */
+void
+act_set_astledger_async(thread_t thread)
+{
+       act_set_ast_async(thread, AST_LEDGER);
+}
+

 void
 act_set_io_telemetry_ast(thread_t thread)
 {
 void
 act_set_io_telemetry_ast(thread_t thread)
 {

diff --git a/osfmk/kern/zalloc.c b/osfmk/kern/zalloc.c
index 699dc3c742488522fab98744992dc56558e35513..0a21dda6a1e6b7fc06ea8e420b9e4a9e3cdc5721 100644 (file)
--- a/osfmk/kern/zalloc.c
+++ b/osfmk/kern/zalloc.c
@@ -87,6 +87,8 @@
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 

+#include <prng/random.h>
+

 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>


@@ -215,6 +217,8 @@ uintptr_t       zp_nopoison_cookie      = 0;
 boolean_t       zone_tagging_on;
 #endif /* VM_MAX_TAG_ZONES */
 
 boolean_t       zone_tagging_on;
 #endif /* VM_MAX_TAG_ZONES */
 

+static struct bool_gen zone_bool_gen;
+

 /*
  * initialize zone poisoning
  * called from zone_bootstrap before any allocations are made from zalloc
 /*
  * initialize zone poisoning
  * called from zone_bootstrap before any allocations are made from zalloc


@@ -342,10 +346,6 @@ vm_offset_t     zone_map_max_address = 0;
 
 /* Globals for random boolean generator for elements in free list */
 #define MAX_ENTROPY_PER_ZCRAM          4
 
 /* Globals for random boolean generator for elements in free list */
 #define MAX_ENTROPY_PER_ZCRAM          4

-#define RANDOM_BOOL_GEN_SEED_COUNT      4
-static unsigned int bool_gen_seed[RANDOM_BOOL_GEN_SEED_COUNT];
-static unsigned int bool_gen_global = 0;
-decl_simple_lock_data(, bool_gen_lock)

 
 /* VM region for all metadata structures */
 vm_offset_t    zone_metadata_region_min = 0;
 
 /* VM region for all metadata structures */
 vm_offset_t    zone_metadata_region_min = 0;


@@ -368,7 +368,7 @@ decl_simple_lock_data(, all_zones_lock)
 unsigned int            num_zones_in_use;
 unsigned int            num_zones;
 
 unsigned int            num_zones_in_use;
 unsigned int            num_zones;
 

-#define MAX_ZONES       288
+#define MAX_ZONES       320

 struct zone             zone_array[MAX_ZONES];
 
 /* Used to keep track of empty slots in the zone_array */
 struct zone             zone_array[MAX_ZONES];
 
 /* Used to keep track of empty slots in the zone_array */


@@ -2543,61 +2543,18 @@ zcram_metadata_init(vm_offset_t newmem, vm_size_t size, struct zone_page_metadat
        return;
 }
 
        return;
 }
 

-
-/*
- * Boolean Random Number Generator for generating booleans to randomize 
- * the order of elements in newly zcram()'ed memory. The algorithm is a 
- * modified version of the KISS RNG proposed in the paper:
- * http://stat.fsu.edu/techreports/M802.pdf
- * The modifications have been documented in the technical paper 
- * paper from UCL:
- * http://www0.cs.ucl.ac.uk/staff/d.jones/GoodPracticeRNG.pdf 
- */
-
-static void random_bool_gen_entropy(
-               int     *buffer,
-               int     count)
-{
-
-       int i, t;
-       simple_lock(&bool_gen_lock);
-       for (i = 0; i < count; i++) {
-               bool_gen_seed[1] ^= (bool_gen_seed[1] << 5);
-               bool_gen_seed[1] ^= (bool_gen_seed[1] >> 7);
-               bool_gen_seed[1] ^= (bool_gen_seed[1] << 22);
-               t = bool_gen_seed[2] + bool_gen_seed[3] + bool_gen_global;
-               bool_gen_seed[2] = bool_gen_seed[3];
-               bool_gen_global = t < 0;
-               bool_gen_seed[3] = t &2147483647;
-               bool_gen_seed[0] += 1411392427;
-               buffer[i] = (bool_gen_seed[0] + bool_gen_seed[1] + bool_gen_seed[3]);
-       }
-       simple_unlock(&bool_gen_lock);
-}
-
-static boolean_t random_bool_gen(
-               int     *buffer,
-               int     index,
-               int     bufsize)
-{
-       int valindex, bitpos;
-       valindex = (index / (8 * sizeof(int))) % bufsize;
-       bitpos = index % (8 * sizeof(int));
-       return (boolean_t)(buffer[valindex] & (1 << bitpos));
-} 
-
-static void 
+static void

 random_free_to_zone(
                        zone_t          zone,
                        vm_offset_t     newmem,
                        vm_offset_t     first_element_offset,
                        int             element_count,
 random_free_to_zone(
                        zone_t          zone,
                        vm_offset_t     newmem,
                        vm_offset_t     first_element_offset,
                        int             element_count,

-                       int             *entropy_buffer)
+                       unsigned int    *entropy_buffer)

 {
        vm_offset_t     last_element_offset;
        vm_offset_t     element_addr;
        vm_size_t       elem_size;
 {
        vm_offset_t     last_element_offset;
        vm_offset_t     element_addr;
        vm_size_t       elem_size;

-       int             index;  
+       int             index;

 
        assert(element_count  <= ZONE_CHUNK_MAXELEMENTS);
        elem_size = zone->elem_size;
 
        assert(element_count  <= ZONE_CHUNK_MAXELEMENTS);
        elem_size = zone->elem_size;


@@ -2608,7 +2565,7 @@ random_free_to_zone(
 #if DEBUG || DEVELOPMENT
                leak_scan_debug_flag || __improbable(zone->tags) ||
 #endif /* DEBUG || DEVELOPMENT */
 #if DEBUG || DEVELOPMENT
                leak_scan_debug_flag || __improbable(zone->tags) ||
 #endif /* DEBUG || DEVELOPMENT */

-               random_bool_gen(entropy_buffer, index, MAX_ENTROPY_PER_ZCRAM)) {
+               random_bool_gen_bits(&zone_bool_gen, entropy_buffer, MAX_ENTROPY_PER_ZCRAM, 1)) {

                        element_addr = newmem + first_element_offset;
                        first_element_offset += elem_size;
                } else {
                        element_addr = newmem + first_element_offset;
                        first_element_offset += elem_size;
                } else {


@@ -2635,7 +2592,7 @@ zcram(
        vm_size_t       elem_size;
        boolean_t   from_zm = FALSE;
        int element_count;
        vm_size_t       elem_size;
        boolean_t   from_zm = FALSE;
        int element_count;

-       int entropy_buffer[MAX_ENTROPY_PER_ZCRAM];
+       unsigned int entropy_buffer[MAX_ENTROPY_PER_ZCRAM] = { 0 };

 
        /* Basic sanity checks */
        assert(zone != ZONE_NULL && newmem != (vm_offset_t)0);
 
        /* Basic sanity checks */
        assert(zone != ZONE_NULL && newmem != (vm_offset_t)0);


@@ -2650,12 +2607,12 @@ zcram(
                from_zm = TRUE;
 
        if (!from_zm) {
                from_zm = TRUE;
 
        if (!from_zm) {

-               /* We cannot support elements larger than page size for foreign memory because we 
-                * put metadata on the page itself for each page of foreign memory. We need to do 
-                * this in order to be able to reach the metadata when any element is freed 
+               /* We cannot support elements larger than page size for foreign memory because we
+                * put metadata on the page itself for each page of foreign memory. We need to do
+                * this in order to be able to reach the metadata when any element is freed

                 */
                assert((zone->allows_foreign == TRUE) && (zone->elem_size <= (PAGE_SIZE - sizeof(struct zone_page_metadata))));
                 */
                assert((zone->allows_foreign == TRUE) && (zone->elem_size <= (PAGE_SIZE - sizeof(struct zone_page_metadata))));

-       } 
+       }

 
        if (zalloc_debug & ZALLOC_DEBUG_ZCRAM)
                kprintf("zcram(%p[%s], 0x%lx%s, 0x%lx)\n", zone, zone->zone_name,
 
        if (zalloc_debug & ZALLOC_DEBUG_ZCRAM)
                kprintf("zcram(%p[%s], 0x%lx%s, 0x%lx)\n", zone, zone->zone_name,


@@ -2663,10 +2620,8 @@ zcram(
 
        ZONE_PAGE_COUNT_INCR(zone, (size / PAGE_SIZE));
 
 
        ZONE_PAGE_COUNT_INCR(zone, (size / PAGE_SIZE));
 

-       random_bool_gen_entropy(entropy_buffer, MAX_ENTROPY_PER_ZCRAM);
-       
-       /* 
-        * Initialize the metadata for all pages. We dont need the zone lock 
+       /*
+        * Initialize the metadata for all pages. We dont need the zone lock

         * here because we are not manipulating any zone related state yet.
         */
 
         * here because we are not manipulating any zone related state yet.
         */
 


@@ -2773,7 +2728,6 @@ void
 zone_bootstrap(void)
 {
        char temp_buf[16];
 zone_bootstrap(void)
 {
        char temp_buf[16];

-       unsigned int i;

 
        if (!PE_parse_boot_argn("zalloc_debug", &zalloc_debug, sizeof(zalloc_debug)))
                zalloc_debug = 0;
 
        if (!PE_parse_boot_argn("zalloc_debug", &zalloc_debug, sizeof(zalloc_debug)))
                zalloc_debug = 0;


@@ -2781,11 +2735,7 @@ zone_bootstrap(void)
        /* Set up zone element poisoning */
        zp_init();
 
        /* Set up zone element poisoning */
        zp_init();
 

-       /* Seed the random boolean generator for elements in zone free list */
-       for (i = 0; i < RANDOM_BOOL_GEN_SEED_COUNT; i++) {
-               bool_gen_seed[i] = (unsigned int)early_random();
-       }
-       simple_lock_init(&bool_gen_lock, 0);
+       random_bool_init(&zone_bool_gen);

 
        /* should zlog log to debug zone corruption instead of leaks? */
        if (PE_parse_boot_argn("-zc", temp_buf, sizeof(temp_buf))) {
 
        /* should zlog log to debug zone corruption instead of leaks? */
        if (PE_parse_boot_argn("-zc", temp_buf, sizeof(temp_buf))) {

diff --git a/osfmk/mach/arm/vm_param.h b/osfmk/mach/arm/vm_param.h
index 715ee1d31f0a0fef59398b17d8f1d3813bb01bbc..915e237e6a1b99379cf09e57d3e493544594aab1 100644 (file)
--- a/osfmk/mach/arm/vm_param.h
+++ b/osfmk/mach/arm/vm_param.h
@@ -154,7 +154,7 @@ extern unsigned             PAGE_SHIFT_CONST;
 
 /* system-wide values */
 #define MACH_VM_MIN_ADDRESS    ((mach_vm_offset_t) 0x0ULL)
 
 /* system-wide values */
 #define MACH_VM_MIN_ADDRESS    ((mach_vm_offset_t) 0x0ULL)

-#define MACH_VM_MAX_ADDRESS    ((mach_vm_offset_t) 0x0000001000000000ULL)
+#define MACH_VM_MAX_ADDRESS    ((mach_vm_offset_t) 0x0000000FC0000000ULL)

 
 #else
 #error architecture not supported
 
 #else
 #error architecture not supported

diff --git a/osfmk/mach/vm_statistics.h b/osfmk/mach/vm_statistics.h
index f6ffb8b0aef0aa2baa2d7de671ac1dd52c207b87..fa0560559023e7c83191d7435af39eed6c71c4ec 100644 (file)
--- a/osfmk/mach/vm_statistics.h
+++ b/osfmk/mach/vm_statistics.h
@@ -362,7 +362,8 @@ typedef struct {
                vmkf_keep_map_locked:1,
                vmkf_fourk:1,
                vmkf_overwrite_immutable:1,
                vmkf_keep_map_locked:1,
                vmkf_fourk:1,
                vmkf_overwrite_immutable:1,

-               __vmkf_unused:19;
+               vmkf_remap_prot_copy:1,
+               __vmkf_unused:18;

 } vm_map_kernel_flags_t;
 #define VM_MAP_KERNEL_FLAGS_NONE (vm_map_kernel_flags_t) {             \
        .vmkf_atomic_entry = 0, /* keep entry atomic (no coalescing) */ \
 } vm_map_kernel_flags_t;
 #define VM_MAP_KERNEL_FLAGS_NONE (vm_map_kernel_flags_t) {             \
        .vmkf_atomic_entry = 0, /* keep entry atomic (no coalescing) */ \


@@ -378,6 +379,7 @@ typedef struct {
        .vmkf_keep_map_locked = 0, /* keep map locked when returning from vm_map_enter() */ \
        .vmkf_fourk = 0,        /* use fourk pager */                   \
        .vmkf_overwrite_immutable = 0,  /* can overwrite immutable mappings */ \
        .vmkf_keep_map_locked = 0, /* keep map locked when returning from vm_map_enter() */ \
        .vmkf_fourk = 0,        /* use fourk pager */                   \
        .vmkf_overwrite_immutable = 0,  /* can overwrite immutable mappings */ \

+       .vmkf_remap_prot_copy = 0, /* vm_remap for VM_PROT_COPY */      \

        .__vmkf_unused = 0                                              \
 }
 #endif /* KERNEL_PRIVATE */
        .__vmkf_unused = 0                                              \
 }
 #endif /* KERNEL_PRIVATE */

diff --git a/osfmk/prng/random.c b/osfmk/prng/random.c
index cb3555d39936c73da0b0acfe4c297902654333ae..5dc056f8abb418871bcfd2e49cda6b8b3ec3ab21 100644 (file)
--- a/osfmk/prng/random.c
+++ b/osfmk/prng/random.c
@@ -626,3 +626,79 @@ write_random(void* buffer, u_int numbytes)
        return 0;
 #endif
 }
        return 0;
 #endif
 }

+
+
+/*
+ * Boolean PRNG for generating booleans to randomize order of elements
+ * in certain kernel data structures. The algorithm is a
+ * modified version of the KISS RNG proposed in the paper:
+ * http://stat.fsu.edu/techreports/M802.pdf
+ * The modifications have been documented in the technical paper
+ * paper from UCL:
+ * http://www0.cs.ucl.ac.uk/staff/d.jones/GoodPracticeRNG.pdf
+ */
+
+/* Initialize the PRNG structures. */
+void random_bool_init(struct bool_gen *bg)
+{
+       /* Seed the random boolean generator */
+       for (int i = 0; i < RANDOM_BOOL_GEN_SEED_COUNT; i++) {
+               bg->seed[i] = (unsigned int)early_random();
+       }
+       bg->state = 0;
+       simple_lock_init(&bg->lock, 0);
+}
+
+/* Generate random bits and add them to an entropy pool. */
+void random_bool_gen_entropy(
+               struct bool_gen *bg,
+               unsigned int    *buffer,
+               int             count)
+{
+
+       simple_lock(&bg->lock);
+       int i, t;
+       for (i = 0; i < count; i++) {
+               bg->seed[1] ^= (bg->seed[1] << 5);
+               bg->seed[1] ^= (bg->seed[1] >> 7);
+               bg->seed[1] ^= (bg->seed[1] << 22);
+               t = bg->seed[2] + bg->seed[3] + bg->state;
+               bg->seed[2] = bg->seed[3];
+               bg->state = t < 0;
+               bg->seed[3] = t & 2147483647;
+               bg->seed[0] += 1411392427;
+               buffer[i] = (bg->seed[0] + bg->seed[1] + bg->seed[3]);
+       }
+       simple_unlock(&bg->lock);
+}
+
+/* Get some number of bits from the entropy pool, refilling if necessary. */
+unsigned int random_bool_gen_bits(
+               struct bool_gen *bg,
+               unsigned int    *buffer,
+               unsigned int    count,
+               unsigned int    numbits)
+{
+       unsigned int index = 0;
+       unsigned int rbits = 0;
+       for (unsigned int bitct = 0; bitct < numbits; bitct++) {
+               /*
+                * Find a portion of the buffer that hasn't been emptied.
+                * We might have emptied our last index in the previous iteration.
+                */
+               while (index < count && buffer[index] == 0)
+                       index++;
+
+               /* If we've exhausted the pool, refill it. */
+               if (index == count) {
+                       random_bool_gen_entropy(bg, buffer, count);
+                       index = 0;
+               }
+
+               /* Collect-a-bit */
+               unsigned int bit = buffer[index] & 1;
+               buffer[index] = buffer[index] >> 1;
+               rbits = bit | (rbits << 1);
+       }
+       return rbits;
+}

diff --git a/osfmk/prng/random.h b/osfmk/prng/random.h
index aa269df35082727761dd1232c436479dd711a7ae..7ba5f00e18d68c117d13631c8ba0c13bd3e92f11 100644 (file)
--- a/osfmk/prng/random.h
+++ b/osfmk/prng/random.h
@@ -115,6 +115,28 @@ MACRO_END
 /* /dev/random's PRNG is reseeded after generating this many bytes: */
 #define        RESEED_BYTES (17597)
 
 /* /dev/random's PRNG is reseeded after generating this many bytes: */
 #define        RESEED_BYTES (17597)
 

+#include <kern/simple_lock.h>
+/* Definitions for boolean PRNG */
+#define RANDOM_BOOL_GEN_SEED_COUNT 4
+struct bool_gen {
+       unsigned int seed[RANDOM_BOOL_GEN_SEED_COUNT];
+       unsigned int state;
+       decl_simple_lock_data(, lock)
+};
+
+extern void random_bool_init(struct bool_gen *bg);
+
+extern void random_bool_gen_entropy(
+               struct bool_gen *bg,
+               unsigned int *buffer,
+               int count);
+
+extern unsigned int random_bool_gen_bits(
+               struct bool_gen *bg,
+               unsigned int *buffer,
+               unsigned int count,
+               unsigned int numbits);
+

 __END_DECLS
 
 #endif /* _PRNG_RANDOM_H_ */
 __END_DECLS
 
 #endif /* _PRNG_RANDOM_H_ */

diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h
index 332c6e9093a0055cda336ce660c44488b670a477..4916313ce81a1fc168278e660e8273016c7e46c8 100644 (file)
--- a/osfmk/vm/pmap.h
+++ b/osfmk/vm/pmap.h
@@ -660,6 +660,9 @@ extern pmap_t       kernel_pmap;                    /* The kernel's map */
 #define VM_MEM_STACK           0x200
 
 #if __x86_64__
 #define VM_MEM_STACK           0x200
 
 #if __x86_64__

+/* N.B. These use the same numerical space as the PMAP_EXPAND_OPTIONS
+ * definitions in i386/pmap_internal.h
+ */

 #define PMAP_CREATE_64BIT      0x1
 #define PMAP_CREATE_EPT                0x2
 #define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_EPT)
 #define PMAP_CREATE_64BIT      0x1
 #define PMAP_CREATE_EPT                0x2
 #define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_EPT)


@@ -710,9 +713,6 @@ extern void         pmap_remove_options(    /* Remove mappings. */
 
 extern void            fillPage(ppnum_t pa, unsigned int fill);
 
 
 extern void            fillPage(ppnum_t pa, unsigned int fill);
 

-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
-

 #if defined(__LP64__)
 void pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr);
 #endif
 #if defined(__LP64__)
 void pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr);
 #endif

diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c
index a0301085edc7436768386e2dab763c6173bdbcf4..a14a10db9a0145440bf2887056a0892f55de4c4c 100644 (file)
--- a/osfmk/vm/vm_kern.c
+++ b/osfmk/vm/vm_kern.c
@@ -905,7 +905,11 @@ kmem_realloc(
        oldmapsize = oldmapmax - oldmapmin;
        newmapsize = vm_map_round_page(newsize,
                                       VM_MAP_PAGE_MASK(map));
        oldmapsize = oldmapmax - oldmapmin;
        newmapsize = vm_map_round_page(newsize,
                                       VM_MAP_PAGE_MASK(map));

-
+       if (newmapsize < newsize) {
+               /* overflow */
+               *newaddrp = 0;
+               return KERN_INVALID_ARGUMENT;
+       }

 
        /*
         *      Find the VM object backing the old region.
 
        /*
         *      Find the VM object backing the old region.


@@ -1075,6 +1079,11 @@ kmem_alloc_pageable(
 #endif
        map_size = vm_map_round_page(size,
                                     VM_MAP_PAGE_MASK(map));
 #endif
        map_size = vm_map_round_page(size,
                                     VM_MAP_PAGE_MASK(map));

+       if (map_size < size) {
+               /* overflow */
+               *addrp = 0;
+               return KERN_INVALID_ARGUMENT;
+       }

 
        kr = vm_map_enter(map, &map_addr, map_size,
                          (vm_map_offset_t) 0, 
 
        kr = vm_map_enter(map, &map_addr, map_size,
                          (vm_map_offset_t) 0, 


@@ -1200,6 +1209,11 @@ kmem_suballoc(
 
        map_size = vm_map_round_page(size,
                                     VM_MAP_PAGE_MASK(parent));
 
        map_size = vm_map_round_page(size,
                                     VM_MAP_PAGE_MASK(parent));

+       if (map_size < size) {
+               /* overflow */
+               *addr = 0;
+               return KERN_INVALID_ARGUMENT;
+       }

 
        /*
         *      Need reference on submap object because it is internal
 
        /*
         *      Need reference on submap object because it is internal

diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c
index dbdf22e41ad0b995026743120e3eeb3a835279fc..28aa9421869da2e4d9e55480d79e5360ae6f4f75 100644 (file)
--- a/osfmk/vm/vm_map.c
+++ b/osfmk/vm/vm_map.c
@@ -276,7 +276,8 @@ static kern_return_t        vm_map_remap_extract(
        vm_prot_t               *max_protection,
        vm_inherit_t            inheritance,
        boolean_t               pageable,
        vm_prot_t               *max_protection,
        vm_inherit_t            inheritance,
        boolean_t               pageable,

-       boolean_t               same_map);
+       boolean_t               same_map,
+       vm_map_kernel_flags_t   vmk_flags);

 
 static kern_return_t   vm_map_remap_range_allocate(
        vm_map_t                map,
 
 static kern_return_t   vm_map_remap_range_allocate(
        vm_map_t                map,


@@ -354,7 +355,11 @@ boolean_t _vmec_reserved = (NEW)->from_reserved_zone;      \
        (NEW)->permanent = FALSE;       \
        (NEW)->used_for_jit = FALSE;    \
        (NEW)->from_reserved_zone = _vmec_reserved;     \
        (NEW)->permanent = FALSE;       \
        (NEW)->used_for_jit = FALSE;    \
        (NEW)->from_reserved_zone = _vmec_reserved;     \

-       (NEW)->iokit_acct = FALSE;      \
+       if ((NEW)->iokit_acct) {                        \
+            assertf(!(NEW)->use_pmap, "old %p new %p\n", (OLD), (NEW)); \
+            (NEW)->iokit_acct = FALSE;                 \
+            (NEW)->use_pmap = TRUE;                    \
+       }                                               \

        (NEW)->vme_resilient_codesign = FALSE; \
        (NEW)->vme_resilient_media = FALSE;     \
        (NEW)->vme_atomic = FALSE;      \
        (NEW)->vme_resilient_codesign = FALSE; \
        (NEW)->vme_resilient_media = FALSE;     \
        (NEW)->vme_atomic = FALSE;      \


@@ -5388,11 +5393,44 @@ vm_map_protect(
        vm_map_entry_t                  entry;
        vm_prot_t                       new_max;
        int                             pmap_options = 0;
        vm_map_entry_t                  entry;
        vm_prot_t                       new_max;
        int                             pmap_options = 0;

+       kern_return_t                   kr;

 
        XPR(XPR_VM_MAP,
            "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
            map, start, end, new_prot, set_max);
 
 
        XPR(XPR_VM_MAP,
            "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
            map, start, end, new_prot, set_max);
 

+       if (new_prot & VM_PROT_COPY) {
+               vm_map_offset_t         new_start;
+               vm_prot_t               cur_prot, max_prot;
+               vm_map_kernel_flags_t   kflags;
+
+               /* LP64todo - see below */
+               if (start >= map->max_offset) {
+                       return KERN_INVALID_ADDRESS;
+               }
+
+               kflags = VM_MAP_KERNEL_FLAGS_NONE;
+               kflags.vmkf_remap_prot_copy = TRUE;
+               new_start = start;
+               kr = vm_map_remap(map,
+                                 &new_start,
+                                 end - start,
+                                 0, /* mask */
+                                 VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
+                                 kflags,
+                                 0,
+                                 map,
+                                 start,
+                                 TRUE, /* copy-on-write remapping! */
+                                 &cur_prot,
+                                 &max_prot,
+                                 VM_INHERIT_DEFAULT);
+               if (kr != KERN_SUCCESS) {
+                       return kr;
+               }
+               new_prot &= ~VM_PROT_COPY;
+       }
+

        vm_map_lock(map);
 
        /* LP64todo - remove this check when vm_map_commpage64()
        vm_map_lock(map);
 
        /* LP64todo - remove this check when vm_map_commpage64()


@@ -5442,17 +5480,9 @@ vm_map_protect(
                }
 
                new_max = current->max_protection;
                }
 
                new_max = current->max_protection;

-               if(new_prot & VM_PROT_COPY) {
-                       new_max |= VM_PROT_WRITE;
-                       if ((new_prot & (new_max | VM_PROT_COPY)) != new_prot) {
-                               vm_map_unlock(map);
-                               return(KERN_PROTECTION_FAILURE);
-                       }
-               } else {
-                       if ((new_prot & new_max) != new_prot) {
-                               vm_map_unlock(map);
-                               return(KERN_PROTECTION_FAILURE);
-                       }
+               if ((new_prot & new_max) != new_prot) {
+                       vm_map_unlock(map);
+                       return(KERN_PROTECTION_FAILURE);

                }
 
 #if CONFIG_EMBEDDED
                }
 
 #if CONFIG_EMBEDDED


@@ -5521,36 +5551,13 @@ vm_map_protect(
 
                old_prot = current->protection;
 
 
                old_prot = current->protection;
 

-               if(new_prot & VM_PROT_COPY) {
-                       /* caller is asking specifically to copy the      */
-                       /* mapped data, this implies that max protection  */
-                       /* will include write.  Caller must be prepared   */
-                       /* for loss of shared memory communication in the */
-                       /* target area after taking this step */
-
-                       if (current->is_sub_map == FALSE &&
-                           VME_OBJECT(current) == VM_OBJECT_NULL) {
-                               VME_OBJECT_SET(current,
-                                              vm_object_allocate(
-                                                      (vm_map_size_t)
-                                                      (current->vme_end -
-                                                       current->vme_start)));
-                               VME_OFFSET_SET(current, 0);
-                               assert(current->use_pmap);
-                       }
-                       assert(current->wired_count == 0);
-                       current->needs_copy = TRUE;
-                       current->max_protection |= VM_PROT_WRITE;
+               if (set_max) {
+                       current->max_protection = new_prot;
+                       current->protection = new_prot & old_prot;
+               } else {
+                       current->protection = new_prot;

                }
 
                }
 

-               if (set_max)
-                       current->protection =
-                               (current->max_protection =
-                                new_prot & ~VM_PROT_COPY) &
-                               old_prot;
-               else
-                       current->protection = new_prot & ~VM_PROT_COPY;
-

                /*
                 *      Update physical map if necessary.
                 *      If the request is to turn off write protection,
                /*
                 *      Update physical map if necessary.
                 *      If the request is to turn off write protection,


@@ -11354,7 +11361,8 @@ vm_map_copy_extract(
                                  max_prot,
                                  VM_INHERIT_SHARE,
                                  TRUE, /* pageable */
                                  max_prot,
                                  VM_INHERIT_SHARE,
                                  TRUE, /* pageable */

-                                 FALSE); /* same_map */
+                                 FALSE, /* same_map */
+                                 VM_MAP_KERNEL_FLAGS_NONE);

        if (kr != KERN_SUCCESS) {
                vm_map_copy_discard(copy);
                return kr;
        if (kr != KERN_SUCCESS) {
                vm_map_copy_discard(copy);
                return kr;


@@ -14780,7 +14788,8 @@ vm_map_remap_extract(
        /* What, no behavior? */
        vm_inherit_t            inheritance,
        boolean_t               pageable,
        /* What, no behavior? */
        vm_inherit_t            inheritance,
        boolean_t               pageable,

-       boolean_t               same_map)
+       boolean_t               same_map,
+       vm_map_kernel_flags_t   vmk_flags)

 {
        kern_return_t           result;
        vm_map_size_t           mapped_size;
 {
        kern_return_t           result;
        vm_map_size_t           mapped_size;


@@ -14961,7 +14970,20 @@ vm_map_remap_extract(
                new_entry->vme_start = map_address;
                new_entry->vme_end = map_address + tmp_size;
                assert(new_entry->vme_start < new_entry->vme_end);
                new_entry->vme_start = map_address;
                new_entry->vme_end = map_address + tmp_size;
                assert(new_entry->vme_start < new_entry->vme_end);

-               new_entry->inheritance = inheritance;
+               if (copy && vmk_flags.vmkf_remap_prot_copy) {
+                       /*
+                        * Remapping for vm_map_protect(VM_PROT_COPY)
+                        * to convert a read-only mapping into a
+                        * copy-on-write version of itself but
+                        * with write access:
+                        * keep the original inheritance and add 
+                        * VM_PROT_WRITE to the max protection.
+                        */
+                       new_entry->inheritance = src_entry->inheritance;
+                       new_entry->max_protection |= VM_PROT_WRITE;
+               } else {
+                       new_entry->inheritance = inheritance;
+               }

                VME_OFFSET_SET(new_entry, offset);
                
                /*
                VME_OFFSET_SET(new_entry, offset);
                
                /*


@@ -15236,7 +15258,8 @@ vm_map_remap(
                                      max_protection,
                                      inheritance,
                                      target_map->hdr.entries_pageable,
                                      max_protection,
                                      inheritance,
                                      target_map->hdr.entries_pageable,

-                                     src_map == target_map);
+                                     src_map == target_map,
+                                     vmk_flags);

 
        if (result != KERN_SUCCESS) {
                return result;
 
        if (result != KERN_SUCCESS) {
                return result;

diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c
index b24e87a9afeccd01be62850a6039cfc2f5f653a2..ada634f90184ce8c39a7c073eb578ffcfb778bb6 100644 (file)
--- a/osfmk/vm/vm_pageout.c
+++ b/osfmk/vm/vm_pageout.c
@@ -420,6 +420,7 @@ unsigned int vm_stat_discard_failure = 0;   /* debugging */
 unsigned int vm_stat_discard_throttle = 0;     /* debugging */
 unsigned int vm_pageout_reactivation_limit_exceeded = 0;       /* debugging */
 unsigned int vm_pageout_inactive_force_reclaim = 0;    /* debugging */
 unsigned int vm_stat_discard_throttle = 0;     /* debugging */
 unsigned int vm_pageout_reactivation_limit_exceeded = 0;       /* debugging */
 unsigned int vm_pageout_inactive_force_reclaim = 0;    /* debugging */

+unsigned int vm_pageout_skipped_external = 0;   /* debugging */

 
 unsigned int vm_pageout_scan_reclaimed_throttled = 0;
 unsigned int vm_pageout_scan_active_throttled = 0;
 
 unsigned int vm_pageout_scan_reclaimed_throttled = 0;
 unsigned int vm_pageout_scan_active_throttled = 0;


@@ -1702,10 +1703,12 @@ int     last_vm_pageout_reactivation_limit_exceeded = 0;
 int    last_vm_pageout_considered_page = 0;
 int    last_vm_compressor_pages_grabbed = 0;
 int    last_vm_compressor_failed = 0;
 int    last_vm_pageout_considered_page = 0;
 int    last_vm_compressor_pages_grabbed = 0;
 int    last_vm_compressor_failed = 0;

+int     last_vm_pageout_skipped_external = 0;
+

 
 void update_vm_info(void)
 {
 
 void update_vm_info(void)
 {

-       int     tmp1, tmp2, tmp3;
+        int    tmp1, tmp2, tmp3, tmp4;

 
        if (!kdebug_enable)
                return;
 
        if (!kdebug_enable)
                return;


@@ -1773,12 +1776,14 @@ void update_vm_info(void)
        tmp1 = vm_pageout_scan_inactive_throttled_internal;
        tmp2 = vm_pageout_freed_after_compression;
        tmp3 = vm_compressor_pages_grabbed;
        tmp1 = vm_pageout_scan_inactive_throttled_internal;
        tmp2 = vm_pageout_freed_after_compression;
        tmp3 = vm_compressor_pages_grabbed;

+       tmp4 = vm_pageout_skipped_external;

 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
                              tmp1 - last_vm_pageout_scan_inactive_throttled_internal,
                              tmp2 - last_vm_pageout_freed_after_compression,
                              tmp3 - last_vm_compressor_pages_grabbed,
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
                              tmp1 - last_vm_pageout_scan_inactive_throttled_internal,
                              tmp2 - last_vm_pageout_freed_after_compression,
                              tmp3 - last_vm_compressor_pages_grabbed,

-                             0, 0);
+                             tmp4 - last_vm_pageout_skipped_external,
+                             0);

                              
        vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal);
        vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression);
                              
        vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal);
        vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression);


@@ -1787,6 +1792,7 @@ void update_vm_info(void)
        last_vm_pageout_scan_inactive_throttled_internal = tmp1;
        last_vm_pageout_freed_after_compression = tmp2;
        last_vm_compressor_pages_grabbed = tmp3;
        last_vm_pageout_scan_inactive_throttled_internal = tmp1;
        last_vm_pageout_freed_after_compression = tmp2;
        last_vm_compressor_pages_grabbed = tmp3;

+       last_vm_pageout_skipped_external = tmp4;

 
 
        if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 &&
 
 
        if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 &&


@@ -2655,6 +2661,9 @@ consider_inactive:
                            ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) {
                                grab_anonymous = TRUE;
                                anons_grabbed = 0;
                            ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) {
                                grab_anonymous = TRUE;
                                anons_grabbed = 0;

+
+                               vm_pageout_skipped_external++;
+                               goto want_anonymous;

                        }
 #if CONFIG_JETSAM
                        /* If the file-backed pool has accumulated
                        }
 #if CONFIG_JETSAM
                        /* If the file-backed pool has accumulated


@@ -2687,6 +2696,7 @@ consider_inactive:
                        }
 #endif /* CONFIG_JETSAM */
 
                        }
 #endif /* CONFIG_JETSAM */
 

+want_anonymous:

                        if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
 
                                if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) {
                        if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
 
                                if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) {


@@ -2801,6 +2811,7 @@ consider_inactive:
                                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
                                m->reference = FALSE;
 
                                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
                                m->reference = FALSE;
 

+#if !CONFIG_EMBEDDED

                                /*
                                 * m->object must be stable since we hold the page queues lock...
                                 * we can update the scan_collisions field sans the object lock
                                /*
                                 * m->object must be stable since we hold the page queues lock...
                                 * we can update the scan_collisions field sans the object lock


@@ -2810,7 +2821,17 @@ consider_inactive:
                                 * is possible for the value to be a bit non-determistic, but that's ok
                                 * since it's only used as a hint
                                 */
                                 * is possible for the value to be a bit non-determistic, but that's ok
                                 * since it's only used as a hint
                                 */

+
+                               /*
+                                * This is not used on EMBEDDED because having this variable set *could* lead
+                                * us to self-cannibalize pages from m_object to fill a UPL for a pagein.
+                                * And, there's a high probability that the object that vm_pageout_scan
+                                * wants and collides on is a very popular object e.g. the shared cache on EMBEDDED.
+                                * The older pages that we cannibalize from the shared cache could be really
+                                * important text pages e.g. the system call stubs.
+                                */

                                m_object->scan_collisions = 1;
                                m_object->scan_collisions = 1;

+#endif /* !CONFIG_EMBEDDED */

 
                                if ( !vm_page_queue_empty(&sq->age_q) )
                                        m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
 
                                if ( !vm_page_queue_empty(&sq->age_q) )
                                        m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);

diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c
index feeaddd1e080901f9adcc99b7f50dfdf90fdadc8..b34f1b2d505715f45977528c534b303631cb69b1 100644 (file)
--- a/osfmk/vm/vm_resident.c
+++ b/osfmk/vm/vm_resident.c
@@ -604,6 +604,8 @@ vm_page_init_lck_grp(void)
        vm_compressor_init_locks();
 }
 
        vm_compressor_init_locks();
 }
 

+#define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1)))
+

 void
 vm_page_init_local_q()
 {
 void
 vm_page_init_local_q()
 {


@@ -619,11 +621,12 @@ vm_page_init_local_q()
        if (num_cpus >= 2) {
 #if KASAN
                /* KASAN breaks the expectation of a size-aligned object by adding a
        if (num_cpus >= 2) {
 #if KASAN
                /* KASAN breaks the expectation of a size-aligned object by adding a

-                * rezone, so explicitly align. */
+                * redzone, so explicitly align. */

                t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq) + VM_PACKED_POINTER_ALIGNMENT);
                t_local_q = (void *)(((uintptr_t)t_local_q + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT-1));
 #else
                t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq) + VM_PACKED_POINTER_ALIGNMENT);
                t_local_q = (void *)(((uintptr_t)t_local_q + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT-1));
 #else

-               t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq));
+               /* round the size up to the nearest power of two */
+               t_local_q = (struct vplq *)kalloc(ROUNDUP_NEXTP2(num_cpus * sizeof(struct vplq)));

 #endif
 
                for (i = 0; i < num_cpus; i++) {
 #endif
 
                for (i = 0; i < num_cpus; i++) {

diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c
index 491d27fe94aec67e03f5997a2aa46ef6f1e865f3..2f9bdc101c6699fb9a829fefc355f6cfbe93efc7 100644 (file)
--- a/osfmk/vm/vm_user.c
+++ b/osfmk/vm/vm_user.c
@@ -3330,6 +3330,9 @@ mach_memory_object_memory_entry_64(
 
        if (pager == MEMORY_OBJECT_NULL && internal) {
                object = vm_object_allocate(size);
 
        if (pager == MEMORY_OBJECT_NULL && internal) {
                object = vm_object_allocate(size);

+               if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
+                       object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+               }

        } else {
                object = memory_object_to_vm_object(pager);
                if (object != VM_OBJECT_NULL) {
        } else {
                object = memory_object_to_vm_object(pager);
                if (object != VM_OBJECT_NULL) {

diff --git a/osfmk/x86_64/bcopy.s b/osfmk/x86_64/bcopy.s
index fc0af58d6cd0296084beddc5f4c3abc7b7d4dc20..8cd909bbb87c0daee3ae7bb01459a7fec5ceeea0 100644 (file)
--- a/osfmk/x86_64/bcopy.s
+++ b/osfmk/x86_64/bcopy.s
@@ -62,7 +62,8 @@
 /*                     rdi,                 rsi,             rdx   */
 /*
  * Note: memcpy does not support overlapping copies
 /*                     rdi,                 rsi,             rdx   */
 /*
  * Note: memcpy does not support overlapping copies

- */
+       */
+       /* TODO: movsb */

 ENTRY(memcpy)
        movq    %rdi, %rax                      /* return destination */
        movq    %rdx,%rcx
 ENTRY(memcpy)
        movq    %rdi, %rax                      /* return destination */
        movq    %rdx,%rcx

diff --git a/osfmk/x86_64/copyio.c b/osfmk/x86_64/copyio.c
index c86d1136cdac40a945c20b9b7c00e175149d1221..dd4b936702bb1a1c6a75482b07f1d349a2e1ac86 100644 (file)
--- a/osfmk/x86_64/copyio.c
+++ b/osfmk/x86_64/copyio.c
@@ -166,7 +166,6 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
        int             debug_type = 0xeff70010;
        debug_type += (copy_type << 2);
 #endif
        int             debug_type = 0xeff70010;
        debug_type += (copy_type << 2);
 #endif

-       boolean_t nopagezero = thread->map->pmap->pagezero_accessible;

 
        if (__improbable(nbytes > copysize_limit_panic))
                panic("%s(%p, %p, %lu) - transfer too large", __func__,
 
        if (__improbable(nbytes > copysize_limit_panic))
                panic("%s(%p, %p, %lu) - transfer too large", __func__,


@@ -179,6 +178,7 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
                goto out;
 
         pmap = thread->map->pmap;
                goto out;
 
         pmap = thread->map->pmap;

+       boolean_t nopagezero = pmap->pagezero_accessible;

 
        if (__improbable((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS))) {
                panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
 
        if (__improbable((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS))) {
                panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);

diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s
index 7c42b1dee6bdf9ef42904ef2277bf8c866f30afd..13f3f021802880fe6a8778ec44f08c61a1a3af2c 100644 (file)
--- a/osfmk/x86_64/idt64.s
+++ b/osfmk/x86_64/idt64.s
@@ -66,31 +66,41 @@
  *   syscall   - synchronous system call request
  *   fatal     - fatal traps
  */
  *   syscall   - synchronous system call request
  *   fatal     - fatal traps
  */

-

 /*
 /*

- * Handlers:
+ * Indices of handlers for each exception type.

  */
  */

-#define        HNDL_ALLINTRS           EXT(hndl_allintrs)
-#define        HNDL_ALLTRAPS           EXT(hndl_alltraps)
-#define        HNDL_SYSENTER           EXT(hndl_sysenter)
-#define        HNDL_SYSCALL            EXT(hndl_syscall)
-#define        HNDL_UNIX_SCALL         EXT(hndl_unix_scall)
-#define        HNDL_MACH_SCALL         EXT(hndl_mach_scall)
-#define        HNDL_MDEP_SCALL         EXT(hndl_mdep_scall)
-#define        HNDL_DOUBLE_FAULT       EXT(hndl_double_fault)
-#define        HNDL_MACHINE_CHECK      EXT(hndl_machine_check)
-
-
-#if 1
-#define PUSH_FUNCTION(func)                     \
-       sub     $8, %rsp                        ;\
-       push    %rax                            ;\
-       leaq    func(%rip), %rax                ;\
-       movq    %rax, 8(%rsp)                   ;\
-       pop     %rax
-#else
-#define PUSH_FUNCTION(func) pushq func
-#endif
+#define        HNDL_ALLINTRS           0
+#define        HNDL_ALLTRAPS           1
+#define        HNDL_SYSENTER           2
+#define        HNDL_SYSCALL            3
+#define        HNDL_UNIX_SCALL         4
+#define        HNDL_MACH_SCALL         5
+#define        HNDL_MDEP_SCALL         6
+#define        HNDL_DOUBLE_FAULT       7
+#define        HNDL_MACHINE_CHECK      8
+
+/* Begin double-mapped descriptor section */
+       
+.section       __HIB, __desc
+.globl EXT(idt64_hndl_table0)
+EXT(idt64_hndl_table0):
+       .quad EXT(ks_dispatch)
+       .quad EXT(ks_64bit_return)
+       .quad 0 /* Populated with CPU shadow displacement*/
+       .quad EXT(ks_return)
+
+EXT(idt64_hndl_table1):
+       .quad   EXT(hndl_allintrs)
+       .quad   EXT(hndl_alltraps)
+       .quad   EXT(hndl_sysenter)
+       .quad   EXT(hndl_syscall)
+       .quad   EXT(hndl_unix_scall)
+       .quad   EXT(hndl_mach_scall)
+       .quad   EXT(hndl_mdep_scall)
+       .quad   EXT(hndl_double_fault)
+       .quad   EXT(hndl_machine_check)
+.text
+

 
 /* The wrapper for all non-special traps/interrupts */
 /* Everything up to PUSH_FUNCTION is just to output 
 
 /* The wrapper for all non-special traps/interrupts */
 /* Everything up to PUSH_FUNCTION is just to output 


@@ -101,12 +111,12 @@
        push    %rax                            ;\
        POSTCODE2(0x6400+n)                     ;\
        pop     %rax                            ;\
        push    %rax                            ;\
        POSTCODE2(0x6400+n)                     ;\
        pop     %rax                            ;\

-       PUSH_FUNCTION(f)                        ;\
+       pushq   $(f)                            ;\

        pushq   $(n)                            ;\
        jmp L_dispatch
 #else
 #define IDT_ENTRY_WRAPPER(n, f)                         \
        pushq   $(n)                            ;\
        jmp L_dispatch
 #else
 #define IDT_ENTRY_WRAPPER(n, f)                         \

-       PUSH_FUNCTION(f)                        ;\
+       pushq   $(f)                            ;\

        pushq   $(n)                            ;\
        jmp L_dispatch
 #endif
        pushq   $(n)                            ;\
        jmp L_dispatch
 #endif


@@ -136,9 +146,175 @@
 #define TRAP_IST2(n, f)
 #define USER_TRAP_SPC(n, f)
 
 #define TRAP_IST2(n, f)
 #define USER_TRAP_SPC(n, f)
 

+/* Begin double-mapped text section */
+.section __HIB, __text

 /* Generate all the stubs */
 #include "idt_table.h"
 
 /* Generate all the stubs */
 #include "idt_table.h"
 

+Entry(idt64_page_fault)
+       pushq   $(HNDL_ALLTRAPS)
+       push    $(T_PAGE_FAULT)
+       jmp     L_dispatch
+
+Entry(idt64_debug)
+       push    $0                      /* error code */
+       pushq   $(HNDL_ALLTRAPS)
+       pushq   $(T_DEBUG)
+       jmp     L_dispatch
+/*
+ * Legacy interrupt gate System call handlers.
+ * These are entered via a syscall interrupt. The system call number in %rax
+ * is saved to the error code slot in the stack frame. We then branch to the
+ * common state saving code.
+ */
+
+#ifndef UNIX_INT
+#error NO UNIX INT!!!
+#endif
+Entry(idt64_unix_scall)
+       pushq   %rax                    /* save system call number */
+       pushq   $(HNDL_UNIX_SCALL)
+       pushq   $(UNIX_INT)
+       jmp     L_dispatch
+       
+Entry(idt64_mach_scall)
+       pushq   %rax                    /* save system call number */
+       pushq   $(HNDL_MACH_SCALL)
+       pushq   $(MACH_INT)
+       jmp     L_dispatch
+       
+Entry(idt64_mdep_scall)
+       pushq   %rax                    /* save system call number */
+       pushq   $(HNDL_MDEP_SCALL)
+       pushq   $(MACHDEP_INT)
+       jmp     L_dispatch
+
+/*
+ * For GP/NP/SS faults, we use the IST1 stack.
+ * For faults from user-space, we have to copy the machine state to the
+ * PCB stack and then dispatch as normal.
+ * For faults in kernel-space, we need to scrub for kernel exit faults and
+ * treat these as user-space faults. But for all other kernel-space faults
+ * we continue to run on the IST1 stack and we dispatch to handle the fault
+ * as fatal.
+ */
+Entry(idt64_gen_prot)
+       pushq   $(HNDL_ALLTRAPS)
+       pushq   $(T_GENERAL_PROTECTION)
+       jmp     L_dispatch
+
+Entry(idt64_stack_fault)
+       pushq   $(HNDL_ALLTRAPS)
+       pushq   $(T_STACK_FAULT)
+       jmp     L_dispatch
+
+Entry(idt64_segnp)
+       pushq   $(HNDL_ALLTRAPS)
+       pushq   $(T_SEGMENT_NOT_PRESENT)
+       jmp     L_dispatch
+
+/*
+ * Fatal exception handlers:
+ */
+Entry(idt64_db_task_dbl_fault)
+       pushq   $(HNDL_DOUBLE_FAULT)
+       pushq   $(T_DOUBLE_FAULT)
+       jmp     L_dispatch
+
+Entry(idt64_db_task_stk_fault)
+       pushq   $(HNDL_DOUBLE_FAULT)
+       pushq   $(T_STACK_FAULT)
+       jmp     L_dispatch
+
+Entry(idt64_mc)
+       push    $(0)                    /* Error */
+       pushq   $(HNDL_MACHINE_CHECK)
+       pushq   $(T_MACHINE_CHECK)
+       jmp     L_dispatch
+
+/*
+ * NMI
+ * This may or may not be fatal but extreme care is required
+ * because it may fall when control was already in another trampoline.
+ *
+ * We get here on IST2 stack which is used for NMIs only.
+ */
+Entry(idt64_nmi)
+       push    %rax                            /* save RAX to ISF64_ERR */
+       push    %rcx                            /* save RCX to ISF64_TRAPFN */
+       push    %rdx                            /* save RDX to ISF64_TRAPNO */
+       jmp     L_dispatch
+
+Entry(idt64_double_fault)
+       pushq   $(HNDL_DOUBLE_FAULT)
+       pushq   $(T_DOUBLE_FAULT)
+       jmp     L_dispatch
+
+Entry(hi64_syscall)
+Entry(idt64_syscall)
+       swapgs
+     /* Use RAX as a temporary by shifting its contents into R11[32:63]
+      * The systemcall number is defined to be a 32-bit quantity, as is
+      * RFLAGS.
+      */
+       shlq    $32, %rax
+       or      %rax, %r11
+.globl EXT(dblsyscall_patch_point)
+EXT(dblsyscall_patch_point):
+//     movabsq $0x12345678ABCDEFFFULL, %rax
+     /* Generate offset to the double-mapped per-CPU data shadow
+      * into RAX
+      */
+       leaq    EXT(idt64_hndl_table0)(%rip), %rax
+       mov     16(%rax), %rax
+       mov     %rsp, %gs:CPU_UBER_TMP(%rax)  /* save user stack */
+       mov     %gs:CPU_ESTACK(%rax), %rsp  /* switch stack to per-cpu estack */
+       sub     $(ISF64_SIZE), %rsp
+
+       /*
+        * Synthesize an ISF frame on the exception stack
+        */
+       movl    $(USER_DS), ISF64_SS(%rsp)
+       mov     %rcx, ISF64_RIP(%rsp)           /* rip */
+
+       mov     %gs:CPU_UBER_TMP(%rax), %rcx
+       mov     %rcx, ISF64_RSP(%rsp)           /* user stack --changed */
+
+       mov     %r11, %rax
+       shrq    $32, %rax               /* Restore RAX */
+       mov     %r11d, %r11d            /* Clear r11[32:63] */
+
+       mov     %r11, ISF64_RFLAGS(%rsp)        /* rflags */
+       movl    $(SYSCALL_CS), ISF64_CS(%rsp)   /* cs - a pseudo-segment */
+       mov     %rax, ISF64_ERR(%rsp)           /* err/rax - syscall code */
+       movq    $(HNDL_SYSCALL), ISF64_TRAPFN(%rsp)
+       movq    $(T_SYSCALL), ISF64_TRAPNO(%rsp)        /* trapno */
+       swapgs
+       jmp     L_dispatch                      /* this can only be 64-bit */
+
+Entry(hi64_sysenter)
+Entry(idt64_sysenter)
+       /* Synthesize an interrupt stack frame onto the
+        * exception stack.
+        */
+       push    $(USER_DS)              /* ss */
+       push    %rcx                    /* uesp */
+       pushf                           /* flags */
+       /*
+        * Clear, among others, the Nested Task (NT) flags bit;
+        * this is zeroed by INT, but not by SYSENTER.
+        */
+       push    $0
+       popf
+       push    $(SYSENTER_CS)          /* cs */ 
+L_sysenter_continue:
+       push    %rdx                    /* eip */
+       push    %rax                    /* err/eax - syscall code */
+       pushq   $(HNDL_SYSENTER)
+       pushq   $(T_SYSENTER)
+       orl     $(EFL_IF), ISF64_RFLAGS(%rsp)
+       jmp L_dispatch
+

 /*
  * Common dispatch point.
  * Determine what mode has been interrupted and save state accordingly.
 /*
  * Common dispatch point.
  * Determine what mode has been interrupted and save state accordingly.


@@ -148,13 +324,113 @@
  *     GSBASE  from user-space:   pthread area, or
  *             from kernel-space: cpu_data
  */
  *     GSBASE  from user-space:   pthread area, or
  *             from kernel-space: cpu_data
  */

+

 L_dispatch:
 L_dispatch:

+       pushq   %rax
+       testb   $3, 8+ISF64_CS(%rsp)
+       jz      1f
+       swapgs
+       leaq    EXT(idt64_hndl_table0)(%rip), %rax
+       mov     16(%rax), %rax
+
+       mov     %gs:CPU_TASK_CR3(%rax), %rax 
+       mov     %rax, %cr3
+#if    DEBUG
+       mov     %rax, %gs:CPU_ENTRY_CR3
+#endif
+1:
+       /* The text/data relationship here must be preserved in the doublemap, and the contents must be remapped */
+       leaq    EXT(idt64_hndl_table0)(%rip), %rax
+       /* Indirect branch to non-doublemapped trampolines */
+       jmp *(%rax)
+/* User return: register restoration and address space switch sequence */
+Entry(ks_64bit_return)
+       mov     R64_R14(%r15), %r14
+       mov     R64_R13(%r15), %r13
+       mov     R64_R12(%r15), %r12
+       mov     R64_R11(%r15), %r11
+       mov     R64_R10(%r15), %r10
+       mov     R64_R9(%r15),  %r9
+       mov     R64_R8(%r15),  %r8
+       mov     R64_RSI(%r15), %rsi
+       mov     R64_RDI(%r15), %rdi
+       mov     R64_RBP(%r15), %rbp
+       mov     R64_RDX(%r15), %rdx
+       mov     R64_RCX(%r15), %rcx
+       mov     R64_RBX(%r15), %rbx
+       mov     R64_RAX(%r15), %rax
+       /* Switch to per-CPU exception stack */
+       mov     %gs:CPU_ESTACK, %rsp
+
+       /* Synthesize interrupt stack frame from PCB savearea to exception stack */
+       push    R64_SS(%r15)
+       push    R64_RSP(%r15)
+       push    R64_RFLAGS(%r15)
+       push    R64_CS(%r15)
+       push    R64_RIP(%r15)
+
+       mov     R64_R15(%r15), %r15
+       cmpq    $(KERNEL64_CS), 8(%rsp)
+       jz      1f
+       /* Discover user cr3/ASID */
+       push    %rax
+       mov     %gs:CPU_UCR3, %rax
+#if    DEBUG
+       mov     %rax, %gs:CPU_EXIT_CR3
+#endif
+       mov     %rax, %cr3
+       /* Continue execution on the shared/doublemapped trampoline */
+       pop     %rax
+       swapgs
+1:
+       cmpl    $(SYSCALL_CS), 8(%rsp) /* test for exit via SYSRET */
+       je      L_sysret
+EXT(ret64_iret):
+        iretq                          /* return from interrupt */
+L_sysret:
+       /*
+        * Here to restore rcx/r11/rsp and perform the sysret back to user-space.
+        *      rcx     user rip
+        *      r11     user rflags
+        *      rsp     user stack pointer
+        */
+       pop     %rcx
+       add     $8, %rsp
+       pop     %r11
+       pop     %rsp
+       sysretq                         /* return from system call */
+/* End of double-mapped TEXT */
+.text
+
+Entry(ks_dispatch)
+       popq    %rax

        cmpl    $(KERNEL64_CS), ISF64_CS(%rsp)
        cmpl    $(KERNEL64_CS), ISF64_CS(%rsp)

-       je      L_dispatch_kernel
+       je      EXT(ks_dispatch_kernel)

 
 

-       swapgs
+       mov     %rax, %gs:CPU_UBER_TMP
+       mov     %gs:CPU_UBER_ISF, %rax
+       add     $(ISF64_SIZE), %rax
+
+       xchg    %rsp, %rax
+/* Memory to memory moves (aint x86 wonderful):
+ * Transfer the exception frame from the per-CPU exception stack to the
+ * 'PCB' stack programmed at cswitch.
+ */
+       push    ISF64_SS(%rax)
+       push    ISF64_RSP(%rax)
+       push    ISF64_RFLAGS(%rax)
+       push    ISF64_CS(%rax)
+       push    ISF64_RIP(%rax)
+       push    ISF64_ERR(%rax)
+       push    ISF64_TRAPFN(%rax)
+       push    ISF64_TRAPNO(%rax)
+       mov     %gs:CPU_UBER_TMP, %rax
+       jmp     EXT(ks_dispatch_user)

 
 

-L_dispatch_user:
+Entry (ks_return)
+     jmp .
+
+Entry(ks_dispatch_user)

        cmpl    $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
        je      L_dispatch_U32          /* 32-bit user task */
 
        cmpl    $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
        je      L_dispatch_U32          /* 32-bit user task */
 


@@ -165,7 +441,7 @@ L_dispatch_U64:
        mov     %gs:CPU_KERNEL_STACK, %rsp
        jmp     L_dispatch_64bit
 
        mov     %gs:CPU_KERNEL_STACK, %rsp
        jmp     L_dispatch_64bit
 

-L_dispatch_kernel:
+Entry(ks_dispatch_kernel)

        subq    $(ISS64_OFFSET), %rsp
        mov     %r15, R64_R15(%rsp)
        mov     %rsp, %r15
        subq    $(ISS64_OFFSET), %rsp
        mov     %r15, R64_R15(%rsp)
        mov     %rsp, %r15


@@ -212,14 +488,11 @@ L_64bit_entry_reject:
        /*
         * Here for a 64-bit user attempting an invalid kernel entry.
         */
        /*
         * Here for a 64-bit user attempting an invalid kernel entry.
         */

-       pushq   %rax
-       leaq    HNDL_ALLTRAPS(%rip), %rax
-       movq    %rax, ISF64_TRAPFN+8(%rsp)
-       popq    %rax
+       movq    $(HNDL_ALLTRAPS), ISF64_TRAPFN(%rsp)

        movq    $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp)
        jmp     L_dispatch_U64
        
        movq    $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp)
        jmp     L_dispatch_U64
        

-L_32bit_entry_check:
+Entry(ks_32bit_entry_check)

        /*
         * Check we're not a confused 64-bit user.
         */
        /*
         * Check we're not a confused 64-bit user.
         */


@@ -315,19 +588,12 @@ L_common_dispatch:
        shr     $32, %rcx
        testl   %ecx, %ecx
        jz      4f
        shr     $32, %rcx
        testl   %ecx, %ecx
        jz      4f

-       testl   $(1<<16), %ecx                  /* Global? */
-       jz      3f

        movl    $0, %gs:CPU_TLB_INVALID
        mov     %cr4, %rcx      /* RMWW CR4, for lack of an alternative*/
        and     $(~CR4_PGE), %rcx
        mov     %rcx, %cr4
        or      $(CR4_PGE), %rcx
        mov     %rcx, %cr4
        movl    $0, %gs:CPU_TLB_INVALID
        mov     %cr4, %rcx      /* RMWW CR4, for lack of an alternative*/
        and     $(~CR4_PGE), %rcx
        mov     %rcx, %cr4
        or      $(CR4_PGE), %rcx
        mov     %rcx, %cr4

-       jmp     4f
-3:
-       movb    $0, %gs:CPU_TLB_INVALID_LOCAL
-       mov     %cr3, %rcx
-       mov     %rcx, %cr3

 4:
        mov     %gs:CPU_ACTIVE_THREAD, %rcx     /* Get the active thread */
        testq   %rcx, %rcx
 4:
        mov     %gs:CPU_ACTIVE_THREAD, %rcx     /* Get the active thread */
        testq   %rcx, %rcx


@@ -340,7 +606,8 @@ L_common_dispatch:
 5:
        incl    %gs:hwIntCnt(,%ebx,4)           // Bump the trap/intr count
        /* Dispatch the designated handler */
 5:
        incl    %gs:hwIntCnt(,%ebx,4)           // Bump the trap/intr count
        /* Dispatch the designated handler */

-       jmp     *%rdx
+       leaq    EXT(idt64_hndl_table1)(%rip), %rax
+       jmp     *(%rax, %rdx, 8)

 
 /*
  * Control is passed here to return to user.
 
 /*
  * Control is passed here to return to user.


@@ -430,6 +697,40 @@ L_32bit_return:
        movl    R32_SS(%r15), %eax
        movl    %eax, R64_SS(%r15)
 
        movl    R32_SS(%r15), %eax
        movl    %eax, R64_SS(%r15)
 

+       /* Validate DS/ES/FS/GS segment selectors with the Load Access Rights instruction prior to restoration */
+       /* Exempt "known good" statically configured selectors, e.g. USER_DS and 0 */
+       cmpl    $(USER_DS), R32_DS(%r15)
+       jz      22f
+       cmpl    $0, R32_DS(%r15)
+       jz      22f
+       larw    R32_DS(%r15), %ax
+       jz      22f
+       movl    $(USER_DS), R32_DS(%r15)
+22:
+       cmpl    $(USER_DS), R32_ES(%r15)
+       jz      33f
+       cmpl    $0, R32_ES(%r15)
+       jz      33f
+       larw    R32_ES(%r15), %ax
+       jz      33f
+       movl    $(USER_DS), R32_ES(%r15)
+33:
+       cmpl    $(USER_DS), R32_FS(%r15)
+       jz      44f
+       cmpl    $0, R32_FS(%r15)
+       jz      44f
+       larw    R32_FS(%r15), %ax
+       jz      44f
+       movl    $(USER_DS), R32_FS(%r15)
+44:
+       cmpl    $(USER_CTHREAD), R32_GS(%r15)
+       jz      55f
+       cmpl    $0, R32_GS(%r15)
+       jz      55f
+       larw    R32_GS(%r15), %ax
+       jz      55f
+       movl    $(USER_CTHREAD), R32_GS(%r15)
+55:

        /*
         * Restore general 32-bit registers
         */
        /*
         * Restore general 32-bit registers
         */


@@ -458,13 +759,13 @@ L_32bit_return:
        xor     %r15, %r15
 
 EXT(ret32_set_ds):     
        xor     %r15, %r15
 
 EXT(ret32_set_ds):     

-       movl    R32_DS(%rsp), %ds
+       movw    R32_DS(%rsp), %ds

 EXT(ret32_set_es):
 EXT(ret32_set_es):

-       movl    R32_ES(%rsp), %es
+       movw    R32_ES(%rsp), %es

 EXT(ret32_set_fs):
 EXT(ret32_set_fs):

-       movl    R32_FS(%rsp), %fs
+       movw    R32_FS(%rsp), %fs

 EXT(ret32_set_gs):
 EXT(ret32_set_gs):

-       movl    R32_GS(%rsp), %gs
+       movw    R32_GS(%rsp), %gs

 
        /* pop compat frame + trapno, trapfn and error */       
        add     $(ISS64_OFFSET)+8+8+8, %rsp
 
        /* pop compat frame + trapno, trapfn and error */       
        add     $(ISS64_OFFSET)+8+8+8, %rsp


@@ -474,7 +775,6 @@ EXT(ret32_set_gs):
 EXT(ret32_iret):
        iretq                           /* return from interrupt */
 
 EXT(ret32_iret):
        iretq                           /* return from interrupt */
 

-       

 L_fast_exit:
        pop     %rdx                    /* user return eip */
        pop     %rcx                    /* pop and toss cs */
 L_fast_exit:
        pop     %rdx                    /* user return eip */
        pop     %rcx                    /* pop and toss cs */


@@ -512,176 +812,10 @@ L_64bit_return:
         * Restore general 64-bit registers.
         * Here on fault stack and PCB address in R15.
         */
         * Restore general 64-bit registers.
         * Here on fault stack and PCB address in R15.
         */

-       mov     R64_R14(%r15), %r14
-       mov     R64_R13(%r15), %r13
-       mov     R64_R12(%r15), %r12
-       mov     R64_R11(%r15), %r11
-       mov     R64_R10(%r15), %r10
-       mov     R64_R9(%r15),  %r9
-       mov     R64_R8(%r15),  %r8
-       mov     R64_RSI(%r15), %rsi
-       mov     R64_RDI(%r15), %rdi
-       mov     R64_RBP(%r15), %rbp
-       mov     R64_RDX(%r15), %rdx
-       mov     R64_RCX(%r15), %rcx
-       mov     R64_RBX(%r15), %rbx
-       mov     R64_RAX(%r15), %rax
-
-       /*
-        * We must swap GS base if we're returning to user-space,
-        * or we're returning from an NMI that occurred in a trampoline
-        * before the user GS had been swapped. In the latter case, the NMI
-        * handler will have flagged the high-order 32-bits of the CS.
-        */
-       cmpq    $(KERNEL64_CS), R64_CS(%r15)
-       jz      1f
-       swapgs
-1:
-       mov     R64_R15(%r15), %rsp
-       xchg    %r15, %rsp
-       add     $(ISS64_OFFSET)+24, %rsp        /* pop saved state       */
-                                               /* + trapno/trapfn/error */     
-       cmpl    $(SYSCALL_CS),ISF64_CS-24(%rsp)
-                                               /* test for fast entry/exit */
-       je      L_sysret
-.globl _dump_iretq
-EXT(ret64_iret):
-        iretq                          /* return from interrupt */
-
-L_sysret:
-       /*
-        * Here to load rcx/r11/rsp and perform the sysret back to user-space.
-        *      rcx     user rip
-        *      r11     user rflags
-        *      rsp     user stack pointer
-        */
-       mov     ISF64_RIP-24(%rsp), %rcx
-       mov     ISF64_RFLAGS-24(%rsp), %r11
-       mov     ISF64_RSP-24(%rsp), %rsp
-        sysretq                                /* return from systen call */
-
-
-
-/*
- * System call handlers.
- * These are entered via a syscall interrupt. The system call number in %rax
- * is saved to the error code slot in the stack frame. We then branch to the
- * common state saving code.
- */
-               
-#ifndef UNIX_INT
-#error NO UNIX INT!!!
-#endif
-Entry(idt64_unix_scall)
-       swapgs                          /* switch to kernel gs (cpu_data) */
-       pushq   %rax                    /* save system call number */
-       PUSH_FUNCTION(HNDL_UNIX_SCALL)
-       pushq   $(UNIX_INT)
-       jmp     L_32bit_entry_check
-
-       
-Entry(idt64_mach_scall)
-       swapgs                          /* switch to kernel gs (cpu_data) */
-       pushq   %rax                    /* save system call number */
-       PUSH_FUNCTION(HNDL_MACH_SCALL)
-       pushq   $(MACH_INT)
-       jmp     L_32bit_entry_check
-
-       
-Entry(idt64_mdep_scall)
-       swapgs                          /* switch to kernel gs (cpu_data) */
-       pushq   %rax                    /* save system call number */
-       PUSH_FUNCTION(HNDL_MDEP_SCALL)
-       pushq   $(MACHDEP_INT)
-       jmp     L_32bit_entry_check
-
-/* Programmed into MSR_IA32_LSTAR by mp_desc.c */
-Entry(hi64_syscall)
-Entry(idt64_syscall)
-L_syscall_continue:
-       swapgs                          /* Kapow! get per-cpu data area */
-       mov     %rsp, %gs:CPU_UBER_TMP  /* save user stack */
-       mov     %gs:CPU_UBER_ISF, %rsp  /* switch stack to pcb */
-
-       /*
-        * Save values in the ISF frame in the PCB
-        * to cons up the saved machine state.
-        */
-       movl    $(USER_DS), ISF64_SS(%rsp)      
-       movl    $(SYSCALL_CS), ISF64_CS(%rsp)   /* cs - a pseudo-segment */
-       mov     %r11, ISF64_RFLAGS(%rsp)        /* rflags */
-       mov     %rcx, ISF64_RIP(%rsp)           /* rip */
-       mov     %gs:CPU_UBER_TMP, %rcx
-       mov     %rcx, ISF64_RSP(%rsp)           /* user stack */
-       mov     %rax, ISF64_ERR(%rsp)           /* err/rax - syscall code */
-       movq    $(T_SYSCALL), ISF64_TRAPNO(%rsp)        /* trapno */
-       leaq    HNDL_SYSCALL(%rip), %r11;
-       movq    %r11, ISF64_TRAPFN(%rsp)
-       mov     ISF64_RFLAGS(%rsp), %r11        /* Avoid leak, restore R11 */
-       jmp     L_dispatch_U64                  /* this can only be 64-bit */
-       
-/*
- * sysenter entry point
- * Requires user code to set up:
- *     edx: user instruction pointer (return address)
- *     ecx: user stack pointer
- *             on which is pushed stub ret addr and saved ebx
- * Return to user-space is made using sysexit.
- * Note: sysenter/sysexit cannot be used for calls returning a value in edx,
- *       or requiring ecx to be preserved.
- */
-Entry(hi64_sysenter)
-Entry(idt64_sysenter)
-       movq    (%rsp), %rsp
-       /*
-        * Push values on to the PCB stack
-        * to cons up the saved machine state.
-        */
-       push    $(USER_DS)              /* ss */
-       push    %rcx                    /* uesp */
-       pushf                           /* flags */
-       /*
-        * Clear, among others, the Nested Task (NT) flags bit;
-        * this is zeroed by INT, but not by SYSENTER.
-        */
-       push    $0
-       popf
-       push    $(SYSENTER_CS)          /* cs */ 
-L_sysenter_continue:
-       swapgs                          /* switch to kernel gs (cpu_data) */
-       push    %rdx                    /* eip */
-       push    %rax                    /* err/eax - syscall code */
-       PUSH_FUNCTION(HNDL_SYSENTER)
-       pushq   $(T_SYSENTER)
-       orl     $(EFL_IF), ISF64_RFLAGS(%rsp)
-       jmp     L_32bit_entry_check
-
-
-Entry(idt64_page_fault)
-       PUSH_FUNCTION(HNDL_ALLTRAPS)
-       push    $(T_PAGE_FAULT)
-       push    %rax                    /* save %rax temporarily */
-       testb   $3, 8+ISF64_CS(%rsp)    /* was trap from kernel? */
-       jz      L_kernel_trap           /* - yes, handle with care */
-       pop     %rax                    /* restore %rax, swapgs, and continue */
-       swapgs
-       jmp     L_dispatch_user
-
-
-/*
- * Debug trap.  Check for single-stepping across system call into
- * kernel.  If this is the case, taking the debug trap has turned
- * off single-stepping - save the flags register with the trace
- * bit set.
- */
-Entry(idt64_debug)
-       push    $0                      /* error code */
-       PUSH_FUNCTION(HNDL_ALLTRAPS)
-       pushq   $(T_DEBUG)
-
-       testb   $3, ISF64_CS(%rsp)
-       jnz     L_dispatch
+       leaq    EXT(idt64_hndl_table0)(%rip), %rax
+       jmp *8(%rax)

 
 

+Entry(ks_idt64_debug_kernel)

        /*
         * trap came from kernel mode
         */
        /*
         * trap came from kernel mode
         */


@@ -690,7 +824,7 @@ Entry(idt64_debug)
        lea     EXT(idt64_sysenter)(%rip), %rax
        cmp     %rax, ISF64_RIP+8(%rsp)
        pop     %rax
        lea     EXT(idt64_sysenter)(%rip), %rax
        cmp     %rax, ISF64_RIP+8(%rsp)
        pop     %rax

-       jne     L_dispatch
+       jne     EXT(ks_dispatch_kernel)

        /*
         * Interrupt stack frame has been pushed on the temporary stack.
         * We have to switch to pcb stack and patch up the saved state.
        /*
         * Interrupt stack frame has been pushed on the temporary stack.
         * We have to switch to pcb stack and patch up the saved state.


@@ -705,37 +839,7 @@ Entry(idt64_debug)
        mov     ISF64_ERR(%rcx),%rcx    /* restore %rcx */
        jmp     L_sysenter_continue     /* continue sysenter entry */
        
        mov     ISF64_ERR(%rcx),%rcx    /* restore %rcx */
        jmp     L_sysenter_continue     /* continue sysenter entry */
        

-
-Entry(idt64_double_fault)
-       PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
-       pushq   $(T_DOUBLE_FAULT)
-       jmp     L_dispatch_kernel
-
-
-/*
- * For GP/NP/SS faults, we use the IST1 stack.
- * For faults from user-space, we have to copy the machine state to the
- * PCB stack and then dispatch as normal.
- * For faults in kernel-space, we need to scrub for kernel exit faults and
- * treat these as user-space faults. But for all other kernel-space faults
- * we continue to run on the IST1 stack and we dispatch to handle the fault
- * as fatal.
- */
-Entry(idt64_gen_prot)
-       PUSH_FUNCTION(HNDL_ALLTRAPS)
-       pushq   $(T_GENERAL_PROTECTION)
-       jmp     trap_check_kernel_exit  /* check for kernel exit sequence */
-
-Entry(idt64_stack_fault)
-       PUSH_FUNCTION(HNDL_ALLTRAPS)
-       pushq   $(T_STACK_FAULT)
-       jmp     trap_check_kernel_exit  /* check for kernel exit sequence */
-
-Entry(idt64_segnp)
-       PUSH_FUNCTION(HNDL_ALLTRAPS)
-       pushq   $(T_SEGMENT_NOT_PRESENT)
-                                       /* indicate fault type */
-trap_check_kernel_exit:
+Entry(ks_trap_check_kernel_exit)

        testb   $3,ISF64_CS(%rsp)
        jz      L_kernel_gpf
 
        testb   $3,ISF64_CS(%rsp)
        jz      L_kernel_gpf
 


@@ -763,7 +867,7 @@ trap_check_kernel_exit:
        pop     %rax
        mov     %gs:CPU_UBER_TMP, %rsp          /* user RCX into RSP */
        xchg    %rcx, %rsp                      /* to PCB stack with user RCX */
        pop     %rax
        mov     %gs:CPU_UBER_TMP, %rsp          /* user RCX into RSP */
        xchg    %rcx, %rsp                      /* to PCB stack with user RCX */

-       jmp     L_dispatch_user
+       jmp     EXT(ks_dispatch_user)

 
 L_kernel_gpf:
        /* Here for GPF from kernel_space. Check for recoverable cases. */
 
 L_kernel_gpf:
        /* Here for GPF from kernel_space. Check for recoverable cases. */


@@ -786,10 +890,10 @@ L_kernel_gpf:
        leaq    EXT(ret32_set_gs)(%rip), %rax
        cmp     %rax, 8+ISF64_RIP(%rsp)
        je      L_32bit_fault_set_seg
        leaq    EXT(ret32_set_gs)(%rip), %rax
        cmp     %rax, 8+ISF64_RIP(%rsp)
        je      L_32bit_fault_set_seg

-
+       jmp     EXT(ks_kernel_trap)

        /* Fall through */
 
        /* Fall through */
 

-L_kernel_trap:
+Entry(ks_kernel_trap)

        /*
         * Here after taking an unexpected trap from kernel mode - perhaps
         * while running in the trampolines hereabouts.
        /*
         * Here after taking an unexpected trap from kernel mode - perhaps
         * while running in the trampolines hereabouts.


@@ -814,7 +918,7 @@ L_kernel_trap:
        cmpq    $(PAGE_SIZE), %rax              /* current stack in PCB? */
        jb      2f                              /*  - yes, deal with it */
        pop     %rax                            /*  - no, restore %rax */
        cmpq    $(PAGE_SIZE), %rax              /* current stack in PCB? */
        jb      2f                              /*  - yes, deal with it */
        pop     %rax                            /*  - no, restore %rax */

-       jmp     L_dispatch_kernel
+       jmp     EXT(ks_dispatch_kernel)

 2:
        /*
         *  Here if %rsp is in the PCB
 2:
        /*
         *  Here if %rsp is in the PCB


@@ -831,7 +935,7 @@ L_kernel_trap:
        pushq   8+ISF64_TRAPFN(%rax)
        pushq   8+ISF64_TRAPNO(%rax)
        movq    (%rax), %rax
        pushq   8+ISF64_TRAPFN(%rax)
        pushq   8+ISF64_TRAPNO(%rax)
        movq    (%rax), %rax

-       jmp     L_dispatch_kernel
+       jmp     EXT(ks_dispatch_kernel)

 
 
 /*
 
 
 /*


@@ -917,46 +1021,8 @@ L_32bit_fault_set_seg:
                                        /* the compatibility frame */
        jmp     L_dispatch_U32_after_fault
 
                                        /* the compatibility frame */
        jmp     L_dispatch_U32_after_fault
 

-/*
- * Fatal exception handlers:
- */
-Entry(idt64_db_task_dbl_fault)
-       PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
-       pushq   $(T_DOUBLE_FAULT)
-       jmp     L_dispatch      
-
-Entry(idt64_db_task_stk_fault)
-       PUSH_FUNCTION(HNDL_DOUBLE_FAULT)
-       pushq   $(T_STACK_FAULT)
-       jmp     L_dispatch      
-
-Entry(idt64_mc)
-       push    $(0)                    /* Error */
-       PUSH_FUNCTION(HNDL_MACHINE_CHECK)
-       pushq   $(T_MACHINE_CHECK)
-       jmp     L_dispatch      
-
-/*
- * NMI
- * This may or may not be fatal but extreme care is required
- * because it may fall when control was already in another trampoline.
- *
- * We get here on IST2 stack which is used for NMIs only.
- * We must be aware of the interrupted state:
- *  - from user-space, we
- *    - copy state to the PCB and continue;
- *  - from kernel-space, we
- *    - copy state to the kernel stack and continue, but
- *    - check what GSBASE was active, set the kernel base and
- *    - ensure that the active state is restored when the NMI is dismissed.
- */
-Entry(idt64_nmi)
-       push    %rax                            /* save RAX to ISF64_ERR */
-       push    %rcx                            /* save RCX to ISF64_TRAPFN */
-       push    %rdx                            /* save RDX to ISF64_TRAPNO */
-       testb   $3, ISF64_CS(%rsp)              /* NMI from user-space? */
-       je      1f

 
 

+Entry(ks_idt64_nmi_kernel)

        /* From user-space: copy interrupt state to user PCB */
        swapgs
        mov     %gs:CPU_UBER_ISF, %rcx          /* PCB stack addr */
        /* From user-space: copy interrupt state to user PCB */
        swapgs
        mov     %gs:CPU_UBER_ISF, %rcx          /* PCB stack addr */


@@ -1015,8 +1081,7 @@ Entry(idt64_nmi)
        push    ISF64_CS(%rcx)
        push    ISF64_RIP(%rcx)
        push    $(0)                            /* error code 0 */
        push    ISF64_CS(%rcx)
        push    ISF64_RIP(%rcx)
        push    $(0)                            /* error code 0 */

-       lea     HNDL_ALLINTRS(%rip), %rax
-       push    %rax                            /* trapfn allintrs */
+       push    $(HNDL_ALLINTRS)                /* trapfn allintrs */

        push    $(T_NMI)                        /* trapno T_NMI */
        mov     ISF64_ERR(%rcx), %rax
        mov     ISF64_TRAPNO(%rcx), %rdx
        push    $(T_NMI)                        /* trapno T_NMI */
        mov     ISF64_ERR(%rcx), %rax
        mov     ISF64_TRAPNO(%rcx), %rdx


@@ -1088,7 +1153,7 @@ L_return_from_trap_with_ast:
        je      2f                      /* not in the PFZ... go service AST */
        movl    %eax, R64_RBX(%r15)     /* let the PFZ know we've pended an AST */
        jmp     EXT(return_to_user)
        je      2f                      /* not in the PFZ... go service AST */
        movl    %eax, R64_RBX(%r15)     /* let the PFZ know we've pended an AST */
        jmp     EXT(return_to_user)

-2:     
+2:

 
        xorq    %rbp, %rbp              /* clear framepointer */
        CCALL(ast_taken_user)           /* handle all ASTs (enables interrupts, may return via continuation) */
 
        xorq    %rbp, %rbp              /* clear framepointer */
        CCALL(ast_taken_user)           /* handle all ASTs (enables interrupts, may return via continuation) */


@@ -1180,7 +1245,7 @@ Entry(hndl_allintrs)
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */
 
 
        CCALL1(interrupt, %r15)         /* call generic interrupt routine */
 

-       .globl  EXT(return_to_iret)
+.globl EXT(return_to_iret)

 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */
 
        decl    %gs:CPU_INTERRUPT_LEVEL
 LEXT(return_to_iret)                   /* (label for kdb_kintr and hardclock) */
 
        decl    %gs:CPU_INTERRUPT_LEVEL


@@ -1442,11 +1507,16 @@ Entry(hndl_diag_scall64)
        sti
        CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
        /* no return */
        sti
        CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
        /* no return */

-
+/* TODO assert at all 'C' entry points that we're never operating on the fault stack's alias mapping */

 Entry(hndl_machine_check)
 Entry(hndl_machine_check)

+       /* Adjust SP and savearea to their canonical, non-aliased addresses */
+       subq    EXT(dblmap_dist)(%rip), %rsp
+       subq    EXT(dblmap_dist)(%rip), %r15

        CCALL1(panic_machine_check64, %r15)
        hlt
 
 Entry(hndl_double_fault)
        CCALL1(panic_machine_check64, %r15)
        hlt
 
 Entry(hndl_double_fault)

+       subq    EXT(dblmap_dist)(%rip), %rsp
+       subq    EXT(dblmap_dist)(%rip), %r15

        CCALL1(panic_double_fault64, %r15)
        hlt
        CCALL1(panic_double_fault64, %r15)
        hlt

diff --git a/osfmk/x86_64/locore.s b/osfmk/x86_64/locore.s
index 0638e5162042f405ff79f01b73b8a39ad133e67f..d88f2a08a32dd77eb9631712b9a2ef0ad68f9923 100644 (file)
--- a/osfmk/x86_64/locore.s
+++ b/osfmk/x86_64/locore.s
@@ -159,6 +159,12 @@ wrmsr_fail:
        ret
 
 #if DEBUG
        ret
 
 #if DEBUG

+#ifndef TERI
+#define TERI 1
+#endif
+#endif
+
+#if TERI

 .globl EXT(thread_exception_return_internal)
 #else
 .globl EXT(thread_exception_return)
 .globl EXT(thread_exception_return_internal)
 #else
 .globl EXT(thread_exception_return)


@@ -169,7 +175,7 @@ LEXT(thread_bootstrap_return)
        call EXT(dtrace_thread_bootstrap)
 #endif
 
        call EXT(dtrace_thread_bootstrap)
 #endif
 

-#if DEBUG
+#if TERI

 LEXT(thread_exception_return_internal)
 #else
 LEXT(thread_exception_return)
 LEXT(thread_exception_return_internal)
 #else
 LEXT(thread_exception_return)

diff --git a/osfmk/x86_64/pmap.c b/osfmk/x86_64/pmap.c
index 9b2a9675eff7faa94360314f2b0313c5d5b6fd5a..ce64b82dbf1486c193fa2f6e54166b4de75cc8d7 100644 (file)
--- a/osfmk/x86_64/pmap.c
+++ b/osfmk/x86_64/pmap.c
@@ -239,6 +239,7 @@ pmap_t              kernel_pmap;
 struct zone    *pmap_zone;             /* zone of pmap structures */
 
 struct zone    *pmap_anchor_zone;
 struct zone    *pmap_zone;             /* zone of pmap structures */
 
 struct zone    *pmap_anchor_zone;

+struct zone    *pmap_uanchor_zone;

 int            pmap_debug = 0;         /* flag for debugging prints */
 
 unsigned int   inuse_ptepages_count = 0;
 int            pmap_debug = 0;         /* flag for debugging prints */
 
 unsigned int   inuse_ptepages_count = 0;


@@ -323,12 +324,8 @@ void
 pmap_cpu_init(void)
 {
        cpu_data_t      *cdp = current_cpu_datap();
 pmap_cpu_init(void)
 {
        cpu_data_t      *cdp = current_cpu_datap();

-       /*
-        * Here early in the life of a processor (from cpu_mode_init()).
-        * Ensure global page feature is disabled at this point.
-        */

 
 

-       set_cr4(get_cr4() &~ CR4_PGE);
+       set_cr4(get_cr4() | CR4_PGE);

 
        /*
         * Initialize the per-cpu, TLB-related fields.
 
        /*
         * Initialize the per-cpu, TLB-related fields.


@@ -337,6 +334,7 @@ pmap_cpu_init(void)
        cdp->cpu_active_cr3 = kernel_pmap->pm_cr3;
        cdp->cpu_tlb_invalid = FALSE;
        cdp->cpu_task_map = TASK_MAP_64BIT;
        cdp->cpu_active_cr3 = kernel_pmap->pm_cr3;
        cdp->cpu_tlb_invalid = FALSE;
        cdp->cpu_task_map = TASK_MAP_64BIT;

+

        pmap_pcid_configure();
        if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
                pmap_smep_enabled = TRUE;
        pmap_pcid_configure();
        if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) {
                pmap_smep_enabled = TRUE;


@@ -415,14 +413,13 @@ pmap_bootstrap(
        kernel_pmap->nx_enabled = TRUE;
        kernel_pmap->pm_task_map = TASK_MAP_64BIT;
        kernel_pmap->pm_obj = (vm_object_t) NULL;
        kernel_pmap->nx_enabled = TRUE;
        kernel_pmap->pm_task_map = TASK_MAP_64BIT;
        kernel_pmap->pm_obj = (vm_object_t) NULL;

-       kernel_pmap->dirbase = (pd_entry_t *)((uintptr_t)IdlePTD);
-       kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT);

        kernel_pmap->pm_pml4 = IdlePML4;
        kernel_pmap->pm_pml4 = IdlePML4;

+       kernel_pmap->pm_upml4 = IdlePML4;

        kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);
        kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);

+       kernel_pmap->pm_ucr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4);

        kernel_pmap->pm_eptp = 0;
        kernel_pmap->pm_eptp = 0;

-       pmap_pcid_initialize_kernel(kernel_pmap);

 
 

-       
+       pmap_pcid_initialize_kernel(kernel_pmap);

 
        current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
 
 
        current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3;
 


@@ -808,6 +805,18 @@ pmap_init(void)
         */
 
        zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);
         */
 
        zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);

+/* TODO: possible general optimisation...pre-allocate via zones commonly created
+ * level3/2 pagetables
+ */
+       pmap_uanchor_zone = zinit(PAGE_SIZE, task_max, PAGE_SIZE, "pagetable user anchors");
+       zone_change(pmap_uanchor_zone, Z_NOENCRYPT, TRUE);
+
+       /* The anchor is required to be page aligned. Zone debugging adds
+        * padding which may violate that requirement. Tell the zone
+        * subsystem that alignment is required.
+        */
+
+       zone_change(pmap_uanchor_zone, Z_ALIGNMENT_REQUIRED, TRUE);

 
        s = (vm_size_t) sizeof(struct pv_hashed_entry);
        pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,
 
        s = (vm_size_t) sizeof(struct pv_hashed_entry);
        pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */,


@@ -1288,6 +1297,7 @@ hv_ept_pmap_create(void **ept_pmap, void **eptp)
  *     the map will be used in software only, and
  *     is bounded by that size.
  */
  *     the map will be used in software only, and
  *     is bounded by that size.
  */

+

 pmap_t
 pmap_create_options(
        ledger_t        ledger,
 pmap_t
 pmap_create_options(
        ledger_t        ledger,


@@ -1314,24 +1324,20 @@ pmap_create_options(
        /*
         *      Return error when unrecognized flags are passed.
         */
        /*
         *      Return error when unrecognized flags are passed.
         */

-       if ((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0) {
+       if (__improbable((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0)) {

                return(PMAP_NULL);
        }
 
        p = (pmap_t) zalloc(pmap_zone);
        if (PMAP_NULL == p)
                panic("pmap_create zalloc");
                return(PMAP_NULL);
        }
 
        p = (pmap_t) zalloc(pmap_zone);
        if (PMAP_NULL == p)
                panic("pmap_create zalloc");

+

        /* Zero all fields */
        bzero(p, sizeof(*p));
        /* init counts now since we'll be bumping some */
        simple_lock_init(&p->lock, 0);
        /* Zero all fields */
        bzero(p, sizeof(*p));
        /* init counts now since we'll be bumping some */
        simple_lock_init(&p->lock, 0);

-#if 00
-       p->stats.resident_count = 0;
-       p->stats.resident_max = 0;
-       p->stats.wired_count = 0;
-#else

        bzero(&p->stats, sizeof (p->stats));
        bzero(&p->stats, sizeof (p->stats));

-#endif
+

        p->ref_count = 1;
        p->nx_enabled = 1;
        p->pm_shared = FALSE;
        p->ref_count = 1;
        p->nx_enabled = 1;
        p->pm_shared = FALSE;


@@ -1347,10 +1353,13 @@ pmap_create_options(
        }
 
        p->pm_pml4 = zalloc(pmap_anchor_zone);
        }
 
        p->pm_pml4 = zalloc(pmap_anchor_zone);

+       p->pm_upml4 = zalloc(pmap_uanchor_zone); //cleanup for EPT

 
        pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0);
 
        pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0);

+       pmap_assert((((uintptr_t)p->pm_upml4) & PAGE_MASK) == 0);

 
        memset((char *)p->pm_pml4, 0, PAGE_SIZE);
 
        memset((char *)p->pm_pml4, 0, PAGE_SIZE);

+       memset((char *)p->pm_upml4, 0, PAGE_SIZE);

 
        if (flags & PMAP_CREATE_EPT) {
                p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;
 
        if (flags & PMAP_CREATE_EPT) {
                p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags;


@@ -1358,6 +1367,7 @@ pmap_create_options(
        } else {
                p->pm_eptp = 0;
                p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);
        } else {
                p->pm_eptp = 0;
                p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4);

+               p->pm_ucr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_upml4);

        }
 
        /* allocate the vm_objs to hold the pdpt, pde and pte pages */
        }
 
        /* allocate the vm_objs to hold the pdpt, pde and pte pages */


@@ -1381,11 +1391,13 @@ pmap_create_options(
                pml4[KERNEL_PML4_INDEX]    = kpml4[KERNEL_PML4_INDEX];
                pml4[KERNEL_KEXTS_INDEX]   = kpml4[KERNEL_KEXTS_INDEX];
                pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];
                pml4[KERNEL_PML4_INDEX]    = kpml4[KERNEL_PML4_INDEX];
                pml4[KERNEL_KEXTS_INDEX]   = kpml4[KERNEL_KEXTS_INDEX];
                pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX];

-
+               pml4[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];

 #if KASAN
                pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0];
                pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1];
 #endif
 #if KASAN
                pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0];
                pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1];
 #endif

+               pml4_entry_t    *pml4u = pmap64_user_pml4(p, 0ULL);
+               pml4u[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX];

        }
 
 #if MACH_ASSERT
        }
 
 #if MACH_ASSERT


@@ -1556,6 +1568,7 @@ pmap_destroy(pmap_t       p)
        int inuse_ptepages = 0;
 
        zfree(pmap_anchor_zone, p->pm_pml4);
        int inuse_ptepages = 0;
 
        zfree(pmap_anchor_zone, p->pm_pml4);

+       zfree(pmap_uanchor_zone, p->pm_upml4);

 
        inuse_ptepages += p->pm_obj_pml4->resident_page_count;
        vm_object_deallocate(p->pm_obj_pml4);
 
        inuse_ptepages += p->pm_obj_pml4->resident_page_count;
        vm_object_deallocate(p->pm_obj_pml4);


@@ -1782,6 +1795,10 @@ pmap_expand_pml4(
 
        DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
 
 
        DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr);
 

+       /* With the exception of the kext "basement", the kernel's level 4
+        * pagetables must not be dynamically expanded.
+        */
+       assert(map != kernel_pmap || (vaddr == KERNEL_BASEMENT));

        /*
         *      Allocate a VM page for the pml4 page
         */
        /*
         *      Allocate a VM page for the pml4 page
         */


@@ -1847,6 +1864,13 @@ pmap_expand_pml4(
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));

+       pml4_entry_t    *upml4p;
+
+       upml4p = pmap64_user_pml4(map, vaddr);
+       pmap_store_pte(upml4p, pa_to_pte(pa)
+                               | PTE_READ(is_ept)
+                               | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+                               | PTE_WRITE(is_ept));

 
        PMAP_UNLOCK(map);
 
 
        PMAP_UNLOCK(map);
 


@@ -1980,12 +2004,16 @@ pmap_expand(
         * which is for kexts and is in the 512GB immediately below the kernel..
         * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
         */
         * which is for kexts and is in the 512GB immediately below the kernel..
         * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT
         */

-       if (map == kernel_pmap && 
-           !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))
-               panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+       if (__improbable(map == kernel_pmap && 
+               !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))) {
+               if ((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0) {
+                       panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr);
+               }
+       }

 
 
        while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
 
 
        while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {

+               assert((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0);

                kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options);
                if (pepkr != KERN_SUCCESS)
                        return pepkr;
                kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options);
                if (pepkr != KERN_SUCCESS)
                        return pepkr;


@@ -2035,7 +2063,7 @@ pmap_expand(
 
                VM_PAGE_FREE(m);
 
 
                VM_PAGE_FREE(m);
 

-               OSAddAtomic(-1,  &inuse_ptepages_count);
+               OSAddAtomic(-1,  &inuse_ptepages_count);//todo replace all with inlines

                PMAP_ZINFO_PFREE(map, PAGE_SIZE);
                return KERN_SUCCESS;
        }
                PMAP_ZINFO_PFREE(map, PAGE_SIZE);
                return KERN_SUCCESS;
        }


@@ -2089,6 +2117,14 @@ pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr)
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));
                                | PTE_READ(is_ept)
                                | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
                                | PTE_WRITE(is_ept));

+
+               pte = pmap64_user_pml4(pmap, vaddr);
+
+               pmap_store_pte(pte, pa_to_pte(i386_ptob(pn))
+                               | PTE_READ(is_ept)
+                               | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER)
+                               | PTE_WRITE(is_ept));
+

        }
 
        if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {
        }
 
        if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) {


@@ -2552,7 +2588,7 @@ pmap_flush(
                                }
                                orig_acks = NMIPI_acks;
                                NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);
                                }
                                orig_acks = NMIPI_acks;
                                NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT);

-                               panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",
+                               panic("Uninterruptible processor(s): CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu",

                                      cpus_to_respond, orig_acks, NMIPI_acks, deadline);
                        }
                }
                                      cpus_to_respond, orig_acks, NMIPI_acks, deadline);
                        }
                }


@@ -2646,7 +2682,7 @@ pmap_flush_tlbs(pmap_t    pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o
                        continue;
                uint64_t        cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
                uint64_t        cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);
                        continue;
                uint64_t        cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu);
                uint64_t        cpu_task_cr3 = CPU_GET_TASK_CR3(cpu);

-
+//recall that the shadowed task cr3 is pre-composed

                if ((pmap_cr3 == cpu_task_cr3) ||
                    (pmap_cr3 == cpu_active_cr3) ||
                    (pmap_is_shared)) {
                if ((pmap_cr3 == cpu_task_cr3) ||
                    (pmap_cr3 == cpu_active_cr3) ||
                    (pmap_is_shared)) {


@@ -2781,16 +2817,9 @@ process_pmap_updates(void)
        pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
        if (pmap_pcid_ncpus) {
                pmap_pcid_validate_current();
        pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
        if (pmap_pcid_ncpus) {
                pmap_pcid_validate_current();

-               if (cpu_datap(ccpu)->cpu_tlb_invalid_global) {
-                       cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
-                       tlb_flush_global();
-               }
-               else {
-                       cpu_datap(ccpu)->cpu_tlb_invalid_local = FALSE;
-                       flush_tlb_raw();
-               }
-       }
-       else {
+               cpu_datap(ccpu)->cpu_tlb_invalid = FALSE;
+               tlb_flush_global();
+       } else {

                current_cpu_datap()->cpu_tlb_invalid = FALSE;
                flush_tlb_raw();
        }
                current_cpu_datap()->cpu_tlb_invalid = FALSE;
                flush_tlb_raw();
        }


@@ -3209,7 +3238,16 @@ pmap_check_ledgers(
        }
 
        if (pmap->stats.resident_count != 0 ||
        }
 
        if (pmap->stats.resident_count != 0 ||

+#if 35156815
+           /*
+            * "wired_count" is unfortunately a bit inaccurate, so let's
+            * tolerate some slight deviation to limit the amount of
+            * somewhat-spurious assertion failures.
+            */
+           pmap->stats.wired_count > 10 ||
+#else /* 35156815 */

            pmap->stats.wired_count != 0 ||
            pmap->stats.wired_count != 0 ||

+#endif /* 35156815 */

            pmap->stats.device != 0 ||
            pmap->stats.internal != 0 ||
            pmap->stats.external != 0 ||
            pmap->stats.device != 0 ||
            pmap->stats.internal != 0 ||
            pmap->stats.external != 0 ||


@@ -3287,11 +3325,3 @@ void pmap_verify_noncacheable(uintptr_t vaddr) {
                return;
        panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
 }
                return;
        panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep);
 }

-
-#if KASAN
-void kasan_map_low_fixed_regions(void) {
-       kasan_map_shadow(MASTER_GDT_ALIAS, PAGE_SIZE, false);
-       kasan_map_shadow(MASTER_IDT_ALIAS, PAGE_SIZE, false);
-       kasan_map_shadow(LOWGLOBAL_ALIAS, PAGE_SIZE, false);
-}
-#endif

diff --git a/osfmk/x86_64/pmap_pcid.c b/osfmk/x86_64/pmap_pcid.c
index 2a7280d41ec4d0707b1c70114a45f21f0f63bb44..3cf4a0e49a2f01c7527268775a934fe0c21f0da6 100644 (file)
--- a/osfmk/x86_64/pmap_pcid.c
+++ b/osfmk/x86_64/pmap_pcid.c
@@ -58,6 +58,7 @@
 
 uint32_t       pmap_pcid_ncpus;
 boolean_t      pmap_pcid_disabled = FALSE;
 
 uint32_t       pmap_pcid_ncpus;
 boolean_t      pmap_pcid_disabled = FALSE;

+pcid_cdata_t pcid_data[MAX_CPUS] __attribute__((aligned(64)));

 
 void   pmap_pcid_configure(void) {
        int ccpu = cpu_number();
 
 void   pmap_pcid_configure(void) {
        int ccpu = cpu_number();


@@ -74,6 +75,7 @@ void  pmap_pcid_configure(void) {
                kprintf("PMAP: PCID feature disabled %u\n", pmap_pcid_disabled);
        }
         /* no_shared_cr3+PCID is currently unsupported */
                kprintf("PMAP: PCID feature disabled %u\n", pmap_pcid_disabled);
        }
         /* no_shared_cr3+PCID is currently unsupported */

+       //todo remove nscr3

 #if    DEBUG
        if (pmap_pcid_disabled == FALSE)
                no_shared_cr3 = FALSE;
 #if    DEBUG
        if (pmap_pcid_disabled == FALSE)
                no_shared_cr3 = FALSE;


@@ -134,7 +136,8 @@ void        pmap_pcid_configure(void) {
                cpu_datap(ccpu)->cpu_pmap_pcid_coherentp =
                    cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel =
                    &(kernel_pmap->pmap_pcid_coherency_vector[ccpu]);
                cpu_datap(ccpu)->cpu_pmap_pcid_coherentp =
                    cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel =
                    &(kernel_pmap->pmap_pcid_coherency_vector[ccpu]);

-               cpu_datap(ccpu)->cpu_pcid_refcounts[0] = 1;
+               cpu_datap(ccpu)->cpu_pcid_data = &pcid_data[ccpu];
+               cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0] = 1;

        }
 }
 
        }
 }
 


@@ -167,13 +170,13 @@ pcid_t    pmap_pcid_allocate_pcid(int ccpu) {
        int i;
        pcid_ref_t      cur_min = 0xFF;
        uint32_t        cur_min_index = ~1;
        int i;
        pcid_ref_t      cur_min = 0xFF;
        uint32_t        cur_min_index = ~1;

-       pcid_ref_t      *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_refcounts[0];
+       pcid_ref_t      *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0];

        pcid_ref_t      old_count;
 
        pcid_ref_t      old_count;
 

-       if ((i = cpu_datap(ccpu)->cpu_pcid_free_hint) != 0) {
+       if ((i = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint) != 0) {

                if (cpu_pcid_refcounts[i] == 0) {
                        (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
                if (cpu_pcid_refcounts[i] == 0) {
                        (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);

-                       cpu_datap(ccpu)->cpu_pcid_free_hint = 0;
+                       cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = 0;

                        return i;
                }
        }
                        return i;
                }
        }


@@ -193,8 +196,7 @@ pcid_t      pmap_pcid_allocate_pcid(int ccpu) {
                if (cur_refcount == 0) {
                        (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
                        return i;
                if (cur_refcount == 0) {
                        (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1);
                        return i;

-               }
-               else {
+               } else {

                        if (cur_refcount < cur_min) {
                                cur_min_index = i;
                                cur_min = cur_refcount;
                        if (cur_refcount < cur_min) {
                                cur_min_index = i;
                                cur_min = cur_refcount;


@@ -208,7 +210,7 @@ pcid_t      pmap_pcid_allocate_pcid(int ccpu) {
 
        old_count = __sync_fetch_and_add(&cpu_pcid_refcounts[cur_min_index], 1);
        pmap_assert(old_count < PMAP_PCID_MAX_REFCOUNT);
 
        old_count = __sync_fetch_and_add(&cpu_pcid_refcounts[cur_min_index], 1);
        pmap_assert(old_count < PMAP_PCID_MAX_REFCOUNT);

-       return cur_min_index;
+       return (cur_min_index);

 }
 
 void   pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {
 }
 
 void   pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {


@@ -221,15 +223,15 @@ void      pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {
        if (pcid == PMAP_PCID_INVALID_PCID)
                return;
 
        if (pcid == PMAP_PCID_INVALID_PCID)
                return;
 

-       lp = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid];
+       lp = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid];

        pmap_assert(pcid > 0 && pcid < PMAP_PCID_MAX_PCID);
        pmap_assert(pcid > 0 && pcid < PMAP_PCID_MAX_PCID);

-       pmap_assert(cpu_datap(ccpu)->cpu_pcid_refcounts[pcid] >= 1);
+       pmap_assert(cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid] >= 1);

 
        if (lp == tpmap)
 
        if (lp == tpmap)

-               (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);
+               (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);

 
 

-       if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_refcounts[pcid], 1)) == 1) {
-                   cpu_datap(ccpu)->cpu_pcid_free_hint = pcid;
+       if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid], 1)) == 1) {
+                   cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = pcid;

        }
        pmap_assert(prior_count <= PMAP_PCID_MAX_REFCOUNT);
 }
        }
        pmap_assert(prior_count <= PMAP_PCID_MAX_REFCOUNT);
 }


@@ -253,6 +255,7 @@ pcid_t      pcid_for_pmap_cpu_tuple(pmap_t cpmap, thread_t cthread, int ccpu) {
 
        return active_pmap->pmap_pcid_cpus[ccpu];
 }
 
        return active_pmap->pmap_pcid_cpus[ccpu];
 }

+int npz = 0;

 
 #if PMAP_ASSERT
 #define PCID_RECORD_SIZE 128
 
 #if PMAP_ASSERT
 #define PCID_RECORD_SIZE 128


@@ -263,6 +266,7 @@ void        pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t
        pcid_t          new_pcid = tpmap->pmap_pcid_cpus[ccpu];
        pmap_t          last_pmap;
        boolean_t       pcid_conflict = FALSE, pending_flush = FALSE;
        pcid_t          new_pcid = tpmap->pmap_pcid_cpus[ccpu];
        pmap_t          last_pmap;
        boolean_t       pcid_conflict = FALSE, pending_flush = FALSE;

+       pcid_cdata_t    *pcdata = cpu_datap(ccpu)->cpu_pcid_data;

 
        pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled);
        if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {
 
        pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled);
        if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) {


@@ -277,14 +281,14 @@ void      pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t
 
        pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
        if (__probable(pending_flush == FALSE)) {
 
        pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
        if (__probable(pending_flush == FALSE)) {

-               last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];
+               last_pmap = pcdata->cpu_pcid_last_pmap_dispatched[new_pcid];

                pcid_conflict = ((last_pmap != NULL) && (tpmap != last_pmap));
        }
        if (__improbable(pending_flush || pcid_conflict)) {
                pmap_pcid_validate_cpu(tpmap, ccpu);
        }
        /* Consider making this a unique id */
                pcid_conflict = ((last_pmap != NULL) && (tpmap != last_pmap));
        }
        if (__improbable(pending_flush || pcid_conflict)) {
                pmap_pcid_validate_cpu(tpmap, ccpu);
        }
        /* Consider making this a unique id */

-       cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
+       pcdata->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;

 
        pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);
        pmap_assert(((tpmap ==  kernel_pmap) && new_pcid == 0) ||
 
        pmap_assert(new_pcid < PMAP_PCID_MAX_PCID);
        pmap_assert(((tpmap ==  kernel_pmap) && new_pcid == 0) ||


@@ -306,9 +310,24 @@ void       pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t
                        ncr3 = kernel_pmap->pm_cr3;
                }
                cpu_datap(ccpu)->cpu_kernel_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];
                        ncr3 = kernel_pmap->pm_cr3;
                }
                cpu_datap(ccpu)->cpu_kernel_pcid = kernel_pmap->pmap_pcid_cpus[ccpu];

+               npz++;

        }
 
        }
 

-       set_cr3_composed(ncr3, new_pcid, !(pending_flush || pcid_conflict));
+       uint64_t preserve = !(pending_flush || pcid_conflict);
+       set_cr3_composed(ncr3, new_pcid, preserve);
+#if DEBUG
+       cpu_datap(ccpu)->cpu_pcid_last_cr3 = ncr3 | new_pcid | preserve << 63;
+#endif
+       uint64_t spcid = (new_pcid + PMAP_PCID_MAX_PCID);
+       if (new_pcid == 0) {
+               spcid = 0;
+       }
+       uint64_t scr3 = tpmap->pm_ucr3 | spcid;
+
+       cpu_datap(ccpu)->cpu_ucr3 = scr3;
+       cpu_shadowp(ccpu)->cpu_ucr3 = scr3;
+
+       cpu_shadowp(ccpu)->cpu_task_cr3 = ncr3 | new_pcid;

 
        if (!pending_flush) {
                /* We did not previously observe a pending invalidation for this
 
        if (!pending_flush) {
                /* We did not previously observe a pending invalidation for this

diff --git a/pexpert/pexpert/GearImage.h b/pexpert/pexpert/GearImage.h
index f074d0092ba77312da20a4af007678c097e3880a..d79fa3ff4891f9506834ad97fcace8893f76fc93 100644 (file)
--- a/pexpert/pexpert/GearImage.h
+++ b/pexpert/pexpert/GearImage.h
@@ -4,7 +4,3466 @@
 #define kGearFrames    (6)
 #define kGearFPS       (24)
 
 #define kGearFrames    (6)
 #define kGearFPS       (24)
 

-
+#if !PEXPERT_NO_3X_IMAGES
+const unsigned char gGearPict3x[9*kGearFrames*kGearWidth*kGearHeight] = {
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0,
+    0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x94,
+    0x94,0xac,0xd6,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xa1,0x76,0x6f,
+    0x6f,0x76,0x9f,0xdc,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xb4,0x7c,0x6d,0x7d,
+    0x7d,0x6e,0x79,0xac,0xe8,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xff,0xfe,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd5,0x90,0x75,0x75,0x79,
+    0x79,0x75,0x74,0x8e,0xd4,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf7,0xe7,0xe6,0xf6,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x82,0x76,0x7a,0x72,
+    0x72,0x79,0x75,0x83,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xeb,0xca,0xae,0xac,0xc4,0xe7,
+    0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x76,
+    0x76,0x79,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xf2,0xed,0xed,0xf3,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa4,0x86,0x82,0x7e,0x81,0xa6,
+    0xdd,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x75,
+    0x75,0x78,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xf8,
+    0xec,0xe4,0xe3,0xe3,0xe5,0xec,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfc,0xbc,0x78,0x70,0x79,0x76,0x71,0x77,
+    0xa8,0xec,0xff,0xfc,0xfe,0xff,0xff,0xff,0xf9,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74,
+    0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfb,0xed,
+    0xe2,0xe0,0xe1,0xe1,0xdf,0xe1,0xf0,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xe7,0xa7,0x76,0x7a,0x7e,0x7d,0x80,0x72,
+    0x81,0xc4,0xf6,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfd,0xf2,0xe4,
+    0xe1,0xe3,0xe2,0xe2,0xe1,0xe0,0xeb,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe2,0xa4,0x78,0x79,0x7f,0x7b,0x7c,0x7c,
+    0x79,0x91,0xd4,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xff,0xf6,0xe7,0xe2,
+    0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xeb,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xf2,0xad,0x72,0x75,0x80,0x77,0x76,0x81,
+    0x79,0x77,0xb4,0xf8,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xfe,0xef,0xe2,0xe1,
+    0xe3,0xe2,0xe2,0xe3,0xe0,0xdf,0xec,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xc1,0x7c,0x7a,0x82,0x79,0x7a,0x7e,
+    0x78,0x77,0x93,0xcb,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf3,0xe8,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa0,0x81,0x7c,0x7d,0x7c,0x78,
+    0x7c,0x7e,0x76,0x9e,0xef,0xff,0xfc,0xfe,0xfc,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfc,0xea,0xe1,0xe4,0xe3,
+    0xe1,0xe2,0xe2,0xe2,0xe2,0xe9,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd0,0x8b,0x73,0x80,0x7e,0x77,
+    0x7e,0x7f,0x6f,0x88,0xd0,0xfb,0xfe,0xff,0xfa,0xff,0xff,0xc9,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xfe,0xf5,0xe5,0xdf,0xe3,0xe3,
+    0xe1,0xe3,0xe3,0xe0,0xe5,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa6,0x7c,0x7f,0x7f,0x7a,
+    0x7b,0x7b,0x7a,0x7e,0x9d,0xda,0xff,0xff,0xf6,0xff,0xff,0xc9,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xf6,0xea,0xe4,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe3,0xe1,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0x99,0x79,0x7b,0x7f,
+    0x79,0x79,0x81,0x76,0x77,0xba,0xff,0xff,0xf4,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74,
+    0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf8,0xff,0xff,0xf0,0xe3,0xe2,0xe4,0xe1,0xe2,
+    0xe3,0xe2,0xe1,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff,
+    0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfa,0xb9,0x7b,0x7a,0x82,
+    0x7a,0x79,0x7f,0x77,0x73,0x99,0xda,0xfd,0xfc,0xff,0xff,0xcb,0x80,0x72,0x78,0x75,
+    0x75,0x78,0x71,0x7f,0xca,0xff,0xff,0xfa,0xff,0xf8,0xe9,0xe2,0xe2,0xe3,0xe2,0xe1,
+    0xe3,0xe1,0xe0,0xee,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xd9,0x9a,0x7e,0x7c,
+    0x7d,0x7c,0x79,0x7c,0x7d,0x7c,0xa8,0xf1,0xff,0xff,0xff,0xcc,0x80,0x75,0x7a,0x73,
+    0x73,0x79,0x74,0x80,0xca,0xff,0xff,0xfc,0xfe,0xef,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe7,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xf1,0xe4,
+    0xf0,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x89,0x73,
+    0x80,0x80,0x77,0x7e,0x7f,0x70,0x8a,0xd4,0xf9,0xff,0xff,0xcf,0x87,0x77,0x79,0x74,
+    0x74,0x78,0x77,0x8a,0xd0,0xff,0xff,0xfb,0xfa,0xe9,0xe0,0xe3,0xe2,0xe1,0xe2,0xe3,
+    0xe0,0xe5,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb,
+    0xf8,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xdd,0xc2,0xb2,0xa9,
+    0xaf,0xc6,0xe2,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfa,0xff,0xec,0xa2,0x7c,
+    0x81,0x7e,0x79,0x7d,0x7d,0x74,0x7f,0xad,0xe4,0xff,0xff,0xde,0xa0,0x74,0x6e,0x7d,
+    0x7d,0x6e,0x73,0xa1,0xe2,0xff,0xff,0xfa,0xf2,0xe6,0xe1,0xe3,0xe3,0xe1,0xe3,0xe3,
+    0xe0,0xe9,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8,
+    0xe7,0xe9,0xed,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xe7,0xa4,0x7b,0x7d,0x80,
+    0x79,0x82,0xa7,0xd7,0xf3,0xfc,0xfe,0xff,0xfe,0xff,0xfd,0xfe,0xff,0xfa,0xcf,0x96,
+    0x7a,0x7c,0x7e,0x79,0x79,0x7c,0x7b,0x8f,0xd3,0xff,0xff,0xf4,0xcd,0x8c,0x6c,0x76,
+    0x76,0x6c,0x89,0xca,0xf7,0xff,0xff,0xf8,0xeb,0xe4,0xe2,0xe3,0xe2,0xe2,0xe2,0xe1,
+    0xe6,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf5,0xe8,0xdd,0xd9,
+    0xda,0xda,0xda,0xe5,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8d,0x7b,0x83,0x84,
+    0x7d,0x7f,0x86,0x90,0xaa,0xd6,0xf7,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfa,0xb5,
+    0x78,0x7c,0x83,0x77,0x77,0x7e,0x79,0x87,0xce,0xff,0xff,0xfe,0xf5,0xc6,0x93,0x7a,
+    0x7a,0x92,0xc3,0xf3,0xff,0xff,0xff,0xf6,0xe7,0xe3,0xe3,0xe2,0xe2,0xe3,0xe1,0xe0,
+    0xed,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xfd,0xf5,0xe9,0xe0,0xdd,0xdb,0xda,
+    0xdb,0xdc,0xd9,0xde,0xee,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe3,0xae,0x8c,0x85,0x89,0x89,
+    0x89,0x8a,0x83,0x79,0x81,0x9f,0xbf,0xde,0xfa,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xd8,
+    0x97,0x7a,0x7b,0x81,0x81,0x7c,0x79,0x92,0xd5,0xff,0xff,0xfb,0xff,0xf3,0xd7,0xc2,
+    0xc2,0xd7,0xf1,0xfe,0xff,0xff,0xff,0xf6,0xe8,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe6,
+    0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf6,0xee,0xe5,0xdc,0xda,0xdc,0xde,0xde,
+    0xdd,0xdc,0xdb,0xdd,0xe6,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe6,0xaf,0x88,0x80,0x86,0x84,
+    0x82,0x84,0x85,0x87,0x85,0x7e,0x81,0x9f,0xcd,0xee,0xfa,0xfe,0xff,0xff,0xfe,0xf7,
+    0xcd,0x91,0x76,0x7c,0x79,0x73,0x86,0xb7,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfa,0xef,0xe4,0xe0,0xe2,0xe2,0xe0,0xe5,0xf3,
+    0xfd,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf1,0xe6,0xdc,0xda,0xdc,0xdd,0xdc,0xdb,0xdb,
+    0xdb,0xdb,0xda,0xdc,0xe7,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8b,0x7e,0x89,0x85,
+    0x80,0x84,0x86,0x88,0x86,0x81,0x80,0x83,0x8b,0xa3,0xd0,0xf9,0xff,0xfc,0xfb,0xff,
+    0xf5,0xc7,0x96,0x7d,0x78,0x82,0xad,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xec,0xe2,0xe0,0xe0,0xe6,0xf2,0xfc,
+    0xff,0xff,0xff,0xff,0xfd,0xf3,0xe7,0xe0,0xdc,0xdb,0xdb,0xdd,0xdd,0xdc,0xdb,0xda,
+    0xdc,0xdd,0xd9,0xdc,0xec,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa6,0x7f,0x7e,0x86,
+    0x8a,0x88,0x83,0x82,0x84,0x87,0x8a,0x83,0x77,0x80,0x9a,0xb8,0xdb,0xf6,0xff,0xff,
+    0xfe,0xf2,0xd8,0xc2,0xbf,0xca,0xe2,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8,
+    0xf8,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf3,0xf1,0xf0,0xf5,0xfc,0xff,
+    0xff,0xff,0xfd,0xf6,0xec,0xe5,0xde,0xda,0xdb,0xdd,0xdd,0xdb,0xda,0xda,0xdc,0xdd,
+    0xdb,0xd8,0xd8,0xe4,0xf6,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd6,0xa7,0x8b,0x85,
+    0x84,0x83,0x86,0x88,0x85,0x83,0x84,0x85,0x87,0x86,0x7d,0x7c,0x9a,0xce,0xf5,0xff,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf3,0xe4,0xdb,0xdc,0xde,0xdd,0xdc,0xdc,0xdb,0xdb,0xdc,0xdc,0xda,0xda,
+    0xda,0xdb,0xe3,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xc9,0x9b,
+    0x82,0x83,0x88,0x88,0x85,0x83,0x84,0x86,0x88,0x84,0x82,0x80,0x7e,0x94,0xc8,0xf3,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfb,0xf0,0xe3,0xdc,0xdb,0xdc,0xdd,0xdd,0xdb,0xda,0xda,0xdb,0xdc,0xdc,0xd9,0xd8,
+    0xe0,0xee,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf7,0xd8,
+    0xb9,0x9c,0x82,0x7c,0x86,0x8a,0x87,0x83,0x81,0x82,0x86,0x89,0x81,0x7a,0x95,0xd8,
+    0xff,0xff,0xf8,0xf9,0xf9,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xff,0xff,
+    0xf5,0xe3,0xdb,0xdc,0xdd,0xdc,0xdb,0xda,0xdb,0xdc,0xdc,0xda,0xd7,0xd9,0xe0,0xe9,
+    0xf3,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,
+    0xf6,0xd2,0xa4,0x8e,0x8a,0x84,0x83,0x87,0x87,0x87,0x83,0x80,0x8b,0x80,0x7b,0xc1,
+    0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xef,0xdb,0xdb,0xdd,0xdb,0xdb,0xdd,0xdc,0xdb,0xda,0xda,0xda,0xdb,0xe3,0xf1,0xfc,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfe,0xfb,0xef,0xd0,0xa5,0x85,0x80,0x89,0x8a,0x88,0x83,0x81,0x8c,0x81,0x79,0xc0,
+    0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xed,0xd9,0xda,0xde,0xda,0xda,0xdc,0xdc,0xda,0xd8,0xda,0xe3,0xef,0xf9,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,
+    0xfb,0xff,0xff,0xf4,0xdc,0xbf,0xa0,0x87,0x7d,0x80,0x86,0x87,0x86,0x7c,0x89,0xcd,
+    0xff,0xff,0xf9,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf1,0xdc,0xd8,0xdc,0xdc,0xda,0xd8,0xd7,0xd9,0xe0,0xeb,0xf4,0xfb,0xff,0xff,0xfd,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfd,0xda,0xad,0x94,0x8b,0x89,0x87,0x84,0x90,0xb6,0xe8,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf9,0xea,0xdf,0xdb,0xdb,0xda,0xda,0xdd,0xe6,0xf4,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xda,0xb9,0xa0,0x98,0xa0,0xc2,0xee,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xee,0xe2,0xdf,0xe1,0xe9,0xf3,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa,
+    0xfa,0xfc,0xfd,0xfb,0xf7,0xf7,0xfc,0xff,0xfa,0xea,0xd8,0xd4,0xdd,0xee,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xfa,0xf5,0xf1,0xf3,0xf9,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xed,0xdf,0xd7,0xd6,0xd6,0xd6,0xd6,0xd6,
+    0xd6,0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xd8,0xdc,0xe7,0xf5,0xfd,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,0xf3,0xf1,0xf0,0xf0,0xf0,0xf0,0xf0,0xef,0xef,
+    0xef,0xef,0xef,0xef,0xef,0xf0,0xf3,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe3,0xbc,0xa6,0x9d,0x9a,0x9a,0x9a,0x9a,0x9a,
+    0x9a,0x9a,0x9a,0x9a,0x9a,0x99,0x98,0x9a,0xa2,0xb3,0xd5,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xee,0xe3,0xde,0xda,0xda,0xda,0xda,0xda,0xda,0xd8,0xd8,
+    0xd8,0xd8,0xd8,0xd8,0xd8,0xd9,0xdd,0xe6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfd,0xe2,0xb2,0x91,0x8e,0x90,0x8c,0x8c,0x8d,0x8d,0x8d,
+    0x8d,0x8d,0x8d,0x8d,0x8c,0x8c,0x8b,0x8d,0x8f,0x8b,0x9f,0xd0,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xeb,0xda,0xd4,0xd5,0xd4,0xd3,0xd4,0xd4,0xd4,0xd4,0xd3,0xd3,
+    0xd3,0xd3,0xd3,0xd3,0xd3,0xd4,0xd4,0xd5,0xe1,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbb,0x8e,0x87,0x8d,0x90,0x90,0x90,0x90,0x90,0x90,
+    0x90,0x90,0x90,0x90,0x90,0x90,0x8f,0x90,0x8f,0x86,0x86,0xa5,0xdf,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf2,0xdc,0xd2,0xd2,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd5,0xd5,
+    0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd3,0xd1,0xd4,0xe6,0xfb,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x86,0x94,0x91,0x8b,0x8e,0x8d,0x8d,0x8d,0x8d,
+    0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8e,0x8c,0x8c,0x94,0x8d,0x90,0xce,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xec,0xd4,0xd4,0xd6,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,
+    0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd4,0xd5,0xd0,0xdd,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa7,0x88,0x95,0x91,0x8c,0x8f,0x8e,0x8e,0x8e,0x8e,
+    0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8f,0x8d,0x8d,0x95,0x8f,0x93,0xcf,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xec,0xd4,0xd2,0xd4,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,
+    0xd1,0xd1,0xd1,0xd1,0xd1,0xd0,0xd2,0xd3,0xcf,0xdc,0xf9,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbd,0x90,0x89,0x8e,0x91,0x91,0x91,0x91,0x91,0x91,
+    0x91,0x91,0x91,0x91,0x91,0x91,0x90,0x91,0x90,0x88,0x89,0xa9,0xe1,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf1,0xd9,0xce,0xce,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,
+    0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd0,0xce,0xd1,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfc,0xe0,0xb1,0x91,0x8f,0x91,0x8d,0x8d,0x8e,0x8e,0x8e,
+    0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8d,0x8f,0x91,0x8e,0xa1,0xd1,0xf6,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfa,0xea,0xd7,0xd0,0xd1,0xd0,0xcf,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,
+    0xd1,0xd1,0xd1,0xd1,0xd1,0xd2,0xd0,0xd2,0xdf,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe3,0xbd,0xa7,0x9e,0x9b,0x9b,0x9b,0x9b,0x9b,
+    0x9b,0x9b,0x9d,0x9d,0x9d,0x9d,0x9c,0x9c,0xa3,0xb4,0xd6,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfb,0xed,0xdf,0xd8,0xd4,0xd3,0xd4,0xd6,0xd6,0xd6,0xd6,0xd6,
+    0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xda,0xe3,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xdd,0xd5,0xd4,0xd4,0xd4,0xd4,0xd4,
+    0xd4,0xd4,0xd5,0xd5,0xd5,0xd5,0xd6,0xd7,0xda,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xff,0xff,0xfb,0xf4,0xef,0xed,0xec,0xec,0xed,0xee,0xee,0xee,0xee,
+    0xee,0xee,0xee,0xee,0xee,0xee,0xf1,0xf7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa,
+    0xfa,0xfb,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xe0,0xdd,0xe5,0xf3,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xf9,0xf1,0xee,0xf0,0xf7,0xfd,0xff,0xfe,0xfc,0xfc,0xfd,0xfe,0xfe,0xfd,
+    0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xc4,0xaf,0xa8,0xb0,0xcd,0xf1,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xf8,0xe5,0xd6,0xd3,0xd8,0xe3,0xf0,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfe,0xe0,0xb9,0xa5,0x9d,0x9a,0x98,0x98,0xa4,0xc4,0xed,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf4,0xde,0xce,0xc8,0xca,0xcc,0xce,0xd1,0xdb,0xf0,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa,
+    0xfb,0xff,0xff,0xf6,0xe3,0xca,0xae,0x97,0x8f,0x92,0x98,0x9a,0x99,0x92,0x9f,0xd7,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xe9,0xca,0xc4,0xc8,0xc9,0xc9,0xc7,0xc6,0xcb,0xd6,0xe5,0xf1,0xfb,0xff,0xff,0xfd,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfc,0xf1,0xd7,0xb1,0x96,0x92,0x9a,0x9a,0x99,0x97,0x97,0xa1,0x97,0x91,0xcb,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe2,0xc1,0xc6,0xcc,0xc7,0xc7,0xc9,0xcb,0xcc,0xc8,0xcb,0xd9,0xec,0xf8,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf7,0xd8,0xb2,0x9f,0x9a,0x95,0x94,0x99,0x9b,0x9b,0x98,0x98,0xa2,0x99,0x95,0xce,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe3,0xc2,0xc6,0xcb,0xc6,0xc7,0xc9,0xc9,0xca,0xc9,0xc9,0xcc,0xcf,0xda,0xec,0xfb,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf9,0xe1,
+    0xc7,0xac,0x94,0x8e,0x98,0x9d,0x9b,0x98,0x97,0x98,0x9c,0xa0,0x9b,0x96,0xaa,0xdf,
+    0xff,0xff,0xf9,0xfb,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfc,0xfc,0xfc,0xff,0xff,
+    0xed,0xce,0xc2,0xc5,0xca,0xca,0xc7,0xc6,0xc7,0xca,0xcc,0xca,0xc6,0xc9,0xd5,0xe3,
+    0xef,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf0,0xd2,0xab,
+    0x95,0x95,0x9a,0x9b,0x99,0x98,0x99,0x9b,0x9e,0x9d,0x9c,0x9a,0x98,0xa9,0xd1,0xf4,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xf9,0xe5,0xcc,0xc2,0xc4,0xc7,0xc8,0xc9,0xc8,0xc7,0xc7,0xc9,0xca,0xca,0xc9,0xc9,
+    0xd3,0xe8,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xdd,0xb6,0x9e,0x97,
+    0x97,0x98,0x9a,0x9b,0x9a,0x9a,0x9a,0x9b,0x9e,0x9f,0x99,0x97,0xad,0xd7,0xf7,0xff,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfa,0xe7,0xcf,0xc2,0xc4,0xc8,0xc9,0xc8,0xc7,0xc7,0xc8,0xc9,0xc9,0xc8,0xc8,
+    0xc8,0xce,0xda,0xed,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe9,0xb5,0x93,0x92,0x99,
+    0x9d,0x9b,0x97,0x97,0x9a,0x9f,0xa1,0x9b,0x93,0x99,0xac,0xc4,0xe0,0xf6,0xff,0xff,
+    0xfe,0xf7,0xe5,0xd6,0xd5,0xde,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,
+    0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xe8,0xe2,0xe3,0xee,0xfb,0xff,
+    0xff,0xff,0xfa,0xee,0xdd,0xce,0xc4,0xc1,0xc6,0xca,0xca,0xc7,0xc6,0xc7,0xca,0xcb,
+    0xc9,0xc7,0xc7,0xd8,0xf3,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xcc,0x9d,0x93,0x9d,0x9a,
+    0x96,0x9a,0x9b,0x9d,0x9c,0x9a,0x9a,0x9d,0xa2,0xb4,0xd8,0xfa,0xff,0xfd,0xfc,0xff,
+    0xf8,0xd8,0xb5,0xa3,0xa1,0xaa,0xc9,0xf2,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd7,0xc2,0xbd,0xbf,0xcd,0xe6,0xfb,
+    0xff,0xfd,0xfe,0xff,0xfc,0xe7,0xd2,0xc8,0xc5,0xc5,0xc7,0xc8,0xca,0xc9,0xc8,0xc6,
+    0xc8,0xcb,0xc6,0xcc,0xe4,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xba,0x9b,0x97,0x9b,0x99,
+    0x98,0x9b,0x9d,0x9e,0x9d,0x99,0x9b,0xb2,0xd7,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf9,
+    0xda,0xaf,0x9d,0xa3,0xa2,0x9f,0xae,0xd1,0xf3,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf5,0xdb,0xc2,0xb9,0xbc,0xbe,0xbc,0xca,0xe7,
+    0xfb,0xfe,0xff,0xff,0xff,0xfc,0xf6,0xe7,0xd2,0xc4,0xc4,0xc8,0xca,0xc9,0xc7,0xc6,
+    0xc7,0xc8,0xc6,0xc9,0xdb,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xba,0x9e,0x9a,0x9e,0x9e,
+    0x9f,0xa1,0x9b,0x94,0x9b,0xb1,0xc9,0xe2,0xfb,0xff,0xfe,0xfc,0xfb,0xff,0xff,0xe4,
+    0xb5,0x9f,0xa1,0xaa,0xab,0xa6,0xa5,0xb9,0xe5,0xff,0xff,0xfd,0xff,0xf9,0xeb,0xe0,
+    0xe1,0xed,0xfa,0xff,0xfd,0xff,0xff,0xea,0xc8,0xba,0xbc,0xc0,0xc2,0xbe,0xbd,0xcd,
+    0xed,0xff,0xff,0xfc,0xfd,0xfe,0xff,0xfd,0xee,0xdf,0xd0,0xc5,0xc3,0xc7,0xca,0xca,
+    0xca,0xc9,0xc8,0xcb,0xda,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xce,0xa1,0x93,0x9b,0x9b,
+    0x97,0x9a,0x9f,0xa5,0xba,0xdd,0xf8,0xff,0xff,0xfd,0xfe,0xff,0xfc,0xff,0xfc,0xca,
+    0x9f,0xa2,0xa9,0xa2,0xa3,0xa8,0xa6,0xb2,0xe0,0xff,0xff,0xfe,0xfa,0xde,0xc2,0xb6,
+    0xb7,0xc6,0xe2,0xfa,0xfe,0xff,0xff,0xe6,0xc1,0xba,0xbd,0xba,0xbc,0xc2,0xbe,0xbc,
+    0xdb,0xfd,0xff,0xfd,0xff,0xff,0xfd,0xfe,0xff,0xfb,0xec,0xd7,0xcc,0xc8,0xc6,0xc5,
+    0xc8,0xc8,0xc4,0xcc,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xba,0x98,0x96,0x98,
+    0x94,0x9c,0xb7,0xdd,0xf5,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfb,0xdc,0xb4,
+    0xa1,0xa4,0xa6,0xa3,0xa4,0xa7,0xa9,0xb7,0xe3,0xff,0xff,0xf8,0xe1,0xbb,0xab,0xb2,
+    0xb2,0xae,0xc1,0xe4,0xf9,0xff,0xff,0xe8,0xc5,0xbb,0xbc,0xbc,0xbc,0xc0,0xbf,0xbe,
+    0xcb,0xe7,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xf9,0xec,0xd6,0xc5,0xc2,
+    0xc6,0xc6,0xc6,0xd9,0xf6,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe6,0xcd,0xbe,0xb8,
+    0xbe,0xcf,0xe6,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf2,0xbc,0xa2,
+    0xa6,0xa4,0xa2,0xa7,0xa8,0xa4,0xab,0xc9,0xed,0xff,0xff,0xed,0xc8,0xad,0xab,0xb5,
+    0xb5,0xad,0xb4,0xcf,0xf0,0xff,0xff,0xf0,0xd4,0xbd,0xb8,0xbc,0xbd,0xbb,0xbe,0xc1,
+    0xbe,0xd1,0xf6,0xff,0xfc,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf0,0xe3,0xd9,
+    0xd6,0xda,0xe3,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf3,0xe9,
+    0xf2,0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xdb,0xab,0x9c,
+    0xa6,0xa7,0xa2,0xa8,0xaa,0xa2,0xb4,0xe3,0xfb,0xff,0xff,0xe2,0xb8,0xae,0xb1,0xb0,
+    0xb0,0xb3,0xb6,0xc1,0xe6,0xff,0xff,0xfb,0xe8,0xc3,0xb5,0xbd,0xbe,0xbb,0xbe,0xc0,
+    0xba,0xc5,0xe6,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,0xf7,
+    0xf1,0xf7,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe3,0xb6,0xa4,0xa3,
+    0xa4,0xa5,0xa4,0xa7,0xa9,0xaa,0xc7,0xf6,0xff,0xff,0xff,0xe1,0xb4,0xad,0xb2,0xb0,
+    0xb0,0xb5,0xb4,0xba,0xe2,0xff,0xff,0xff,0xf7,0xd0,0xba,0xbc,0xbd,0xbb,0xbe,0xbf,
+    0xbe,0xbf,0xce,0xed,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff,
+    0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xfd,0xce,0xa1,0xa2,0xa9,
+    0xa4,0xa4,0xa9,0xa4,0xa2,0xbc,0xe6,0xfe,0xfd,0xff,0xff,0xe0,0xb3,0xab,0xb1,0xb0,
+    0xb0,0xb4,0xb2,0xb9,0xe2,0xff,0xff,0xfd,0xfe,0xea,0xca,0xb8,0xb9,0xbe,0xbc,0xbd,
+    0xc2,0xbe,0xbf,0xde,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe,
+    0xff,0xfe,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe1,0xb8,0xa1,0xa2,0xa7,
+    0xa4,0xa4,0xab,0xa5,0xa6,0xd2,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb4,0xac,0xb2,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xf9,0xff,0xff,0xdb,0xb8,0xb8,0xbf,0xbb,0xbc,
+    0xbf,0xbe,0xbe,0xcd,0xe9,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xbf,0xa2,0xa6,0xa6,0xa4,
+    0xa6,0xa6,0xa7,0xab,0xbf,0xe7,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb4,0xac,0xb2,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfa,0xff,0xff,0xea,0xca,0xbd,0xbc,0xbc,0xbd,
+    0xbc,0xbe,0xc0,0xbf,0xd3,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xad,0x9c,0xa7,0xa7,0xa3,
+    0xa9,0xa9,0xa0,0xaf,0xdc,0xfb,0xff,0xff,0xfc,0xff,0xff,0xdf,0xb4,0xad,0xb2,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfa,0xe3,0xc0,0xb5,0xbd,0xbe,
+    0xbb,0xbe,0xbf,0xba,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xc0,0xa8,0xa3,0xa6,0xa6,0xa3,
+    0xa7,0xa9,0xa5,0xbe,0xf1,0xff,0xfd,0xff,0xfd,0xff,0xff,0xdf,0xb4,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xfe,0xff,0xf4,0xcc,0xb8,0xbd,0xbe,
+    0xbc,0xbd,0xbd,0xbd,0xc1,0xd2,0xef,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd4,0xa4,0xa3,0xa9,0xa4,0xa5,0xa7,
+    0xa4,0xa5,0xb8,0xdd,0xfb,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe4,0xc7,0xb9,0xbb,
+    0xbe,0xbe,0xbe,0xc2,0xbd,0xbe,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf7,0xc6,0x9d,0xa0,0xa8,0xa2,0xa3,0xaa,
+    0xa5,0xa5,0xcd,0xfa,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb2,0xba,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xfa,0xd7,0xb8,0xba,
+    0xbe,0xba,0xbc,0xc2,0xbb,0xba,0xd8,0xfa,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xeb,0xbf,0xa1,0xa4,0xa8,0xa4,0xa6,0xa8,
+    0xa7,0xb7,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0,
+    0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe8,0xc5,0xba,
+    0xbc,0xbd,0xbc,0xc0,0xbe,0xbd,0xd3,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xed,0xc1,0xa1,0xa4,0xa8,0xa7,0xa9,0xa1,
+    0xad,0xd8,0xf9,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0,
+    0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfa,0xe0,0xbe,
+    0xb7,0xbf,0xbd,0xbe,0xbd,0xbb,0xd4,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xcf,0xa2,0x9d,0xa5,0xa3,0xa0,0xa3,
+    0xc4,0xf2,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0,
+    0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd1,
+    0xb8,0xb7,0xba,0xbc,0xb7,0xbb,0xdd,0xfe,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe8,0xc1,0xad,0xab,0xa8,0xa9,0xc2,
+    0xe7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0,
+    0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xec,
+    0xcf,0xbd,0xbe,0xc1,0xc2,0xd0,0xee,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf0,0xd9,0xc7,0xc7,0xd6,0xee,
+    0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0,
+    0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe,
+    0xf1,0xdf,0xd5,0xd6,0xe3,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xed,0xed,0xf8,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb6,0xb0,0xb3,0xaf,
+    0xaf,0xb3,0xb2,0xbb,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfa,0xf1,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xbf,0xaf,0xb0,0xb2,
+    0xb2,0xb1,0xb2,0xc1,0xe7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb4,0xac,0xb5,
+    0xb5,0xad,0xb4,0xd1,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xec,0xc7,0xaf,0xac,
+    0xad,0xb1,0xc8,0xeb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe7,0xce,0xc0,
+    0xc1,0xd0,0xe7,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf5,
+    0xf5,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0,
+    0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x94,
+    0x94,0xad,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xa1,0x76,0x6f,
+    0x6f,0x76,0x9f,0xdc,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xb4,0x7c,0x6d,0x7d,
+    0x7d,0x6d,0x78,0xac,0xe8,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xff,0xfe,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd5,0x90,0x75,0x75,0x79,
+    0x79,0x75,0x74,0x8e,0xd4,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf7,0xe7,0xe6,0xf6,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x82,0x76,0x7a,0x72,
+    0x72,0x79,0x75,0x83,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xeb,0xca,0xae,0xac,0xc4,0xe7,
+    0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x76,
+    0x76,0x79,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xf2,0xed,0xed,0xf3,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa4,0x86,0x82,0x7e,0x81,0xa6,
+    0xdd,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x75,
+    0x75,0x78,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xf8,
+    0xec,0xe4,0xe3,0xe3,0xe5,0xec,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfc,0xbc,0x78,0x70,0x79,0x75,0x70,0x77,
+    0xa8,0xec,0xff,0xfc,0xfe,0xff,0xff,0xff,0xf9,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74,
+    0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfb,0xed,
+    0xe2,0xe0,0xe1,0xe1,0xdf,0xe1,0xf0,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xe7,0xa7,0x76,0x7a,0x7e,0x7d,0x80,0x72,
+    0x81,0xc4,0xf6,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfd,0xf2,0xe4,
+    0xe1,0xe3,0xe2,0xe2,0xe1,0xe0,0xeb,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe2,0xa4,0x78,0x79,0x7f,0x7b,0x7c,0x7c,
+    0x79,0x91,0xd4,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xff,0xf5,0xe7,0xe3,
+    0xe3,0xe2,0xe2,0xe3,0xe1,0xe1,0xeb,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xf2,0xad,0x72,0x74,0x80,0x77,0x76,0x81,
+    0x79,0x77,0xb4,0xf8,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xfe,0xef,0xe2,0xe2,
+    0xe4,0xe1,0xe1,0xe3,0xe0,0xdf,0xec,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xc1,0x7c,0x7a,0x82,0x79,0x7a,0x7e,
+    0x78,0x77,0x93,0xcb,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf3,0xe8,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa0,0x81,0x7c,0x7d,0x7c,0x78,
+    0x7c,0x7e,0x76,0x9e,0xef,0xff,0xfc,0xfe,0xfc,0xff,0xff,0xca,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfc,0xea,0xe1,0xe4,0xe3,
+    0xe1,0xe2,0xe2,0xe2,0xe2,0xe9,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xd0,0x89,0x72,0x80,0x7e,0x77,
+    0x7f,0x7f,0x6f,0x88,0xd0,0xfb,0xfe,0xff,0xfa,0xff,0xff,0xc9,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xfe,0xf5,0xe5,0xdf,0xe3,0xe3,
+    0xe1,0xe3,0xe3,0xe0,0xe5,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x7b,0x7f,0x7d,0x79,
+    0x7c,0x7b,0x7a,0x7e,0x9d,0xda,0xff,0xff,0xf6,0xff,0xff,0xc9,0x81,0x74,0x79,0x74,
+    0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xf6,0xea,0xe4,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe3,0xe1,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0x99,0x79,0x7a,0x7e,
+    0x79,0x79,0x81,0x76,0x77,0xba,0xff,0xff,0xf4,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74,
+    0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf8,0xff,0xff,0xf0,0xe2,0xe1,0xe3,0xe2,0xe2,
+    0xe3,0xe2,0xe1,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff,
+    0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfa,0xb9,0x7b,0x7b,0x83,
+    0x79,0x79,0x7f,0x77,0x73,0x99,0xda,0xfd,0xfc,0xff,0xff,0xcb,0x81,0x72,0x78,0x75,
+    0x75,0x78,0x71,0x7f,0xca,0xff,0xff,0xfa,0xff,0xf8,0xe9,0xe1,0xe1,0xe3,0xe1,0xe1,
+    0xe4,0xe2,0xe0,0xee,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xd9,0x9a,0x7f,0x7d,
+    0x7d,0x7c,0x79,0x7c,0x7d,0x7c,0xa8,0xf1,0xff,0xff,0xff,0xcc,0x80,0x75,0x7a,0x73,
+    0x73,0x79,0x74,0x80,0xca,0xff,0xff,0xfc,0xfe,0xef,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe1,0xe7,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xf1,0xe4,
+    0xf0,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x89,0x74,
+    0x80,0x7e,0x76,0x7e,0x7f,0x70,0x8a,0xd4,0xf9,0xff,0xff,0xce,0x86,0x77,0x79,0x74,
+    0x74,0x78,0x77,0x8a,0xd0,0xff,0xff,0xfc,0xfa,0xe8,0xe0,0xe4,0xe3,0xe1,0xe3,0xe3,
+    0xdf,0xe3,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb,
+    0xf8,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xdd,0xc2,0xb2,0xa9,
+    0xaf,0xc6,0xe2,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xec,0xa2,0x7c,
+    0x81,0x7c,0x77,0x7d,0x7d,0x74,0x7f,0xad,0xe4,0xff,0xff,0xde,0xa0,0x74,0x6e,0x7d,
+    0x7d,0x6e,0x73,0xa0,0xe1,0xff,0xff,0xf9,0xf1,0xe5,0xe0,0xe3,0xe2,0xe1,0xe3,0xe3,
+    0xe0,0xe8,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8,
+    0xe7,0xe9,0xed,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xe7,0xa4,0x7b,0x7d,0x80,
+    0x79,0x82,0xa7,0xd7,0xf3,0xfc,0xfe,0xff,0xfe,0xff,0xfd,0xfe,0xff,0xfa,0xcf,0x96,
+    0x7a,0x7c,0x7e,0x79,0x79,0x7c,0x7b,0x8f,0xd3,0xff,0xff,0xf4,0xcd,0x8c,0x6c,0x76,
+    0x76,0x6c,0x89,0xc9,0xf7,0xff,0xff,0xf8,0xeb,0xe4,0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,
+    0xe6,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf5,0xe8,0xdd,0xd9,
+    0xda,0xda,0xda,0xe5,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8d,0x7b,0x83,0x84,
+    0x7d,0x7f,0x87,0x90,0xaa,0xd6,0xf7,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfa,0xb5,
+    0x78,0x7c,0x84,0x78,0x77,0x7e,0x79,0x87,0xce,0xff,0xff,0xfe,0xf5,0xc6,0x93,0x7a,
+    0x7a,0x92,0xc3,0xf2,0xff,0xff,0xff,0xf7,0xe8,0xe3,0xe3,0xe2,0xe1,0xe4,0xe2,0xe0,
+    0xed,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xfd,0xf5,0xe9,0xe0,0xdd,0xdb,0xda,
+    0xdb,0xdc,0xd9,0xde,0xee,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe3,0xae,0x8c,0x86,0x89,0x89,
+    0x8a,0x89,0x81,0x78,0x82,0x9f,0xbf,0xde,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xd8,
+    0x96,0x79,0x7c,0x83,0x81,0x7c,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xf3,0xd7,0xc2,
+    0xc2,0xd7,0xf1,0xfe,0xff,0xff,0xff,0xf7,0xe9,0xe2,0xe2,0xe3,0xe3,0xe1,0xdf,0xe5,
+    0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe5,0xdc,0xd9,0xdb,0xde,0xde,
+    0xdd,0xdc,0xdb,0xdd,0xe6,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe6,0xaf,0x88,0x81,0x86,0x84,
+    0x82,0x83,0x85,0x87,0x85,0x7e,0x81,0x9f,0xcd,0xee,0xfa,0xfe,0xff,0xff,0xfe,0xf7,
+    0xcd,0x91,0x77,0x7d,0x7a,0x73,0x86,0xb7,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfa,0xef,0xe4,0xe0,0xe2,0xe2,0xdf,0xe4,0xf2,
+    0xfd,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf1,0xe6,0xdc,0xda,0xdc,0xdd,0xdc,0xdb,0xda,
+    0xda,0xdb,0xda,0xdc,0xe7,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8b,0x7e,0x89,0x85,
+    0x80,0x84,0x86,0x88,0x86,0x81,0x7f,0x82,0x8b,0xa4,0xd0,0xf9,0xff,0xfc,0xfb,0xff,
+    0xf5,0xc7,0x96,0x7d,0x78,0x82,0xad,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xec,0xe3,0xe1,0xe0,0xe5,0xf1,0xfc,
+    0xff,0xff,0xfe,0xff,0xfd,0xf3,0xe7,0xe0,0xdc,0xdb,0xdb,0xdd,0xdd,0xdd,0xdc,0xd9,
+    0xdb,0xdd,0xd9,0xdc,0xec,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa6,0x7f,0x7e,0x86,
+    0x8a,0x88,0x83,0x81,0x83,0x87,0x8a,0x82,0x77,0x80,0x9a,0xb8,0xdb,0xf6,0xff,0xff,
+    0xfe,0xf2,0xd8,0xc2,0xbf,0xca,0xe2,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8,
+    0xf8,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf3,0xf0,0xf0,0xf6,0xfd,0xff,
+    0xff,0xff,0xfd,0xf5,0xec,0xe5,0xde,0xda,0xdb,0xdd,0xdd,0xdb,0xda,0xda,0xdc,0xdd,
+    0xdb,0xd8,0xd8,0xe4,0xf6,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd6,0xa7,0x8b,0x85,
+    0x84,0x83,0x86,0x88,0x85,0x83,0x84,0x85,0x87,0x86,0x7d,0x7b,0x99,0xcf,0xf5,0xff,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf3,0xe4,0xdb,0xdc,0xde,0xdd,0xdc,0xdc,0xdb,0xdb,0xdc,0xdc,0xda,0xda,
+    0xda,0xdb,0xe3,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xc9,0x9b,
+    0x82,0x83,0x88,0x88,0x85,0x83,0x84,0x86,0x87,0x84,0x81,0x7f,0x7d,0x94,0xc8,0xf3,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfb,0xf0,0xe3,0xdc,0xdb,0xdc,0xdd,0xdd,0xdb,0xda,0xda,0xdb,0xdc,0xdc,0xd9,0xd8,
+    0xe0,0xee,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf7,0xd8,
+    0xb9,0x9c,0x82,0x7b,0x85,0x8a,0x88,0x83,0x81,0x82,0x86,0x89,0x81,0x7a,0x96,0xd8,
+    0xff,0xff,0xf8,0xf9,0xf9,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff,
+    0xf5,0xe3,0xdb,0xdc,0xdd,0xdb,0xdb,0xdb,0xdb,0xdc,0xdc,0xda,0xd7,0xd9,0xe0,0xe9,
+    0xf3,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,
+    0xf5,0xd2,0xa5,0x8e,0x89,0x84,0x83,0x87,0x87,0x87,0x83,0x82,0x8c,0x80,0x7b,0xc2,
+    0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xef,0xdb,0xdb,0xde,0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xda,0xda,0xdb,0xe3,0xf1,0xfc,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfe,0xfb,0xef,0xd0,0xa5,0x85,0x80,0x89,0x8a,0x88,0x83,0x82,0x8d,0x80,0x78,0xbf,
+    0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xed,0xd9,0xda,0xdd,0xda,0xdb,0xdb,0xda,0xd9,0xd8,0xda,0xe3,0xef,0xf9,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,
+    0xfb,0xff,0xff,0xf4,0xdc,0xbf,0xa0,0x87,0x7d,0x7f,0x85,0x87,0x86,0x7c,0x88,0xcd,
+    0xff,0xff,0xf9,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf0,0xdc,0xd8,0xda,0xda,0xda,0xd7,0xd5,0xd8,0xe1,0xeb,0xf4,0xfc,0xff,0xff,0xfd,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfd,0xda,0xad,0x94,0x8b,0x88,0x87,0x84,0x91,0xb7,0xe8,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf8,0xea,0xdf,0xda,0xd9,0xda,0xdb,0xde,0xe6,0xf3,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xda,0xb9,0xa0,0x98,0xa0,0xc2,0xee,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xee,0xe2,0xdf,0xe1,0xea,0xf4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa,
+    0xfa,0xfc,0xfd,0xfb,0xf7,0xf7,0xfc,0xff,0xfa,0xea,0xd8,0xd4,0xdd,0xee,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xfa,0xf5,0xf1,0xf3,0xf9,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfd,
+    0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xed,0xdf,0xd7,0xd6,0xd6,0xd6,0xd6,0xd6,
+    0xd6,0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xd8,0xdc,0xe7,0xf5,0xfd,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,0xf3,0xf1,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,
+    0xef,0xef,0xef,0xef,0xef,0xf0,0xf3,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe3,0xbc,0xa6,0x9d,0x9a,0x9a,0x9a,0x9a,0x9a,
+    0x9a,0x9a,0x9a,0x9a,0x9a,0x99,0x98,0x9a,0xa2,0xb3,0xd5,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xee,0xe3,0xde,0xda,0xda,0xda,0xda,0xda,0xda,0xda,0xd9,
+    0xd8,0xd8,0xd8,0xd8,0xd8,0xd9,0xdd,0xe6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfd,0xe2,0xb2,0x91,0x8e,0x90,0x8c,0x8c,0x8d,0x8d,0x8d,
+    0x8c,0x8c,0x8d,0x8d,0x8c,0x8c,0x8b,0x8d,0x8f,0x8b,0x9f,0xd0,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xeb,0xda,0xd4,0xd5,0xd4,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,
+    0xd3,0xd3,0xd3,0xd3,0xd3,0xd4,0xd4,0xd5,0xe1,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbb,0x8e,0x87,0x8d,0x90,0x90,0x90,0x90,0x90,0x90,
+    0x90,0x90,0x90,0x90,0x90,0x90,0x8f,0x90,0x8f,0x86,0x86,0xa5,0xdf,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf2,0xdc,0xd2,0xd2,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,
+    0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd3,0xd1,0xd4,0xe6,0xfb,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x86,0x94,0x91,0x8b,0x8e,0x8d,0x8d,0x8d,0x8d,
+    0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8e,0x8c,0x8c,0x94,0x8d,0x90,0xce,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xec,0xd4,0xd4,0xd6,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,
+    0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd4,0xd5,0xd0,0xdd,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa7,0x88,0x95,0x91,0x8c,0x8f,0x8e,0x8e,0x8e,0x8e,
+    0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8f,0x8d,0x8d,0x95,0x8f,0x93,0xcf,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xec,0xd4,0xd2,0xd4,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,
+    0xd1,0xd1,0xd1,0xd1,0xd1,0xd0,0xd2,0xd3,0xcf,0xdc,0xf9,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbd,0x90,0x89,0x8e,0x91,0x91,0x91,0x91,0x91,0x91,
+    0x91,0x91,0x91,0x91,0x91,0x91,0x90,0x91,0x90,0x88,0x89,0xa9,0xe1,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf1,0xd9,0xce,0xce,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,
+    0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd0,0xce,0xd1,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfc,0xe0,0xb1,0x91,0x8f,0x91,0x8d,0x8d,0x8e,0x8e,0x8e,
+    0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8d,0x8f,0x91,0x8e,0xa1,0xd1,0xf6,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfa,0xea,0xd7,0xd0,0xd1,0xd0,0xcf,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,
+    0xd1,0xd1,0xd1,0xd1,0xd1,0xd2,0xd0,0xd2,0xdf,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe3,0xbd,0xa7,0x9e,0x9b,0x9b,0x9b,0x9b,0x9b,
+    0x9c,0x9d,0x9d,0x9d,0x9d,0x9d,0x9c,0x9c,0xa3,0xb4,0xd6,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfb,0xed,0xdf,0xd8,0xd4,0xd3,0xd4,0xd6,0xd6,0xd6,0xd6,0xd6,
+    0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xda,0xe3,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xdd,0xd5,0xd4,0xd4,0xd4,0xd4,0xd4,
+    0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd6,0xd7,0xda,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xff,0xff,0xfb,0xf4,0xef,0xed,0xec,0xec,0xed,0xee,0xee,0xee,0xee,
+    0xee,0xee,0xee,0xee,0xee,0xee,0xf1,0xf7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa,
+    0xfa,0xfb,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xee,0xe1,0xdd,0xe5,0xf3,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xf9,0xf1,0xee,0xf0,0xf7,0xfd,0xff,0xfe,0xfc,0xfc,0xfd,0xfe,0xfe,0xfd,
+    0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xc4,0xaf,0xa8,0xb0,0xcd,0xf1,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xf8,0xe5,0xd6,0xd3,0xd8,0xe2,0xef,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfe,0xe0,0xb9,0xa5,0x9d,0x9a,0x99,0x97,0xa3,0xc4,0xed,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf5,0xde,0xcd,0xc8,0xca,0xcc,0xcd,0xd1,0xdc,0xef,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa,
+    0xfb,0xff,0xff,0xf6,0xe3,0xca,0xae,0x97,0x8f,0x92,0x98,0x9a,0x99,0x93,0x9f,0xd7,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xe9,0xca,0xc4,0xc8,0xc9,0xc9,0xc8,0xc7,0xcb,0xd6,0xe5,0xf1,0xfb,0xff,0xff,0xfd,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfc,0xf2,0xd7,0xb1,0x96,0x93,0x9a,0x9a,0x9a,0x98,0x97,0xa2,0x99,0x92,0xcb,
+    0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe3,0xc2,0xc6,0xcb,0xc6,0xc8,0xcb,0xcc,0xcc,0xc8,0xcb,0xd9,0xec,0xf8,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf7,0xd9,0xb4,0xa0,0x9a,0x95,0x95,0x9a,0x9b,0x9b,0x99,0x98,0xa1,0x99,0x96,0xce,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe3,0xc2,0xc6,0xcb,0xc6,0xc7,0xc9,0xc9,0xca,0xc9,0xc9,0xcc,0xcf,0xda,0xec,0xfb,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf9,0xe1,
+    0xc7,0xac,0x94,0x8f,0x98,0x9d,0x9b,0x98,0x97,0x98,0x9c,0xa1,0x9b,0x95,0xaa,0xdf,
+    0xff,0xff,0xf9,0xfc,0xfc,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfc,0xfc,0xff,0xff,
+    0xec,0xcd,0xc2,0xc6,0xcb,0xc9,0xc6,0xc6,0xc7,0xca,0xcc,0xca,0xc6,0xc9,0xd5,0xe3,
+    0xef,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf0,0xd2,0xab,
+    0x95,0x95,0x99,0x9b,0x9a,0x98,0x99,0x9b,0x9e,0x9d,0x9c,0x9a,0x98,0xa9,0xd1,0xf4,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xf9,0xe5,0xcc,0xc3,0xc6,0xc7,0xc8,0xc9,0xc8,0xc7,0xc7,0xc9,0xca,0xca,0xc9,0xca,
+    0xd4,0xe8,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xdd,0xb6,0x9e,0x97,
+    0x97,0x97,0x99,0x9a,0x9a,0x9a,0x9a,0x9b,0x9e,0x9f,0x99,0x97,0xad,0xd7,0xf7,0xff,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfa,0xe7,0xcf,0xc2,0xc4,0xc8,0xc9,0xc8,0xc7,0xc7,0xc8,0xc9,0xc9,0xc8,0xca,
+    0xcb,0xce,0xda,0xee,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe9,0xb5,0x93,0x92,0x99,
+    0x9d,0x9b,0x98,0x98,0x9a,0x9f,0xa2,0x9b,0x93,0x99,0xac,0xc4,0xe0,0xf6,0xff,0xff,
+    0xfe,0xf7,0xe5,0xd7,0xd6,0xdf,0xed,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfc,
+    0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xe8,0xe2,0xe3,0xee,0xfb,0xff,
+    0xff,0xff,0xfa,0xee,0xdd,0xce,0xc3,0xc2,0xc7,0xcb,0xc9,0xc7,0xc7,0xc7,0xc9,0xcb,
+    0xca,0xc7,0xc8,0xda,0xf3,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xcc,0x9d,0x92,0x9d,0x9a,
+    0x96,0x9a,0x9d,0x9e,0x9c,0x9a,0x9b,0x9c,0xa2,0xb4,0xd8,0xfa,0xff,0xfd,0xfc,0xff,
+    0xf8,0xd8,0xb4,0xa4,0xa3,0xab,0xc9,0xf2,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd7,0xc2,0xbd,0xbf,0xcc,0xe6,0xfb,
+    0xff,0xfd,0xfe,0xff,0xfc,0xe8,0xd3,0xc8,0xc6,0xc6,0xc7,0xc9,0xc9,0xc9,0xc8,0xc6,
+    0xc8,0xcb,0xc7,0xcd,0xe5,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xba,0x9b,0x97,0x9b,0x99,
+    0x98,0x9b,0x9d,0x9e,0x9d,0x99,0x9b,0xb2,0xd7,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf9,
+    0xdb,0xb1,0x9e,0xa3,0xa3,0xa0,0xae,0xd2,0xf4,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf5,0xdb,0xc2,0xb9,0xbc,0xbe,0xbc,0xca,0xe7,
+    0xfb,0xfe,0xff,0xff,0xff,0xfc,0xf6,0xe7,0xd1,0xc5,0xc6,0xc8,0xca,0xc9,0xc7,0xc6,
+    0xc7,0xc8,0xc7,0xcb,0xdb,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xba,0x9e,0x9a,0x9e,0x9e,
+    0x9f,0xa1,0x9b,0x94,0x9b,0xb1,0xc9,0xe2,0xfb,0xff,0xfe,0xfc,0xfb,0xff,0xff,0xe4,
+    0xb6,0xa0,0xa2,0xa9,0xaa,0xa6,0xa6,0xb9,0xe5,0xff,0xff,0xfd,0xff,0xf9,0xeb,0xe0,
+    0xe0,0xec,0xfa,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbd,0xc1,0xc2,0xbe,0xbe,0xcd,
+    0xec,0xff,0xff,0xfc,0xfd,0xfe,0xff,0xfd,0xee,0xe0,0xd2,0xc5,0xc2,0xc7,0xca,0xca,
+    0xca,0xc9,0xc8,0xcb,0xda,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xce,0xa1,0x93,0x9b,0x9b,
+    0x97,0x9a,0x9f,0xa5,0xba,0xdd,0xf8,0xff,0xff,0xfd,0xfe,0xff,0xfc,0xff,0xfc,0xca,
+    0x9f,0xa2,0xa9,0xa2,0xa3,0xa9,0xa8,0xb2,0xe0,0xff,0xff,0xfe,0xfa,0xde,0xc2,0xb6,
+    0xb7,0xc6,0xe2,0xfa,0xfe,0xff,0xff,0xe6,0xc2,0xbb,0xbe,0xbb,0xbc,0xc3,0xbf,0xbc,
+    0xdc,0xfe,0xff,0xfd,0xff,0xff,0xfd,0xfe,0xff,0xfb,0xec,0xd7,0xcc,0xc8,0xc6,0xc5,
+    0xc8,0xc8,0xc4,0xcc,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xba,0x98,0x96,0x98,
+    0x94,0x9c,0xb7,0xdd,0xf5,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfb,0xdd,0xb4,
+    0xa1,0xa5,0xa7,0xa3,0xa4,0xa7,0xa9,0xb7,0xe3,0xff,0xff,0xf8,0xe1,0xbb,0xaa,0xb2,
+    0xb2,0xaf,0xc2,0xe5,0xfa,0xff,0xff,0xe9,0xc6,0xbb,0xbc,0xbc,0xbc,0xc0,0xbf,0xbd,
+    0xcd,0xe9,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xf9,0xec,0xd6,0xc5,0xc2,
+    0xc6,0xc6,0xc6,0xd9,0xf6,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe6,0xcd,0xbe,0xb8,
+    0xbe,0xcf,0xe6,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf2,0xbd,0xa2,
+    0xa6,0xa5,0xa3,0xa7,0xa8,0xa4,0xab,0xc9,0xed,0xff,0xff,0xed,0xc8,0xad,0xab,0xb5,
+    0xb5,0xae,0xb5,0xce,0xef,0xff,0xff,0xf1,0xd4,0xbd,0xb8,0xbd,0xbe,0xbb,0xbe,0xc1,
+    0xbf,0xd2,0xf6,0xff,0xfc,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf0,0xe3,0xd9,
+    0xd6,0xda,0xe3,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf3,0xe9,
+    0xf2,0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xdb,0xab,0x9c,
+    0xa6,0xa7,0xa2,0xa8,0xaa,0xa2,0xb4,0xe3,0xfb,0xff,0xff,0xe2,0xb8,0xae,0xb1,0xb0,
+    0xb0,0xb4,0xb7,0xc2,0xe7,0xff,0xff,0xfb,0xe7,0xc3,0xb6,0xbe,0xbe,0xbb,0xbe,0xc0,
+    0xba,0xc5,0xe6,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,0xf7,
+    0xf1,0xf7,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe3,0xb6,0xa4,0xa3,
+    0xa4,0xa6,0xa5,0xa7,0xa9,0xaa,0xc7,0xf6,0xff,0xff,0xff,0xe1,0xb3,0xac,0xb2,0xb0,
+    0xb0,0xb5,0xb5,0xbc,0xe3,0xff,0xff,0xff,0xf7,0xd2,0xbc,0xbd,0xbd,0xbb,0xbe,0xbf,
+    0xbf,0xc0,0xce,0xed,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff,
+    0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xfd,0xce,0xa1,0xa2,0xa9,
+    0xa4,0xa5,0xa9,0xa4,0xa3,0xbd,0xe6,0xfe,0xfd,0xff,0xff,0xe1,0xb5,0xad,0xb1,0xb0,
+    0xb1,0xb5,0xb3,0xbb,0xe3,0xff,0xff,0xfd,0xfe,0xeb,0xca,0xb7,0xb9,0xbe,0xbc,0xbd,
+    0xc2,0xbe,0xbf,0xde,0xfd,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe,
+    0xff,0xfe,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe1,0xb8,0xa1,0xa3,0xa7,
+    0xa4,0xa4,0xab,0xa6,0xa8,0xd3,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb5,0xaf,0xb3,0xb0,
+    0xb1,0xb5,0xb3,0xba,0xe1,0xff,0xff,0xf9,0xff,0xff,0xdb,0xb8,0xb9,0xc0,0xbc,0xbc,
+    0xbf,0xbe,0xbe,0xcd,0xe9,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xbf,0xa2,0xa6,0xa6,0xa4,
+    0xa6,0xa6,0xa7,0xab,0xbf,0xe7,0xff,0xff,0xf9,0xff,0xff,0xe0,0xb5,0xad,0xb2,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfa,0xff,0xff,0xea,0xca,0xbd,0xbc,0xbc,0xbd,
+    0xbd,0xc0,0xc0,0xbe,0xd3,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xad,0x9c,0xa7,0xa6,0xa3,
+    0xa9,0xa9,0xa1,0xb0,0xdc,0xfa,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb2,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfa,0xe3,0xc0,0xb5,0xbd,0xbe,
+    0xbb,0xbf,0xc0,0xbb,0xc7,0xe9,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xc0,0xa8,0xa3,0xa6,0xa6,0xa3,
+    0xa6,0xa9,0xa6,0xc0,0xf2,0xff,0xfe,0xff,0xfd,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xfe,0xff,0xf4,0xcc,0xb8,0xbd,0xbe,
+    0xbc,0xbe,0xbf,0xbf,0xc2,0xd2,0xef,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd4,0xa4,0xa3,0xa9,0xa4,0xa5,0xa7,
+    0xa4,0xa5,0xb8,0xdd,0xfb,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe4,0xc7,0xb9,0xbb,
+    0xbe,0xbc,0xbc,0xc0,0xbd,0xc0,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf7,0xc7,0x9d,0x9f,0xa9,0xa3,0xa2,0xa9,
+    0xa6,0xa5,0xcc,0xf9,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xfa,0xd7,0xb8,0xba,
+    0xbf,0xbb,0xbb,0xc0,0xbc,0xbb,0xd8,0xfa,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xeb,0xbf,0xa1,0xa4,0xa8,0xa5,0xa6,0xa8,
+    0xa7,0xb7,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe8,0xc5,0xba,
+    0xbd,0xbe,0xbc,0xc0,0xbe,0xbc,0xd2,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xed,0xc1,0xa1,0xa4,0xa7,0xa8,0xab,0xa3,
+    0xad,0xd8,0xf9,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfa,0xe0,0xbe,
+    0xb7,0xbf,0xbe,0xbf,0xbd,0xbb,0xd4,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xcf,0xa2,0x9e,0xa4,0xa3,0xa2,0xa5,
+    0xc3,0xf2,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd1,
+    0xb8,0xb7,0xba,0xbc,0xb7,0xbc,0xde,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe8,0xc1,0xae,0xab,0xa8,0xab,0xc3,
+    0xe7,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xec,
+    0xcf,0xbe,0xbe,0xc0,0xc2,0xd1,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf0,0xd9,0xc7,0xc7,0xd7,0xef,
+    0xfd,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0,
+    0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe,
+    0xf1,0xdf,0xd5,0xd6,0xe3,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xed,0xed,0xf8,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb7,0xb1,0xb3,0xaf,
+    0xaf,0xb4,0xb4,0xbc,0xe3,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xf9,0xf1,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xbf,0xb0,0xb1,0xb2,
+    0xb2,0xb2,0xb4,0xc2,0xe7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb4,0xac,0xb5,
+    0xb5,0xad,0xb4,0xd1,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xeb,0xc7,0xb0,0xad,
+    0xad,0xb1,0xc8,0xeb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe7,0xcf,0xc2,
+    0xc2,0xcf,0xe8,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf5,
+    0xf5,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0,
+    0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xb1,0x99,
+    0x99,0xb1,0xd9,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa5,0x7b,0x74,
+    0x74,0x7b,0xa4,0xde,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xb8,0x82,0x73,0x82,
+    0x81,0x74,0x7f,0xaf,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd7,0x97,0x7d,0x7b,0x7d,
+    0x7d,0x7b,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xe9,0xe8,0xf6,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xce,0x8b,0x7e,0x7f,0x77,
+    0x77,0x7f,0x7b,0x87,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xf6,0xe5,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xed,0xce,0xb4,0xb2,0xc8,0xe9,
+    0xfe,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7f,0x7b,
+    0x7b,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe,
+    0xe7,0xc2,0xa9,0xa9,0xc6,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe1,0xab,0x90,0x8c,0x87,0x89,0xab,
+    0xdf,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7e,0x7a,
+    0x7a,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xdc,
+    0xa2,0x7c,0x79,0x7d,0x81,0xa1,0xdd,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc0,0x81,0x7a,0x84,0x80,0x7a,0x7f,
+    0xad,0xee,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7c,0x80,0x79,
+    0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xff,0xfd,0xfc,0xff,0xec,0xa6,
+    0x72,0x6b,0x70,0x75,0x6a,0x71,0xb9,0xfd,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xea,0xac,0x7f,0x84,0x88,0x86,0x88,0x7a,
+    0x89,0xc8,0xf7,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xf6,0xc2,0x7d,
+    0x6d,0x7b,0x78,0x7a,0x75,0x6f,0xa3,0xe8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe3,0xa9,0x81,0x83,0x88,0x84,0x85,0x85,
+    0x81,0x98,0xd7,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xff,0xd3,0x8e,0x74,
+    0x77,0x77,0x75,0x7a,0x75,0x70,0x9e,0xe1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf3,0xb3,0x7c,0x7f,0x89,0x81,0x81,0x89,
+    0x81,0x7e,0xb7,0xf7,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xce,0x89,0x7b,0x80,0x7a,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xf8,0xb2,0x72,0x74,
+    0x7c,0x71,0x72,0x7b,0x6e,0x6a,0xa7,0xf0,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xc5,0x84,0x82,0x8a,0x83,0x85,0x87,
+    0x81,0x7f,0x99,0xce,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xce,0x89,0x7c,0x81,0x7b,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xff,0xff,0xf9,0xca,0x90,0x72,0x73,
+    0x79,0x75,0x74,0x7c,0x71,0x72,0xbb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa6,0x89,0x84,0x85,0x86,0x82,
+    0x85,0x86,0x7e,0xa4,0xf0,0xff,0xfc,0xff,0xfc,0xff,0xff,0xcd,0x88,0x7c,0x81,0x7b,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfc,0xff,0xfc,0xff,0xed,0x9a,0x71,0x78,0x77,
+    0x73,0x77,0x76,0x75,0x78,0x98,0xd9,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd3,0x92,0x7d,0x89,0x87,0x81,
+    0x88,0x88,0x79,0x8f,0xd2,0xfb,0xfe,0xff,0xfb,0xff,0xff,0xcd,0x88,0x7b,0x80,0x7a,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xfe,0xfb,0xce,0x83,0x6a,0x7a,0x7a,
+    0x72,0x79,0x79,0x6a,0x81,0xcb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfc,0xff,0xf1,0xab,0x84,0x88,0x87,0x83,
+    0x86,0x86,0x83,0x85,0xa3,0xdd,0xff,0xff,0xf7,0xff,0xff,0xcd,0x89,0x7c,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf6,0xff,0xff,0xda,0x9b,0x7a,0x75,0x76,0x77,
+    0x74,0x78,0x78,0x72,0x9c,0xed,0xff,0xfb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xfb,0xd6,0xa1,0x82,0x83,0x88,
+    0x83,0x83,0x89,0x7f,0x80,0xbf,0xff,0xff,0xf5,0xff,0xff,0xcd,0x89,0x7d,0x80,0x79,
+    0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf4,0xff,0xff,0xb9,0x74,0x72,0x7c,0x74,0x74,
+    0x79,0x74,0x71,0x92,0xcf,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff,
+    0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfb,0xbe,0x84,0x83,0x8b,
+    0x83,0x83,0x89,0x81,0x7b,0xa0,0xdd,0xfe,0xfc,0xff,0xff,0xce,0x89,0x7c,0x7f,0x7a,
+    0x7a,0x7d,0x77,0x84,0xcc,0xff,0xff,0xfb,0xfe,0xda,0x97,0x6d,0x71,0x7a,0x74,0x73,
+    0x7d,0x72,0x71,0xb3,0xfa,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xdc,0xa1,0x87,0x85,
+    0x85,0x85,0x84,0x86,0x85,0x82,0xad,0xf1,0xff,0xff,0xff,0xcf,0x89,0x7d,0x80,0x79,
+    0x78,0x7e,0x7a,0x86,0xce,0xff,0xff,0xff,0xf0,0xa7,0x78,0x78,0x77,0x74,0x76,0x76,
+    0x75,0x75,0x90,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf2,0xe7,
+    0xf1,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xcf,0x90,0x7b,
+    0x88,0x88,0x82,0x87,0x86,0x78,0x91,0xd6,0xf9,0xff,0xff,0xd2,0x8f,0x80,0x80,0x7a,
+    0x79,0x7d,0x7d,0x8f,0xd2,0xff,0xff,0xf9,0xd2,0x87,0x6c,0x7a,0x79,0x72,0x7a,0x7a,
+    0x69,0x7e,0xc6,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,
+    0xf9,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfb,0xe0,0xc7,0xb9,0xb1,
+    0xb6,0xcb,0xe5,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xed,0xa7,0x84,
+    0x8a,0x86,0x81,0x86,0x86,0x7d,0x86,0xb2,0xe5,0xff,0xff,0xe0,0xa5,0x7d,0x77,0x84,
+    0x81,0x73,0x7a,0xa5,0xe0,0xff,0xff,0xe3,0xab,0x7b,0x6f,0x78,0x79,0x72,0x76,0x79,
+    0x72,0x99,0xea,0xff,0xfb,0xfd,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,0xf2,0xec,
+    0xeb,0xed,0xf1,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe9,0xab,0x86,0x89,0x8a,
+    0x83,0x8d,0xb0,0xda,0xf4,0xfd,0xff,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd2,0x9d,
+    0x82,0x84,0x87,0x83,0x84,0x85,0x82,0x96,0xd6,0xff,0xff,0xf4,0xcf,0x92,0x74,0x7d,
+    0x7c,0x72,0x8f,0xcc,0xf3,0xff,0xff,0xd2,0x8d,0x77,0x77,0x74,0x74,0x78,0x75,0x71,
+    0x8e,0xca,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xf6,0xec,0xe3,0xe0,
+    0xe1,0xe1,0xdf,0xe9,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc9,0x96,0x84,0x8c,0x8c,
+    0x88,0x8b,0x91,0x98,0xb1,0xda,0xf8,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfb,0xbb,
+    0x80,0x84,0x8b,0x82,0x82,0x87,0x81,0x8c,0xd0,0xff,0xff,0xfe,0xf5,0xc8,0x97,0x80,
+    0x80,0x96,0xc6,0xf5,0xfe,0xff,0xff,0xce,0x85,0x76,0x79,0x72,0x71,0x7c,0x73,0x6f,
+    0xaf,0xf9,0xff,0xfb,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf7,0xec,0xe6,0xe5,0xe3,0xe1,
+    0xe1,0xe1,0xe0,0xe4,0xf2,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xb5,0x95,0x8f,0x92,0x92,
+    0x92,0x92,0x8c,0x84,0x8c,0xa7,0xc4,0xe0,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xda,
+    0x9e,0x82,0x84,0x8a,0x8b,0x86,0x82,0x98,0xd7,0xff,0xff,0xfc,0xff,0xf3,0xd9,0xc5,
+    0xc5,0xd9,0xf2,0xff,0xfc,0xff,0xff,0xd4,0x8f,0x75,0x77,0x7d,0x7c,0x73,0x70,0x8e,
+    0xd5,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf8,0xf1,0xea,0xe3,0xe0,0xe2,0xe4,0xe4,
+    0xe3,0xe3,0xe3,0xe4,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x91,0x8b,0x90,0x8e,
+    0x8b,0x8c,0x8e,0x91,0x8f,0x89,0x8b,0xa8,0xd3,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf8,
+    0xd0,0x98,0x7e,0x84,0x83,0x7d,0x8e,0xbb,0xec,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xea,0xb4,0x82,0x6d,0x73,0x73,0x6c,0x88,0xc8,
+    0xf6,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf5,0xea,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,
+    0xe2,0xe2,0xe1,0xe3,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc7,0x93,0x88,0x93,0x8f,
+    0x8b,0x8e,0x8f,0x91,0x8f,0x8b,0x8a,0x8d,0x95,0xac,0xd5,0xf9,0xff,0xfd,0xfc,0xff,
+    0xf5,0xcb,0x9d,0x85,0x81,0x8b,0xb2,0xe9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe8,0xa9,0x7b,0x6f,0x72,0x8d,0xc3,0xf3,
+    0xff,0xfd,0xfe,0xff,0xfc,0xf5,0xeb,0xe5,0xe3,0xe2,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,
+    0xe2,0xe3,0xe0,0xe3,0xf0,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe5,0xad,0x88,0x88,0x90,
+    0x94,0x91,0x8d,0x8c,0x8c,0x90,0x93,0x8c,0x82,0x8b,0xa4,0xbf,0xdf,0xf7,0xff,0xff,
+    0xfe,0xf2,0xda,0xc6,0xc3,0xce,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8,
+    0xf8,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xc7,0xba,0xbc,0xd4,0xf1,0xfe,
+    0xff,0xff,0xfe,0xf8,0xf0,0xe9,0xe2,0xe0,0xe2,0xe4,0xe4,0xe1,0xe1,0xe1,0xe3,0xe3,
+    0xe3,0xe1,0xe0,0xe9,0xf8,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xda,0xaf,0x95,0x8e,
+    0x8d,0x8d,0x90,0x91,0x90,0x8d,0x8d,0x8d,0x8f,0x90,0x89,0x87,0xa3,0xd3,0xf6,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xfd,0xf6,0xeb,0xe4,0xe3,0xe4,0xe4,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe1,0xe1,
+    0xe2,0xe3,0xe9,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xef,0xce,0xa3,
+    0x8c,0x8d,0x91,0x92,0x8f,0x8d,0x8d,0x8e,0x90,0x8e,0x8d,0x8a,0x88,0x9e,0xcc,0xf3,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xf4,0xe7,0xe2,0xe2,0xe3,0xe3,0xe3,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,
+    0xe6,0xf1,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf8,0xdb,
+    0xbf,0xa5,0x8c,0x85,0x8f,0x94,0x92,0x8d,0x8b,0x8b,0x8f,0x93,0x8b,0x85,0x9f,0xdc,
+    0xff,0xff,0xf8,0xfa,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf9,0xf8,0xf8,0xfa,0xff,0xff,
+    0xf7,0xea,0xe2,0xe1,0xe2,0xe3,0xe2,0xe1,0xe2,0xe3,0xe4,0xe2,0xdf,0xe0,0xe7,0xee,
+    0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf6,0xd5,0xad,0x97,0x92,0x8d,0x8c,0x90,0x91,0x91,0x8c,0x8b,0x95,0x8b,0x87,0xc8,
+    0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf3,0xe4,0xe3,0xe4,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,0xe3,0xe3,0xe8,0xf3,0xfc,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfe,0xfb,0xf0,0xd4,0xac,0x8f,0x8a,0x91,0x92,0x91,0x8d,0x8c,0x96,0x8a,0x82,0xc4,
+    0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf1,0xe1,0xe2,0xe4,0xe1,0xe2,0xe2,0xe3,0xe3,0xdf,0xdf,0xe8,0xf4,0xfb,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa,
+    0xfb,0xff,0xff,0xf5,0xe0,0xc5,0xa6,0x90,0x87,0x89,0x8e,0x90,0x8f,0x86,0x92,0xd1,
+    0xff,0xff,0xfa,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff,
+    0xf3,0xe3,0xe0,0xe3,0xe3,0xe2,0xe1,0xe0,0xe1,0xe6,0xed,0xf6,0xfd,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfd,0xdd,0xb4,0x9e,0x95,0x91,0x90,0x8e,0x9a,0xbd,0xea,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xee,0xe4,0xe2,0xe2,0xe2,0xe3,0xe5,0xea,0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf3,0xdd,0xc1,0xaa,0xa1,0xa8,0xc8,0xf0,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf0,0xe7,0xe5,0xe7,0xee,0xf6,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfa,
+    0xfb,0xfc,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xde,0xd8,0xe0,0xf0,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xfa,0xf6,0xf4,0xf5,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xef,0xe2,0xdb,0xda,0xda,0xda,0xda,0xda,
+    0xda,0xda,0xda,0xda,0xd9,0xd9,0xda,0xdb,0xdf,0xe9,0xf5,0xfd,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfc,0xf8,0xf4,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,
+    0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf6,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc2,0xaf,0xa8,0xa3,0xa3,0xa3,0xa3,0xa3,
+    0xa3,0xa3,0xa3,0xa3,0xa3,0xa2,0xa1,0xa3,0xaa,0xb9,0xd7,0xf8,0xff,0xfe,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xf3,0xe9,0xe4,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,
+    0xe1,0xe1,0xe1,0xe1,0xe1,0xe2,0xe5,0xeb,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfd,0xe4,0xb8,0x9a,0x98,0x9a,0x96,0x96,0x97,0x97,0x97,
+    0x97,0x97,0x97,0x97,0x97,0x97,0x96,0x98,0x9a,0x96,0xa8,0xd5,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xf0,0xe3,0xde,0xdf,0xde,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe7,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x99,0x92,0x97,0x9a,0x9a,0x9b,0x9b,0x9a,0x9a,
+    0x9a,0x9b,0x9b,0x9a,0x9a,0x9b,0x9a,0x9a,0x98,0x92,0x92,0xad,0xe2,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe4,0xdb,0xdb,0xde,0xde,0xdd,0xdd,0xdd,0xdd,0xdd,0xdc,0xdc,
+    0xdc,0xdd,0xdd,0xdd,0xdd,0xdc,0xda,0xd9,0xdc,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xac,0x92,0xa0,0x9c,0x96,0x99,0x98,0x98,0x99,0x99,
+    0x99,0x98,0x98,0x99,0x99,0x98,0x98,0x96,0x96,0x9e,0x99,0x9a,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xf0,0xde,0xdc,0xde,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xdc,0xdd,0xda,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xae,0x94,0xa1,0x9d,0x97,0x9a,0x99,0x9a,0x9b,0x9b,
+    0x9a,0x99,0x99,0x9a,0x9b,0x99,0x99,0x98,0x99,0xa0,0x9a,0x9c,0xd3,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xef,0xdc,0xdb,0xdd,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,
+    0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xdc,0xdd,0xd9,0xe3,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x9c,0x97,0x9b,0x9d,0x9d,0x9d,0x9e,0x9e,0x9e,
+    0x9e,0x9d,0x9d,0x9e,0x9e,0x9e,0x9c,0x9e,0x9d,0x96,0x95,0xb0,0xe3,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf4,0xe1,0xd8,0xd9,0xda,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xda,0xd9,
+    0xda,0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xd9,0xdb,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfc,0xe3,0xb9,0x9e,0x9b,0x9d,0x9b,0x9b,0x9b,0x9b,0x9b,
+    0x9b,0x9b,0x9b,0x9b,0x9b,0x9b,0x9a,0x9c,0x9e,0x9a,0xaa,0xd5,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xda,0xdb,0xdb,0xdb,0xdb,0xd9,0xd8,
+    0xd9,0xdb,0xdb,0xdb,0xdb,0xdc,0xdb,0xdb,0xe5,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe5,0xc3,0xb0,0xa9,0xa6,0xa6,0xa6,0xa6,0xa6,
+    0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa8,0xae,0xbc,0xd8,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xf1,0xe5,0xdf,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde,
+    0xde,0xde,0xde,0xde,0xde,0xde,0xe0,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xd9,0xd9,0xd9,
+    0xd9,0xd9,0xd8,0xd8,0xd8,0xd8,0xd9,0xdb,0xde,0xe8,0xf5,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xff,0xff,0xfb,0xf6,0xf2,0xf2,0xf2,0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,
+    0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,0xf3,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfb,0xf8,0xf8,0xfc,0xff,0xfb,0xf0,0xe3,0xe0,0xe7,0xf3,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xf9,0xf3,0xf2,0xf4,0xf9,0xfe,0xff,0xfd,0xfc,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe3,0xca,0xb7,0xb0,0xb7,0xd3,0xf3,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xf8,0xea,0xde,0xdc,0xe0,0xe8,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfc,0xff,0xfe,0xe4,0xc0,0xad,0xa7,0xa6,0xa4,0xa3,0xad,0xca,0xee,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf6,0xe4,0xd8,0xd4,0xd4,0xd5,0xd6,0xd9,0xe2,0xf2,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfb,
+    0xfc,0xff,0xff,0xf7,0xe6,0xd0,0xb7,0xa3,0x9b,0x9e,0xa4,0xa6,0xa5,0x9e,0xa9,0xdb,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xec,0xd2,0xcf,0xd3,0xd4,0xd3,0xd1,0xd0,0xd4,0xdd,0xe9,0xf4,0xfc,0xff,0xff,0xfe,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfc,0xf4,0xdd,0xba,0xa2,0x9f,0xa6,0xa6,0xa5,0xa3,0xa2,0xab,0xa4,0xa0,0xd2,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe7,0xcc,0xcf,0xd5,0xd1,0xd3,0xd5,0xd5,0xd5,0xd1,0xd3,0xe0,0xef,0xf9,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf8,0xdc,0xba,0xaa,0xa6,0xa2,0xa1,0xa5,0xa7,0xa7,0xa4,0xa3,0xab,0xa4,0xa1,0xd4,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe8,0xce,0xd0,0xd4,0xd0,0xd2,0xd4,0xd4,0xd4,0xd2,0xd2,0xd5,0xd8,0xe0,0xf0,0xfc,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xe4,
+    0xcd,0xb6,0x9f,0x9b,0xa5,0xa9,0xa7,0xa3,0xa2,0xa3,0xa7,0xaa,0xa6,0xa1,0xb4,0xe3,
+    0xff,0xff,0xfa,0xfc,0xfb,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfd,0xfd,0xff,0xff,
+    0xf0,0xd8,0xce,0xd0,0xd3,0xd2,0xd0,0xd0,0xd3,0xd6,0xd6,0xd4,0xd1,0xd4,0xdd,0xe7,
+    0xf2,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf2,0xd6,0xb4,
+    0xa1,0xa2,0xa6,0xa6,0xa5,0xa4,0xa4,0xa6,0xa8,0xa7,0xa6,0xa7,0xa5,0xb3,0xd6,0xf6,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xea,0xd7,0xcf,0xd1,0xd1,0xd1,0xd2,0xd3,0xd3,0xd3,0xd4,0xd5,0xd5,0xd3,0xd2,
+    0xdc,0xed,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xbe,0xaa,0xa3,
+    0xa2,0xa2,0xa5,0xa7,0xa6,0xa4,0xa4,0xa6,0xa9,0xaa,0xa5,0xa4,0xb8,0xdb,0xf7,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xec,0xd9,0xcf,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd5,0xd4,0xd3,0xd3,
+    0xd3,0xd6,0xe0,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xeb,0xbd,0x9f,0x9f,0xa6,
+    0xa8,0xa6,0xa3,0xa3,0xa3,0xa7,0xaa,0xa6,0xa0,0xa6,0xb8,0xcb,0xe3,0xf7,0xff,0xff,
+    0xfe,0xf8,0xe9,0xdc,0xdb,0xe2,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,
+    0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xec,0xe7,0xe7,0xf1,0xfb,0xff,
+    0xff,0xff,0xfb,0xf1,0xe4,0xd9,0xd0,0xcd,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd6,
+    0xd4,0xd1,0xd1,0xe0,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd2,0xa9,0x9f,0xa8,0xa5,
+    0xa2,0xa5,0xa6,0xa9,0xa7,0xa4,0xa5,0xa9,0xae,0xbd,0xdd,0xfa,0xff,0xfd,0xfd,0xff,
+    0xf9,0xde,0xbe,0xaf,0xad,0xb5,0xd0,0xf3,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf7,0xdf,0xcc,0xc8,0xcb,0xd6,0xe9,0xfb,
+    0xff,0xfd,0xfe,0xff,0xfc,0xec,0xdb,0xd4,0xd1,0xd0,0xd0,0xd1,0xd1,0xd1,0xd2,0xd1,
+    0xd3,0xd5,0xd2,0xd6,0xe9,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc3,0xa7,0xa2,0xa6,0xa5,
+    0xa3,0xa3,0xa6,0xaa,0xa9,0xa5,0xa6,0xbb,0xdd,0xf3,0xfa,0xfe,0xff,0xff,0xfe,0xf9,
+    0xdf,0xba,0xa9,0xad,0xae,0xac,0xb8,0xd6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf6,0xe1,0xce,0xc5,0xc7,0xca,0xc8,0xd3,0xeb,
+    0xfc,0xfe,0xff,0xff,0xff,0xfd,0xf8,0xec,0xdb,0xd0,0xcf,0xd2,0xd2,0xd1,0xd0,0xd1,
+    0xd3,0xd4,0xd2,0xd5,0xe2,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc2,0xa9,0xa5,0xa8,0xa8,
+    0xa8,0xa9,0xa6,0xa1,0xa7,0xba,0xce,0xe5,0xfb,0xff,0xfe,0xfc,0xfc,0xff,0xff,0xe8,
+    0xbf,0xaa,0xac,0xb4,0xb5,0xb1,0xb1,0xc1,0xe8,0xff,0xff,0xfd,0xff,0xfa,0xee,0xe4,
+    0xe4,0xef,0xfb,0xff,0xfd,0xff,0xff,0xed,0xd0,0xc7,0xc8,0xcb,0xcb,0xc9,0xc9,0xd6,
+    0xf0,0xff,0xff,0xfd,0xfd,0xff,0xff,0xfd,0xf1,0xe5,0xda,0xd0,0xce,0xd1,0xd3,0xd3,
+    0xd3,0xd3,0xd1,0xd3,0xe1,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd3,0xab,0x9f,0xa6,0xa5,
+    0xa1,0xa4,0xaa,0xb0,0xc2,0xe0,0xf8,0xff,0xfe,0xfd,0xff,0xff,0xfc,0xff,0xfc,0xd2,
+    0xab,0xac,0xb3,0xaf,0xaf,0xb3,0xb3,0xbc,0xe4,0xff,0xff,0xfe,0xfa,0xe3,0xcb,0xc0,
+    0xc1,0xce,0xe6,0xfa,0xfe,0xff,0xff,0xea,0xcb,0xc6,0xc9,0xc6,0xc6,0xcc,0xcb,0xca,
+    0xe1,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xfc,0xf0,0xdf,0xd5,0xd3,0xd1,0xcf,
+    0xd1,0xd1,0xce,0xd4,0xe9,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf0,0xc2,0xa2,0xa1,0xa4,
+    0xa1,0xa6,0xbe,0xe0,0xf6,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfc,0xe1,0xbd,
+    0xab,0xae,0xb2,0xaf,0xb0,0xb4,0xb6,0xc1,0xe7,0xff,0xff,0xfa,0xe7,0xc6,0xb7,0xbd,
+    0xbd,0xba,0xca,0xe9,0xfb,0xff,0xff,0xec,0xce,0xc5,0xc7,0xc8,0xc7,0xc9,0xca,0xca,
+    0xd5,0xeb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfb,0xef,0xdd,0xd1,0xce,
+    0xcf,0xce,0xcf,0xe0,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe9,0xd2,0xc4,0xc1,
+    0xc5,0xd3,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xc6,0xad,
+    0xb0,0xb1,0xb0,0xb2,0xb2,0xaf,0xb8,0xd1,0xef,0xff,0xff,0xef,0xd0,0xba,0xb7,0xc0,
+    0xc0,0xb9,0xc0,0xd7,0xf2,0xff,0xff,0xf3,0xda,0xc7,0xc4,0xc9,0xc9,0xc6,0xc9,0xcb,
+    0xca,0xdb,0xf9,0xff,0xfd,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf3,0xe9,0xe1,
+    0xde,0xe0,0xe8,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf4,0xea,
+    0xf4,0xff,0xff,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe0,0xb5,0xa7,
+    0xb0,0xb2,0xaf,0xb2,0xb3,0xad,0xbe,0xe7,0xfb,0xff,0xff,0xe6,0xc2,0xbb,0xbd,0xbb,
+    0xbc,0xc0,0xc1,0xcb,0xea,0xff,0xff,0xfc,0xeb,0xcc,0xc1,0xc9,0xc9,0xc6,0xc9,0xca,
+    0xc6,0xd0,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf9,
+    0xf4,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xbf,0xae,0xae,
+    0xb0,0xb1,0xb0,0xb2,0xb4,0xb5,0xcf,0xf7,0xff,0xff,0xff,0xe4,0xbe,0xba,0xbe,0xbb,
+    0xbe,0xc2,0xbf,0xc4,0xe7,0xff,0xff,0xff,0xf9,0xd8,0xc5,0xc7,0xc8,0xc7,0xc8,0xc9,
+    0xc9,0xca,0xd6,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff,
+    0xfe,0xfc,0xfd,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xfd,0xd4,0xad,0xac,0xb4,
+    0xb1,0xb0,0xb3,0xb1,0xb1,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xe5,0xbf,0xb9,0xbc,0xbc,
+    0xbd,0xc0,0xbd,0xc4,0xe7,0xff,0xff,0xfe,0xfe,0xed,0xd1,0xc2,0xc5,0xc9,0xc7,0xc8,
+    0xcb,0xc8,0xca,0xe3,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe5,0xc0,0xab,0xae,0xb3,
+    0xb0,0xb0,0xb6,0xb2,0xb3,0xd8,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfa,0xff,0xff,0xe1,0xc4,0xc5,0xca,0xc7,0xc7,
+    0xc9,0xc9,0xc9,0xd6,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf5,0xc8,0xae,0xaf,0xb1,0xb0,
+    0xb1,0xb2,0xb3,0xb5,0xc6,0xe9,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc,
+    0xbe,0xc1,0xbe,0xc5,0xe6,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc7,0xc7,0xc8,0xc8,
+    0xc7,0xc9,0xca,0xc9,0xda,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe4,0xb7,0xa8,0xb1,0xb2,0xae,
+    0xb3,0xb4,0xae,0xba,0xe1,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc,
+    0xbd,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xfc,0xe7,0xc9,0xc1,0xc9,0xc9,
+    0xc6,0xc9,0xc9,0xc5,0xd0,0xed,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc8,0xb1,0xae,0xb2,0xb2,0xaf,
+    0xb2,0xb5,0xb2,0xc8,0xf5,0xff,0xfe,0xff,0xfe,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfe,0xff,0xfe,0xff,0xf6,0xd3,0xc3,0xc9,0xc9,
+    0xc7,0xc8,0xc8,0xc9,0xcc,0xda,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xd9,0xaf,0xad,0xb4,0xb0,0xb0,0xb2,
+    0xb0,0xb2,0xc2,0xe1,0xfc,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe8,0xd0,0xc6,0xc7,
+    0xc9,0xc7,0xc7,0xcb,0xc8,0xc9,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf9,0xcd,0xa8,0xac,0xb4,0xae,0xae,0xb4,
+    0xb0,0xb1,0xd4,0xfa,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xfb,0xde,0xc4,0xc6,
+    0xca,0xc6,0xc7,0xca,0xc5,0xc5,0xde,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc6,0xab,0xaf,0xb3,0xb0,0xb1,0xb2,
+    0xb1,0xc1,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xeb,0xce,0xc5,
+    0xc9,0xc9,0xc8,0xca,0xc8,0xc6,0xd9,0xf4,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf0,0xc9,0xac,0xb0,0xb3,0xb2,0xb4,0xae,
+    0xb8,0xdd,0xfa,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfb,0xe6,0xca,
+    0xc4,0xca,0xc9,0xca,0xc7,0xc5,0xdb,0xf7,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd5,0xae,0xab,0xb1,0xae,0xab,0xaf,
+    0xcc,0xf4,0xff,0xfd,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf7,0xd8,
+    0xc4,0xc3,0xc5,0xc7,0xc3,0xc6,0xe3,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc8,0xb7,0xb5,0xb3,0xb5,0xca,
+    0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,
+    0xd7,0xc9,0xc9,0xcb,0xcc,0xd8,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xde,0xce,0xcd,0xdc,0xf1,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xf4,0xe5,0xdd,0xdd,0xe8,0xf6,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xef,0xef,0xfa,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc2,0xbc,0xbe,0xba,
+    0xbb,0xc0,0xbf,0xc6,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf4,0xf4,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbc,0xbe,
+    0xbe,0xbe,0xbf,0xcb,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd8,0xbd,0xb8,0xc1,
+    0xc1,0xba,0xc1,0xd9,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xee,0xcf,0xbb,0xb9,
+    0xb9,0xbc,0xd1,0xef,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,0xd6,0xca,
+    0xca,0xd7,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf6,
+    0xf6,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0,
+    0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xb1,0x99,
+    0x99,0xb1,0xd9,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa5,0x7b,0x74,
+    0x74,0x7b,0xa4,0xde,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xb8,0x82,0x73,0x82,
+    0x81,0x74,0x7f,0xaf,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd7,0x97,0x7d,0x7b,0x7d,
+    0x7d,0x7b,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xe9,0xe8,0xf6,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xce,0x8b,0x7e,0x7f,0x77,
+    0x77,0x7f,0x7b,0x87,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xf6,0xe5,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xed,0xce,0xb4,0xb2,0xc8,0xe9,
+    0xfe,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7f,0x7b,
+    0x7b,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe,
+    0xe7,0xc2,0xa9,0xa9,0xc6,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe1,0xab,0x90,0x8c,0x87,0x89,0xab,
+    0xdf,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7e,0x7a,
+    0x7a,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xdc,
+    0xa2,0x7c,0x79,0x7d,0x81,0xa1,0xdd,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc0,0x81,0x7a,0x84,0x80,0x7a,0x7f,
+    0xad,0xee,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7c,0x80,0x79,
+    0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xff,0xfd,0xfc,0xff,0xec,0xa6,
+    0x72,0x6b,0x70,0x75,0x6a,0x71,0xb9,0xfd,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xea,0xac,0x7f,0x84,0x88,0x86,0x88,0x7a,
+    0x89,0xc8,0xf7,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xf6,0xc2,0x7d,
+    0x6d,0x7b,0x78,0x7a,0x75,0x6f,0xa3,0xe8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe3,0xa9,0x81,0x83,0x88,0x84,0x85,0x85,
+    0x81,0x98,0xd7,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xff,0xd3,0x8e,0x74,
+    0x77,0x77,0x75,0x7a,0x75,0x70,0x9e,0xe1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf3,0xb3,0x7c,0x7f,0x89,0x81,0x81,0x89,
+    0x81,0x7e,0xb7,0xf7,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xce,0x89,0x7b,0x80,0x7a,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xf8,0xb2,0x72,0x74,
+    0x7c,0x71,0x72,0x7b,0x6e,0x6a,0xa7,0xf0,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xc5,0x84,0x82,0x8a,0x83,0x85,0x87,
+    0x81,0x7f,0x99,0xce,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xce,0x89,0x7c,0x81,0x7b,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xff,0xff,0xf9,0xca,0x90,0x72,0x73,
+    0x79,0x75,0x74,0x7c,0x71,0x72,0xbb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa6,0x89,0x84,0x85,0x86,0x82,
+    0x85,0x86,0x7e,0xa4,0xf0,0xff,0xfc,0xff,0xfc,0xff,0xff,0xcd,0x88,0x7c,0x81,0x7b,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfc,0xff,0xfc,0xff,0xed,0x9a,0x71,0x78,0x77,
+    0x73,0x77,0x76,0x75,0x78,0x98,0xd9,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd3,0x92,0x7d,0x89,0x87,0x81,
+    0x88,0x88,0x79,0x8f,0xd2,0xfb,0xfe,0xff,0xfb,0xff,0xff,0xcd,0x88,0x7b,0x80,0x7a,
+    0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xfe,0xfb,0xce,0x83,0x6a,0x7a,0x7a,
+    0x72,0x79,0x79,0x6a,0x81,0xcb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfc,0xff,0xf1,0xab,0x84,0x88,0x87,0x83,
+    0x86,0x86,0x83,0x85,0xa3,0xdd,0xff,0xff,0xf7,0xff,0xff,0xcd,0x89,0x7c,0x7f,0x79,
+    0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf6,0xff,0xff,0xda,0x9b,0x7a,0x75,0x76,0x77,
+    0x74,0x78,0x78,0x72,0x9c,0xed,0xff,0xfb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xfb,0xd6,0xa1,0x82,0x83,0x88,
+    0x83,0x83,0x89,0x7f,0x80,0xbf,0xff,0xff,0xf5,0xff,0xff,0xcd,0x89,0x7d,0x80,0x79,
+    0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf4,0xff,0xff,0xb9,0x74,0x72,0x7c,0x74,0x74,
+    0x79,0x74,0x71,0x92,0xcf,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff,
+    0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfb,0xbe,0x84,0x83,0x8b,
+    0x83,0x83,0x89,0x81,0x7b,0xa0,0xdd,0xfe,0xfc,0xff,0xff,0xce,0x89,0x7c,0x7f,0x7a,
+    0x7a,0x7d,0x77,0x84,0xcc,0xff,0xff,0xfb,0xfe,0xda,0x97,0x6d,0x71,0x7a,0x74,0x73,
+    0x7d,0x72,0x71,0xb3,0xfa,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xdc,0xa1,0x87,0x85,
+    0x85,0x85,0x84,0x86,0x85,0x82,0xad,0xf1,0xff,0xff,0xff,0xcf,0x89,0x7d,0x80,0x79,
+    0x78,0x7e,0x7a,0x86,0xce,0xff,0xff,0xff,0xf0,0xa7,0x78,0x78,0x77,0x74,0x76,0x76,
+    0x75,0x75,0x90,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf2,0xe7,
+    0xf1,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xcf,0x90,0x7b,
+    0x88,0x88,0x82,0x87,0x86,0x78,0x91,0xd6,0xf9,0xff,0xff,0xd2,0x8f,0x80,0x80,0x7a,
+    0x79,0x7d,0x7d,0x8f,0xd2,0xff,0xff,0xf9,0xd2,0x87,0x6c,0x7a,0x79,0x72,0x7a,0x7a,
+    0x69,0x7e,0xc6,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,
+    0xf9,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfb,0xe0,0xc7,0xb9,0xb1,
+    0xb6,0xcb,0xe5,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xed,0xa7,0x84,
+    0x8a,0x86,0x81,0x86,0x86,0x7d,0x86,0xb2,0xe5,0xff,0xff,0xe0,0xa5,0x7d,0x77,0x84,
+    0x81,0x73,0x7a,0xa5,0xe0,0xff,0xff,0xe3,0xab,0x7b,0x6f,0x78,0x79,0x72,0x76,0x79,
+    0x72,0x99,0xea,0xff,0xfb,0xfd,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,0xf2,0xec,
+    0xeb,0xed,0xf1,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe9,0xab,0x86,0x89,0x8a,
+    0x83,0x8d,0xb0,0xda,0xf4,0xfd,0xff,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd2,0x9d,
+    0x82,0x84,0x87,0x83,0x84,0x85,0x82,0x96,0xd6,0xff,0xff,0xf4,0xcf,0x92,0x74,0x7d,
+    0x7c,0x72,0x8f,0xcc,0xf3,0xff,0xff,0xd2,0x8d,0x77,0x77,0x74,0x74,0x78,0x75,0x71,
+    0x8e,0xca,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xf6,0xec,0xe3,0xe0,
+    0xe1,0xe1,0xdf,0xe9,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc9,0x96,0x84,0x8c,0x8c,
+    0x88,0x8b,0x91,0x98,0xb1,0xda,0xf8,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfb,0xbb,
+    0x80,0x84,0x8b,0x82,0x82,0x87,0x81,0x8c,0xd0,0xff,0xff,0xfe,0xf5,0xc8,0x97,0x80,
+    0x80,0x96,0xc6,0xf5,0xfe,0xff,0xff,0xce,0x85,0x76,0x79,0x72,0x71,0x7c,0x73,0x6f,
+    0xaf,0xf9,0xff,0xfb,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf7,0xec,0xe6,0xe5,0xe3,0xe1,
+    0xe1,0xe1,0xe0,0xe4,0xf2,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xb5,0x95,0x8f,0x92,0x92,
+    0x92,0x92,0x8c,0x84,0x8c,0xa7,0xc4,0xe0,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xda,
+    0x9e,0x82,0x84,0x8a,0x8b,0x86,0x82,0x98,0xd7,0xff,0xff,0xfc,0xff,0xf3,0xd9,0xc5,
+    0xc5,0xd9,0xf2,0xff,0xfc,0xff,0xff,0xd4,0x8f,0x75,0x77,0x7d,0x7c,0x73,0x70,0x8e,
+    0xd5,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf8,0xf1,0xea,0xe3,0xe0,0xe2,0xe4,0xe4,
+    0xe3,0xe3,0xe3,0xe4,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x91,0x8b,0x90,0x8e,
+    0x8b,0x8c,0x8e,0x91,0x8f,0x89,0x8b,0xa8,0xd3,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf8,
+    0xd0,0x98,0x7e,0x84,0x83,0x7d,0x8e,0xbb,0xec,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xea,0xb4,0x82,0x6d,0x73,0x73,0x6c,0x88,0xc8,
+    0xf6,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf5,0xea,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,
+    0xe2,0xe2,0xe1,0xe3,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc7,0x93,0x88,0x93,0x8f,
+    0x8b,0x8e,0x8f,0x91,0x8f,0x8b,0x8a,0x8d,0x95,0xac,0xd5,0xf9,0xff,0xfd,0xfc,0xff,
+    0xf5,0xcb,0x9d,0x85,0x81,0x8b,0xb2,0xe9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe8,0xa9,0x7b,0x6f,0x72,0x8d,0xc3,0xf3,
+    0xff,0xfd,0xfe,0xff,0xfc,0xf5,0xeb,0xe5,0xe3,0xe2,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,
+    0xe2,0xe3,0xe0,0xe3,0xf0,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe5,0xad,0x88,0x88,0x90,
+    0x94,0x91,0x8d,0x8c,0x8c,0x90,0x93,0x8c,0x82,0x8b,0xa4,0xbf,0xdf,0xf7,0xff,0xff,
+    0xfe,0xf2,0xda,0xc6,0xc3,0xce,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8,
+    0xf8,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xc7,0xba,0xbc,0xd4,0xf1,0xfe,
+    0xff,0xff,0xfe,0xf8,0xf0,0xe9,0xe2,0xe0,0xe2,0xe4,0xe4,0xe1,0xe1,0xe1,0xe3,0xe3,
+    0xe3,0xe1,0xe0,0xe9,0xf8,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xda,0xaf,0x95,0x8e,
+    0x8d,0x8d,0x90,0x91,0x90,0x8d,0x8d,0x8d,0x8f,0x90,0x89,0x87,0xa3,0xd3,0xf6,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xfd,0xf6,0xeb,0xe4,0xe3,0xe4,0xe4,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe1,0xe1,
+    0xe2,0xe3,0xe9,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xef,0xce,0xa3,
+    0x8c,0x8d,0x91,0x92,0x8f,0x8d,0x8d,0x8e,0x90,0x8e,0x8d,0x8a,0x88,0x9e,0xcc,0xf3,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xf4,0xe7,0xe2,0xe2,0xe3,0xe3,0xe3,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,
+    0xe6,0xf1,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf8,0xdb,
+    0xbf,0xa5,0x8c,0x85,0x8f,0x94,0x92,0x8d,0x8b,0x8b,0x8f,0x93,0x8b,0x85,0x9f,0xdc,
+    0xff,0xff,0xf8,0xfa,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf9,0xf8,0xf8,0xfa,0xff,0xff,
+    0xf7,0xea,0xe2,0xe1,0xe2,0xe3,0xe2,0xe1,0xe2,0xe3,0xe4,0xe2,0xdf,0xe0,0xe7,0xee,
+    0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf6,0xd5,0xad,0x97,0x92,0x8d,0x8c,0x90,0x91,0x91,0x8c,0x8b,0x95,0x8b,0x87,0xc8,
+    0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf3,0xe4,0xe3,0xe4,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,0xe3,0xe3,0xe8,0xf3,0xfc,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfe,0xfb,0xf0,0xd4,0xac,0x8f,0x8a,0x91,0x92,0x91,0x8d,0x8c,0x96,0x8a,0x82,0xc4,
+    0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf1,0xe1,0xe2,0xe4,0xe1,0xe2,0xe2,0xe3,0xe3,0xdf,0xdf,0xe8,0xf4,0xfb,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa,
+    0xfb,0xff,0xff,0xf5,0xe0,0xc5,0xa6,0x90,0x87,0x89,0x8e,0x90,0x8f,0x86,0x92,0xd1,
+    0xff,0xff,0xfa,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff,
+    0xf3,0xe3,0xe0,0xe3,0xe3,0xe2,0xe1,0xe0,0xe1,0xe6,0xed,0xf6,0xfd,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xfd,0xdd,0xb4,0x9e,0x95,0x91,0x90,0x8e,0x9a,0xbd,0xea,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xee,0xe4,0xe2,0xe2,0xe2,0xe3,0xe5,0xea,0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf3,0xdd,0xc1,0xaa,0xa1,0xa8,0xc8,0xf0,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf0,0xe7,0xe5,0xe7,0xee,0xf6,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfa,
+    0xfb,0xfc,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xde,0xd8,0xe0,0xf0,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xfa,0xf6,0xf4,0xf5,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xef,0xe2,0xdb,0xda,0xda,0xda,0xda,0xda,
+    0xda,0xda,0xda,0xda,0xd9,0xd9,0xda,0xdb,0xdf,0xe9,0xf5,0xfd,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfc,0xf8,0xf4,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,
+    0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf6,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc2,0xaf,0xa8,0xa3,0xa3,0xa3,0xa3,0xa3,
+    0xa3,0xa3,0xa3,0xa3,0xa3,0xa2,0xa1,0xa3,0xaa,0xb9,0xd7,0xf8,0xff,0xfe,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xf3,0xe9,0xe4,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,
+    0xe1,0xe1,0xe1,0xe1,0xe1,0xe2,0xe5,0xeb,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfd,0xe4,0xb8,0x9a,0x98,0x9a,0x96,0x96,0x97,0x97,0x97,
+    0x97,0x97,0x97,0x97,0x97,0x97,0x96,0x98,0x9a,0x96,0xa8,0xd5,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xf0,0xe3,0xde,0xdf,0xde,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe7,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x99,0x92,0x97,0x9a,0x9a,0x9b,0x9b,0x9a,0x9a,
+    0x9a,0x9b,0x9b,0x9a,0x9a,0x9b,0x9a,0x9a,0x98,0x92,0x92,0xad,0xe2,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe4,0xdb,0xdb,0xde,0xde,0xdd,0xdd,0xdd,0xdd,0xdd,0xdc,0xdc,
+    0xdc,0xdd,0xdd,0xdd,0xdd,0xdc,0xda,0xd9,0xdc,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xac,0x92,0xa0,0x9c,0x96,0x99,0x98,0x98,0x99,0x99,
+    0x99,0x98,0x98,0x99,0x99,0x98,0x98,0x96,0x96,0x9e,0x99,0x9a,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xf0,0xde,0xdc,0xde,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xdc,0xdd,0xda,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xae,0x94,0xa1,0x9d,0x97,0x9a,0x99,0x9a,0x9b,0x9b,
+    0x9a,0x99,0x99,0x9a,0x9b,0x99,0x99,0x98,0x99,0xa0,0x9a,0x9c,0xd3,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfd,0xff,0xff,0xef,0xdc,0xdb,0xdd,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,
+    0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xdc,0xdd,0xd9,0xe3,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x9c,0x97,0x9b,0x9d,0x9d,0x9d,0x9e,0x9e,0x9e,
+    0x9e,0x9d,0x9d,0x9e,0x9e,0x9e,0x9c,0x9e,0x9d,0x96,0x95,0xb0,0xe3,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf4,0xe1,0xd8,0xd9,0xda,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xda,0xd9,
+    0xda,0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xd9,0xdb,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfc,0xe3,0xb9,0x9e,0x9b,0x9d,0x9b,0x9b,0x9b,0x9b,0x9b,
+    0x9b,0x9b,0x9b,0x9b,0x9b,0x9b,0x9a,0x9c,0x9e,0x9a,0xaa,0xd5,0xf7,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xda,0xdb,0xdb,0xdb,0xdb,0xd9,0xd8,
+    0xd9,0xdb,0xdb,0xdb,0xdb,0xdc,0xdb,0xdb,0xe5,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe5,0xc3,0xb0,0xa9,0xa6,0xa6,0xa6,0xa6,0xa6,
+    0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa8,0xae,0xbc,0xd8,0xf7,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfc,0xf1,0xe5,0xdf,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde,
+    0xde,0xde,0xde,0xde,0xde,0xde,0xe0,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xd9,0xd9,0xd9,
+    0xd9,0xd9,0xd8,0xd8,0xd8,0xd8,0xd9,0xdb,0xde,0xe8,0xf5,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xff,0xff,0xfb,0xf6,0xf2,0xf2,0xf2,0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,
+    0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,0xf3,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfb,0xf8,0xf8,0xfc,0xff,0xfb,0xf0,0xe3,0xe0,0xe7,0xf3,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfe,0xf9,0xf3,0xf2,0xf4,0xf9,0xfe,0xff,0xfd,0xfc,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe3,0xca,0xb7,0xb0,0xb7,0xd3,0xf3,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xf8,0xea,0xde,0xdc,0xe0,0xe8,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfc,0xff,0xfe,0xe4,0xc0,0xad,0xa7,0xa6,0xa4,0xa3,0xad,0xca,0xee,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf6,0xe4,0xd8,0xd4,0xd4,0xd5,0xd6,0xd9,0xe2,0xf2,0xfe,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfb,
+    0xfc,0xff,0xff,0xf7,0xe6,0xd0,0xb7,0xa3,0x9b,0x9e,0xa4,0xa6,0xa5,0x9e,0xa9,0xdb,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xec,0xd2,0xcf,0xd3,0xd4,0xd3,0xd1,0xd0,0xd4,0xdd,0xe9,0xf4,0xfc,0xff,0xff,0xfe,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xfc,0xf4,0xdd,0xba,0xa2,0x9f,0xa6,0xa6,0xa5,0xa3,0xa2,0xab,0xa4,0xa0,0xd2,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe7,0xcc,0xcf,0xd5,0xd1,0xd3,0xd5,0xd5,0xd5,0xd1,0xd3,0xe0,0xef,0xf9,0xfd,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xf8,0xdc,0xba,0xaa,0xa6,0xa2,0xa1,0xa5,0xa7,0xa7,0xa4,0xa3,0xab,0xa4,0xa1,0xd4,
+    0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,
+    0xe8,0xce,0xd0,0xd4,0xd0,0xd2,0xd4,0xd4,0xd4,0xd2,0xd2,0xd5,0xd8,0xe0,0xf0,0xfc,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xe4,
+    0xcd,0xb6,0x9f,0x9b,0xa5,0xa9,0xa7,0xa3,0xa2,0xa3,0xa7,0xaa,0xa6,0xa1,0xb4,0xe3,
+    0xff,0xff,0xfa,0xfc,0xfb,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfd,0xfd,0xff,0xff,
+    0xf0,0xd8,0xce,0xd0,0xd3,0xd2,0xd0,0xd0,0xd3,0xd6,0xd6,0xd4,0xd1,0xd4,0xdd,0xe7,
+    0xf2,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf2,0xd6,0xb4,
+    0xa1,0xa2,0xa6,0xa6,0xa5,0xa4,0xa4,0xa6,0xa8,0xa7,0xa6,0xa7,0xa5,0xb3,0xd6,0xf6,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfa,0xea,0xd7,0xcf,0xd1,0xd1,0xd1,0xd2,0xd3,0xd3,0xd3,0xd4,0xd5,0xd5,0xd3,0xd2,
+    0xdc,0xed,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xbe,0xaa,0xa3,
+    0xa2,0xa2,0xa5,0xa7,0xa6,0xa4,0xa4,0xa6,0xa9,0xaa,0xa5,0xa4,0xb8,0xdb,0xf7,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xec,0xd9,0xcf,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd5,0xd4,0xd3,0xd3,
+    0xd3,0xd6,0xe0,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xeb,0xbd,0x9f,0x9f,0xa6,
+    0xa8,0xa6,0xa3,0xa3,0xa3,0xa7,0xaa,0xa6,0xa0,0xa6,0xb8,0xcb,0xe3,0xf7,0xff,0xff,
+    0xfe,0xf8,0xe9,0xdc,0xdb,0xe2,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,
+    0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xec,0xe7,0xe7,0xf1,0xfb,0xff,
+    0xff,0xff,0xfb,0xf1,0xe4,0xd9,0xd0,0xcd,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd6,
+    0xd4,0xd1,0xd1,0xe0,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd2,0xa9,0x9f,0xa8,0xa5,
+    0xa2,0xa5,0xa6,0xa9,0xa7,0xa4,0xa5,0xa9,0xae,0xbd,0xdd,0xfa,0xff,0xfd,0xfd,0xff,
+    0xf9,0xde,0xbe,0xaf,0xad,0xb5,0xd0,0xf3,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf7,0xdf,0xcc,0xc8,0xcb,0xd6,0xe9,0xfb,
+    0xff,0xfd,0xfe,0xff,0xfc,0xec,0xdb,0xd4,0xd1,0xd0,0xd0,0xd1,0xd1,0xd1,0xd2,0xd1,
+    0xd3,0xd5,0xd2,0xd6,0xe9,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc3,0xa7,0xa2,0xa6,0xa5,
+    0xa3,0xa3,0xa6,0xaa,0xa9,0xa5,0xa6,0xbb,0xdd,0xf3,0xfa,0xfe,0xff,0xff,0xfe,0xf9,
+    0xdf,0xba,0xa9,0xad,0xae,0xac,0xb8,0xd6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf6,0xe1,0xce,0xc5,0xc7,0xca,0xc8,0xd3,0xeb,
+    0xfc,0xfe,0xff,0xff,0xff,0xfd,0xf8,0xec,0xdb,0xd0,0xcf,0xd2,0xd2,0xd1,0xd0,0xd1,
+    0xd3,0xd4,0xd2,0xd5,0xe2,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc2,0xa9,0xa5,0xa8,0xa8,
+    0xa8,0xa9,0xa6,0xa1,0xa7,0xba,0xce,0xe5,0xfb,0xff,0xfe,0xfc,0xfc,0xff,0xff,0xe8,
+    0xbf,0xaa,0xac,0xb4,0xb5,0xb1,0xb1,0xc1,0xe8,0xff,0xff,0xfd,0xff,0xfa,0xee,0xe4,
+    0xe4,0xef,0xfb,0xff,0xfd,0xff,0xff,0xed,0xd0,0xc7,0xc8,0xcb,0xcb,0xc9,0xc9,0xd6,
+    0xf0,0xff,0xff,0xfd,0xfd,0xff,0xff,0xfd,0xf1,0xe5,0xda,0xd0,0xce,0xd1,0xd3,0xd3,
+    0xd3,0xd3,0xd1,0xd3,0xe1,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd3,0xab,0x9f,0xa6,0xa5,
+    0xa1,0xa4,0xaa,0xb0,0xc2,0xe0,0xf8,0xff,0xfe,0xfd,0xff,0xff,0xfc,0xff,0xfc,0xd2,
+    0xab,0xac,0xb3,0xaf,0xaf,0xb3,0xb3,0xbc,0xe4,0xff,0xff,0xfe,0xfa,0xe3,0xcb,0xc0,
+    0xc1,0xce,0xe6,0xfa,0xfe,0xff,0xff,0xea,0xcb,0xc6,0xc9,0xc6,0xc6,0xcc,0xcb,0xca,
+    0xe1,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xfc,0xf0,0xdf,0xd5,0xd3,0xd1,0xcf,
+    0xd1,0xd1,0xce,0xd4,0xe9,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf0,0xc2,0xa2,0xa1,0xa4,
+    0xa1,0xa6,0xbe,0xe0,0xf6,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfc,0xe1,0xbd,
+    0xab,0xae,0xb2,0xaf,0xb0,0xb4,0xb6,0xc1,0xe7,0xff,0xff,0xfa,0xe7,0xc6,0xb7,0xbd,
+    0xbd,0xba,0xca,0xe9,0xfb,0xff,0xff,0xec,0xce,0xc5,0xc7,0xc8,0xc7,0xc9,0xca,0xca,
+    0xd5,0xeb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfb,0xef,0xdd,0xd1,0xce,
+    0xcf,0xce,0xcf,0xe0,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe9,0xd2,0xc4,0xc1,
+    0xc5,0xd3,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xc6,0xad,
+    0xb0,0xb1,0xb0,0xb2,0xb2,0xaf,0xb8,0xd1,0xef,0xff,0xff,0xef,0xd0,0xba,0xb7,0xc0,
+    0xc0,0xb9,0xc0,0xd7,0xf2,0xff,0xff,0xf3,0xda,0xc7,0xc4,0xc9,0xc9,0xc6,0xc9,0xcb,
+    0xca,0xdb,0xf9,0xff,0xfd,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf3,0xe9,0xe1,
+    0xde,0xe0,0xe8,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf4,0xea,
+    0xf4,0xff,0xff,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe0,0xb5,0xa7,
+    0xb0,0xb2,0xaf,0xb2,0xb3,0xad,0xbe,0xe7,0xfb,0xff,0xff,0xe6,0xc2,0xbb,0xbd,0xbb,
+    0xbc,0xc0,0xc1,0xcb,0xea,0xff,0xff,0xfc,0xeb,0xcc,0xc1,0xc9,0xc9,0xc6,0xc9,0xca,
+    0xc6,0xd0,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf9,
+    0xf4,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xbf,0xae,0xae,
+    0xb0,0xb1,0xb0,0xb2,0xb4,0xb5,0xcf,0xf7,0xff,0xff,0xff,0xe4,0xbe,0xba,0xbe,0xbb,
+    0xbe,0xc2,0xbf,0xc4,0xe7,0xff,0xff,0xff,0xf9,0xd8,0xc5,0xc7,0xc8,0xc7,0xc8,0xc9,
+    0xc9,0xca,0xd6,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff,
+    0xfe,0xfc,0xfd,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xfd,0xd4,0xad,0xac,0xb4,
+    0xb1,0xb0,0xb3,0xb1,0xb1,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xe5,0xbf,0xb9,0xbc,0xbc,
+    0xbd,0xc0,0xbd,0xc4,0xe7,0xff,0xff,0xfe,0xfe,0xed,0xd1,0xc2,0xc5,0xc9,0xc7,0xc8,
+    0xcb,0xc8,0xca,0xe3,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe,
+    0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe5,0xc0,0xab,0xae,0xb3,
+    0xb0,0xb0,0xb6,0xb2,0xb3,0xd8,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfa,0xff,0xff,0xe1,0xc4,0xc5,0xca,0xc7,0xc7,
+    0xc9,0xc9,0xc9,0xd6,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf5,0xc8,0xae,0xaf,0xb1,0xb0,
+    0xb1,0xb2,0xb3,0xb5,0xc6,0xe9,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc,
+    0xbe,0xc1,0xbe,0xc5,0xe6,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc7,0xc7,0xc8,0xc8,
+    0xc7,0xc9,0xca,0xc9,0xda,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe4,0xb7,0xa8,0xb1,0xb2,0xae,
+    0xb3,0xb4,0xae,0xba,0xe1,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc,
+    0xbd,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xfc,0xe7,0xc9,0xc1,0xc9,0xc9,
+    0xc6,0xc9,0xc9,0xc5,0xd0,0xed,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc8,0xb1,0xae,0xb2,0xb2,0xaf,
+    0xb2,0xb5,0xb2,0xc8,0xf5,0xff,0xfe,0xff,0xfe,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfe,0xff,0xfe,0xff,0xf6,0xd3,0xc3,0xc9,0xc9,
+    0xc7,0xc8,0xc8,0xc9,0xcc,0xda,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xd9,0xaf,0xad,0xb4,0xb0,0xb0,0xb2,
+    0xb0,0xb2,0xc2,0xe1,0xfc,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe8,0xd0,0xc6,0xc7,
+    0xc9,0xc7,0xc7,0xcb,0xc8,0xc9,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf9,0xcd,0xa8,0xac,0xb4,0xae,0xae,0xb4,
+    0xb0,0xb1,0xd4,0xfa,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xfb,0xde,0xc4,0xc6,
+    0xca,0xc6,0xc7,0xca,0xc5,0xc5,0xde,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc6,0xab,0xaf,0xb3,0xb0,0xb1,0xb2,
+    0xb1,0xc1,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xeb,0xce,0xc5,
+    0xc9,0xc9,0xc8,0xca,0xc8,0xc6,0xd9,0xf4,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf0,0xc9,0xac,0xb0,0xb3,0xb2,0xb4,0xae,
+    0xb8,0xdd,0xfa,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfb,0xe6,0xca,
+    0xc4,0xca,0xc9,0xca,0xc7,0xc5,0xdb,0xf7,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd5,0xae,0xab,0xb1,0xae,0xab,0xaf,
+    0xcc,0xf4,0xff,0xfd,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc,
+    0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf7,0xd8,
+    0xc4,0xc3,0xc5,0xc7,0xc3,0xc6,0xe3,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc8,0xb7,0xb5,0xb3,0xb5,0xca,
+    0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,
+    0xd7,0xc9,0xc9,0xcb,0xcc,0xd8,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xde,0xce,0xcd,0xdc,0xf1,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb,
+    0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xf4,0xe5,0xdd,0xdd,0xe8,0xf6,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xef,0xef,0xfa,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc2,0xbc,0xbe,0xba,
+    0xbb,0xc0,0xbf,0xc6,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf4,0xf4,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbc,0xbe,
+    0xbe,0xbe,0xbf,0xcb,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd8,0xbd,0xb8,0xc1,
+    0xc1,0xba,0xc1,0xd9,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xee,0xcf,0xbb,0xb9,
+    0xb9,0xbc,0xd1,0xef,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,0xd6,0xca,
+    0xca,0xd7,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf6,
+    0xf6,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf2,
+    0xf1,0xf6,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xea,0xc4,0xa3,
+    0x9e,0xb2,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xb8,0x8d,0x80,
+    0x7d,0x83,0xa6,0xd9,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x8c,0x79,0x8b,
+    0x8b,0x7f,0x84,0xa9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xaf,0x88,0x81,0x87,
+    0x87,0x86,0x7f,0x8b,0xcf,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xe8,0xf3,0xfe,
+    0xff,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa5,0x8c,0x87,0x81,
+    0x81,0x89,0x81,0x82,0xc7,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfb,0xe6,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xd7,0xbb,0xb5,0xc6,0xe7,
+    0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xde,0xa5,0x88,0x84,0x85,
+    0x85,0x89,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,
+    0xee,0xcb,0xae,0xac,0xc6,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc0,0x9d,0x92,0x92,0x94,0xa9,
+    0xd5,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xde,0xa4,0x86,0x84,0x85,
+    0x84,0x88,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xed,
+    0xb3,0x83,0x7f,0x87,0x83,0x94,0xd3,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd0,0x93,0x85,0x8b,0x8c,0x88,0x86,
+    0xaa,0xe7,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf9,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf4,0xb8,
+    0x81,0x71,0x76,0x7c,0x6e,0x6e,0xae,0xf3,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfb,0xbc,0x85,0x8c,0x95,0x8f,0x91,0x8a,
+    0x92,0xbe,0xee,0xff,0xff,0xfd,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf8,0xff,0xff,0xff,0xfe,0xfd,0xff,0xd6,0x88,
+    0x72,0x83,0x7e,0x7c,0x7c,0x79,0x9b,0xd5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf4,0xb8,0x87,0x8c,0x94,0x8d,0x8d,0x91,
+    0x8b,0x96,0xd2,0xff,0xff,0xfa,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf8,0xff,0xff,0xfc,0xff,0xff,0xe4,0xaa,0x80,
+    0x7a,0x7f,0x7b,0x7c,0x7c,0x7a,0x97,0xd0,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc2,0x89,0x89,0x93,0x8c,0x8b,0x93,
+    0x8b,0x87,0xb6,0xf0,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf8,0xff,0xff,0xf9,0xff,0xff,0xc4,0x83,0x7e,
+    0x82,0x77,0x78,0x7e,0x74,0x70,0xa1,0xe4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xd3,0x98,0x8e,0x92,0x8e,0x8f,0x90,
+    0x8d,0x8e,0x9b,0xc5,0xf7,0xff,0xfe,0xff,0xfd,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xfe,0xe6,0xa5,0x75,0x78,
+    0x81,0x79,0x78,0x82,0x77,0x74,0xb5,0xfb,0xff,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xbf,0x97,0x8b,0x90,0x90,0x8b,
+    0x8f,0x91,0x85,0xa1,0xe7,0xff,0xfd,0xfe,0xfe,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfb,0xff,0xfd,0xff,0xf4,0xb3,0x85,0x7d,0x7b,
+    0x7a,0x7b,0x7a,0x7d,0x7c,0x90,0xd1,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xe3,0xa1,0x86,0x92,0x91,0x8c,
+    0x91,0x90,0x84,0x92,0xc6,0xf2,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfa,0xff,0xfd,0xff,0xdd,0x92,0x76,0x82,0x7e,
+    0x76,0x7d,0x7e,0x71,0x81,0xbe,0xf2,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbc,0x94,0x8e,0x8f,0x8f,
+    0x8f,0x8e,0x8f,0x8e,0x9e,0xd6,0xff,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc4,0xff,0xff,0xf7,0xff,0xff,0xe6,0xb0,0x84,0x78,0x7d,0x7c,
+    0x78,0x7d,0x7e,0x72,0x95,0xe6,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xe8,0xb1,0x89,0x8c,0x93,
+    0x8d,0x8c,0x92,0x89,0x87,0xbc,0xf8,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf4,0xff,0xff,0xc6,0x83,0x7b,0x81,0x79,0x7a,
+    0x7d,0x7a,0x79,0x8d,0xc3,0xf7,0xff,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,0xfd,0xfe,
+    0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xcc,0x93,0x8d,0x93,
+    0x8d,0x8e,0x90,0x8b,0x88,0x9f,0xd1,0xfa,0xff,0xff,0xff,0xdf,0xa4,0x87,0x84,0x84,
+    0x84,0x89,0x7d,0x7d,0xc5,0xff,0xff,0xf8,0xff,0xea,0xa6,0x76,0x7a,0x82,0x78,0x79,
+    0x81,0x77,0x75,0xaf,0xf1,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xfd,0xf9,0xfa,
+    0xfe,0xfe,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x92,0x8b,
+    0x90,0x90,0x8d,0x91,0x91,0x88,0xaa,0xee,0xff,0xff,0xff,0xdf,0xa4,0x8a,0x86,0x84,
+    0x82,0x89,0x7f,0x7d,0xc6,0xff,0xff,0xff,0xf6,0xba,0x88,0x7e,0x7d,0x7c,0x7a,0x7a,
+    0x7e,0x79,0x88,0xcd,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf6,0xee,
+    0xf5,0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xde,0x9e,0x86,
+    0x92,0x92,0x8c,0x91,0x91,0x85,0x96,0xcd,0xf4,0xff,0xff,0xe2,0xac,0x8d,0x85,0x83,
+    0x83,0x88,0x81,0x85,0xcb,0xff,0xff,0xfe,0xe1,0x95,0x76,0x82,0x7e,0x78,0x7e,0x7d,
+    0x75,0x82,0xb5,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfb,0xfc,0xff,0xff,0xf9,
+    0xed,0xeb,0xf7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xea,0xd1,0xbf,0xba,
+    0xc1,0xd0,0xe4,0xf6,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf4,0xb9,0x94,
+    0x91,0x8e,0x8c,0x8f,0x8f,0x8c,0x8f,0xa9,0xdf,0xff,0xff,0xee,0xc0,0x89,0x7a,0x8a,
+    0x8b,0x82,0x80,0x98,0xd8,0xff,0xff,0xef,0xc1,0x85,0x74,0x81,0x7f,0x76,0x7d,0x7f,
+    0x72,0x90,0xdf,0xff,0xfa,0xfd,0xff,0xfe,0xfb,0xfb,0xfe,0xff,0xfc,0xeb,0xcd,0xb4,
+    0xa8,0xa7,0xb3,0xd5,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf3,0xc0,0x99,0x93,0x94,
+    0x90,0x95,0xb1,0xd8,0xf2,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfd,0xe2,0xae,
+    0x8c,0x8e,0x92,0x8c,0x8b,0x91,0x8b,0x93,0xd2,0xff,0xff,0xfb,0xe0,0x9f,0x7e,0x86,
+    0x85,0x7d,0x91,0xc1,0xec,0xff,0xff,0xde,0xa4,0x83,0x7c,0x7c,0x7a,0x7a,0x7c,0x7a,
+    0x83,0xb4,0xf4,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xf7,0xde,0xae,0x7f,0x6b,
+    0x70,0x70,0x6a,0x91,0xdb,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xda,0xa5,0x91,0x99,0x98,
+    0x92,0x95,0x9a,0xa0,0xb4,0xd9,0xf6,0xff,0xff,0xfc,0xfe,0xff,0xfb,0xff,0xff,0xc8,
+    0x91,0x90,0x95,0x8b,0x8b,0x92,0x89,0x8b,0xcc,0xff,0xff,0xfc,0xfa,0xd8,0xa9,0x8d,
+    0x86,0x94,0xbf,0xee,0xfd,0xff,0xff,0xda,0x9b,0x81,0x7d,0x78,0x77,0x7f,0x78,0x71,
+    0xa4,0xe8,0xff,0xfe,0xff,0xfd,0xfd,0xfc,0xfe,0xfe,0xe3,0xaf,0x8a,0x7d,0x74,0x6f,
+    0x72,0x73,0x6e,0x7b,0xac,0xe7,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc8,0xa2,0x97,0x9e,0x9e,
+    0x9d,0x9e,0x99,0x8f,0x95,0xad,0xc6,0xdf,0xf9,0xff,0xfe,0xfd,0xfb,0xff,0xff,0xe6,
+    0xb2,0x8e,0x8b,0x94,0x94,0x93,0x8a,0x91,0xd0,0xff,0xff,0xfa,0xff,0xfa,0xe2,0xcb,
+    0xc7,0xd5,0xed,0xfd,0xfe,0xff,0xff,0xe2,0xa6,0x7e,0x79,0x81,0x81,0x7d,0x75,0x82,
+    0xc8,0xff,0xff,0xf7,0xfb,0xfe,0xff,0xfb,0xe2,0xc2,0x9f,0x7c,0x6c,0x71,0x7a,0x7d,
+    0x7b,0x79,0x7a,0x7b,0x93,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc9,0xa1,0x95,0x9c,0x9c,
+    0x98,0x97,0x99,0x9b,0x99,0x95,0x94,0xa9,0xd0,0xee,0xfa,0xfe,0xff,0xfe,0xfd,0xfe,
+    0xdf,0xa6,0x8a,0x8e,0x8c,0x89,0x94,0xb4,0xe5,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf4,0xc6,0x8c,0x73,0x79,0x79,0x73,0x86,0xb6,
+    0xeb,0xff,0xff,0xfe,0xff,0xfb,0xf1,0xd6,0xa6,0x7b,0x6e,0x76,0x7a,0x77,0x74,0x73,
+    0x74,0x76,0x74,0x76,0x93,0xd7,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xd9,0xa2,0x93,0xa1,0x9d,
+    0x98,0x9b,0x9a,0x9a,0x99,0x97,0x95,0x97,0x9f,0xb0,0xd3,0xf7,0xff,0xfd,0xfc,0xff,
+    0xfa,0xda,0xae,0x91,0x8a,0x91,0xb4,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xee,0xb7,0x88,0x79,0x78,0x88,0xb8,0xee,
+    0xff,0xfc,0xfd,0xff,0xfa,0xd4,0xa4,0x85,0x7a,0x73,0x71,0x75,0x79,0x78,0x75,0x72,
+    0x76,0x79,0x71,0x78,0xa7,0xe4,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xef,0xbb,0x98,0x95,0x9c,
+    0xa1,0x9f,0x9a,0x98,0x98,0x9b,0x9e,0x99,0x90,0x94,0xa8,0xc0,0xdb,0xf2,0xfe,0xff,
+    0xff,0xfa,0xe4,0xcc,0xc7,0xcf,0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xf9,
+    0xf9,0xfb,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe8,0xcf,0xbf,0xbf,0xcf,0xe9,0xfc,
+    0xff,0xff,0xfb,0xe0,0xb8,0x97,0x79,0x6a,0x71,0x7c,0x7b,0x75,0x72,0x73,0x78,0x7b,
+    0x7a,0x71,0x6c,0x91,0xd4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe4,0xbc,0xa0,0x9b,
+    0x9b,0x9a,0x9c,0x9e,0x9d,0x9a,0x98,0x98,0x9a,0x9c,0x96,0x91,0xa3,0xce,0xf3,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xff,0xf8,0xd6,0x9c,0x74,0x70,0x79,0x7a,0x76,0x74,0x74,0x76,0x78,0x77,0x75,0x72,
+    0x72,0x7a,0x93,0xc6,0xf8,0xff,0xfd,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf3,0xd6,0xaf,
+    0x99,0x99,0x9d,0x9e,0x9d,0x9a,0x99,0x99,0x9c,0x9b,0x98,0x95,0x94,0xa3,0xc6,0xee,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xf9,0xd2,0x92,0x72,0x74,0x75,0x76,0x79,0x77,0x74,0x73,0x75,0x78,0x78,0x73,0x6e,
+    0x80,0xae,0xe0,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,
+    0xc7,0xb1,0x9b,0x93,0x9a,0x9f,0x9f,0x9b,0x97,0x96,0x9a,0x9d,0x9a,0x91,0x9d,0xd5,
+    0xff,0xff,0xf9,0xfa,0xfa,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfa,0xf9,0xf8,0xf8,0xff,0xff,
+    0xe2,0xa1,0x71,0x6e,0x7d,0x79,0x72,0x71,0x73,0x78,0x7c,0x78,0x6b,0x6d,0x87,0xa7,
+    0xc8,0xe9,0xfc,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xf8,0xdd,0xba,0xa5,0x9f,0x9a,0x99,0x9d,0x9c,0x9c,0x99,0x97,0x9f,0x95,0x8f,0xc5,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,
+    0xc8,0x81,0x77,0x7b,0x74,0x74,0x77,0x79,0x78,0x73,0x71,0x76,0x7b,0x90,0xbd,0xec,
+    0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,
+    0xfe,0xfc,0xf5,0xdf,0xba,0x9c,0x96,0x9d,0x9e,0x9e,0x9b,0x99,0x9f,0x97,0x91,0xc2,
+    0xfb,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,
+    0xc5,0x7b,0x75,0x7c,0x73,0x75,0x78,0x7a,0x79,0x6f,0x6e,0x8a,0xbd,0xe6,0xf7,0xfd,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xfb,
+    0xfc,0xff,0xff,0xf9,0xe6,0xcd,0xb4,0xa1,0x95,0x95,0x9b,0x9d,0x9e,0x94,0x98,0xcd,
+    0xff,0xff,0xfb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xfa,0xff,0xff,
+    0xd4,0x8c,0x6f,0x72,0x78,0x77,0x70,0x6a,0x6f,0x8b,0xaf,0xce,0xee,0xff,0xff,0xfa,
+    0xf9,0xfd,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfc,0xff,0xff,0xe8,0xc3,0xaa,0xa1,0x9e,0x9b,0x9c,0xa3,0xb9,0xe5,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xed,0xb9,0x86,0x72,0x76,0x76,0x77,0x7f,0x96,0xca,0xf7,0xff,0xfd,0xfd,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe5,0xcc,0xb5,0xa8,0xad,0xc9,0xec,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xee,0xbe,0x96,0x89,0x8d,0xa4,0xc9,0xeb,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfc,0xf9,0xf8,0xfb,0xff,0xfe,0xf1,0xe1,0xda,0xdf,0xee,0xfd,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xee,0xd9,0xcc,0xd0,0xe3,0xf6,0xfe,0xfe,0xfb,0xfa,0xfc,0xff,0xff,0xfe,
+    0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe9,0xe0,0xdf,0xdf,0xdf,0xdf,0xdf,
+    0xdf,0xdf,0xde,0xdd,0xdd,0xdd,0xdd,0xdf,0xe3,0xe9,0xf4,0xfe,0xff,0xfd,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfb,0xfa,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf4,0xf4,0xf4,0xf4,0xf4,
+    0xf4,0xf5,0xf4,0xf4,0xf5,0xf5,0xf7,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xd2,0xbe,0xb3,0xaf,0xaf,0xaf,0xaf,0xaf,
+    0xaf,0xaf,0xad,0xac,0xac,0xac,0xaa,0xac,0xb4,0xbe,0xd7,0xf6,0xff,0xfe,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfa,0xf2,0xea,0xe7,0xe7,0xe7,0xe6,0xe6,0xe6,0xe6,0xe6,
+    0xe6,0xe5,0xe6,0xe6,0xe5,0xe6,0xe9,0xee,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xf1,0xc9,0xa8,0xa5,0xa7,0xa3,0xa4,0xa4,0xa3,0xa3,
+    0xa3,0xa3,0xa3,0xa3,0xa3,0xa3,0xa1,0xa3,0xa5,0xa4,0xaf,0xd0,0xf2,0xff,0xff,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xf8,0xea,0xe3,0xe4,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,0xe3,0xe9,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x9c,0xa2,0xa8,0xa7,0xa7,0xa7,0xa6,0xa6,
+    0xa6,0xa6,0xa6,0xa7,0xa7,0xa7,0xa7,0xa7,0xa5,0xa2,0x9f,0xae,0xde,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfa,0xec,0xe1,0xde,0xe2,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe1,0xe0,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xa9,0xa6,0xa3,0xa4,0xa4,0xa4,0xa4,0xa5,
+    0xa5,0xa4,0xa4,0xa4,0xa4,0xa4,0xa5,0xa3,0xa3,0xab,0xa3,0x9f,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe6,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xaa,0xa8,0xa4,0xa5,0xa5,0xa5,0xa5,0xa5,
+    0xa5,0xa5,0xa5,0xa5,0xa5,0xa5,0xa6,0xa4,0xa4,0xac,0xa3,0x9f,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe5,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0xac,0x9e,0xa3,0xa8,0xa8,0xa8,0xaa,0xa9,0xa7,
+    0xa7,0xa8,0xa9,0xa9,0xa9,0xa9,0xa9,0xa9,0xa7,0xa6,0xa1,0xad,0xdd,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xf9,0xeb,0xe0,0xdf,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe2,0xe2,0xe0,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc7,0xa8,0xa6,0xa8,0xa5,0xa6,0xa8,0xa7,0xa6,
+    0xa7,0xa8,0xa8,0xa8,0xa8,0xa8,0xa7,0xa8,0xa9,0xa8,0xb1,0xd0,0xf1,0xff,0xff,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xf5,0xe7,0xe1,0xe2,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,
+    0xe1,0xe2,0xe2,0xe1,0xe2,0xe2,0xe2,0xe2,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xec,0xd0,0xbd,0xb3,0xb0,0xb0,0xb0,0xb0,0xb2,
+    0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb6,0xbe,0xd6,0xf4,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xf6,0xec,0xe6,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe4,0xe3,0xe3,0xe4,0xe4,0xe5,0xeb,0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xe7,0xde,0xdd,0xdd,0xdd,0xdd,0xde,
+    0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xdf,0xe1,0xe7,0xf2,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfd,0xf8,0xf4,0xf3,0xf3,0xf3,0xf2,0xf2,0xf3,0xf3,0xf3,
+    0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf4,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfd,0xfa,0xf9,0xfc,0xff,0xfe,0xf6,0xeb,0xe5,0xe9,0xf4,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfc,0xf8,0xf5,0xf6,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xea,0xd9,0xc6,0xba,0xbf,0xd8,0xf3,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf1,0xe7,0xe4,0xe6,0xec,0xf5,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,0xcf,0xba,0xb3,0xb0,0xad,0xae,0xb7,0xca,0xec,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfb,0xee,0xe1,0xdc,0xdc,0xdd,0xde,0xe2,0xe9,0xf4,0xfd,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,
+    0xfc,0xff,0xff,0xfc,0xef,0xdb,0xc5,0xb3,0xa7,0xa8,0xae,0xb1,0xb1,0xab,0xaf,0xda,
+    0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf4,0xe1,0xd9,0xda,0xdc,0xdb,0xda,0xda,0xdd,0xe4,0xed,0xf4,0xfb,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,
+    0xff,0xfe,0xf9,0xe8,0xca,0xb0,0xa9,0xae,0xb0,0xb2,0xb1,0xaf,0xb4,0xaf,0xad,0xd2,
+    0xfc,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xef,0xdc,0xdb,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe3,0xf0,0xfa,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfc,0xe9,0xcc,0xb8,0xb1,0xad,0xad,0xb0,0xb2,0xb3,0xb0,0xaf,0xb5,0xb0,0xae,0xd4,
+    0xfe,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf0,0xdc,0xd9,0xdc,0xdb,0xdb,0xdb,0xdc,0xdc,0xdb,0xdb,0xdd,0xdf,0xe5,0xf1,0xfb,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,
+    0xd7,0xc3,0xb0,0xa8,0xad,0xb3,0xb3,0xb0,0xaf,0xaf,0xb1,0xb4,0xb5,0xaf,0xb6,0xde,
+    0xff,0xff,0xfb,0xfd,0xfc,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfd,0xfd,0xff,0xff,
+    0xf7,0xe4,0xd7,0xd7,0xdc,0xdb,0xd9,0xda,0xdb,0xdc,0xdd,0xdd,0xda,0xdb,0xe2,0xea,
+    0xf3,0xfc,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf7,0xe2,0xc3,
+    0xad,0xab,0xb1,0xb3,0xb0,0xb0,0xb1,0xb1,0xb2,0xb2,0xb3,0xb2,0xb3,0xbb,0xd0,0xef,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xf2,0xe0,0xd7,0xd8,0xd9,0xda,0xdc,0xdb,0xdb,0xdb,0xdc,0xdd,0xdd,0xdb,0xdb,
+    0xe1,0xed,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xce,0xb4,0xae,
+    0xad,0xad,0xb1,0xb3,0xb2,0xb0,0xb0,0xb1,0xb2,0xb5,0xb3,0xb1,0xbb,0xd7,0xf3,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xfd,0xf2,0xe2,0xd7,0xd7,0xdb,0xdc,0xdb,0xdb,0xdb,0xdb,0xdc,0xdd,0xdb,0xda,
+    0xdb,0xde,0xe4,0xf0,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf3,0xcd,0xaf,0xa9,0xae,
+    0xb4,0xb3,0xb0,0xae,0xaf,0xb2,0xb5,0xb4,0xaf,0xb0,0xbd,0xce,0xe1,0xf2,0xfe,0xff,
+    0xff,0xfd,0xf1,0xe2,0xe0,0xe6,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xf9,0xf1,0xec,0xec,0xf2,0xfa,0xff,
+    0xff,0xff,0xfd,0xf5,0xe9,0xe1,0xda,0xd7,0xd9,0xdb,0xdb,0xda,0xda,0xda,0xdc,0xdd,
+    0xdd,0xdb,0xd9,0xe3,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe1,0xb5,0xa9,0xb4,0xb3,
+    0xae,0xb1,0xb2,0xb2,0xb2,0xb1,0xb1,0xb4,0xb9,0xc4,0xdb,0xf5,0xff,0xfe,0xfd,0xff,
+    0xfc,0xea,0xd0,0xbe,0xb9,0xc0,0xd6,0xf4,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe8,0xd9,0xd3,0xd3,0xda,0xeb,0xfa,
+    0xff,0xfe,0xfe,0xff,0xfd,0xf2,0xe5,0xde,0xda,0xd8,0xd9,0xdb,0xdc,0xdb,0xdb,0xdb,
+    0xdb,0xdc,0xdb,0xde,0xea,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xd5,0xb5,0xac,0xb2,0xb1,
+    0xae,0xb0,0xb2,0xb3,0xb4,0xb3,0xb2,0xbf,0xda,0xf1,0xfb,0xfe,0xff,0xfe,0xfe,0xfe,
+    0xeb,0xc9,0xb9,0xbb,0xba,0xba,0xc4,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,0xed,0xd7,0xce,0xd1,0xd2,0xd2,0xd9,0xe8,
+    0xf9,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf2,0xe5,0xda,0xd7,0xdb,0xdd,0xdc,0xdb,0xdb,
+    0xdb,0xdb,0xdb,0xdd,0xe4,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb6,0xaf,0xb4,0xb3,
+    0xb3,0xb5,0xb3,0xad,0xb0,0xc1,0xd3,0xe5,0xf8,0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,
+    0xcf,0xb9,0xb7,0xbd,0xbf,0xc0,0xbd,0xc4,0xe6,0xff,0xff,0xfc,0xff,0xfe,0xf4,0xea,
+    0xea,0xf0,0xf9,0xff,0xff,0xff,0xff,0xf5,0xe0,0xd1,0xce,0xd3,0xd6,0xd4,0xd1,0xd8,
+    0xee,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe3,0xda,0xd7,0xd9,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xe3,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xe1,0xb9,0xaa,0xb1,0xb2,
+    0xae,0xb0,0xb3,0xb8,0xc6,0xdf,0xf7,0xff,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,
+    0xba,0xba,0xbe,0xba,0xbb,0xc1,0xbd,0xbf,0xe3,0xff,0xff,0xfd,0xfd,0xef,0xd9,0xcd,
+    0xcd,0xd5,0xe7,0xfa,0xff,0xff,0xff,0xf1,0xda,0xd2,0xd2,0xd1,0xd2,0xd5,0xd3,0xd2,
+    0xe3,0xf8,0xff,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf5,0xe6,0xdd,0xda,0xd8,0xd8,
+    0xdb,0xdb,0xd8,0xdc,0xe9,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd0,0xb1,0xab,0xaf,
+    0xad,0xb0,0xc3,0xe0,0xf5,0xfd,0xfe,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xed,0xcd,
+    0xb8,0xba,0xbe,0xbb,0xbd,0xc2,0xbe,0xc2,0xe5,0xff,0xff,0xfd,0xf1,0xd3,0xc4,0xc8,
+    0xc8,0xc7,0xd2,0xe7,0xf8,0xff,0xff,0xf2,0xdd,0xd3,0xd1,0xd1,0xd1,0xd3,0xd4,0xd3,
+    0xda,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf4,0xe6,0xda,0xd7,
+    0xda,0xda,0xd8,0xe2,0xf6,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,0xdc,0xcd,0xc7,
+    0xcb,0xd7,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfd,0xff,0xf8,0xd3,0xbd,
+    0xbc,0xbb,0xbb,0xbe,0xbf,0xbf,0xc1,0xcf,0xed,0xff,0xff,0xf8,0xe2,0xc6,0xc1,0xcb,
+    0xca,0xc7,0xca,0xd5,0xef,0xff,0xff,0xf8,0xe8,0xd2,0xcd,0xd2,0xd2,0xcf,0xd2,0xd4,
+    0xd3,0xde,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8,
+    0xe5,0xe6,0xeb,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xec,
+    0xf3,0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xea,0xc2,0xb4,
+    0xbd,0xbf,0xbb,0xbe,0xc0,0xbb,0xc5,0xe4,0xf9,0xff,0xff,0xf1,0xd7,0xc7,0xc6,0xc8,
+    0xc8,0xca,0xca,0xce,0xeb,0xff,0xff,0xff,0xf4,0xd7,0xcc,0xd3,0xd2,0xcf,0xd2,0xd4,
+    0xd2,0xd7,0xea,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb,
+    0xf6,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xd4,0xbd,0xb9,
+    0xbd,0xbd,0xbd,0xc0,0xc0,0xbc,0xcf,0xf5,0xff,0xff,0xff,0xf0,0xd3,0xc7,0xc6,0xc7,
+    0xc7,0xcb,0xc9,0xca,0xe8,0xff,0xff,0xff,0xfc,0xe4,0xd3,0xd1,0xd1,0xd1,0xd1,0xd2,
+    0xd5,0xd3,0xd8,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,
+    0xfe,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe2,0xbf,0xba,0xbf,
+    0xbc,0xbc,0xbf,0xbf,0xbe,0xc7,0xe2,0xfc,0xff,0xff,0xff,0xf0,0xd2,0xc5,0xc5,0xc7,
+    0xc8,0xc9,0xc7,0xca,0xe8,0xff,0xff,0xfd,0xff,0xf6,0xdf,0xcf,0xd0,0xd3,0xd0,0xd1,
+    0xd5,0xd3,0xd1,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd1,0xb6,0xb8,0xbf,
+    0xbc,0xbc,0xc0,0xbd,0xbd,0xd7,0xf8,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc7,
+    0xc7,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfb,0xff,0xff,0xea,0xd2,0xd0,0xd3,0xd0,0xd0,
+    0xd2,0xd3,0xd3,0xda,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xc0,0xba,0xbb,0xbc,
+    0xbd,0xbe,0xc0,0xbf,0xc8,0xe8,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfc,0xff,0xff,0xf5,0xe1,0xd3,0xd0,0xd1,0xd1,
+    0xd1,0xd4,0xd5,0xd1,0xdc,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf0,0xc7,0xb6,0xbe,0xbe,0xbb,
+    0xbe,0xc0,0xbb,0xc2,0xdd,0xf7,0xff,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xf1,0xd6,0xcd,0xd2,0xd2,
+    0xd0,0xd3,0xd4,0xd0,0xd7,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd9,0xbf,0xb8,0xbd,0xbe,0xbb,
+    0xbe,0xc0,0xbb,0xcb,0xf1,0xff,0xfe,0xff,0xfe,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xfe,0xff,0xfa,0xe0,0xd1,0xd2,0xd1,
+    0xd0,0xd1,0xd3,0xd4,0xd5,0xdd,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc1,0xba,0xbe,0xbc,0xbc,0xbe,
+    0xbf,0xbe,0xc6,0xe0,0xfb,0xff,0xfe,0xff,0xfe,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xfe,0xf2,0xdd,0xcf,0xd0,
+    0xd2,0xd0,0xd0,0xd4,0xd2,0xd3,0xe8,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfe,0xda,0xb6,0xb7,0xbf,0xbc,0xbb,0xc0,
+    0xbd,0xbb,0xd4,0xf6,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc7,0xc9,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xe8,0xd1,0xd1,
+    0xd3,0xcf,0xcf,0xd3,0xd1,0xd0,0xe0,0xf7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf8,0xd4,0xb7,0xba,0xc0,0xbc,0xbd,0xc1,
+    0xbe,0xc2,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xf4,0xde,0xd1,
+    0xd0,0xd2,0xd1,0xd2,0xd3,0xd4,0xde,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd7,0xb8,0xbc,0xc0,0xbd,0xc0,0xbe,
+    0xc2,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xef,0xd3,
+    0xcd,0xd3,0xd2,0xd1,0xd2,0xd3,0xdf,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xbb,0xb7,0xbd,0xbd,0xbc,0xba,
+    0xcc,0xef,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe3,
+    0xd1,0xcd,0xcf,0xd1,0xce,0xcf,0xe5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf0,0xd3,0xc3,0xc0,0xc1,0xc2,0xcb,
+    0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf7,
+    0xe3,0xd2,0xd2,0xd4,0xd3,0xdb,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe4,0xd5,0xd3,0xde,0xef,
+    0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc6,0xc8,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xf7,0xeb,0xe3,0xe1,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xf1,0xf0,0xf8,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xef,0xd4,0xc9,0xc8,0xc7,
+    0xc6,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf6,0xf4,0xf9,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xda,0xc8,0xc6,0xca,
+    0xc9,0xc9,0xc9,0xcf,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe7,0xca,0xc2,0xcb,
+    0xcb,0xc6,0xc9,0xd9,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf5,0xdc,0xc8,0xc4,
+    0xc4,0xc5,0xd3,0xec,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xd2,
+    0xd2,0xd9,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,
+    0xf7,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,
+    0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf2,
+    0xf1,0xf6,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xea,0xc4,0xa3,
+    0x9e,0xb2,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xb8,0x8d,0x80,
+    0x7d,0x83,0xa6,0xd9,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x8c,0x79,0x8b,
+    0x8b,0x7f,0x84,0xa9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe,
+    0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xaf,0x88,0x81,0x87,
+    0x87,0x86,0x7f,0x8b,0xcf,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xe8,0xf3,0xfe,
+    0xff,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa5,0x8c,0x87,0x81,
+    0x81,0x89,0x81,0x82,0xc7,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xfb,0xe6,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xd7,0xbb,0xb5,0xc6,0xe7,
+    0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xde,0xa5,0x88,0x84,0x85,
+    0x85,0x89,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,
+    0xee,0xcb,0xae,0xac,0xc6,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc0,0x9d,0x92,0x92,0x94,0xa9,
+    0xd5,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xde,0xa4,0x86,0x84,0x85,
+    0x84,0x88,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xed,
+    0xb3,0x83,0x7f,0x87,0x83,0x94,0xd3,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd0,0x93,0x85,0x8b,0x8c,0x88,0x86,
+    0xaa,0xe7,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf9,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf4,0xb8,
+    0x81,0x71,0x76,0x7c,0x6e,0x6e,0xae,0xf3,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfb,0xbc,0x85,0x8c,0x95,0x8f,0x91,0x8a,
+    0x92,0xbe,0xee,0xff,0xff,0xfd,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf8,0xff,0xff,0xff,0xfe,0xfd,0xff,0xd6,0x88,
+    0x72,0x83,0x7e,0x7c,0x7c,0x79,0x9b,0xd5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf4,0xb8,0x87,0x8c,0x94,0x8d,0x8d,0x91,
+    0x8b,0x96,0xd2,0xff,0xff,0xfa,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf8,0xff,0xff,0xfc,0xff,0xff,0xe4,0xaa,0x80,
+    0x7a,0x7f,0x7b,0x7c,0x7c,0x7a,0x97,0xd0,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc2,0x89,0x89,0x93,0x8c,0x8b,0x93,
+    0x8b,0x87,0xb6,0xf0,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf8,0xff,0xff,0xf9,0xff,0xff,0xc4,0x83,0x7e,
+    0x82,0x77,0x78,0x7e,0x74,0x70,0xa1,0xe4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xd3,0x98,0x8e,0x92,0x8e,0x8f,0x90,
+    0x8d,0x8e,0x9b,0xc5,0xf7,0xff,0xfe,0xff,0xfd,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xfe,0xe6,0xa5,0x75,0x78,
+    0x81,0x79,0x78,0x82,0x77,0x74,0xb5,0xfb,0xff,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xbf,0x97,0x8b,0x90,0x90,0x8b,
+    0x8f,0x91,0x85,0xa1,0xe7,0xff,0xfd,0xfe,0xfe,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfb,0xff,0xfd,0xff,0xf4,0xb3,0x85,0x7d,0x7b,
+    0x7a,0x7b,0x7a,0x7d,0x7c,0x90,0xd1,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xe3,0xa1,0x86,0x92,0x91,0x8c,
+    0x91,0x90,0x84,0x92,0xc6,0xf2,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfa,0xff,0xfd,0xff,0xdd,0x92,0x76,0x82,0x7e,
+    0x76,0x7d,0x7e,0x71,0x81,0xbe,0xf2,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbc,0x94,0x8e,0x8f,0x8f,
+    0x8f,0x8e,0x8f,0x8e,0x9e,0xd6,0xff,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc4,0xff,0xff,0xf7,0xff,0xff,0xe6,0xb0,0x84,0x78,0x7d,0x7c,
+    0x78,0x7d,0x7e,0x72,0x95,0xe6,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xe8,0xb1,0x89,0x8c,0x93,
+    0x8d,0x8c,0x92,0x89,0x87,0xbc,0xf8,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84,
+    0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf4,0xff,0xff,0xc6,0x83,0x7b,0x81,0x79,0x7a,
+    0x7d,0x7a,0x79,0x8d,0xc3,0xf7,0xff,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,0xfd,0xfe,
+    0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xcc,0x93,0x8d,0x93,
+    0x8d,0x8e,0x90,0x8b,0x88,0x9f,0xd1,0xfa,0xff,0xff,0xff,0xdf,0xa4,0x87,0x84,0x84,
+    0x84,0x89,0x7d,0x7d,0xc5,0xff,0xff,0xf8,0xff,0xea,0xa6,0x76,0x7a,0x82,0x78,0x79,
+    0x81,0x77,0x75,0xaf,0xf1,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xfd,0xf9,0xfa,
+    0xfe,0xfe,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x92,0x8b,
+    0x90,0x90,0x8d,0x91,0x91,0x88,0xaa,0xee,0xff,0xff,0xff,0xdf,0xa4,0x8a,0x86,0x84,
+    0x82,0x89,0x7f,0x7d,0xc6,0xff,0xff,0xff,0xf6,0xba,0x88,0x7e,0x7d,0x7c,0x7a,0x7a,
+    0x7e,0x79,0x88,0xcd,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf6,0xee,
+    0xf5,0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xde,0x9e,0x86,
+    0x92,0x92,0x8c,0x91,0x91,0x85,0x96,0xcd,0xf4,0xff,0xff,0xe2,0xac,0x8d,0x85,0x83,
+    0x83,0x88,0x81,0x85,0xcb,0xff,0xff,0xfe,0xe1,0x95,0x76,0x82,0x7e,0x78,0x7e,0x7d,
+    0x75,0x82,0xb5,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfb,0xfc,0xff,0xff,0xf9,
+    0xed,0xeb,0xf7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xea,0xd1,0xbf,0xba,
+    0xc1,0xd0,0xe4,0xf6,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf4,0xb9,0x94,
+    0x91,0x8e,0x8c,0x8f,0x8f,0x8c,0x8f,0xa9,0xdf,0xff,0xff,0xee,0xc0,0x89,0x7a,0x8a,
+    0x8b,0x82,0x80,0x98,0xd8,0xff,0xff,0xef,0xc1,0x85,0x74,0x81,0x7f,0x76,0x7d,0x7f,
+    0x72,0x90,0xdf,0xff,0xfa,0xfd,0xff,0xfe,0xfb,0xfb,0xfe,0xff,0xfc,0xeb,0xcd,0xb4,
+    0xa8,0xa7,0xb3,0xd5,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf3,0xc0,0x99,0x93,0x94,
+    0x90,0x95,0xb1,0xd8,0xf2,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfd,0xe2,0xae,
+    0x8c,0x8e,0x92,0x8c,0x8b,0x91,0x8b,0x93,0xd2,0xff,0xff,0xfb,0xe0,0x9f,0x7e,0x86,
+    0x85,0x7d,0x91,0xc1,0xec,0xff,0xff,0xde,0xa4,0x83,0x7c,0x7c,0x7a,0x7a,0x7c,0x7a,
+    0x83,0xb4,0xf4,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xf7,0xde,0xae,0x7f,0x6b,
+    0x70,0x70,0x6a,0x91,0xdb,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xda,0xa5,0x91,0x99,0x98,
+    0x92,0x95,0x9a,0xa0,0xb4,0xd9,0xf6,0xff,0xff,0xfc,0xfe,0xff,0xfb,0xff,0xff,0xc8,
+    0x91,0x90,0x95,0x8b,0x8b,0x92,0x89,0x8b,0xcc,0xff,0xff,0xfc,0xfa,0xd8,0xa9,0x8d,
+    0x86,0x94,0xbf,0xee,0xfd,0xff,0xff,0xda,0x9b,0x81,0x7d,0x78,0x77,0x7f,0x78,0x71,
+    0xa4,0xe8,0xff,0xfe,0xff,0xfd,0xfd,0xfc,0xfe,0xfe,0xe3,0xaf,0x8a,0x7d,0x74,0x6f,
+    0x72,0x73,0x6e,0x7b,0xac,0xe7,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc8,0xa2,0x97,0x9e,0x9e,
+    0x9d,0x9e,0x99,0x8f,0x95,0xad,0xc6,0xdf,0xf9,0xff,0xfe,0xfd,0xfb,0xff,0xff,0xe6,
+    0xb2,0x8e,0x8b,0x94,0x94,0x93,0x8a,0x91,0xd0,0xff,0xff,0xfa,0xff,0xfa,0xe2,0xcb,
+    0xc7,0xd5,0xed,0xfd,0xfe,0xff,0xff,0xe2,0xa6,0x7e,0x79,0x81,0x81,0x7d,0x75,0x82,
+    0xc8,0xff,0xff,0xf7,0xfb,0xfe,0xff,0xfb,0xe2,0xc2,0x9f,0x7c,0x6c,0x71,0x7a,0x7d,
+    0x7b,0x79,0x7a,0x7b,0x93,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc9,0xa1,0x95,0x9c,0x9c,
+    0x98,0x97,0x99,0x9b,0x99,0x95,0x94,0xa9,0xd0,0xee,0xfa,0xfe,0xff,0xfe,0xfd,0xfe,
+    0xdf,0xa6,0x8a,0x8e,0x8c,0x89,0x94,0xb4,0xe5,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf4,0xc6,0x8c,0x73,0x79,0x79,0x73,0x86,0xb6,
+    0xeb,0xff,0xff,0xfe,0xff,0xfb,0xf1,0xd6,0xa6,0x7b,0x6e,0x76,0x7a,0x77,0x74,0x73,
+    0x74,0x76,0x74,0x76,0x93,0xd7,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xd9,0xa2,0x93,0xa1,0x9d,
+    0x98,0x9b,0x9a,0x9a,0x99,0x97,0x95,0x97,0x9f,0xb0,0xd3,0xf7,0xff,0xfd,0xfc,0xff,
+    0xfa,0xda,0xae,0x91,0x8a,0x91,0xb4,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xee,0xb7,0x88,0x79,0x78,0x88,0xb8,0xee,
+    0xff,0xfc,0xfd,0xff,0xfa,0xd4,0xa4,0x85,0x7a,0x73,0x71,0x75,0x79,0x78,0x75,0x72,
+    0x76,0x79,0x71,0x78,0xa7,0xe4,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xef,0xbb,0x98,0x95,0x9c,
+    0xa1,0x9f,0x9a,0x98,0x98,0x9b,0x9e,0x99,0x90,0x94,0xa8,0xc0,0xdb,0xf2,0xfe,0xff,
+    0xff,0xfa,0xe4,0xcc,0xc7,0xcf,0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xf9,
+    0xf9,0xfb,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe8,0xcf,0xbf,0xbf,0xcf,0xe9,0xfc,
+    0xff,0xff,0xfb,0xe0,0xb8,0x97,0x79,0x6a,0x71,0x7c,0x7b,0x75,0x72,0x73,0x78,0x7b,
+    0x7a,0x71,0x6c,0x91,0xd4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe4,0xbc,0xa0,0x9b,
+    0x9b,0x9a,0x9c,0x9e,0x9d,0x9a,0x98,0x98,0x9a,0x9c,0x96,0x91,0xa3,0xce,0xf3,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xff,0xf8,0xd6,0x9c,0x74,0x70,0x79,0x7a,0x76,0x74,0x74,0x76,0x78,0x77,0x75,0x72,
+    0x72,0x7a,0x93,0xc6,0xf8,0xff,0xfd,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf3,0xd6,0xaf,
+    0x99,0x99,0x9d,0x9e,0x9d,0x9a,0x99,0x99,0x9c,0x9b,0x98,0x95,0x94,0xa3,0xc6,0xee,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xf9,0xd2,0x92,0x72,0x74,0x75,0x76,0x79,0x77,0x74,0x73,0x75,0x78,0x78,0x73,0x6e,
+    0x80,0xae,0xe0,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,
+    0xc7,0xb1,0x9b,0x93,0x9a,0x9f,0x9f,0x9b,0x97,0x96,0x9a,0x9d,0x9a,0x91,0x9d,0xd5,
+    0xff,0xff,0xf9,0xfa,0xfa,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfa,0xf9,0xf8,0xf8,0xff,0xff,
+    0xe2,0xa1,0x71,0x6e,0x7d,0x79,0x72,0x71,0x73,0x78,0x7c,0x78,0x6b,0x6d,0x87,0xa7,
+    0xc8,0xe9,0xfc,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xf8,0xdd,0xba,0xa5,0x9f,0x9a,0x99,0x9d,0x9c,0x9c,0x99,0x97,0x9f,0x95,0x8f,0xc5,
+    0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,
+    0xc8,0x81,0x77,0x7b,0x74,0x74,0x77,0x79,0x78,0x73,0x71,0x76,0x7b,0x90,0xbd,0xec,
+    0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,
+    0xfe,0xfc,0xf5,0xdf,0xba,0x9c,0x96,0x9d,0x9e,0x9e,0x9b,0x99,0x9f,0x97,0x91,0xc2,
+    0xfb,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,
+    0xc5,0x7b,0x75,0x7c,0x73,0x75,0x78,0x7a,0x79,0x6f,0x6e,0x8a,0xbd,0xe6,0xf7,0xfd,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xfb,
+    0xfc,0xff,0xff,0xf9,0xe6,0xcd,0xb4,0xa1,0x95,0x95,0x9b,0x9d,0x9e,0x94,0x98,0xcd,
+    0xff,0xff,0xfb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xfa,0xff,0xff,
+    0xd4,0x8c,0x6f,0x72,0x78,0x77,0x70,0x6a,0x6f,0x8b,0xaf,0xce,0xee,0xff,0xff,0xfa,
+    0xf9,0xfd,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfd,0xfc,0xff,0xff,0xe8,0xc3,0xaa,0xa1,0x9e,0x9b,0x9c,0xa3,0xb9,0xe5,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xed,0xb9,0x86,0x72,0x76,0x76,0x77,0x7f,0x96,0xca,0xf7,0xff,0xfd,0xfd,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe5,0xcc,0xb5,0xa8,0xad,0xc9,0xec,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xee,0xbe,0x96,0x89,0x8d,0xa4,0xc9,0xeb,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfc,0xf9,0xf8,0xfb,0xff,0xfe,0xf1,0xe1,0xda,0xdf,0xee,0xfd,0xff,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xee,0xd9,0xcc,0xd0,0xe3,0xf6,0xfe,0xfe,0xfb,0xfa,0xfc,0xff,0xff,0xfe,
+    0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe9,0xe0,0xdf,0xdf,0xdf,0xdf,0xdf,
+    0xdf,0xdf,0xde,0xdd,0xdd,0xdd,0xdd,0xdf,0xe3,0xe9,0xf4,0xfe,0xff,0xfd,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfb,0xfa,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf4,0xf4,0xf4,0xf4,0xf4,
+    0xf4,0xf5,0xf4,0xf4,0xf5,0xf5,0xf7,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xd2,0xbe,0xb3,0xaf,0xaf,0xaf,0xaf,0xaf,
+    0xaf,0xaf,0xad,0xac,0xac,0xac,0xaa,0xac,0xb4,0xbe,0xd7,0xf6,0xff,0xfe,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfa,0xf2,0xea,0xe7,0xe7,0xe7,0xe6,0xe6,0xe6,0xe6,0xe6,
+    0xe6,0xe5,0xe6,0xe6,0xe5,0xe6,0xe9,0xee,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xf1,0xc9,0xa8,0xa5,0xa7,0xa3,0xa4,0xa4,0xa3,0xa3,
+    0xa3,0xa3,0xa3,0xa3,0xa3,0xa3,0xa1,0xa3,0xa5,0xa4,0xaf,0xd0,0xf2,0xff,0xff,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xf8,0xea,0xe3,0xe4,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,0xe3,0xe9,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x9c,0xa2,0xa8,0xa7,0xa7,0xa7,0xa6,0xa6,
+    0xa6,0xa6,0xa6,0xa7,0xa7,0xa7,0xa7,0xa7,0xa5,0xa2,0x9f,0xae,0xde,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfa,0xec,0xe1,0xde,0xe2,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe1,0xe0,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xa9,0xa6,0xa3,0xa4,0xa4,0xa4,0xa4,0xa5,
+    0xa5,0xa4,0xa4,0xa4,0xa4,0xa4,0xa5,0xa3,0xa3,0xab,0xa3,0x9f,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe6,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xaa,0xa8,0xa4,0xa5,0xa5,0xa5,0xa5,0xa5,
+    0xa5,0xa5,0xa5,0xa5,0xa5,0xa5,0xa6,0xa4,0xa4,0xac,0xa3,0x9f,0xd2,0xff,0xff,0xfa,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,
+    0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe5,0xfa,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0xac,0x9e,0xa3,0xa8,0xa8,0xa8,0xaa,0xa9,0xa7,
+    0xa7,0xa8,0xa9,0xa9,0xa9,0xa9,0xa9,0xa9,0xa7,0xa6,0xa1,0xad,0xdd,0xff,0xff,0xfc,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xf9,0xeb,0xe0,0xdf,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe2,0xe2,0xe0,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc7,0xa8,0xa6,0xa8,0xa5,0xa6,0xa8,0xa7,0xa6,
+    0xa7,0xa8,0xa8,0xa8,0xa8,0xa8,0xa7,0xa8,0xa9,0xa8,0xb1,0xd0,0xf1,0xff,0xff,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xf5,0xe7,0xe1,0xe2,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,
+    0xe1,0xe2,0xe2,0xe1,0xe2,0xe2,0xe2,0xe2,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xec,0xd0,0xbd,0xb3,0xb0,0xb0,0xb0,0xb0,0xb2,
+    0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb6,0xbe,0xd6,0xf4,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xfe,0xf6,0xec,0xe6,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,
+    0xe3,0xe4,0xe3,0xe3,0xe4,0xe4,0xe5,0xeb,0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xe7,0xde,0xdd,0xdd,0xdd,0xdd,0xde,
+    0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xdf,0xe1,0xe7,0xf2,0xfe,0xff,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfd,0xf8,0xf4,0xf3,0xf3,0xf3,0xf2,0xf2,0xf3,0xf3,0xf3,
+    0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf4,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,0xfb,
+    0xfb,0xfc,0xfd,0xfd,0xfa,0xf9,0xfc,0xff,0xfe,0xf6,0xeb,0xe5,0xe9,0xf4,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfc,0xf8,0xf5,0xf6,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,
+    0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xea,0xd9,0xc6,0xba,0xbf,0xd8,0xf3,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfb,0xf1,0xe7,0xe4,0xe6,0xec,0xf5,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,0xcf,0xba,0xb3,0xb0,0xad,0xae,0xb7,0xca,0xec,
+    0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfb,0xee,0xe1,0xdc,0xdc,0xdd,0xde,0xe2,0xe9,0xf4,0xfd,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,
+    0xfc,0xff,0xff,0xfc,0xef,0xdb,0xc5,0xb3,0xa7,0xa8,0xae,0xb1,0xb1,0xab,0xaf,0xda,
+    0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,
+    0xf4,0xe1,0xd9,0xda,0xdc,0xdb,0xda,0xda,0xdd,0xe4,0xed,0xf4,0xfb,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,
+    0xff,0xfe,0xf9,0xe8,0xca,0xb0,0xa9,0xae,0xb0,0xb2,0xb1,0xaf,0xb4,0xaf,0xad,0xd2,
+    0xfc,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xef,0xdc,0xdb,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe3,0xf0,0xfa,0xfe,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,
+    0xfc,0xe9,0xcc,0xb8,0xb1,0xad,0xad,0xb0,0xb2,0xb3,0xb0,0xaf,0xb5,0xb0,0xae,0xd4,
+    0xfe,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,
+    0xf0,0xdc,0xd9,0xdc,0xdb,0xdb,0xdb,0xdc,0xdc,0xdb,0xdb,0xdd,0xdf,0xe5,0xf1,0xfb,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,
+    0xd7,0xc3,0xb0,0xa8,0xad,0xb3,0xb3,0xb0,0xaf,0xaf,0xb1,0xb4,0xb5,0xaf,0xb6,0xde,
+    0xff,0xff,0xfb,0xfd,0xfc,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfd,0xfd,0xff,0xff,
+    0xf7,0xe4,0xd7,0xd7,0xdc,0xdb,0xd9,0xda,0xdb,0xdc,0xdd,0xdd,0xda,0xdb,0xe2,0xea,
+    0xf3,0xfc,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf7,0xe2,0xc3,
+    0xad,0xab,0xb1,0xb3,0xb0,0xb0,0xb1,0xb1,0xb2,0xb2,0xb3,0xb2,0xb3,0xbb,0xd0,0xef,
+    0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xfe,0xf2,0xe0,0xd7,0xd8,0xd9,0xda,0xdc,0xdb,0xdb,0xdb,0xdc,0xdd,0xdd,0xdb,0xdb,
+    0xe1,0xed,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xce,0xb4,0xae,
+    0xad,0xad,0xb1,0xb3,0xb2,0xb0,0xb0,0xb1,0xb2,0xb5,0xb3,0xb1,0xbb,0xd7,0xf3,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xff,0xfd,0xf2,0xe2,0xd7,0xd7,0xdb,0xdc,0xdb,0xdb,0xdb,0xdb,0xdc,0xdd,0xdb,0xda,
+    0xdb,0xde,0xe4,0xf0,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf3,0xcd,0xaf,0xa9,0xae,
+    0xb4,0xb3,0xb0,0xae,0xaf,0xb2,0xb5,0xb4,0xaf,0xb0,0xbd,0xce,0xe1,0xf2,0xfe,0xff,
+    0xff,0xfd,0xf1,0xe2,0xe0,0xe6,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xf9,0xf1,0xec,0xec,0xf2,0xfa,0xff,
+    0xff,0xff,0xfd,0xf5,0xe9,0xe1,0xda,0xd7,0xd9,0xdb,0xdb,0xda,0xda,0xda,0xdc,0xdd,
+    0xdd,0xdb,0xd9,0xe3,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe1,0xb5,0xa9,0xb4,0xb3,
+    0xae,0xb1,0xb2,0xb2,0xb2,0xb1,0xb1,0xb4,0xb9,0xc4,0xdb,0xf5,0xff,0xfe,0xfd,0xff,
+    0xfc,0xea,0xd0,0xbe,0xb9,0xc0,0xd6,0xf4,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe8,0xd9,0xd3,0xd3,0xda,0xeb,0xfa,
+    0xff,0xfe,0xfe,0xff,0xfd,0xf2,0xe5,0xde,0xda,0xd8,0xd9,0xdb,0xdc,0xdb,0xdb,0xdb,
+    0xdb,0xdc,0xdb,0xde,0xea,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xd5,0xb5,0xac,0xb2,0xb1,
+    0xae,0xb0,0xb2,0xb3,0xb4,0xb3,0xb2,0xbf,0xda,0xf1,0xfb,0xfe,0xff,0xfe,0xfe,0xfe,
+    0xeb,0xc9,0xb9,0xbb,0xba,0xba,0xc4,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,0xed,0xd7,0xce,0xd1,0xd2,0xd2,0xd9,0xe8,
+    0xf9,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf2,0xe5,0xda,0xd7,0xdb,0xdd,0xdc,0xdb,0xdb,
+    0xdb,0xdb,0xdb,0xdd,0xe4,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb6,0xaf,0xb4,0xb3,
+    0xb3,0xb5,0xb3,0xad,0xb0,0xc1,0xd3,0xe5,0xf8,0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,
+    0xcf,0xb9,0xb7,0xbd,0xbf,0xc0,0xbd,0xc4,0xe6,0xff,0xff,0xfc,0xff,0xfe,0xf4,0xea,
+    0xea,0xf0,0xf9,0xff,0xff,0xff,0xff,0xf5,0xe0,0xd1,0xce,0xd3,0xd6,0xd4,0xd1,0xd8,
+    0xee,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe3,0xda,0xd7,0xd9,0xdc,0xdc,
+    0xdc,0xdc,0xdc,0xdc,0xe3,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xe1,0xb9,0xaa,0xb1,0xb2,
+    0xae,0xb0,0xb3,0xb8,0xc6,0xdf,0xf7,0xff,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,
+    0xba,0xba,0xbe,0xba,0xbb,0xc1,0xbd,0xbf,0xe3,0xff,0xff,0xfd,0xfd,0xef,0xd9,0xcd,
+    0xcd,0xd5,0xe7,0xfa,0xff,0xff,0xff,0xf1,0xda,0xd2,0xd2,0xd1,0xd2,0xd5,0xd3,0xd2,
+    0xe3,0xf8,0xff,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf5,0xe6,0xdd,0xda,0xd8,0xd8,
+    0xdb,0xdb,0xd8,0xdc,0xe9,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd0,0xb1,0xab,0xaf,
+    0xad,0xb0,0xc3,0xe0,0xf5,0xfd,0xfe,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xed,0xcd,
+    0xb8,0xba,0xbe,0xbb,0xbd,0xc2,0xbe,0xc2,0xe5,0xff,0xff,0xfd,0xf1,0xd3,0xc4,0xc8,
+    0xc8,0xc7,0xd2,0xe7,0xf8,0xff,0xff,0xf2,0xdd,0xd3,0xd1,0xd1,0xd1,0xd3,0xd4,0xd3,
+    0xda,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf4,0xe6,0xda,0xd7,
+    0xda,0xda,0xd8,0xe2,0xf6,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,0xdc,0xcd,0xc7,
+    0xcb,0xd7,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfd,0xff,0xf8,0xd3,0xbd,
+    0xbc,0xbb,0xbb,0xbe,0xbf,0xbf,0xc1,0xcf,0xed,0xff,0xff,0xf8,0xe2,0xc6,0xc1,0xcb,
+    0xca,0xc7,0xca,0xd5,0xef,0xff,0xff,0xf8,0xe8,0xd2,0xcd,0xd2,0xd2,0xcf,0xd2,0xd4,
+    0xd3,0xde,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8,
+    0xe5,0xe6,0xeb,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xec,
+    0xf3,0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xea,0xc2,0xb4,
+    0xbd,0xbf,0xbb,0xbe,0xc0,0xbb,0xc5,0xe4,0xf9,0xff,0xff,0xf1,0xd7,0xc7,0xc6,0xc8,
+    0xc8,0xca,0xca,0xce,0xeb,0xff,0xff,0xff,0xf4,0xd7,0xcc,0xd3,0xd2,0xcf,0xd2,0xd4,
+    0xd2,0xd7,0xea,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb,
+    0xf6,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xd4,0xbd,0xb9,
+    0xbd,0xbd,0xbd,0xc0,0xc0,0xbc,0xcf,0xf5,0xff,0xff,0xff,0xf0,0xd3,0xc7,0xc6,0xc7,
+    0xc7,0xcb,0xc9,0xca,0xe8,0xff,0xff,0xff,0xfc,0xe4,0xd3,0xd1,0xd1,0xd1,0xd1,0xd2,
+    0xd5,0xd3,0xd8,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,
+    0xfe,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe2,0xbf,0xba,0xbf,
+    0xbc,0xbc,0xbf,0xbf,0xbe,0xc7,0xe2,0xfc,0xff,0xff,0xff,0xf0,0xd2,0xc5,0xc5,0xc7,
+    0xc8,0xc9,0xc7,0xca,0xe8,0xff,0xff,0xfd,0xff,0xf6,0xdf,0xcf,0xd0,0xd3,0xd0,0xd1,
+    0xd5,0xd3,0xd1,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,
+    0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd1,0xb6,0xb8,0xbf,
+    0xbc,0xbc,0xc0,0xbd,0xbd,0xd7,0xf8,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc7,
+    0xc7,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfb,0xff,0xff,0xea,0xd2,0xd0,0xd3,0xd0,0xd0,
+    0xd2,0xd3,0xd3,0xda,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xc0,0xba,0xbb,0xbc,
+    0xbd,0xbe,0xc0,0xbf,0xc8,0xe8,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfc,0xff,0xff,0xf5,0xe1,0xd3,0xd0,0xd1,0xd1,
+    0xd1,0xd4,0xd5,0xd1,0xdc,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf0,0xc7,0xb6,0xbe,0xbe,0xbb,
+    0xbe,0xc0,0xbb,0xc2,0xdd,0xf7,0xff,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xf1,0xd6,0xcd,0xd2,0xd2,
+    0xd0,0xd3,0xd4,0xd0,0xd7,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd9,0xbf,0xb8,0xbd,0xbe,0xbb,
+    0xbe,0xc0,0xbb,0xcb,0xf1,0xff,0xfe,0xff,0xfe,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8,
+    0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xfe,0xff,0xfa,0xe0,0xd1,0xd2,0xd1,
+    0xd0,0xd1,0xd3,0xd4,0xd5,0xdd,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc1,0xba,0xbe,0xbc,0xbc,0xbe,
+    0xbf,0xbe,0xc6,0xe0,0xfb,0xff,0xfe,0xff,0xfe,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xfe,0xf2,0xdd,0xcf,0xd0,
+    0xd2,0xd0,0xd0,0xd4,0xd2,0xd3,0xe8,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfe,0xda,0xb6,0xb7,0xbf,0xbc,0xbb,0xc0,
+    0xbd,0xbb,0xd4,0xf6,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc7,0xc9,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xe8,0xd1,0xd1,
+    0xd3,0xcf,0xcf,0xd3,0xd1,0xd0,0xe0,0xf7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf8,0xd4,0xb7,0xba,0xc0,0xbc,0xbd,0xc1,
+    0xbe,0xc2,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xf4,0xde,0xd1,
+    0xd0,0xd2,0xd1,0xd2,0xd3,0xd4,0xde,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd7,0xb8,0xbc,0xc0,0xbd,0xc0,0xbe,
+    0xc2,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xef,0xd3,
+    0xcd,0xd3,0xd2,0xd1,0xd2,0xd3,0xdf,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xbb,0xb7,0xbd,0xbd,0xbc,0xba,
+    0xcc,0xef,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe3,
+    0xd1,0xcd,0xcf,0xd1,0xce,0xcf,0xe5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf0,0xd3,0xc3,0xc0,0xc1,0xc2,0xcb,
+    0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf7,
+    0xe3,0xd2,0xd2,0xd4,0xd3,0xdb,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe4,0xd5,0xd3,0xde,0xef,
+    0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8,
+    0xc7,0xca,0xc6,0xc8,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xf7,0xeb,0xe3,0xe1,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xf1,0xf0,0xf8,0xfe,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xef,0xd4,0xc9,0xc8,0xc7,
+    0xc6,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xfd,0xf6,0xf4,0xf9,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xda,0xc8,0xc6,0xca,
+    0xc9,0xc9,0xc9,0xcf,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe7,0xca,0xc2,0xcb,
+    0xcb,0xc6,0xc9,0xd9,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf5,0xdc,0xc8,0xc4,
+    0xc4,0xc5,0xd3,0xec,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xd2,
+    0xd2,0xd9,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,
+    0xf7,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,
+    0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,
+    0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+    0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
+};
+#endif /* !PEXPERT_NO_3X_IMAGES */

 
 const unsigned char gGearPict2x[4*kGearFrames*kGearWidth*kGearHeight] = {
     0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
 
 const unsigned char gGearPict2x[4*kGearFrames*kGearWidth*kGearHeight] = {
     0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,

diff --git a/pexpert/pexpert/arm/board_config.h b/pexpert/pexpert/arm/board_config.h
index a6d930e414771d487b17dda6c44121151ce76abb..13ccdb597fe5427a4a2c4e1c4ebed7a839412641 100644 (file)
--- a/pexpert/pexpert/arm/board_config.h
+++ b/pexpert/pexpert/arm/board_config.h
@@ -12,29 +12,26 @@
 
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 18
 
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 18

-

 #define ARM_BOARD_CLASS_S7002
 #define ARM_BOARD_CLASS_S7002

+#define PEXPERT_NO_3X_IMAGES   1

 #endif  /* ARM_BOARD_CONFIG_S7002 */
 
 #ifdef ARM_BOARD_CONFIG_T8002
 #define ARMA7
 #include <pexpert/arm/T8002.h>
 #endif  /* ARM_BOARD_CONFIG_S7002 */
 
 #ifdef ARM_BOARD_CONFIG_T8002
 #define ARMA7
 #include <pexpert/arm/T8002.h>

-

 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 19
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 19

-

 #define ARM_BOARD_CLASS_T8002
 #define ARM_BOARD_CLASS_T8002

+#define PEXPERT_NO_3X_IMAGES   1

 #endif  /* ARM_BOARD_CONFIG_T8002 */
 
 #ifdef ARM_BOARD_CONFIG_T8004
 #define ARMA7
 #include <pexpert/arm/T8002.h>
 #endif  /* ARM_BOARD_CONFIG_T8002 */
 
 #ifdef ARM_BOARD_CONFIG_T8004
 #define ARMA7
 #include <pexpert/arm/T8002.h>

-

 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 20
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define __ARM_L2CACHE_SIZE_LOG__ 20

-

 #define ARM_BOARD_CLASS_T8002
 #define ARM_BOARD_CLASS_T8002

+#define PEXPERT_NO_3X_IMAGES   1

 #endif  /* ARM_BOARD_CONFIG_T8004 */
 
 #endif  /* ARM_BOARD_CONFIG_T8004 */
 

-

 #endif /* ! _PEXPERT_ARM_BOARD_CONFIG_H */
 #endif /* ! _PEXPERT_ARM_BOARD_CONFIG_H */

diff --git a/pexpert/pexpert/arm64/board_config.h b/pexpert/pexpert/arm64/board_config.h
index ecf6fe739fd4cf4e248a7c893d4b7778f3d4e5cc..0aaefb89856c575060f04ef5c2d54873753d85d2 100644 (file)
--- a/pexpert/pexpert/arm64/board_config.h
+++ b/pexpert/pexpert/arm64/board_config.h
@@ -14,6 +14,7 @@
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_S5L8960X
 #define KERNEL_INTEGRITY_WT 1
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_S5L8960X
 #define KERNEL_INTEGRITY_WT 1

+#define PEXPERT_NO_3X_IMAGES   1

 #endif  /* ARM64_BOARD_CONFIG_S5L8960X */
 
 #ifdef ARM64_BOARD_CONFIG_T7000
 #endif  /* ARM64_BOARD_CONFIG_S5L8960X */
 
 #ifdef ARM64_BOARD_CONFIG_T7000


@@ -24,7 +25,6 @@
 #define __ARM_L2CACHE_SIZE_LOG__ 20
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_T7000
 #define __ARM_L2CACHE_SIZE_LOG__ 20
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_T7000

-#define PEXPERT_3X_IMAGES      1

 #define KERNEL_INTEGRITY_WT 1
 #endif  /* ARM64_BOARD_CONFIG_T7000 */
 
 #define KERNEL_INTEGRITY_WT 1
 #endif  /* ARM64_BOARD_CONFIG_T7000 */
 


@@ -36,7 +36,6 @@
 #define __ARM_L2CACHE_SIZE_LOG__ 21
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_T7000
 #define __ARM_L2CACHE_SIZE_LOG__ 21
 #define ARM_BOARD_WFE_TIMEOUT_NS 1000
 #define ARM_BOARD_CLASS_T7000

-#define PEXPERT_3X_IMAGES      1

 #define KERNEL_INTEGRITY_WT 1
 #define CPU_COUNT 3
 #endif  /* ARM64_BOARD_CONFIG_T7001 */
 #define KERNEL_INTEGRITY_WT 1
 #define CPU_COUNT 3
 #endif  /* ARM64_BOARD_CONFIG_T7001 */

diff --git a/pexpert/pexpert/pe_images.h b/pexpert/pexpert/pe_images.h
index c7a0597b168d8e3bfcffb266bdbf8e1e9525d1af..35f179a190307bb5d6d814039851b623ef8dda5b 100644 (file)
--- a/pexpert/pexpert/pe_images.h
+++ b/pexpert/pexpert/pe_images.h
@@ -42,7 +42,11 @@ static const unsigned char * default_noroot_data;
 
 static const unsigned char * default_progress_data1x = gGearPict;
 static const unsigned char * default_progress_data2x = gGearPict2x;
 
 static const unsigned char * default_progress_data1x = gGearPict;
 static const unsigned char * default_progress_data2x = gGearPict2x;

+#if !PEXPERT_NO_3X_IMAGES
+static const unsigned char * default_progress_data3x = gGearPict3x;
+#else

 static const unsigned char * default_progress_data3x = NULL;
 static const unsigned char * default_progress_data3x = NULL;

+#endif

 
 static vc_progress_element default_progress = 
        {   0, 4|1, 1000 / kGearFPS, kGearFrames, {0, 0, 0}, 
 
 static vc_progress_element default_progress = 
        {   0, 4|1, 1000 / kGearFPS, kGearFrames, {0, 0, 0}, 

diff --git a/pexpert/pexpert/pexpert.h b/pexpert/pexpert/pexpert.h
index 33b07cda2d0c148b14df6a461657ede139f6e144..61ac16c9ca348a209c8a0327cd4589437831a9c8 100644 (file)
--- a/pexpert/pexpert/pexpert.h
+++ b/pexpert/pexpert/pexpert.h
@@ -390,8 +390,21 @@ extern void PE_arm_debug_enable_trace(void);
 #endif
 
 #if KERNEL_PRIVATE
 #endif
 
 #if KERNEL_PRIVATE

+#if defined(__arm64__)
+extern uint8_t PE_smc_stashed_x86_power_state;
+extern uint8_t PE_smc_stashed_x86_efi_boot_state;
+extern uint8_t PE_smc_stashed_x86_system_state;
+extern uint32_t PE_pcie_stashed_link_state;
+#endif
+

 boolean_t PE_reboot_on_panic(void);
 void PE_sync_panic_buffers(void);
 boolean_t PE_reboot_on_panic(void);
 void PE_sync_panic_buffers(void);

+
+typedef struct PE_panic_save_context {
+       void *psc_buffer;
+       uint32_t psc_offset;
+       uint32_t psc_length;
+} PE_panic_save_context_t;

 #endif
 
 __END_DECLS
 #endif
 
 __END_DECLS

diff --git a/san/kasan-arm64.c b/san/kasan-arm64.c
index 3fbb9827b64e1ff7a59294671c7716082958a46c..22bb738a17d70f3f8cc4af519e65a94dae6a02a5 100644 (file)
--- a/san/kasan-arm64.c
+++ b/san/kasan-arm64.c
@@ -301,8 +301,9 @@ kasan_bootstrap(boot_args *args, vm_offset_t pgtable)
        kernel_vbase = args->virtBase;
        kernel_vtop = args->virtBase + ptop - pbase;
 
        kernel_vbase = args->virtBase;
        kernel_vtop = args->virtBase + ptop - pbase;
 

-       /* Steal ~15% of physical memory */
-       tosteal = vm_map_trunc_page(args->memSize / 6, ARM_PGMASK);
+       tosteal = (args->memSize * STOLEN_MEM_PERCENT) / 100 + STOLEN_MEM_BYTES;
+       tosteal = vm_map_trunc_page(tosteal, ARM_PGMASK);
+

        args->memSize -= tosteal;
 
        /* Initialize the page allocator */
        args->memSize -= tosteal;
 
        /* Initialize the page allocator */

diff --git a/san/kasan-blacklist-x86_64 b/san/kasan-blacklist-x86_64
index 189a3076d3109dfd36a9a43829caa46f52ad2d07..9313e400310ed7957739bafe686317eeca215c8f 100644 (file)
--- a/san/kasan-blacklist-x86_64
+++ b/san/kasan-blacklist-x86_64
@@ -68,4 +68,6 @@ src:./osfmk/i386/startup64.c
 src:./osfmk/i386/lapic_native.c
 src:./osfmk/i386/fpu.c
 src:./osfmk/vm/vm_compressor.c
 src:./osfmk/i386/lapic_native.c
 src:./osfmk/i386/fpu.c
 src:./osfmk/vm/vm_compressor.c

-
+fun:doublemap_init
+fun:getsegbynamefromheader
+fun:getsectbynamefromheader

diff --git a/san/kasan-x86_64.c b/san/kasan-x86_64.c
index 72339d8cdc2218ad3bad20869404d17d8325dafd..93e8e416e6aadef7698e86096bae899199adbe0a 100644 (file)
--- a/san/kasan-x86_64.c
+++ b/san/kasan-x86_64.c
@@ -274,11 +274,6 @@ kasan_arch_init(void)
 
        /* Map the physical aperture */
        kasan_map_shadow_superpage_zero(physmap_base, physmap_max - physmap_base);
 
        /* Map the physical aperture */
        kasan_map_shadow_superpage_zero(physmap_base, physmap_max - physmap_base);

-       /* Establish shadow mappings for the x86 descriptor tables and
-        * "low global" page; these are specially alias-mapped at fixed VAs
-        * early in boot
-        */
-       kasan_map_low_fixed_regions();

 }
 
 /*
 }
 
 /*


@@ -311,8 +306,7 @@ kasan_reserve_memory(void *_args)
                total_pages += mptr_tmp->NumberOfPages;
        }
 
                total_pages += mptr_tmp->NumberOfPages;
        }
 

-       /* steal 25% of physical memory */
-       to_steal = total_pages / 4;
+       to_steal = (total_pages * STOLEN_MEM_PERCENT) / 100 + (STOLEN_MEM_BYTES / I386_PGBYTES);

 
        /* Search for a range large enough to steal from */
        for (i = 0, mptr_tmp = mptr; i < mcount; i++, mptr_tmp = (EfiMemoryRange *)(((vm_offset_t)mptr_tmp) + msize)) {
 
        /* Search for a range large enough to steal from */
        for (i = 0, mptr_tmp = mptr; i < mcount; i++, mptr_tmp = (EfiMemoryRange *)(((vm_offset_t)mptr_tmp) + msize)) {

diff --git a/san/kasan.c b/san/kasan.c
index 960dc25b4bf0ed440a61ba4a0d8d8aab37f009e4..1723fcef197e8f75f9301f8115b59c2e438b92e9 100644 (file)
--- a/san/kasan.c
+++ b/san/kasan.c
@@ -309,17 +309,16 @@ static const char *shadow_strings[] = {
        [ASAN_PARTIAL5] =       "PARTIAL5",
        [ASAN_PARTIAL6] =       "PARTIAL6",
        [ASAN_PARTIAL7] =       "PARTIAL7",
        [ASAN_PARTIAL5] =       "PARTIAL5",
        [ASAN_PARTIAL6] =       "PARTIAL6",
        [ASAN_PARTIAL7] =       "PARTIAL7",

-       [ASAN_STACK_RZ] =       "<invalid>",

        [ASAN_STACK_LEFT_RZ] =  "STACK_LEFT_RZ",
        [ASAN_STACK_MID_RZ] =   "STACK_MID_RZ",
        [ASAN_STACK_RIGHT_RZ] = "STACK_RIGHT_RZ",
        [ASAN_STACK_FREED] =    "STACK_FREED",
        [ASAN_STACK_LEFT_RZ] =  "STACK_LEFT_RZ",
        [ASAN_STACK_MID_RZ] =   "STACK_MID_RZ",
        [ASAN_STACK_RIGHT_RZ] = "STACK_RIGHT_RZ",
        [ASAN_STACK_FREED] =    "STACK_FREED",

+       [ASAN_STACK_OOSCOPE] =  "STACK_OOSCOPE",

        [ASAN_GLOBAL_RZ] =      "GLOBAL_RZ",
        [ASAN_GLOBAL_RZ] =      "GLOBAL_RZ",

-       [ASAN_HEAP_RZ] =        "<invalid>",

        [ASAN_HEAP_LEFT_RZ] =   "HEAP_LEFT_RZ",
        [ASAN_HEAP_RIGHT_RZ] =  "HEAP_RIGHT_RZ",
        [ASAN_HEAP_FREED] =     "HEAP_FREED",
        [ASAN_HEAP_LEFT_RZ] =   "HEAP_LEFT_RZ",
        [ASAN_HEAP_RIGHT_RZ] =  "HEAP_RIGHT_RZ",
        [ASAN_HEAP_FREED] =     "HEAP_FREED",

-       [0xff] =                "<invalid>",
+       [0xff] =                NULL

 };
 
 #define CRASH_CONTEXT_BEFORE 5
 };
 
 #define CRASH_CONTEXT_BEFORE 5


@@ -370,6 +369,9 @@ kasan_crash_report(uptr p, uptr width, unsigned access_type)
        uint8_t *shadow_ptr = SHADOW_FOR_ADDRESS(p);
        uint8_t shadow_type = *shadow_ptr;
        const char *shadow_str = shadow_strings[shadow_type];
        uint8_t *shadow_ptr = SHADOW_FOR_ADDRESS(p);
        uint8_t shadow_type = *shadow_ptr;
        const char *shadow_str = shadow_strings[shadow_type];

+       if (!shadow_str) {
+               shadow_str = "<invalid>";
+       }

 
        kasan_handle_test();
 
 
        kasan_handle_test();
 

diff --git a/san/kasan.h b/san/kasan.h
index a4d985d5691a2d535751984eb68d4c952977e9c8..bb048d860fda90e4592068c08ba35b5293906f09 100644 (file)
--- a/san/kasan.h
+++ b/san/kasan.h
@@ -73,6 +73,7 @@ typedef uintptr_t uptr;
 #define ASAN_STACK_MID_RZ   0xf2
 #define ASAN_STACK_RIGHT_RZ 0xf3
 #define ASAN_STACK_FREED    0xf5
 #define ASAN_STACK_MID_RZ   0xf2
 #define ASAN_STACK_RIGHT_RZ 0xf3
 #define ASAN_STACK_FREED    0xf5

+#define ASAN_STACK_OOSCOPE  0xf8

 #define ASAN_GLOBAL_RZ      0xf9
 #define ASAN_HEAP_RZ        0xe9
 #define ASAN_HEAP_LEFT_RZ   0xfa
 #define ASAN_GLOBAL_RZ      0xf9
 #define ASAN_HEAP_RZ        0xe9
 #define ASAN_HEAP_LEFT_RZ   0xfa

diff --git a/san/kasan_internal.h b/san/kasan_internal.h
index 36ae0b234dffa7b20665902244567b5bfed82bd3..805255178145914f97e33e9eacfb847857b9a6bb 100644 (file)
--- a/san/kasan_internal.h
+++ b/san/kasan_internal.h
@@ -35,6 +35,8 @@
 
 typedef uintptr_t uptr;
 
 
 typedef uintptr_t uptr;
 

+#define MiB(x) ((x) * 1024UL * 1024)
+

 /*
  * KASAN features and config
  */
 /*
  * KASAN features and config
  */


@@ -46,7 +48,20 @@ typedef uintptr_t uptr;
 #define FAKESTACK_QUARANTINE (1 && FAKESTACK)
 
 #define QUARANTINE_ENTRIES 5000
 #define FAKESTACK_QUARANTINE (1 && FAKESTACK)
 
 #define QUARANTINE_ENTRIES 5000

-#define QUARANTINE_MAXSIZE (10UL * 1024 * 1024)
+#define QUARANTINE_MAXSIZE MiB(10)
+
+/*
+ * The amount of physical memory stolen by KASan at boot to back the shadow memory
+ * and page tables. Larger memory systems need to steal proportionally less.
+ */
+#ifdef __arm64__
+/* Works out at about 25% of 512 MiB and 15% of 3GiB system */
+# define STOLEN_MEM_PERCENT  13UL
+# define STOLEN_MEM_BYTES    MiB(62)
+#else
+# define STOLEN_MEM_PERCENT  25UL
+# define STOLEN_MEM_BYTES    0
+#endif

 
 #ifndef KASAN
 # error KASAN undefined
 
 #ifndef KASAN
 # error KASAN undefined


@@ -138,9 +153,12 @@ struct asan_global {
 
 #if defined(__x86_64__)
 # define _JBLEN ((9 * 2) + 3 + 16)
 
 #if defined(__x86_64__)
 # define _JBLEN ((9 * 2) + 3 + 16)

+#elif defined(__arm64__)
+# define _JBLEN ((14 + 8 + 2) * 2)
+#else
+# error "Unknown arch"

 #endif
 
 #endif
 

-

 typedef int jmp_buf[_JBLEN];
 void _longjmp(jmp_buf env, int val);
 int _setjmp(jmp_buf env);
 typedef int jmp_buf[_JBLEN];
 void _longjmp(jmp_buf env, int val);
 int _setjmp(jmp_buf env);

diff --git a/san/tools/kasan_install b/san/tools/kasan_install
index 5052dc945ddf8dfcf52a6d1333bcf5b833bfcc41..01e7920c3eb0766cc6590c436b476c0fff420d44 100755 (executable)
--- a/san/tools/kasan_install
+++ b/san/tools/kasan_install
@@ -4,16 +4,10 @@
 # kasan_install: set up a system to run the KASan kernel. Run with "--uninstall"
 # to reverse the setup.
 #
 # kasan_install: set up a system to run the KASan kernel. Run with "--uninstall"
 # to reverse the setup.
 #

-# Installs a symlink to the kernel kasan in /System/Library/Kernels/kernel.kasan
-# and adds kcsuffix=kasan to boot-args.
+# Adds kcsuffix=kasan to boot-args.

 #
 
 
 #
 
 

-kernel_name=kernel.kasan
-kernel_src=/AppleInternal/CoreOS/xnu_kasan/${kernel_name}
-SLK=/System/Library/Kernels/
-kernel_dst=${SLK}${kernel_name}
-

 if [[ `whoami` != root ]] ; then
        echo "Re-running with sudo"
        sudo "$0" "$@"
 if [[ `whoami` != root ]] ; then
        echo "Re-running with sudo"
        sudo "$0" "$@"


@@ -35,38 +29,8 @@ prompt() {
 
 kasan_install() {
 
 
 kasan_install() {
 

-       dosymlink=0

        dobootargs=0
 
        dobootargs=0
 

-       if [[ ! -f $kernel_src ]] ; then
-               echo "No KASan kernel found at $kernel_src"
-               exit 1
-       fi
-
-       echo -n "Installing KASan kernel... "
-
-       if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then
-               echo "already installed."
-       elif [[ -f $kernel_dst ]] ; then
-               prompt "file exists. Overwrite?" && {
-                       echo -n "Overwriting KASan kernel... "
-                       dosymlink=1
-               }
-       else
-               dosymlink=1
-       fi
-
-       # Use a temporary directory with a symlink to kernel.kasan. We can ditto
-       # from there into /S/L/K, even with SIP enabled.
-       [[ $dosymlink -eq 1 ]] && {
-               tmp=$(mktemp -d) || exit $?
-               ln -s "$kernel_src" "$tmp" || exit $?
-               ditto "$tmp" "$SLK" || exit $?
-               rm -r "$tmp"
-               echo "done."
-       }
-
-

        echo -n "Checking KASan boot args... "
 
        bootargs=$(nvram boot-args | cut -f2)
        echo -n "Checking KASan boot args... "
 
        bootargs=$(nvram boot-args | cut -f2)


@@ -97,42 +61,11 @@ kasan_install() {
                echo "done."
        }
 
                echo "done."
        }
 

-       [[ $dosymlink -eq 1 ]] && {
-               echo -n "Triggering kernel cache rebuild... "
-               touch /System/Library/Extensions || exit $?
-               echo "done."
-       }
-

 }
 
 
 kasan_uninstall() {
 
 }
 
 
 kasan_uninstall() {
 

-       echo -n "Removing kasan kernel... "
-
-       dorm=0
-
-       if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then
-               dorm=1
-       elif [[ -f $kernel_dst ]] ; then
-               prompt "unexpected file. Remove anyway?" && {
-                       dorm=1
-               }
-       else
-               echo "not installed."
-       fi
-
-       [[ $dorm -eq 1 ]] && {
-               if rm "$kernel_dst" ; then
-                       echo "done."
-               else
-                       if sip_enabled ; then
-                               echo "failed due to SIP - this is normal."
-                       fi
-               fi
-       }
-
-

        echo -n "Removing boot args... "
 
        bootargs=$(nvram boot-args | cut -f2)
        echo -n "Removing boot args... "
 
        bootargs=$(nvram boot-args | cut -f2)

diff --git a/tools/lldbmacros/kcdata.py b/tools/lldbmacros/kcdata.py
index ecc899a0125a11e19cc1540dba50b6e867fb87ae..8732d5e947f4bb3b99fc91415f99a7696a3f658c 100755 (executable)
--- a/tools/lldbmacros/kcdata.py
+++ b/tools/lldbmacros/kcdata.py
@@ -978,7 +978,9 @@ KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_WAITINFO')] = KCT
 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP'),
             (
                         KCSubTypeElement.FromBasicCtype('tgs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),
 KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP'),
             (
                         KCSubTypeElement.FromBasicCtype('tgs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0),

-                        KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1)
+                        KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1),
+                            8, 1),
+                        KCSubTypeElement.FromBasicCtype('tgs_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 24),

             ),
             'thread_group_snapshot')
 
             ),
             'thread_group_snapshot')
 


@@ -1331,8 +1333,14 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete):
     if not ss:
         print "No KCDATA_BUFFER_BEGIN_STACKSHOT object found. Skipping writing report."
         return
     if not ss:
         print "No KCDATA_BUFFER_BEGIN_STACKSHOT object found. Skipping writing report."
         return

+

     timestamp = ss.get('usecs_since_epoch', int(time.time()))
     timestamp = ss.get('usecs_since_epoch', int(time.time()))

-    timestamp = time.strftime("%Y-%m-%d %H:%M:%S %z",time.gmtime(timestamp))
+    try:
+        timestamp = time.strftime("%Y-%m-%d %H:%M:%S %z",time.gmtime(timestamp))
+    except ValueError, e:
+        print "couldn't convert timestamp:", str(e)
+        timestamp = None
+

     os_version = ss.get('osversion', 'Unknown')
     timebase = ss.get('mach_timebase_info', {"denom": 1, "numer": 1})
     if not dsc_uuid and 'imageSlidBaseAddress' not in ss.get('shared_cache_dyld_load_info'):
     os_version = ss.get('osversion', 'Unknown')
     timebase = ss.get('mach_timebase_info', {"denom": 1, "numer": 1})
     if not dsc_uuid and 'imageSlidBaseAddress' not in ss.get('shared_cache_dyld_load_info'):


@@ -1373,7 +1381,8 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete):
     AllImageCatalog = []
     obj = {}
     obj["kernel"] = os_version
     AllImageCatalog = []
     obj = {}
     obj["kernel"] = os_version

-    obj["date"] = timestamp
+    if timestamp is not None:
+        obj["date"] = timestamp

     obj["reason"] = "kernel panic stackshot"
     obj["incident"] = "ABCDEFGH-1234-56IJ-789K-0LMNOPQRSTUV"
     obj["crashReporterKey"] = "12ab34cd45aabbccdd6712ab34cd45aabbccdd67"
     obj["reason"] = "kernel panic stackshot"
     obj["incident"] = "ABCDEFGH-1234-56IJ-789K-0LMNOPQRSTUV"
     obj["crashReporterKey"] = "12ab34cd45aabbccdd6712ab34cd45aabbccdd67"


@@ -1486,7 +1495,15 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete):
         fh = sys.stdout
     else:
         fh = open(outfile_name, "w")
         fh = sys.stdout
     else:
         fh = open(outfile_name, "w")

-    fh.write('{"bug_type":"288", "timestamp":"'+ timestamp +'", "os_version":"'+ os_version +'"}\n')
+
+    header = {}
+    header['bug_type'] = 288
+    if timestamp is not None:
+        header['timestamp'] = timestamp
+    header['os_version'] = os_version
+    fh.write(json.dumps(header))
+    fh.write("\n")
+

     fh.write(json.dumps(obj, sort_keys=False, indent=2, separators=(',', ': ')))
     fh.close()
 
     fh.write(json.dumps(obj, sort_keys=False, indent=2, separators=(',', ': ')))
     fh.close()
 

diff --git a/tools/lldbmacros/mbufs.py b/tools/lldbmacros/mbufs.py
index fb41b65ea0d7ab1765da862beac37ba59f77674c..548f9870fec75cac11e2400afeb5d5e8a227c048 100755 (executable)
--- a/tools/lldbmacros/mbufs.py
+++ b/tools/lldbmacros/mbufs.py
@@ -626,6 +626,28 @@ def GetMbufTraceLeak(trace):
                 cnt += 1
     return out_string
 
                 cnt += 1
     return out_string
 

+@lldb_command('mbuf_largefailures')
+def MbufLargeFailures(cmd_args=None):
+    """ Print the largest allocation failures
+    """
+    topcnt = 0
+    if (int(len(cmd_args)) > 0 and int(cmd_args[0]) < 5):
+        maxcnt = cmd_args[0]
+    else:
+        maxcnt = 5
+    while (topcnt < maxcnt):
+        trace = kern.globals.mtracelarge_table[topcnt]
+        if (trace.size == 0):
+            topcnt += 1
+            continue
+        print str(trace.size)
+        if (trace.depth != 0):
+            cnt = 0
+            while (cnt < trace.depth):
+                print str(cnt + 1) + ": " + GetPc(trace.addr[cnt])
+                cnt += 1
+        topcnt += 1
+

 
 # Macro: mbuf_traceleak
 @lldb_command('mbuf_traceleak')
 
 # Macro: mbuf_traceleak
 @lldb_command('mbuf_traceleak')

diff --git a/tools/tests/darwintests/proc_info.c b/tools/tests/darwintests/proc_info.c
index 11b042d0dd35ea196876f11aee91afffe0930ffe..9c2e16f0505f2c250e942f5c4e8f065ec7075af6 100644 (file)
--- a/tools/tests/darwintests/proc_info.c
+++ b/tools/tests/darwintests/proc_info.c
@@ -145,6 +145,10 @@ next:;
                        T_PASS("found up-pointer for %s", uptr_names[found]);
                }
        }
                        T_PASS("found up-pointer for %s", uptr_names[found]);
                }
        }

+
+       uint64_t up_overflow[2] = {0};
+       uptrs_count = proc_list_uptrs(getpid(), up_overflow, sizeof(uint64_t)+1);
+       T_ASSERT_EQ(up_overflow[1], 0 , "overflow check");

 }
 
 #pragma mark dynamic kqueue info
 }
 
 #pragma mark dynamic kqueue info

diff --git a/tools/tests/darwintests/stackshot.m b/tools/tests/darwintests/stackshot.m
index eb17dc1c5093cd235e17c92972ff2d359860cbdc..8c6219ab7e66e7e45e8f8fcc903424ff78d220be 100644 (file)
--- a/tools/tests/darwintests/stackshot.m
+++ b/tools/tests/darwintests/stackshot.m
@@ -15,6 +15,7 @@ T_GLOBAL_META(
 
 static const char *current_process_name(void);
 static void parse_stackshot(bool delta, void *ssbuf, size_t sslen);
 
 static const char *current_process_name(void);
 static void parse_stackshot(bool delta, void *ssbuf, size_t sslen);

+static void parse_thread_group_stackshot(void **sbuf, size_t sslen);

 static uint64_t stackshot_timestamp(void *ssbuf, size_t sslen);
 static void initialize_thread(void);
 
 static uint64_t stackshot_timestamp(void *ssbuf, size_t sslen);
 static void initialize_thread(void);
 


@@ -231,6 +232,34 @@ T_DECL(instrs_cycles, "test a getting instructions and cycles in stackshot")
        });
 }
 
        });
 }
 

+static void
+check_thread_groups_supported()
+{
+       int err;
+       int supported = 0;
+       size_t supported_size = sizeof(supported);
+       err = sysctlbyname("kern.thread_groups_supported", &supported, &supported_size, NULL, 0);
+
+       if (err || !supported)
+               T_SKIP("thread groups not supported on this system");
+}
+
+T_DECL(thread_groups, "test getting thread groups in stackshot")
+{
+       check_thread_groups_supported();
+
+       struct scenario scenario = {
+               .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_THREAD_GROUP
+                               | STACKSHOT_KCDATA_FORMAT)
+       };
+
+       T_LOG("attempting to take stackshot with thread group flag");
+       take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) {
+               parse_thread_group_stackshot(ssbuf, sslen);
+       });
+
+}
+

 #pragma mark performance tests
 
 #define SHOULD_REUSE_SIZE_HINT 0x01
 #pragma mark performance tests
 
 #define SHOULD_REUSE_SIZE_HINT 0x01


@@ -318,6 +347,79 @@ stackshot_timestamp(void *ssbuf, size_t sslen)
 
 #define TEST_THREAD_NAME "stackshot_test_thread"
 
 
 #define TEST_THREAD_NAME "stackshot_test_thread"
 

+static void
+parse_thread_group_stackshot(void **ssbuf, size_t sslen)
+{
+       bool seen_thread_group_snapshot = false;
+       kcdata_iter_t iter = kcdata_iter(ssbuf, sslen);
+       T_ASSERT_EQ(kcdata_iter_type(iter), KCDATA_BUFFER_BEGIN_STACKSHOT,
+                       "buffer provided is a stackshot");
+
+       NSMutableSet *thread_groups = [[NSMutableSet alloc] init];
+
+       iter = kcdata_iter_next(iter);
+       KCDATA_ITER_FOREACH(iter) {
+               switch (kcdata_iter_type(iter)) {
+               case KCDATA_TYPE_ARRAY: {
+                       T_QUIET;
+                       T_ASSERT_TRUE(kcdata_iter_array_valid(iter),
+                                       "checked that array is valid");
+
+                       if (kcdata_iter_array_elem_type(iter) != STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT) {
+                               continue;
+                       }
+
+                       seen_thread_group_snapshot = true;
+
+                       if (kcdata_iter_array_elem_size(iter) >= sizeof(struct thread_group_snapshot_v2)) {
+                               struct thread_group_snapshot_v2 *tgs_array = kcdata_iter_payload(iter);
+                               for (uint32_t j = 0; j < kcdata_iter_array_elem_count(iter); j++) {
+                                       struct thread_group_snapshot_v2 *tgs = tgs_array + j;
+                                       [thread_groups addObject:@(tgs->tgs_id)];
+                               }
+
+                       }
+                       else {
+                               struct thread_group_snapshot *tgs_array = kcdata_iter_payload(iter);
+                               for (uint32_t j = 0; j < kcdata_iter_array_elem_count(iter); j++) {
+                                       struct thread_group_snapshot *tgs = tgs_array + j;
+                                       [thread_groups addObject:@(tgs->tgs_id)];
+                               }
+                       }
+                       break;
+               }
+               }
+       }
+       KCDATA_ITER_FOREACH(iter) {
+               NSError *error = nil;
+
+               switch (kcdata_iter_type(iter)) {
+
+               case KCDATA_TYPE_CONTAINER_BEGIN: {
+                       T_QUIET;
+                       T_ASSERT_TRUE(kcdata_iter_container_valid(iter),
+                                       "checked that container is valid");
+
+                       NSDictionary *container = parseKCDataContainer(&iter, &error);
+                       T_QUIET; T_ASSERT_NOTNULL(container, "parsed container from stackshot");
+                       T_QUIET; T_ASSERT_NULL(error, "error unset after parsing container");
+
+                       if (kcdata_iter_container_type(iter) != STACKSHOT_KCCONTAINER_THREAD) {
+                               break;
+                       }
+
+                       int tg = [container[@"thread_snapshots"][@"thread_group"] intValue];
+
+                       T_ASSERT_TRUE([thread_groups containsObject:@(tg)], "check that the thread group the thread is in exists");
+
+                       break;
+               };
+
+               }
+       }
+       T_ASSERT_TRUE(seen_thread_group_snapshot, "check that we have seen a thread group snapshot");
+}
+

 static void
 parse_stackshot(bool delta, void *ssbuf, size_t sslen)
 {
 static void
 parse_stackshot(bool delta, void *ssbuf, size_t sslen)
 {