From 5c9f46613a83ebfc29a5b1f099448259e96a98f0 Mon Sep 17 00:00:00 2001 From: Apple Date: Sat, 3 Feb 2018 22:20:03 +0000 Subject: [PATCH] xnu-4570.31.3.tar.gz --- bsd/kern/bsd_stubs.c | 4 +- bsd/kern/kern_control.c | 237 +- bsd/kern/kern_event.c | 10 +- bsd/kern/kern_exec.c | 34 +- bsd/kern/kern_exit.c | 3 + bsd/kern/kern_memorystatus.c | 82 +- bsd/kern/kern_mib.c | 3 +- bsd/kern/kern_proc.c | 4 +- bsd/kern/kern_sysctl.c | 7 +- bsd/kern/kern_xxx.c | 2 +- bsd/kern/posix_shm.c | 9 +- bsd/kern/proc_info.c | 5 +- bsd/kern/process_policy.c | 2 +- bsd/kern/sys_coalition.c | 5 + bsd/kern/sys_reason.c | 8 +- bsd/kern/syscalls.master | 2 +- bsd/kern/trace_codes | 2 + bsd/kern/tty_dev.c | 34 +- bsd/kern/tty_ptmx.c | 246 +- bsd/kern/uipc_mbuf.c | 91 +- bsd/kern/uipc_socket2.c | 14 +- bsd/kern/uipc_syscalls.c | 4 +- bsd/miscfs/specfs/spec_vnops.c | 8 +- bsd/net/bpf.c | 5 + bsd/net/classq/classq_fq_codel.c | 1 + bsd/net/dlil.c | 33 +- bsd/net/if.c | 95 + bsd/net/if_bond.c | 6 +- bsd/net/if_bridge.c | 8 + bsd/net/if_ipsec.c | 604 ++- bsd/net/if_ipsec.h | 6 + bsd/net/if_loop.c | 7 + bsd/net/if_utun.c | 1193 ++++-- bsd/net/if_utun.h | 7 + bsd/net/if_var.h | 2 + bsd/net/if_vlan.c | 7 +- bsd/net/necp_client.c | 12 +- bsd/net/packet_mangler.c | 6 +- bsd/net/pf_ioctl.c | 2 +- bsd/net/pktap.c | 7 + bsd/netinet/flow_divert.c | 5 + bsd/netinet/in_pcb.c | 4 - bsd/netinet/ip_icmp.c | 12 +- bsd/netinet/ip_input.c | 2 +- bsd/netinet/ip_output.c | 6 +- bsd/netinet/mptcp.c | 86 +- bsd/netinet/mptcp_opt.c | 32 +- bsd/netinet/mptcp_subr.c | 64 +- bsd/netinet/mptcp_var.h | 5 +- bsd/netinet/tcp_input.c | 30 +- bsd/netinet/tcp_subr.c | 2 - bsd/netinet/tcp_timer.c | 4 +- bsd/netinet/tcp_usrreq.c | 3 + bsd/netinet/tcp_var.h | 7 +- bsd/netinet6/icmp6.c | 8 +- bsd/netinet6/in6_proto.c | 2 +- bsd/netinet6/ip6_output.c | 4 +- bsd/netinet6/ipsec.c | 26 +- bsd/netinet6/ipsec.h | 2 +- bsd/netinet6/nd6.c | 8 +- bsd/netkey/key.c | 2 +- bsd/netkey/keydb.h | 2 +- bsd/nfs/krpc_subr.c | 7 +- bsd/nfs/nfs.h | 3 + bsd/nfs/nfs_serv.c | 5 +- bsd/nfs/nfs_syscalls.c | 6 +- bsd/nfs/nfs_vfsops.c | 6 +- bsd/nfs/nfsm_subs.h | 2 + bsd/nfs/nfsmount.h | 2 +- bsd/sys/event.h | 3 +- bsd/sys/kern_control.h | 26 + bsd/sys/mbuf.h | 1 + bsd/sys/reason.h | 5 +- bsd/sys/reboot.h | 5 + bsd/sys/socketvar.h | 3 +- bsd/sys/spawn_internal.h | 25 +- bsd/vfs/vfs_attrlist.c | 2 +- bsd/vfs/vfs_syscalls.c | 2 + config/MasterVersion | 2 +- config/Private.arm64.exports | 4 + config/Unsupported.arm.exports | 2 + config/Unsupported.arm64.exports | 3 +- config/Unsupported.exports | 17 +- config/Unsupported.x86_64.exports | 11 +- iokit/IOKit/IOCPU.h | 2 +- iokit/IOKit/IOLocks.h | 68 + iokit/IOKit/IOPlatformExpert.h | 19 +- iokit/IOKit/IOReturn.h | 4 +- iokit/IOKit/IOService.h | 4 +- iokit/IOKit/pwr_mgt/IOPM.h | 2 + iokit/IOKit/pwr_mgt/RootDomain.h | 2 + iokit/IOKit/rtc/IORTCController.h | 41 +- iokit/Kernel/IOCPU.cpp | 10 +- iokit/Kernel/IOHibernateIO.cpp | 31 +- iokit/Kernel/IOLocks.cpp | 25 + iokit/Kernel/IOPMrootDomain.cpp | 201 +- iokit/Kernel/IOPlatformExpert.cpp | 70 +- iokit/Kernel/IOPolledInterface.cpp | 161 +- iokit/Kernel/IORTC.cpp | 68 + iokit/Kernel/IOService.cpp | 86 +- iokit/Tests/Tests.cpp | 1 + iokit/bsddev/IOKitBSDInit.cpp | 23 +- iokit/conf/files | 1 + libkdd/kcdata.h | 13 +- libkdd/kcdtypes.c | 5 +- libkdd/kdd.xcodeproj/project.pbxproj | 8 + libkdd/tests/Tests.swift | 4 + .../stackshot-sample-thread-groups-flags | Bin 0 -> 8240 bytes ...ckshot-sample-thread-groups-flags.plist.gz | Bin 0 -> 4175 bytes .../Libsyscall.xcodeproj/project.pbxproj | 4 + libsyscall/wrappers/libproc/libproc.h | 1 + libsyscall/wrappers/mach_absolute_time.s | 7 +- libsyscall/wrappers/reboot.c | 40 + libsyscall/wrappers/spawn/posix_spawn.c | 66 +- makedefs/MakeInc.def | 14 +- osfmk/arm/commpage/commpage.c | 18 +- osfmk/arm/cpu_capabilities.h | 9 +- osfmk/arm/cpu_data_internal.h | 4 +- osfmk/arm/machine_cpuid.c | 16 +- osfmk/arm/machine_routines.h | 7 +- osfmk/arm/machine_routines_asm.s | 12 - osfmk/arm/model_dep.c | 44 +- osfmk/arm/pmap.c | 304 +- osfmk/arm/pmap.h | 88 +- osfmk/arm/proc_reg.h | 23 +- osfmk/arm64/arm_vm_init.c | 289 +- osfmk/arm64/cpu.c | 7 + osfmk/arm64/genassym.c | 4 + osfmk/arm64/locore.s | 301 +- osfmk/arm64/machine_routines.c | 5 + osfmk/arm64/machine_routines_asm.s | 135 +- osfmk/arm64/platform_tests.c | 4 +- osfmk/arm64/proc_reg.h | 145 +- osfmk/arm64/sleh.c | 7 +- osfmk/arm64/start.s | 31 + osfmk/bank/bank.c | 103 +- osfmk/i386/AT386/model_dep.c | 79 +- osfmk/i386/cpu_data.h | 65 +- osfmk/i386/cpu_topology.c | 14 + osfmk/i386/genassym.c | 23 +- osfmk/i386/i386_init.c | 145 +- osfmk/i386/ktss.c | 4 +- osfmk/i386/ldt.c | 7 +- osfmk/i386/mp.c | 9 +- osfmk/i386/mp_desc.c | 266 +- osfmk/i386/mp_desc.h | 17 +- osfmk/i386/mp_native.c | 12 + osfmk/i386/pcb_native.c | 12 +- osfmk/i386/pmap.h | 86 +- osfmk/i386/pmap_common.c | 2 +- osfmk/i386/pmap_internal.h | 41 +- osfmk/i386/pmap_pcid.h | 9 +- osfmk/i386/pmap_x86_common.c | 75 +- osfmk/i386/proc_reg.h | 12 +- osfmk/i386/rtclock_asm.h | 2 +- osfmk/i386/rtclock_asm_native.h | 18 +- osfmk/i386/seg.h | 11 +- osfmk/i386/trap.c | 14 +- osfmk/i386/tss.h | 3 +- osfmk/i386/user_ldt.c | 11 +- osfmk/ipc/ipc_entry.c | 28 +- osfmk/ipc/ipc_entry.h | 70 +- osfmk/ipc/ipc_init.c | 1 - osfmk/ipc/ipc_kmsg.c | 6 + osfmk/ipc/ipc_pset.c | 4 +- osfmk/ipc/ipc_space.c | 120 +- osfmk/ipc/ipc_space.h | 15 +- osfmk/kdp/kdp_core.c | 19 +- osfmk/kdp/processor_core.c | 4 +- osfmk/kern/debug.c | 14 +- osfmk/kern/debug.h | 18 +- osfmk/kern/gzalloc.c | 2 +- osfmk/kern/kalloc.c | 91 +- osfmk/kern/kcdata.h | 13 +- osfmk/kern/ledger.c | 46 +- osfmk/kern/ledger.h | 6 +- osfmk/kern/sched_prim.c | 35 +- osfmk/kern/task.c | 40 +- osfmk/kern/task_policy.c | 4 +- osfmk/kern/thread.c | 2 +- osfmk/kern/thread.h | 3 +- osfmk/kern/thread_act.c | 53 +- osfmk/kern/zalloc.c | 84 +- osfmk/mach/arm/vm_param.h | 2 +- osfmk/mach/vm_statistics.h | 4 +- osfmk/prng/random.c | 76 + osfmk/prng/random.h | 22 + osfmk/vm/pmap.h | 6 +- osfmk/vm/vm_kern.c | 16 +- osfmk/vm/vm_map.c | 113 +- osfmk/vm/vm_pageout.c | 25 +- osfmk/vm/vm_resident.c | 7 +- osfmk/vm/vm_user.c | 3 + osfmk/x86_64/bcopy.s | 3 +- osfmk/x86_64/copyio.c | 2 +- osfmk/x86_64/idt64.s | 664 ++-- osfmk/x86_64/locore.s | 8 +- osfmk/x86_64/pmap.c | 112 +- osfmk/x86_64/pmap_pcid.c | 49 +- pexpert/pexpert/GearImage.h | 3461 ++++++++++++++++- pexpert/pexpert/arm/board_config.h | 9 +- pexpert/pexpert/arm64/board_config.h | 3 +- pexpert/pexpert/pe_images.h | 4 + pexpert/pexpert/pexpert.h | 13 + san/kasan-arm64.c | 5 +- san/kasan-blacklist-x86_64 | 4 +- san/kasan-x86_64.c | 8 +- san/kasan.c | 8 +- san/kasan.h | 1 + san/kasan_internal.h | 22 +- san/tools/kasan_install | 69 +- tools/lldbmacros/kcdata.py | 25 +- tools/lldbmacros/mbufs.py | 22 + tools/tests/darwintests/proc_info.c | 4 + tools/tests/darwintests/stackshot.m | 102 + 215 files changed, 9853 insertions(+), 2446 deletions(-) create mode 100644 iokit/Kernel/IORTC.cpp create mode 100644 libkdd/tests/stackshot-sample-thread-groups-flags create mode 100644 libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz create mode 100644 libsyscall/wrappers/reboot.c diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c index 1867151e9..aa2ed85ad 100644 --- a/bsd/kern/bsd_stubs.c +++ b/bsd/kern/bsd_stubs.c @@ -64,9 +64,9 @@ kmem_mb_alloc(vm_map_t mbmap, int size, int physContig, kern_return_t *err) kern_return_t kr = KERN_SUCCESS; if (!physContig) - kr = kernel_memory_allocate(mbmap, &addr, size, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF); + kr = kernel_memory_allocate(mbmap, &addr, size, 0, KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF); else - kr = kmem_alloc_contig(mbmap, &addr, size, PAGE_MASK, 0xfffff, 0, KMA_NOPAGEWAIT | KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF); + kr = kmem_alloc_contig(mbmap, &addr, size, PAGE_MASK, 0xfffff, 0, KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF); if (kr != KERN_SUCCESS) addr = 0; diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c index cda6f1046..099201dac 100644 --- a/bsd/kern/kern_control.c +++ b/bsd/kern/kern_control.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999-2015 Apple Inc. All rights reserved. + * Copyright (c) 1999-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -72,6 +72,7 @@ struct kctl { u_int32_t sendbufsize; /* request more than the default buffer size */ /* Dispatch functions */ + ctl_bind_func bind; /* Prepare contact */ ctl_connect_func connect; /* Make contact */ ctl_disconnect_func disconnect; /* Break contact */ ctl_send_func send; /* Send data to nke */ @@ -90,7 +91,7 @@ struct ctl_cb { struct socket *so; /* controlling socket */ struct kctl *kctl; /* back pointer to controller */ void *userdata; - u_int32_t unit; + struct sockaddr_ctl sac; u_int32_t usecount; }; @@ -125,6 +126,7 @@ TAILQ_HEAD(kctl_list, kctl) ctl_head; static int ctl_attach(struct socket *, int, struct proc *); static int ctl_detach(struct socket *); static int ctl_sofreelastref(struct socket *so); +static int ctl_bind(struct socket *, struct sockaddr *, struct proc *); static int ctl_connect(struct socket *, struct sockaddr *, struct proc *); static int ctl_disconnect(struct socket *); static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data, @@ -151,6 +153,7 @@ static lck_mtx_t * ctl_getlock(struct socket *, int); static struct pr_usrreqs ctl_usrreqs = { .pru_attach = ctl_attach, + .pru_bind = ctl_bind, .pru_connect = ctl_connect, .pru_control = ctl_ioctl, .pru_detach = ctl_detach, @@ -354,28 +357,46 @@ ctl_detach(struct socket *so) if (kcb == 0) return (0); + if (kcb->kctl != NULL && kcb->kctl->bind != NULL && + kcb->userdata != NULL && !(so->so_state & SS_ISCONNECTED)) { + // The unit was bound, but not connected + // Invoke the disconnected call to cleanup + if (kcb->kctl->disconnect != NULL) { + socket_unlock(so, 0); + (*kcb->kctl->disconnect)(kcb->kctl->kctlref, + kcb->sac.sc_unit, kcb->userdata); + socket_lock(so, 0); + } + } + soisdisconnected(so); so->so_flags |= SOF_PCBCLEARING; return (0); } static int -ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) +ctl_setup_kctl(struct socket *so, struct sockaddr *nam, struct proc *p) { -#pragma unused(p) - struct kctl *kctl; - int error = 0; + struct kctl *kctl = NULL; + int error = 0; struct sockaddr_ctl sa; - struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; - struct ctl_cb *kcb_next = NULL; - u_quad_t sbmaxsize; - u_int32_t recvbufsize, sendbufsize; + struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; + struct ctl_cb *kcb_next = NULL; + u_quad_t sbmaxsize; + u_int32_t recvbufsize, sendbufsize; - if (kcb == 0) - panic("ctl_connect so_pcb null\n"); + if (kcb == 0) { + panic("ctl_setup_kctl so_pcb null\n"); + } + + if (kcb->kctl != NULL) { + // Already set up, skip + return (0); + } - if (nam->sa_len != sizeof(struct sockaddr_ctl)) + if (nam->sa_len != sizeof(struct sockaddr_ctl)) { return (EINVAL); + } bcopy(nam, &sa, sizeof(struct sockaddr_ctl)); @@ -387,12 +408,12 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) } if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) && - (so->so_type != SOCK_STREAM)) || + (so->so_type != SOCK_STREAM)) || (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) && - (so->so_type != SOCK_DGRAM))) { - lck_mtx_unlock(ctl_mtx); - return (EPROTOTYPE); - } + (so->so_type != SOCK_DGRAM))) { + lck_mtx_unlock(ctl_mtx); + return (EPROTOTYPE); + } if (kctl->flags & CTL_FLAG_PRIVILEGED) { if (p == 0) { @@ -412,16 +433,17 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) } } else { /* Find an unused ID, assumes control IDs are in order */ - u_int32_t unit = 1; + u_int32_t unit = 1; TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) { - if (kcb_next->unit > unit) { + if (kcb_next->sac.sc_unit > unit) { /* Found a gap, lets fill it in */ break; } - unit = kcb_next->unit + 1; - if (unit == ctl_maxunit) + unit = kcb_next->sac.sc_unit + 1; + if (unit == ctl_maxunit) { break; + } } if (unit == ctl_maxunit) { @@ -432,7 +454,7 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) sa.sc_unit = unit; } - kcb->unit = sa.sc_unit; + bcopy(&sa, &kcb->sac, sizeof(struct sockaddr_ctl)); kcb->kctl = kctl; if (kcb_next != NULL) { TAILQ_INSERT_BEFORE(kcb_next, kcb, next); @@ -450,36 +472,102 @@ ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) */ sbmaxsize = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); - if (kctl->sendbufsize > sbmaxsize) + if (kctl->sendbufsize > sbmaxsize) { sendbufsize = sbmaxsize; - else + } else { sendbufsize = kctl->sendbufsize; + } - if (kctl->recvbufsize > sbmaxsize) + if (kctl->recvbufsize > sbmaxsize) { recvbufsize = sbmaxsize; - else + } else { recvbufsize = kctl->recvbufsize; + } error = soreserve(so, sendbufsize, recvbufsize); if (error) { if (ctl_debug) printf("%s - soreserve(%llx, %u, %u) error %d\n", - __func__, (uint64_t)VM_KERNEL_ADDRPERM(so), - sendbufsize, recvbufsize, error); + __func__, (uint64_t)VM_KERNEL_ADDRPERM(so), + sendbufsize, recvbufsize, error); goto done; } - soisconnecting(so); + +done: + if (error) { + soisdisconnected(so); + lck_mtx_lock(ctl_mtx); + TAILQ_REMOVE(&kctl->kcb_head, kcb, next); + kcb->kctl = NULL; + kcb->sac.sc_unit = 0; + kctlstat.kcs_pcbcount--; + kctlstat.kcs_gencnt++; + kctlstat.kcs_conn_fail++; + lck_mtx_unlock(ctl_mtx); + } + return (error); +} + +static int +ctl_bind(struct socket *so, struct sockaddr *nam, struct proc *p) +{ + int error = 0; + struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; + + if (kcb == NULL) { + panic("ctl_bind so_pcb null\n"); + } + + error = ctl_setup_kctl(so, nam, p); + if (error) { + return (error); + } + + if (kcb->kctl == NULL) { + panic("ctl_bind kctl null\n"); + } + + if (kcb->kctl->bind == NULL) { + return (EINVAL); + } socket_unlock(so, 0); - error = (*kctl->connect)(kctl->kctlref, &sa, &kcb->userdata); + error = (*kcb->kctl->bind)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata); socket_lock(so, 0); - if (error) - goto end; + return (error); +} + +static int +ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p) +{ + int error = 0; + struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; + + if (kcb == NULL) { + panic("ctl_connect so_pcb null\n"); + } + + error = ctl_setup_kctl(so, nam, p); + if (error) { + return (error); + } + + if (kcb->kctl == NULL) { + panic("ctl_connect kctl null\n"); + } + + soisconnecting(so); + socket_unlock(so, 0); + error = (*kcb->kctl->connect)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata); + socket_lock(so, 0); + if (error) { + goto end; + } soisconnected(so); end: - if (error && kctl->disconnect) { + if (error && kcb->kctl->disconnect) { /* * XXX Make sure we Don't check the return value * of disconnect here. @@ -490,16 +578,15 @@ end: * ipsec/utun_ctl_disconnect. */ socket_unlock(so, 0); - (*kctl->disconnect)(kctl->kctlref, kcb->unit, kcb->userdata); + (*kcb->kctl->disconnect)(kcb->kctl->kctlref, kcb->sac.sc_unit, kcb->userdata); socket_lock(so, 0); } -done: if (error) { soisdisconnected(so); lck_mtx_lock(ctl_mtx); - kcb->kctl = 0; - kcb->unit = 0; - TAILQ_REMOVE(&kctl->kcb_head, kcb, next); + TAILQ_REMOVE(&kcb->kctl->kcb_head, kcb, next); + kcb->kctl = NULL; + kcb->sac.sc_unit = 0; kctlstat.kcs_pcbcount--; kctlstat.kcs_gencnt++; kctlstat.kcs_conn_fail++; @@ -518,7 +605,7 @@ ctl_disconnect(struct socket *so) if (kctl && kctl->disconnect) { socket_unlock(so, 0); - (*kctl->disconnect)(kctl->kctlref, kcb->unit, + (*kctl->disconnect)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata); socket_lock(so, 0); } @@ -528,7 +615,7 @@ ctl_disconnect(struct socket *so) socket_unlock(so, 0); lck_mtx_lock(ctl_mtx); kcb->kctl = 0; - kcb->unit = 0; + kcb->sac.sc_unit = 0; while (kcb->usecount != 0) { msleep(&kcb->usecount, ctl_mtx, 0, "kcb->usecount", 0); } @@ -559,7 +646,7 @@ ctl_peeraddr(struct socket *so, struct sockaddr **nam) sc.sc_family = AF_SYSTEM; sc.ss_sysaddr = AF_SYS_CONTROL; sc.sc_id = kctl->id; - sc.sc_unit = kcb->unit; + sc.sc_unit = kcb->sac.sc_unit; *nam = dup_sockaddr((struct sockaddr *)&sc, 1); @@ -609,7 +696,7 @@ ctl_usr_rcvd(struct socket *so, int flags) if (kctl->rcvd) { socket_unlock(so, 0); - (*kctl->rcvd)(kctl->kctlref, kcb->unit, kcb->userdata, flags); + (*kctl->rcvd)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, flags); socket_lock(so, 0); } @@ -640,7 +727,7 @@ ctl_send(struct socket *so, int flags, struct mbuf *m, if (error == 0 && kctl->send) { so_tc_update_stats(m, so, m_get_service_class(m)); socket_unlock(so, 0); - error = (*kctl->send)(kctl->kctlref, kcb->unit, kcb->userdata, + error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, m, flags); socket_lock(so, 0); } else { @@ -678,7 +765,7 @@ ctl_send_list(struct socket *so, int flags, struct mbuf *m, so_tc_update_stats(nxt, so, m_get_service_class(nxt)); socket_unlock(so, 0); - error = (*kctl->send_list)(kctl->kctlref, kcb->unit, + error = (*kctl->send_list)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, m, flags); socket_lock(so, 0); } else if (error == 0 && kctl->send) { @@ -688,7 +775,7 @@ ctl_send_list(struct socket *so, int flags, struct mbuf *m, m->m_nextpkt = NULL; so_tc_update_stats(m, so, m_get_service_class(m)); socket_unlock(so, 0); - error = (*kctl->send)(kctl->kctlref, kcb->unit, + error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, m, flags); socket_lock(so, 0); m = nextpkt; @@ -1089,7 +1176,7 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt) struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb; struct kctl *kctl; int error = 0; - void *data; + void *data = NULL; size_t len; if (sopt->sopt_level != SYSPROTO_CONTROL) { @@ -1106,11 +1193,9 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt) case SOPT_SET: if (kctl->setopt == NULL) return (ENOTSUP); - if (sopt->sopt_valsize == 0) { - data = NULL; - } else { + if (sopt->sopt_valsize != 0) { MALLOC(data, void *, sopt->sopt_valsize, M_TEMP, - M_WAITOK); + M_WAITOK | M_ZERO); if (data == NULL) return (ENOMEM); error = sooptcopyin(sopt, data, @@ -1119,20 +1204,22 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt) if (error == 0) { socket_unlock(so, 0); error = (*kctl->setopt)(kctl->kctlref, - kcb->unit, kcb->userdata, sopt->sopt_name, + kcb->sac.sc_unit, kcb->userdata, sopt->sopt_name, data, sopt->sopt_valsize); socket_lock(so, 0); } - FREE(data, M_TEMP); + + if (data != NULL) + FREE(data, M_TEMP); break; case SOPT_GET: if (kctl->getopt == NULL) return (ENOTSUP); - data = NULL; + if (sopt->sopt_valsize && sopt->sopt_val) { MALLOC(data, void *, sopt->sopt_valsize, M_TEMP, - M_WAITOK); + M_WAITOK | M_ZERO); if (data == NULL) return (ENOMEM); /* @@ -1142,22 +1229,25 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt) error = sooptcopyin(sopt, data, sopt->sopt_valsize, sopt->sopt_valsize); } - len = sopt->sopt_valsize; - socket_unlock(so, 0); - error = (*kctl->getopt)(kctl->kctlref, kcb->unit, - kcb->userdata, sopt->sopt_name, - data, &len); - if (data != NULL && len > sopt->sopt_valsize) - panic_plain("ctl_ctloutput: ctl %s returned " - "len (%lu) > sopt_valsize (%lu)\n", - kcb->kctl->name, len, - sopt->sopt_valsize); - socket_lock(so, 0); + if (error == 0) { - if (data != NULL) - error = sooptcopyout(sopt, data, len); - else - sopt->sopt_valsize = len; + len = sopt->sopt_valsize; + socket_unlock(so, 0); + error = (*kctl->getopt)(kctl->kctlref, kcb->sac.sc_unit, + kcb->userdata, sopt->sopt_name, + data, &len); + if (data != NULL && len > sopt->sopt_valsize) + panic_plain("ctl_ctloutput: ctl %s returned " + "len (%lu) > sopt_valsize (%lu)\n", + kcb->kctl->name, len, + sopt->sopt_valsize); + socket_lock(so, 0); + if (error == 0) { + if (data != NULL) + error = sooptcopyout(sopt, data, len); + else + sopt->sopt_valsize = len; + } } if (data != NULL) FREE(data, M_TEMP); @@ -1496,6 +1586,7 @@ ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref) kctl->recvbufsize = userkctl->ctl_recvsize; } + kctl->bind = userkctl->ctl_bind; kctl->connect = userkctl->ctl_connect; kctl->disconnect = userkctl->ctl_disconnect; kctl->send = userkctl->ctl_send; @@ -1643,7 +1734,7 @@ kcb_find(struct kctl *kctl, u_int32_t unit) lck_mtx_assert(ctl_mtx, LCK_MTX_ASSERT_OWNED); TAILQ_FOREACH(kcb, &kctl->kcb_head, next) - if (kcb->unit == unit) + if (kcb->sac.sc_unit == unit) return (kcb); return (NULL); @@ -2014,7 +2105,7 @@ kctl_pcblist SYSCTL_HANDLER_ARGS xk->xkp_len = sizeof(struct xkctlpcb); xk->xkp_kind = XSO_KCB; - xk->xkp_unit = kcb->unit; + xk->xkp_unit = kcb->sac.sc_unit; xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb); xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl); xk->xkp_kctlid = kctl->id; @@ -2099,6 +2190,6 @@ kctl_fill_socketinfo(struct socket *so, struct socket_info *si) kcsi->kcsi_flags = kctl->flags; kcsi->kcsi_recvbufsize = kctl->recvbufsize; kcsi->kcsi_sendbufsize = kctl->sendbufsize; - kcsi->kcsi_unit = kcb->unit; + kcsi->kcsi_unit = kcb->sac.sc_unit; strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME); } diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c index f64bef436..12885133e 100644 --- a/bsd/kern/kern_event.c +++ b/bsd/kern/kern_event.c @@ -382,6 +382,7 @@ SECURITY_READ_ONLY_EARLY(static struct filterops) workloop_filtops = { extern const struct filterops pipe_rfiltops; extern const struct filterops pipe_wfiltops; extern const struct filterops ptsd_kqops; +extern const struct filterops ptmx_kqops; extern const struct filterops soread_filtops; extern const struct filterops sowrite_filtops; extern const struct filterops sock_filtops; @@ -449,7 +450,8 @@ SECURITY_READ_ONLY_EARLY(static struct filterops *) sysfilt_ops[EVFILTID_MAX] = [EVFILTID_NECP_FD] = &necp_fd_rfiltops, [EVFILTID_FSEVENT] = &fsevent_filtops, [EVFILTID_VN] = &vnode_filtops, - [EVFILTID_TTY] = &tty_filtops + [EVFILTID_TTY] = &tty_filtops, + [EVFILTID_PTMX] = &ptmx_kqops, }; /* waitq prepost callback */ @@ -2107,7 +2109,8 @@ filt_wldebounce( #if DEBUG || DEVELOPMENT if ((kev->fflags & NOTE_WL_THREAD_REQUEST) && !(kev->flags & EV_DELETE)) { - if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0) { + if ((udata & DISPATCH_QUEUE_ENQUEUED) == 0 && + (udata >> 48) != 0 && (udata >> 48) != 0xffff) { panic("kevent: workloop %#016llx is not enqueued " "(kev:%p dq_state:%#016llx)", kev->udata, kev, udata); } @@ -2807,7 +2810,8 @@ filt_wlprocess( uint64_t val; if (addr && task_is_active(t) && !task_is_halting(t) && copyin_word(addr, &val, sizeof(val)) == 0 && - val && (val & DISPATCH_QUEUE_ENQUEUED) == 0) { + val && (val & DISPATCH_QUEUE_ENQUEUED) == 0 && + (val >> 48) != 0 && (val >> 48) != 0xffff) { panic("kevent: workloop %#016llx is not enqueued " "(kn:%p dq_state:%#016llx kev.dq_state:%#016llx)", kn->kn_udata, kn, val, diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c index df43f2013..3ccb12119 100644 --- a/bsd/kern/kern_exec.c +++ b/bsd/kern/kern_exec.c @@ -1320,12 +1320,6 @@ badtoolate: } done: - if (!spawn) { - /* notify only if it has not failed due to FP Key error */ - if ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0) - proc_knote(p, NOTE_EXEC); - } - if (load_result.threadstate) { kfree(load_result.threadstate, load_result.threadstate_sz); load_result.threadstate = NULL; @@ -1991,7 +1985,8 @@ spawn_copyin_macpolicyinfo(const struct user__posix_spawn_args_desc *px_args, _p if ((error = copyin(px_args->mac_extensions, psmx, px_args->mac_extensions_size)) != 0) goto bad; - if (PS_MAC_EXTENSIONS_SIZE(psmx->psmx_count) > px_args->mac_extensions_size) { + size_t extsize = PS_MAC_EXTENSIONS_SIZE(psmx->psmx_count); + if (extsize == 0 || extsize > px_args->mac_extensions_size) { error = EINVAL; goto bad; } @@ -2304,8 +2299,9 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval) if (px_args.file_actions_size != 0) { /* Limit file_actions to allowed number of open files */ int maxfa = (p->p_limit ? p->p_rlimit[RLIMIT_NOFILE].rlim_cur : NOFILE); + size_t maxfa_size = PSF_ACTIONS_SIZE(maxfa); if (px_args.file_actions_size < PSF_ACTIONS_SIZE(1) || - px_args.file_actions_size > PSF_ACTIONS_SIZE(maxfa)) { + maxfa_size == 0 || px_args.file_actions_size > maxfa_size) { error = EINVAL; goto bad; } @@ -2321,7 +2317,8 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval) goto bad; /* Verify that the action count matches the struct size */ - if (PSF_ACTIONS_SIZE(px_sfap->psfa_act_count) != px_args.file_actions_size) { + size_t psfsize = PSF_ACTIONS_SIZE(px_sfap->psfa_act_count); + if (psfsize == 0 || psfsize != px_args.file_actions_size) { error = EINVAL; goto bad; } @@ -2347,7 +2344,8 @@ posix_spawn(proc_t ap, struct posix_spawn_args *uap, int32_t *retval) goto bad; /* Verify that the action count matches the struct size */ - if (PS_PORT_ACTIONS_SIZE(px_spap->pspa_count) != px_args.port_actions_size) { + size_t pasize = PS_PORT_ACTIONS_SIZE(px_spap->pspa_count); + if (pasize == 0 || pasize != px_args.port_actions_size) { error = EINVAL; goto bad; } @@ -2944,12 +2942,13 @@ bad: /* flag the 'fork' has occurred */ proc_knote(p->p_pptr, NOTE_FORK | p->p_pid); - /* then flag exec has occurred */ - /* notify only if it has not failed due to FP Key error */ - if ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0) - proc_knote(p, NOTE_EXEC); } + /* flag exec has occurred, notify only if it has not failed due to FP Key error */ + if (!error && ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0)) + proc_knote(p, NOTE_EXEC); + + if (error == 0) { /* * We need to initialize the bank context behind the protection of @@ -3522,7 +3521,12 @@ __mac_execve(proc_t p, struct __mac_execve_args *uap, int32_t *retval) exec_resettextvp(p, imgp); error = check_for_signature(p, imgp); - } + } + + /* flag exec has occurred, notify only if it has not failed due to FP Key error */ + if (exec_done && ((p->p_lflag & P_LTERM_DECRYPTFAIL) == 0)) + proc_knote(p, NOTE_EXEC); + if (imgp->ip_vp != NULLVP) vnode_put(imgp->ip_vp); if (imgp->ip_scriptvp != NULLVP) diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c index b8f5def79..fdbee03d1 100644 --- a/bsd/kern/kern_exit.c +++ b/bsd/kern/kern_exit.c @@ -2668,6 +2668,9 @@ vfork_exit_internal(proc_t p, int rv, int forceexit) __private_extern__ void munge_user64_rusage(struct rusage *a_rusage_p, struct user64_rusage *a_user_rusage_p) { + /* Zero-out struct so that padding is cleared */ + bzero(a_user_rusage_p, sizeof(struct user64_rusage)); + /* timeval changes size, so utime and stime need special handling */ a_user_rusage_p->ru_utime.tv_sec = a_rusage_p->ru_utime.tv_sec; a_user_rusage_p->ru_utime.tv_usec = a_rusage_p->ru_utime.tv_usec; diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c index 74a80dc55..54e431d05 100644 --- a/bsd/kern/kern_memorystatus.c +++ b/bsd/kern/kern_memorystatus.c @@ -4123,11 +4123,87 @@ memorystatus_get_task_memory_region_count(task_t task, uint64_t *count) *count = get_task_memory_region_count(task); } +#if DEVELOPMENT || DEBUG + +/* + * Sysctl only used to test memorystatus_allowed_vm_map_fork() path. + * set a new pidwatch value + * or + * get the current pidwatch value + */ + +uint64_t memorystatus_vm_map_fork_pidwatch_val = 0; +#define MEMORYSTATUS_VM_MAP_FORK_ALLOWED 0x100000000 +#define MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED 0x200000000 + +static int sysctl_memorystatus_vm_map_fork_pidwatch SYSCTL_HANDLER_ARGS { +#pragma unused(oidp, arg1, arg2) + + uint64_t new_value = 0; + uint64_t old_value = 0; + int error = 0; + + /* + * The pid is held in the low 32 bits. + * The 'allowed' flags are in the upper 32 bits. + */ + old_value = memorystatus_vm_map_fork_pidwatch_val; + + error = sysctl_io_number(req, old_value, sizeof(old_value), &new_value, NULL); + + if (error || !req->newptr) { + /* + * No new value passed in. + */ + return(error); + } + + /* + * A new pid was passed in via req->newptr. + * Ignore any attempt to set the higher order bits. + */ + memorystatus_vm_map_fork_pidwatch_val = new_value & 0xFFFFFFFF; + printf("memorystatus: pidwatch old_value = 0x%llx, new_value = 0x%llx \n", old_value, new_value); + + return(error); +} + +SYSCTL_PROC(_kern, OID_AUTO, memorystatus_vm_map_fork_pidwatch, CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED| CTLFLAG_MASKED, + 0, 0, sysctl_memorystatus_vm_map_fork_pidwatch, "Q", "get/set pid watched for in vm_map_fork"); + + +#define SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task) \ +MACRO_BEGIN \ +if (memorystatus_vm_map_fork_pidwatch_val != 0) { \ + proc_t p = get_bsdtask_info(task); \ + if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) { \ + memorystatus_vm_map_fork_pidwatch_val |= MEMORYSTATUS_VM_MAP_FORK_ALLOWED; \ + } \ +} \ +MACRO_END + +#define SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task) \ +MACRO_BEGIN \ +if (memorystatus_vm_map_fork_pidwatch_val != 0) { \ + proc_t p = get_bsdtask_info(task); \ + if (p && (memorystatus_vm_map_fork_pidwatch_val == (uint64_t)p->p_pid)) { \ + memorystatus_vm_map_fork_pidwatch_val |= MEMORYSTATUS_VM_MAP_FORK_NOT_ALLOWED; \ + } \ +} \ +MACRO_END + +#else /* DEVELOPMENT || DEBUG */ + +#define SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task) +#define SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task) + +#endif /* DEVELOPMENT || DEBUG */ + /* * Called during EXC_RESOURCE handling when a process exceeds a soft * memory limit. This is the corpse fork path and here we decide if * vm_map_fork will be allowed when creating the corpse. - * The current task is suspended. + * The task being considered is suspended. * * By default, a vm_map_fork is allowed to proceed. * @@ -4156,6 +4232,7 @@ memorystatus_allowed_vm_map_fork(__unused task_t task) uint64_t max_allowed_bytes = 0; if (max_task_footprint_mb == 0) { + SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task); return (is_allowed); } @@ -4172,14 +4249,17 @@ memorystatus_allowed_vm_map_fork(__unused task_t task) } if (footprint_in_bytes <= max_allowed_bytes) { + SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task); return (is_allowed); } else { printf("memorystatus disallowed vm_map_fork %lld %lld\n", footprint_in_bytes, max_allowed_bytes); + SET_VM_MAP_FORK_PIDWATCH_NOT_ALLOWED(task); return (!is_allowed); } #else /* CONFIG_EMBEDDED */ + SET_VM_MAP_FORK_PIDWATCH_ALLOWED(task); return (is_allowed); #endif /* CONFIG_EMBEDDED */ diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c index dd57cd722..d3a9e2c06 100644 --- a/bsd/kern/kern_mib.c +++ b/bsd/kern/kern_mib.c @@ -594,8 +594,7 @@ sysctl_mib_init(void) watchpoint_flag = arm_debug_info()->num_watchpoint_pairs; breakpoint_flag = arm_debug_info()->num_breakpoint_pairs; - arm_mvfp_info_t *mvfp_info; - mvfp_info = arm_mvfp_info(); + arm_mvfp_info_t *mvfp_info = arm_mvfp_info(); gNeon = mvfp_info->neon; gNeonHpfp = mvfp_info->neon_hpfp; diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c index c599a4bc7..83bd75cd2 100644 --- a/bsd/kern/kern_proc.c +++ b/bsd/kern/kern_proc.c @@ -163,7 +163,9 @@ extern int cs_enforcement_enable; #endif #if CONFIG_COREDUMP /* Name to give to core files */ -#if CONFIG_EMBEDDED +#if defined(XNU_TARGET_OS_BRIDGE) +__XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/internal/%N.core"}; +#elif CONFIG_EMBEDDED __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/private/var/cores/%N.core"}; #else __XNU_PRIVATE_EXTERN char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"}; diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c index af90ce9ff..63bcc0443 100644 --- a/bsd/kern/kern_sysctl.c +++ b/bsd/kern/kern_sysctl.c @@ -1622,8 +1622,13 @@ SYSCTL_STRING(_kern, KERN_VERSION, version, SYSCTL_STRING(_kern, OID_AUTO, uuid, CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED, &kernel_uuid_string[0], 0, ""); - #if DEBUG +#ifndef DKPR +#define DKPR 1 +#endif +#endif + +#if DKPR int debug_kprint_syscall = 0; char debug_kprint_syscall_process[MAXCOMLEN+1]; diff --git a/bsd/kern/kern_xxx.c b/bsd/kern/kern_xxx.c index c4674fc33..c922c1641 100644 --- a/bsd/kern/kern_xxx.c +++ b/bsd/kern/kern_xxx.c @@ -91,7 +91,7 @@ int psem_cache_purge_all(proc_t p); int reboot(struct proc *p, struct reboot_args *uap, __unused int32_t *retval) { - char message[128]; + char message[256]; int error=0; size_t dummy=0; #if CONFIG_MACF diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c index 169eba5c5..1fe787812 100644 --- a/bsd/kern/posix_shm.c +++ b/bsd/kern/posix_shm.c @@ -904,11 +904,16 @@ pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct PSHM_SUBSYS_UNLOCK(); return(EINVAL); } - if ((off_t)user_size > pinfo->pshm_length) { + if (user_size > (vm_map_size_t)pinfo->pshm_length) { PSHM_SUBSYS_UNLOCK(); return(EINVAL); } - if ((off_t)(user_size + file_pos) > pinfo->pshm_length) { + vm_map_size_t end_pos = 0; + if (os_add_overflow(user_size, file_pos, &end_pos)) { + PSHM_SUBSYS_UNLOCK(); + return(EINVAL); + } + if (end_pos > (vm_map_size_t)pinfo->pshm_length) { PSHM_SUBSYS_UNLOCK(); return(EINVAL); } diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c index 9efb0b1ca..1f88ae54e 100644 --- a/bsd/kern/proc_info.c +++ b/bsd/kern/proc_info.c @@ -2472,11 +2472,14 @@ proc_pidlistuptrs(proc_t p, user_addr_t buffer, uint32_t buffersize, int32_t *re count = buffersize / sizeof(uint64_t); if (count > MAX_UPTRS) { count = MAX_UPTRS; - buffersize = count * sizeof(uint64_t); } if (count > 0) { + buffersize = count * sizeof(uint64_t); kbuf = kalloc(buffersize); + bzero(kbuf, buffersize); assert(kbuf != NULL); + } else { + buffersize = 0; } } else { buffersize = 0; diff --git a/bsd/kern/process_policy.c b/bsd/kern/process_policy.c index 0a0b43e66..8c2b5b230 100644 --- a/bsd/kern/process_policy.c +++ b/bsd/kern/process_policy.c @@ -252,7 +252,7 @@ handle_lowresource(__unused int scope, int action, __unused int policy, int poli static int handle_cpuuse(int action, user_addr_t attrp, proc_t proc, __unused uint64_t target_threadid) { - proc_policy_cpuusage_attr_t cpuattr; + proc_policy_cpuusage_attr_t cpuattr = { }; #if CONFIG_MACF || !CONFIG_EMBEDDED proc_t curp = current_proc(); #endif diff --git a/bsd/kern/sys_coalition.c b/bsd/kern/sys_coalition.c index 85100962e..5cb3daa6b 100644 --- a/bsd/kern/sys_coalition.c +++ b/bsd/kern/sys_coalition.c @@ -484,6 +484,11 @@ static int sysctl_coalition_get_pid_list SYSCTL_HANDLER_ARGS out: proc_rele(tproc); + if (npids < 0) { + /* npids is a negative errno */ + return -npids; + } + if (npids == 0) return ENOENT; diff --git a/bsd/kern/sys_reason.c b/bsd/kern/sys_reason.c index 16ea4cfe4..ea260cdc3 100644 --- a/bsd/kern/sys_reason.c +++ b/bsd/kern/sys_reason.c @@ -278,7 +278,9 @@ os_reason_ref(os_reason_t cur_reason) lck_mtx_lock(&cur_reason->osr_lock); assert(cur_reason->osr_refcount > 0); - cur_reason->osr_refcount++; + if (os_add_overflow(cur_reason->osr_refcount, 1, &cur_reason->osr_refcount)) { + panic("os reason refcount overflow"); + } lck_mtx_unlock(&cur_reason->osr_lock); @@ -298,7 +300,9 @@ os_reason_free(os_reason_t cur_reason) lck_mtx_lock(&cur_reason->osr_lock); - assert(cur_reason->osr_refcount > 0); + if (cur_reason->osr_refcount == 0) { + panic("os_reason_free called on reason with zero refcount"); + } cur_reason->osr_refcount--; if (cur_reason->osr_refcount != 0) { diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master index f18cc9121..fd4edb968 100644 --- a/bsd/kern/syscalls.master +++ b/bsd/kern/syscalls.master @@ -102,7 +102,7 @@ 52 AUE_SIGPENDING ALL { int sigpending(struct sigvec *osv); } 53 AUE_SIGALTSTACK ALL { int sigaltstack(struct sigaltstack *nss, struct sigaltstack *oss) NO_SYSCALL_STUB ; } 54 AUE_IOCTL ALL { int ioctl(int fd, u_long com, caddr_t data) NO_SYSCALL_STUB; } -55 AUE_REBOOT ALL { int reboot(int opt, char *command); } +55 AUE_REBOOT ALL { int reboot(int opt, char *command) NO_SYSCALL_STUB; } 56 AUE_REVOKE ALL { int revoke(char *path); } 57 AUE_SYMLINK ALL { int symlink(char *path, char *link); } 58 AUE_READLINK ALL { int readlink(char *path, char *buf, int count); } diff --git a/bsd/kern/trace_codes b/bsd/kern/trace_codes index 981ab7f57..2e174231c 100644 --- a/bsd/kern/trace_codes +++ b/bsd/kern/trace_codes @@ -348,6 +348,8 @@ 0x1800000 MACH_CLOCK_EPOCH_CHANGE 0x1800004 MACH_CLOCK_BRIDGE_RCV_TS 0x1800008 MACH_CLOCK_BRIDGE_REMOTE_TIME +0x180000c MACH_CLOCK_BRIDGE_RESET_TS +0x1800010 MACH_CLOCK_BRIDGE_TS_PARAMS 0x1900000 MP_TLB_FLUSH 0x1900004 MP_CPUS_CALL 0x1900008 MP_CPUS_CALL_LOCAL diff --git a/bsd/kern/tty_dev.c b/bsd/kern/tty_dev.c index 87f3cd7ee..dac57968b 100644 --- a/bsd/kern/tty_dev.c +++ b/bsd/kern/tty_dev.c @@ -496,10 +496,12 @@ ptcwakeup(struct tty *tp, int flag) if (flag & FREAD) { selwakeup(&pti->pt_selr); wakeup(TSA_PTC_READ(tp)); + KNOTE(&pti->pt_selr.si_note, 1); } if (flag & FWRITE) { selwakeup(&pti->pt_selw); wakeup(TSA_PTC_WRITE(tp)); + KNOTE(&pti->pt_selw.si_note, 1); } } @@ -776,10 +778,10 @@ ptcselect(dev_t dev, int rw, void *wql, proc_t p) } /* FALLTHROUGH */ - case 0: /* exceptional */ + case 0: /* exceptional */ if ((tp->t_state&TS_ISOPEN) && - ((pti->pt_flags & PF_PKT && pti->pt_send) || - (pti->pt_flags & PF_UCNTL && pti->pt_ucntl))) { + (((pti->pt_flags & PF_PKT) && pti->pt_send) || + ((pti->pt_flags & PF_UCNTL) && pti->pt_ucntl))) { retval = 1; break; } @@ -790,21 +792,21 @@ ptcselect(dev_t dev, int rw, void *wql, proc_t p) case FWRITE: if (tp->t_state&TS_ISOPEN) { if (pti->pt_flags & PF_REMOTE) { - if (tp->t_canq.c_cc == 0) { - retval = (driver->fix_7828447) ? (TTYHOG - 1) : 1; - break; + if (tp->t_canq.c_cc == 0) { + retval = (driver->fix_7828447) ? (TTYHOG - 1) : 1; + break; } } else { - retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc); - if (retval > 0) { - retval = (driver->fix_7828447) ? retval : 1; - break; - } - if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) { - retval = 1; - break; - } - retval = 0; + retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc); + if (retval > 0) { + retval = (driver->fix_7828447) ? retval : 1; + break; + } + if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) { + retval = 1; + break; + } + retval = 0; } } selrecord(p, &pti->pt_selw, wql); diff --git a/bsd/kern/tty_ptmx.c b/bsd/kern/tty_ptmx.c index cf89b93eb..aa583d857 100644 --- a/bsd/kern/tty_ptmx.c +++ b/bsd/kern/tty_ptmx.c @@ -142,7 +142,7 @@ static struct cdevsw ptsd_cdev = { /* * System-wide limit on the max number of cloned ptys */ -#define PTMX_MAX_DEFAULT 127 /* 128 entries */ +#define PTMX_MAX_DEFAULT 511 /* 512 entries */ #define PTMX_MAX_HARD 999 /* 1000 entries, due to PTSD_TEMPLATE */ static int ptmx_max = PTMX_MAX_DEFAULT; /* default # of clones we allow */ @@ -695,3 +695,247 @@ ptsd_revoke_knotes(__unused int minor, struct tty *tp) tty_unlock(tp); } + +/* + * kevent filter routines for the master side of a pty, a ptmx. + * + * Stuff the ptmx_ioctl structure into the hook for ptmx knotes. Use the + * embedded tty's lock for synchronization. + */ + +int ptmx_kqfilter(dev_t dev, struct knote *kn); +static void ptmx_kqops_detach(struct knote *); +static int ptmx_kqops_event(struct knote *, long); +static int ptmx_kqops_touch(struct knote *kn, struct kevent_internal_s *kev); +static int ptmx_kqops_process(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev); +static int ptmx_kqops_common(struct knote *kn, struct ptmx_ioctl *pti, struct tty *tp); + +SECURITY_READ_ONLY_EARLY(struct filterops) ptmx_kqops = { + .f_isfd = 1, + /* attach is handled by ptmx_kqfilter -- the dev node must be passed in */ + .f_detach = ptmx_kqops_detach, + .f_event = ptmx_kqops_event, + .f_touch = ptmx_kqops_touch, + .f_process = ptmx_kqops_process, +}; + +static struct ptmx_ioctl * +ptmx_knote_ioctl(struct knote *kn) +{ + return (struct ptmx_ioctl *)kn->kn_hook; +} + +static struct tty * +ptmx_knote_tty(struct knote *kn) +{ + struct ptmx_ioctl *pti = kn->kn_hook; + return pti->pt_tty; +} + +int +ptmx_kqfilter(dev_t dev, struct knote *kn) +{ + struct tty *tp = NULL; + struct ptmx_ioctl *pti = NULL; + int ret; + + /* make sure we're talking about the right device type */ + if (cdevsw[major(dev)].d_open != ptcopen) { + knote_set_error(kn, ENODEV); + return 0; + } + + if ((pti = ptmx_get_ioctl(minor(dev), 0)) == NULL) { + knote_set_error(kn, ENXIO); + return 0; + } + + tp = pti->pt_tty; + tty_lock(tp); + + kn->kn_filtid = EVFILTID_PTMX; + kn->kn_hook = pti; + + /* + * Attach to the ptmx's selinfo structures. This is the major difference + * to the ptsd filtops, which use the selinfo structures in the tty + * structure. + */ + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_ATTACH(&pti->pt_selr.si_note, kn); + break; + case EVFILT_WRITE: + KNOTE_ATTACH(&pti->pt_selw.si_note, kn); + break; + default: + panic("ptmx kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } + + /* capture current event state */ + ret = ptmx_kqops_common(kn, pti, tp); + + /* take a reference on the TTY */ + ttyhold(tp); + tty_unlock(tp); + + return ret; +} + +static void +ptmx_kqops_detach(struct knote *kn) +{ + struct ptmx_ioctl *pti = kn->kn_hook; + struct tty *tp = pti->pt_tty; + + assert(tp != NULL); + + tty_lock(tp); + + switch (kn->kn_filter) { + case EVFILT_READ: + KNOTE_DETACH(&pti->pt_selr.si_note, kn); + break; + + case EVFILT_WRITE: + KNOTE_DETACH(&pti->pt_selw.si_note, kn); + break; + + default: + panic("invalid knote %p detach, filter: %d", kn, kn->kn_filter); + break; + } + + kn->kn_hook = NULL; + tty_unlock(tp); + + ttyfree(tp); +} + +static int +ptmx_kqops_common(struct knote *kn, struct ptmx_ioctl *pti, struct tty *tp) +{ + int retval = 0; + + TTY_LOCK_OWNED(tp); + + /* disconnects should force a wakeup (EOF) */ + if (!(tp->t_state & TS_CONNECTED)) { + return 1; + } + + switch (kn->kn_filter) { + case EVFILT_READ: + /* there's data on the TTY and it's not stopped */ + if (tp->t_outq.c_cc && !(tp->t_state & TS_TTSTOP)) { + retval = tp->t_outq.c_cc; + } else if (((pti->pt_flags & PF_PKT) && pti->pt_send) || + ((pti->pt_flags & PF_UCNTL) && pti->pt_ucntl)) { + retval = 1; + } + break; + + case EVFILT_WRITE: + if (pti->pt_flags & PF_REMOTE) { + if (tp->t_canq.c_cc == 0) { + retval = TTYHOG - 1; + } + } else { + retval = (TTYHOG - 2) - (tp->t_rawq.c_cc + tp->t_canq.c_cc); + if (tp->t_canq.c_cc == 0 && (tp->t_lflag & ICANON)) { + retval = 1; + } + if (retval < 0) { + retval = 0; + } + } + break; + + default: + panic("ptmx kevent: unexpected filter: %d, kn = %p, tty = %p", + kn->kn_filter, kn, tp); + break; + } + + if (tp->t_state & TS_ZOMBIE) { + kn->kn_flags |= EV_EOF; + retval = 1; + } + + return retval; +} + +static int +ptmx_kqops_event(struct knote *kn, long hint) +{ + struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn); + struct tty *tp = ptmx_knote_tty(kn); + int ret; + bool revoked = hint & NOTE_REVOKE; + hint &= ~NOTE_REVOKE; + + if (!hint) { + tty_lock(tp); + } + + if (revoked) { + kn->kn_flags |= EV_EOF | EV_ONESHOT; + ret = 1; + } else { + ret = ptmx_kqops_common(kn, pti, tp); + } + + if (!hint) { + tty_unlock(tp); + } + + return ret; +} + +static int +ptmx_kqops_touch(struct knote *kn, struct kevent_internal_s *kev) +{ + struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn); + struct tty *tp = ptmx_knote_tty(kn); + int ret; + + tty_lock(tp); + + /* accept new kevent state */ + kn->kn_sfflags = kev->fflags; + kn->kn_sdata = kev->data; + if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0) { + kn->kn_udata = kev->udata; + } + + /* recapture fired state of knote */ + ret = ptmx_kqops_common(kn, pti, tp); + + tty_unlock(tp); + + return ret; +} + +static int +ptmx_kqops_process(struct knote *kn, __unused struct filt_process_s *data, + struct kevent_internal_s *kev) +{ + struct ptmx_ioctl *pti = ptmx_knote_ioctl(kn); + struct tty *tp = ptmx_knote_tty(kn); + int ret; + + tty_lock(tp); + ret = ptmx_kqops_common(kn, pti, tp); + if (ret) { + *kev = kn->kn_kevent; + if (kn->kn_flags & EV_CLEAR) { + kn->kn_fflags = 0; + kn->kn_data = 0; + } + } + tty_unlock(tp); + + return ret; +} diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c index 57915610c..ca8fde048 100644 --- a/bsd/kern/uipc_mbuf.c +++ b/bsd/kern/uipc_mbuf.c @@ -582,6 +582,18 @@ static lck_attr_t *mleak_lock_attr; static lck_grp_t *mleak_lock_grp; static lck_grp_attr_t *mleak_lock_grp_attr; +/* *Failed* large allocations. */ +struct mtracelarge { + uint64_t size; + uint64_t depth; + uintptr_t addr[MLEAK_STACK_DEPTH]; +}; + +#define MTRACELARGE_NUM_TRACES 5 +static struct mtracelarge mtracelarge_table[MTRACELARGE_NUM_TRACES]; + +static void mtracelarge_register(size_t size); + /* Lock to protect the completion callback table */ static lck_grp_attr_t *mbuf_tx_compl_tbl_lck_grp_attr = NULL; static lck_attr_t *mbuf_tx_compl_tbl_lck_attr = NULL; @@ -681,7 +693,7 @@ boolean_t mb_peak_firstreport = FALSE; static struct timeval mb_wdtstart; /* watchdog start timestamp */ static char *mbuf_dump_buf; -#define MBUF_DUMP_BUF_SIZE 3072 +#define MBUF_DUMP_BUF_SIZE 4096 /* * mbuf watchdog is enabled by default on embedded platforms. It is @@ -2954,12 +2966,14 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) /* Try for 1 page if failed */ size = PAGE_SIZE; page = kmem_mb_alloc(mb_map, size, 0, &error); + if (page == 0) { + m_vm_error_stats(&mb_kmem_one_failed, + &mb_kmem_one_failed_ts, + NULL, size, error); + } } if (page == 0) { - m_vm_error_stats(&mb_kmem_one_failed, - &mb_kmem_one_failed_ts, - NULL, size, error); lck_mtx_lock(mbuf_mlock); goto out; } @@ -3108,6 +3122,8 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) out: LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + mtracelarge_register(size); + /* We're done; let others enter */ mb_clalloc_busy = FALSE; if (mb_clalloc_waiters > 0) { @@ -7324,7 +7340,7 @@ mleak_update_stats() mltr = &mleak_stat->ml_trace[0]; bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES); for (i = 0; i < MLEAK_NUM_TRACES; i++) { - int j; + int j; if (mleak_top_trace[i] == NULL || mleak_top_trace[i]->allocs == 0) @@ -7386,7 +7402,7 @@ mbuf_dump(void) mb_class_stat_t *sp; mleak_trace_stat_t *mltr; char *c = mbuf_dump_buf; - int i, k, clen = MBUF_DUMP_BUF_SIZE; + int i, j, k, clen = MBUF_DUMP_BUF_SIZE; mbuf_dump_buf[0] = '\0'; @@ -7531,6 +7547,32 @@ mbuf_dump(void) MBUF_DUMP_BUF_CHK(); } + k = snprintf(c, clen, "\nlargest allocation failure backtraces:\n"); + MBUF_DUMP_BUF_CHK(); + + for (j = 0; j < MTRACELARGE_NUM_TRACES; j++) { + struct mtracelarge *trace = &mtracelarge_table[j]; + if (trace->size == 0 || trace->depth == 0) + continue; + k = snprintf(c, clen, "size %llu: < ", trace->size); + MBUF_DUMP_BUF_CHK(); + for (i = 0; i < trace->depth; i++) { + if (mleak_stat->ml_isaddr64) { + k = snprintf(c, clen, "0x%0llx ", + (uint64_t)VM_KERNEL_UNSLIDE( + trace->addr[i])); + } else { + k = snprintf(c, clen, + "0x%08x ", + (uint32_t)VM_KERNEL_UNSLIDE( + trace->addr[i])); + } + MBUF_DUMP_BUF_CHK(); + } + k = snprintf(c, clen, ">\n"); + MBUF_DUMP_BUF_CHK(); + } + /* mbuf leak detection statistics */ mleak_update_stats(); @@ -7575,7 +7617,6 @@ mbuf_dump(void) MBUF_DUMP_BUF_CHK(); for (i = 0; i < MLEAK_STACK_DEPTH; i++) { - int j; k = snprintf(c, clen, "%2d: ", (i + 1)); MBUF_DUMP_BUF_CHK(); for (j = 0; j < mleak_stat->ml_cnt; j++) { @@ -8160,8 +8201,12 @@ m_drain_force_sysctl SYSCTL_HANDLER_ARGS err = sysctl_handle_int(oidp, &val, 0, req); if (err != 0 || req->newptr == USER_ADDR_NULL) return (err); - if (val) + if (val) { + lck_mtx_lock(mbuf_mlock); + printf("%s\n", mbuf_dump()); + lck_mtx_unlock(mbuf_mlock); m_drain(); + } return (err); } @@ -8237,6 +8282,36 @@ mbtest SYSCTL_HANDLER_ARGS } #endif + +static void +mtracelarge_register(size_t size) +{ + int i; + struct mtracelarge *trace; + uintptr_t bt[MLEAK_STACK_DEPTH]; + unsigned int depth; + + depth = backtrace(bt, MLEAK_STACK_DEPTH); + /* Check if this entry is already on the list. */ + for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) { + trace = &mtracelarge_table[i]; + if (trace->size == size && trace->depth == depth && + memcmp(bt, trace->addr, depth * sizeof(uintptr_t)) == 0) { + return; + } + + } + for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) { + trace = &mtracelarge_table[i]; + if (size > trace->size) { + trace->depth = depth; + memcpy(trace->addr, bt, depth * sizeof(uintptr_t)); + trace->size = size; + break; + } + } +} + SYSCTL_DECL(_kern_ipc); #if DEBUG || DEVELOPMENT SYSCTL_PROC(_kern_ipc, OID_AUTO, mbtest, diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c index 08ec21e97..ce89864cc 100644 --- a/bsd/kern/uipc_socket2.c +++ b/bsd/kern/uipc_socket2.c @@ -1421,7 +1421,11 @@ sbappendstream_rcvdemux(struct socket *so, struct mbuf *m, uint32_t seqnum, { int ret = 0; - if ((m != NULL) && (m_pktlen(m) <= 0)) { + if ((m != NULL) && + m_pktlen(m) <= 0 && + !((so->so_flags & SOF_MP_SUBFLOW) && + (m->m_flags & M_PKTHDR) && + (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN))) { m_freem(m); return (ret); } @@ -1453,8 +1457,14 @@ sbappendmptcpstream_rcv(struct sockbuf *sb, struct mbuf *m) if (m == NULL || m_pktlen(m) == 0 || (sb->sb_flags & SB_DROP) || (so->so_state & SS_CANTRCVMORE)) { - if (m != NULL) + if (m && m_pktlen(m) == 0 && + (m->m_flags & M_PKTHDR) && + (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) { + mptcp_input(tptomptp(sototcpcb(so))->mpt_mpte, m); + return (1); + } else if (m != NULL) { m_freem(m); + } return (0); } /* the socket is not closed, so SOF_MP_SUBFLOW must be set */ diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c index 73c6ba447..f51596c37 100644 --- a/bsd/kern/uipc_syscalls.c +++ b/bsd/kern/uipc_syscalls.c @@ -90,6 +90,7 @@ #include #include #include +#include #include @@ -328,7 +329,8 @@ bind(__unused proc_t p, struct bind_args *uap, __unused int32_t *retval) goto out; AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa); #if CONFIG_MACF_SOCKET_SUBSET - if ((error = mac_socket_check_bind(kauth_cred_get(), so, sa)) == 0) + if ((sa != NULL && sa->sa_family == AF_SYSTEM) || + (error = mac_socket_check_bind(kauth_cred_get(), so, sa)) == 0) error = sobindlock(so, sa, 1); /* will lock socket */ #else error = sobindlock(so, sa, 1); /* will lock socket */ diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c index 625e9736f..11c3ac25e 100644 --- a/bsd/miscfs/specfs/spec_vnops.c +++ b/bsd/miscfs/specfs/spec_vnops.c @@ -104,6 +104,7 @@ extern dev_t chrtoblk(dev_t dev); extern boolean_t iskmemdev(dev_t dev); extern int bpfkqfilter(dev_t dev, struct knote *kn); extern int ptsd_kqfilter(dev_t, struct knote *); +extern int ptmx_kqfilter(dev_t, struct knote *); struct vnode *speclisth[SPECHSZ]; @@ -768,9 +769,10 @@ spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_internal_s *kev) if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) { kn->kn_filtid = EVFILTID_PTSD; return ptsd_kqfilter(dev, kn); - } else if (cdevsw[major(dev)].d_type == D_TTY && - !(cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) && - kn->kn_vnode_kqok) { + } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) { + kn->kn_filtid = EVFILTID_PTMX; + return ptmx_kqfilter(dev, kn); + } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) { /* * TTYs from drivers that use struct ttys use their own filter * routines. The PTC driver doesn't use the tty for character diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c index 3bf795149..1448fc02b 100644 --- a/bsd/net/bpf.c +++ b/bsd/net/bpf.c @@ -1979,6 +1979,11 @@ bpf_setif(struct bpf_d *d, ifnet_t theywant) */ if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) continue; + /* + * Skip the coprocessor interface + */ + if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) + continue; /* * We found the requested interface. * Allocate the packet buffers. diff --git a/bsd/net/classq/classq_fq_codel.c b/bsd/net/classq/classq_fq_codel.c index 3710ad112..f762add42 100644 --- a/bsd/net/classq/classq_fq_codel.c +++ b/bsd/net/classq/classq_fq_codel.c @@ -253,6 +253,7 @@ fq_addq(fq_if_t *fqs, pktsched_pkt_t *pkt, fq_if_classq_t *fq_cl) if (fqs->fqs_large_flow == NULL) { droptype = DTYPE_FORCED; fq_cl->fcl_stat.fcl_drop_overflow++; + ret = CLASSQEQ_DROP; /* * if this fq was freshly created and there diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c index 50e85b274..9219ab940 100644 --- a/bsd/net/dlil.c +++ b/bsd/net/dlil.c @@ -2723,7 +2723,7 @@ dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head, static void -ifnet_start_common(struct ifnet *ifp, int resetfc) +ifnet_start_common(struct ifnet *ifp, boolean_t resetfc) { if (!(ifp->if_eflags & IFEF_TXSTART)) return; @@ -2731,7 +2731,8 @@ ifnet_start_common(struct ifnet *ifp, int resetfc) * If the starter thread is inactive, signal it to do work, * unless the interface is being flow controlled from below, * e.g. a virtual interface being flow controlled by a real - * network interface beneath it. + * network interface beneath it, or it's been disabled via + * a call to ifnet_disable_output(). */ lck_mtx_lock_spin(&ifp->if_start_lock); if (resetfc) { @@ -2754,7 +2755,7 @@ ifnet_start_common(struct ifnet *ifp, int resetfc) void ifnet_start(struct ifnet *ifp) { - ifnet_start_common(ifp, 0); + ifnet_start_common(ifp, FALSE); } static void @@ -2868,9 +2869,14 @@ ifnet_start_thread_fn(void *v, wait_result_t w) lck_mtx_lock_spin(&ifp->if_start_lock); - /* if there's no pending request, we're done */ - if (req == ifp->if_start_req) + /* + * If there's no pending request or if the + * interface has been disabled, we're done. + */ + if (req == ifp->if_start_req || + (ifp->if_start_flags & IFSF_FLOW_CONTROLLED)) { break; + } } ifp->if_start_req = 0; @@ -3041,8 +3047,9 @@ ifnet_poll_thread_fn(void *v, wait_result_t w) lck_mtx_lock_spin(&ifp->if_poll_lock); /* if there's no pending request, we're done */ - if (req == ifp->if_poll_req) + if (req == ifp->if_poll_req) { break; + } } ifp->if_poll_req = 0; ifp->if_poll_active = 0; @@ -6076,8 +6083,14 @@ ifnet_detach(ifnet_t ifp) ifp->if_refflags |= IFRF_DETACHING; lck_mtx_unlock(&ifp->if_ref_lock); - if (dlil_verbose) + if (dlil_verbose) { printf("%s: detaching\n", if_name(ifp)); + } + + /* clean up flow control entry object if there's any */ + if (ifp->if_eflags & IFEF_TXSTART) { + ifnet_flowadv(ifp->if_flowhash); + } /* Reset ECN enable/disable flags */ ifp->if_eflags &= ~IFEF_ECN_DISABLE; @@ -6438,6 +6451,7 @@ ifnet_detach_final(struct ifnet *ifp) wakeup_one((caddr_t)&inp->input_waiting); } lck_mtx_unlock(&inp->input_lck); + ifnet_lock_done(ifp); /* wait for the input thread to terminate */ lck_mtx_lock_spin(&inp->input_lck); @@ -6447,6 +6461,7 @@ ifnet_detach_final(struct ifnet *ifp) (PZERO - 1) | PSPIN, inp->input_name, NULL); } lck_mtx_unlock(&inp->input_lck); + ifnet_lock_exclusive(ifp); /* clean-up input thread state */ dlil_clean_threading_info(inp); @@ -7834,7 +7849,7 @@ ifnet_enable_output(struct ifnet *ifp) return (ENXIO); } - ifnet_start_common(ifp, 1); + ifnet_start_common(ifp, TRUE); return (0); } @@ -7904,7 +7919,7 @@ ifnet_fc_add(struct ifnet *ifp) /* become regular mutex */ lck_mtx_convert_spin(&ifnet_fc_lock); - ifce = zalloc_noblock(ifnet_fc_zone); + ifce = zalloc(ifnet_fc_zone); if (ifce == NULL) { /* memory allocation failed */ lck_mtx_unlock(&ifnet_fc_lock); diff --git a/bsd/net/if.c b/bsd/net/if.c index 53df43d54..6b00d6cad 100644 --- a/bsd/net/if.c +++ b/bsd/net/if.c @@ -243,6 +243,8 @@ static uint32_t if_verbose = 0; SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_verbose, CTLFLAG_RW | CTLFLAG_LOCKED, &if_verbose, 0, ""); +boolean_t intcoproc_unrestricted; + /* Eventhandler context for interface events */ struct eventhandler_lists_ctxt ifnet_evhdlr_ctxt; @@ -270,6 +272,9 @@ ifa_init(void) lck_mtx_init(&ifma_trash_lock, ifa_mtx_grp, ifa_mtx_attr); TAILQ_INIT(&ifma_trash_head); + + PE_parse_boot_argn("intcoproc_unrestricted", &intcoproc_unrestricted, + sizeof (intcoproc_unrestricted)); } /* @@ -2302,6 +2307,88 @@ ifioctl_nat64prefix(struct ifnet *ifp, u_long cmd, caddr_t data) #endif +/* + * List the ioctl()s we can perform on restricted INTCOPROC interfaces. + */ +static bool +ifioctl_restrict_intcoproc(unsigned long cmd, const char *ifname, + struct ifnet *ifp, struct proc *p) +{ + + if (intcoproc_unrestricted == TRUE) { + return (false); + } + if (proc_pid(p) == 0) { + return (false); + } + if (ifname) { + ifp = ifunit(ifname); + } + if (ifp == NULL) { + return (false); + } + if (!IFNET_IS_INTCOPROC(ifp)) { + return (false); + } + switch (cmd) { + case SIOCGIFBRDADDR: + case SIOCGIFCONF32: + case SIOCGIFCONF64: + case SIOCGIFFLAGS: + case SIOCGIFEFLAGS: + case SIOCGIFCAP: + case SIOCGIFMAC: + case SIOCGIFMETRIC: + case SIOCGIFMTU: + case SIOCGIFPHYS: + case SIOCGIFTYPE: + case SIOCGIFFUNCTIONALTYPE: + case SIOCGIFPSRCADDR: + case SIOCGIFPDSTADDR: + case SIOCGIFGENERIC: + case SIOCGIFDEVMTU: + case SIOCGIFVLAN: + case SIOCGIFBOND: + case SIOCGIFWAKEFLAGS: + case SIOCGIFGETRTREFCNT: + case SIOCGIFOPPORTUNISTIC: + case SIOCGIFLINKQUALITYMETRIC: + case SIOCGIFLOG: + case SIOCGIFDELEGATE: + case SIOCGIFEXPENSIVE: + case SIOCGIFINTERFACESTATE: + case SIOCGIFPROBECONNECTIVITY: + case SIOCGIFTIMESTAMPENABLED: + case SIOCGECNMODE: + case SIOCGQOSMARKINGMODE: + case SIOCGQOSMARKINGENABLED: + case SIOCGIFLOWINTERNET: + case SIOCGIFSTATUS: + case SIOCGIFMEDIA32: + case SIOCGIFMEDIA64: + case SIOCGIFDESC: + case SIOCGIFLINKPARAMS: + case SIOCGIFQUEUESTATS: + case SIOCGIFTHROTTLE: + case SIOCGIFAGENTIDS32: + case SIOCGIFAGENTIDS64: + case SIOCGIFNETSIGNATURE: + case SIOCGIFINFO_IN6: + case SIOCGIFAFLAG_IN6: + case SIOCGNBRINFO_IN6: + case SIOCGIFALIFETIME_IN6: + case SIOCGIFNETMASK_IN6: + return (false); + default: +#if (DEBUG || DEVELOPMENT) + printf("%s: cmd 0x%lx not allowed (pid %u)\n", + __func__, cmd, proc_pid(p)); +#endif + return (true); + } + return (false); +} + /* * Interface ioctls. * @@ -2434,6 +2521,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) bcopy(data, &ifr, sizeof (ifr)); ifr.ifr_name[IFNAMSIZ - 1] = '\0'; bcopy(&ifr.ifr_name, ifname, IFNAMSIZ); + if (ifioctl_restrict_intcoproc(cmd, ifname, NULL, p) == true) { + error = EPERM; + goto done; + } error = ifioctl_ifreq(so, cmd, &ifr, p); bcopy(&ifr, data, sizeof (ifr)); goto done; @@ -2551,6 +2642,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p) goto done; } + if (ifioctl_restrict_intcoproc(cmd, NULL, ifp, p) == true) { + error = EPERM; + goto done; + } switch (cmd) { case SIOCSIFPHYADDR: /* struct {if,in_}aliasreq */ #if INET6 diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c index 202f6c092..204a04c0e 100644 --- a/bsd/net/if_bond.c +++ b/bsd/net/if_bond.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2014 Apple Inc. All rights reserved. + * Copyright (c) 2004-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1957,6 +1957,10 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp) bondport_ref p; int progress = 0; + if (IFNET_IS_INTCOPROC(port_ifp)) { + return (EINVAL); + } + /* pre-allocate space for new port */ p = bondport_create(port_ifp, 0x8000, 1, 0, &error); if (p == NULL) { diff --git a/bsd/net/if_bridge.c b/bsd/net/if_bridge.c index 76de0c2bb..133d9af30 100644 --- a/bsd/net/if_bridge.c +++ b/bsd/net/if_bridge.c @@ -2313,6 +2313,10 @@ bridge_ioctl_add(struct bridge_softc *sc, void *arg) if (ifs->if_ioctl == NULL) /* must be supported */ return (EINVAL); + if (IFNET_IS_INTCOPROC(ifs)) { + return (EINVAL); + } + if (bsd_mode) { /* If it's in the span list, it can't be a member. */ TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) @@ -3104,6 +3108,10 @@ bridge_ioctl_addspan(struct bridge_softc *sc, void *arg) if (ifs == NULL) return (ENOENT); + if (IFNET_IS_INTCOPROC(ifs)) { + return (EINVAL); + } + TAILQ_FOREACH(bif, &sc->sc_spanlist, bif_next) if (ifs == bif->bif_ifp) return (EBUSY); diff --git a/bsd/net/if_ipsec.c b/bsd/net/if_ipsec.c index 8ffac41fe..4c330d639 100644 --- a/bsd/net/if_ipsec.c +++ b/bsd/net/if_ipsec.c @@ -61,6 +61,8 @@ extern int net_qos_policy_restrict_avapps; extern unsigned int if_enable_netagent; /* Kernel Control functions */ +static errno_t ipsec_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, + void **unitinfo); static errno_t ipsec_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo); static errno_t ipsec_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit, @@ -73,9 +75,7 @@ static errno_t ipsec_ctl_setopt(kern_ctl_ref kctlref, u_int32_t unit, void *unit int opt, void *data, size_t len); /* Network Interface functions */ -#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface); -#endif // !IPSEC_NEXUS static errno_t ipsec_output(ifnet_t interface, mbuf_t data); static errno_t ipsec_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); @@ -110,7 +110,7 @@ SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG #define IPSEC_IF_VERIFY(_e) if (unlikely(if_ipsec_verify_interface_creation)) { VERIFY(_e); } -#define IPSEC_IF_DEFAULT_SLOT_SIZE 4096 +#define IPSEC_IF_DEFAULT_SLOT_SIZE 2048 #define IPSEC_IF_DEFAULT_RING_SIZE 64 #define IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE 64 #define IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE 128 @@ -118,6 +118,9 @@ SYSCTL_INT(_net_ipsec, OID_AUTO, verify_interface_creation, CTLFLAG_RW | CTLFLAG #define IPSEC_IF_MIN_RING_SIZE 16 #define IPSEC_IF_MAX_RING_SIZE 1024 +#define IPSEC_IF_MIN_SLOT_SIZE 1024 +#define IPSEC_IF_MAX_SLOT_SIZE 4096 + static int sysctl_if_ipsec_ring_size SYSCTL_HANDLER_ARGS; static int sysctl_if_ipsec_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; static int sysctl_if_ipsec_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; @@ -168,7 +171,6 @@ struct ipsec_pcb { char ipsec_unique_name[IFXNAMSIZ]; // PCB lock protects state fields, like ipsec_kpipe_enabled decl_lck_rw_data(, ipsec_pcb_lock); - bool ipsec_output_disabled; #if IPSEC_NEXUS lck_mtx_t ipsec_input_chain_lock; @@ -186,6 +188,13 @@ struct ipsec_pcb { void * ipsec_netif_rxring; void * ipsec_netif_txring; uint64_t ipsec_netif_txring_size; + + u_int32_t ipsec_slot_size; + u_int32_t ipsec_netif_ring_size; + u_int32_t ipsec_tx_fsw_ring_size; + u_int32_t ipsec_rx_fsw_ring_size; + bool ipsec_use_netif; + #endif // IPSEC_NEXUS }; @@ -298,6 +307,7 @@ ipsec_register_control(void) kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED; /* Require root */ kern_ctl.ctl_sendsize = 64 * 1024; kern_ctl.ctl_recvsize = 64 * 1024; + kern_ctl.ctl_bind = ipsec_ctl_bind; kern_ctl.ctl_connect = ipsec_ctl_connect; kern_ctl.ctl_disconnect = ipsec_ctl_disconnect; kern_ctl.ctl_send = ipsec_ctl_send; @@ -734,7 +744,7 @@ ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); length = MIN(kern_packet_get_data_length(tx_ph), - IPSEC_IF_DEFAULT_SLOT_SIZE); + pcb->ipsec_slot_size); // Increment TX stats tx_ring_stats.kcrsi_slots_transferred++; @@ -828,13 +838,10 @@ ipsec_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, (void)kern_channel_reclaim(tx_ring); } - if (pcb->ipsec_output_disabled) { - errno_t error = ifnet_enable_output(pcb->ipsec_ifp); - if (error != 0) { - printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); - } else { - pcb->ipsec_output_disabled = false; - } + /* always reenable output */ + errno_t error = ifnet_enable_output(pcb->ipsec_ifp); + if (error != 0) { + printf("ipsec_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); } // Unlock first, then exit ring @@ -985,7 +992,7 @@ ipsec_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, bpf_tap_packet_out(pcb->ipsec_ifp, DLT_RAW, tx_ph, NULL, 0); length = MIN(kern_packet_get_data_length(tx_ph), - IPSEC_IF_DEFAULT_SLOT_SIZE); + pcb->ipsec_slot_size); if (length > 0) { errno_t error = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_HEADER, &data); @@ -1056,19 +1063,23 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, { #pragma unused(nxprov) struct ipsec_pcb *pcb = kern_nexus_get_context(nexus); - - lck_rw_lock_shared(&pcb->ipsec_pcb_lock); - boolean_t more = false; errno_t rc = 0; - do { - rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); - if (rc != 0 && rc != EAGAIN && rc != EBUSY) { - printf("%s, tx refill failed %d\n", __func__, rc); - } - } while ((rc == 0) && more); - if (pcb->ipsec_kpipe_enabled && !pcb->ipsec_output_disabled) { + /* + * Refill and sync the ring; we may be racing against another thread doing + * an RX sync that also wants to do kr_enter(), and so use the blocking + * variant here. + */ + rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + + (void) kr_enter(ring, TRUE); + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); + + if (pcb->ipsec_kpipe_enabled) { uint32_t tx_available = kern_channel_available_slot_count(ring); if (pcb->ipsec_netif_txring_size > 0 && tx_available >= pcb->ipsec_netif_txring_size - 1) { @@ -1076,14 +1087,11 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, errno_t error = ifnet_disable_output(pcb->ipsec_ifp); if (error != 0) { printf("ipsec_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); - } else { - pcb->ipsec_output_disabled = true; } } } - if (pcb->ipsec_kpipe_enabled && - (((rc != 0) && (rc != EAGAIN)) || pcb->ipsec_output_disabled)) { + if (pcb->ipsec_kpipe_enabled) { kern_channel_ring_t rx_ring = pcb->ipsec_kpipe_rxring; // Unlock while calling notify @@ -1092,11 +1100,12 @@ ipsec_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, if (rx_ring != NULL) { kern_channel_notify(rx_ring, 0); } - lck_rw_lock_shared(&pcb->ipsec_pcb_lock); } else { lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); } + kr_exit(ring); + return (0); } @@ -1145,8 +1154,6 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, if (unlikely(error != 0)) { STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); STATS_INC(nifs, NETIF_STATS_DROPPED); - printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", - pcb->ipsec_ifp->if_xname); lck_mtx_unlock(&pcb->ipsec_input_chain_lock); break; } @@ -1366,8 +1373,6 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, if (unlikely(error != 0)) { STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); STATS_INC(nifs, NETIF_STATS_DROPPED); - printf("ipsec_netif_sync_rx %s: failed to allocate packet\n", - pcb->ipsec_ifp->if_xname); break; } @@ -1388,7 +1393,7 @@ ipsec_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, tx_baddr += kern_buflet_get_data_offset(tx_buf); length = MIN(kern_packet_get_data_length(tx_ph), - IPSEC_IF_DEFAULT_SLOT_SIZE); + pcb->ipsec_slot_size); // Increment TX stats tx_ring_stats.kcrsi_slots_transferred++; @@ -1595,12 +1600,12 @@ ipsec_nexus_ifattach(struct ipsec_pcb *pcb, goto failed; } - uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + uint64_t slot_buffer_size = pcb->ipsec_slot_size; err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); VERIFY(err == 0); // Reset ring size for netif nexus to limit memory usage - uint64_t ring_size = if_ipsec_ring_size; + uint64_t ring_size = pcb->ipsec_netif_ring_size; err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); VERIFY(err == 0); err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); @@ -1712,7 +1717,8 @@ ipsec_nexus_detach(ipsec_nx_t nx) } static errno_t -ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, +ipsec_create_fs_provider_and_instance(struct ipsec_pcb *pcb, + uint32_t subtype, const char *type_name, const char *ifname, uuid_t *provider, uuid_t *instance) { @@ -1743,15 +1749,15 @@ ipsec_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); VERIFY(err == 0); - uint64_t slot_buffer_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + uint64_t slot_buffer_size = pcb->ipsec_slot_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); VERIFY(err == 0); // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. - uint64_t tx_ring_size = if_ipsec_tx_fsw_ring_size; + uint64_t tx_ring_size = pcb->ipsec_tx_fsw_ring_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); VERIFY(err == 0); - uint64_t rx_ring_size = if_ipsec_rx_fsw_ring_size; + uint64_t rx_ring_size = pcb->ipsec_rx_fsw_ring_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); VERIFY(err == 0); @@ -1798,7 +1804,8 @@ ipsec_multistack_attach(struct ipsec_pcb *pcb) ipsec_nx_t nx = &pcb->ipsec_nx; // Allocate multistack flowswitch - err = ipsec_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + err = ipsec_create_fs_provider_and_instance(pcb, + NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, "multistack", pcb->ipsec_ifp->if_xname, &nx->ms_provider, @@ -2063,27 +2070,26 @@ done: /* Kernel control functions */ static inline void -ipsec_free_pcb(struct ipsec_pcb *pcb) +ipsec_free_pcb(struct ipsec_pcb *pcb, bool in_list) { #if IPSEC_NEXUS mbuf_freem_list(pcb->ipsec_input_chain); lck_mtx_destroy(&pcb->ipsec_input_chain_lock, ipsec_lck_grp); #endif // IPSEC_NEXUS lck_rw_destroy(&pcb->ipsec_pcb_lock, ipsec_lck_grp); - lck_mtx_lock(&ipsec_lock); - TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain); - lck_mtx_unlock(&ipsec_lock); + if (in_list) { + lck_mtx_lock(&ipsec_lock); + TAILQ_REMOVE(&ipsec_head, pcb, ipsec_chain); + lck_mtx_unlock(&ipsec_lock); + } zfree(ipsec_pcb_zone, pcb); } static errno_t -ipsec_ctl_connect(kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +ipsec_ctl_bind(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams ipsec_init = {}; - errno_t result = 0; - struct ipsec_pcb *pcb = zalloc(ipsec_pcb_zone); memset(pcb, 0, sizeof(*pcb)); @@ -2093,6 +2099,36 @@ ipsec_ctl_connect(kern_ctl_ref kctlref, pcb->ipsec_unit = sac->sc_unit; pcb->ipsec_output_service_class = MBUF_SC_OAM; +#if IPSEC_NEXUS + pcb->ipsec_use_netif = false; + pcb->ipsec_slot_size = IPSEC_IF_DEFAULT_SLOT_SIZE; + pcb->ipsec_netif_ring_size = IPSEC_IF_DEFAULT_RING_SIZE; + pcb->ipsec_tx_fsw_ring_size = IPSEC_IF_DEFAULT_TX_FSW_RING_SIZE; + pcb->ipsec_rx_fsw_ring_size = IPSEC_IF_DEFAULT_RX_FSW_RING_SIZE; +#endif // IPSEC_NEXUS + + lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr); +#if IPSEC_NEXUS + lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr); +#endif // IPSEC_NEXUS + + return (0); +} + +static errno_t +ipsec_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) +{ + struct ifnet_init_eparams ipsec_init = {}; + errno_t result = 0; + + if (*unitinfo == NULL) { + (void)ipsec_ctl_bind(kctlref, sac, unitinfo); + } + + struct ipsec_pcb *pcb = *unitinfo; + lck_mtx_lock(&ipsec_lock); /* Find some open interface id */ @@ -2134,22 +2170,20 @@ ipsec_ctl_connect(kern_ctl_ref kctlref, snprintf(pcb->ipsec_unique_name, sizeof(pcb->ipsec_unique_name), "ipsecid%d", pcb->ipsec_unique_id - 1); printf("ipsec_ctl_connect: creating interface %s (id %s)\n", pcb->ipsec_if_xname, pcb->ipsec_unique_name); - lck_rw_init(&pcb->ipsec_pcb_lock, ipsec_lck_grp, ipsec_lck_attr); -#if IPSEC_NEXUS - lck_mtx_init(&pcb->ipsec_input_chain_lock, ipsec_lck_grp, ipsec_lck_attr); -#endif // IPSEC_NEXUS - /* Create the interface */ bzero(&ipsec_init, sizeof(ipsec_init)); ipsec_init.ver = IFNET_INIT_CURRENT_VERSION; ipsec_init.len = sizeof (ipsec_init); #if IPSEC_NEXUS - ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); -#else // IPSEC_NEXUS - ipsec_init.flags = IFNET_INIT_NX_NOAUTO; - ipsec_init.start = ipsec_start; + if (pcb->ipsec_use_netif) { + ipsec_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); + } else #endif // IPSEC_NEXUS + { + ipsec_init.flags = IFNET_INIT_NX_NOAUTO; + ipsec_init.start = ipsec_start; + } ipsec_init.name = "ipsec"; ipsec_init.unit = pcb->ipsec_unit - 1; ipsec_init.uniqueid = pcb->ipsec_unique_name; @@ -2165,44 +2199,49 @@ ipsec_ctl_connect(kern_ctl_ref kctlref, ipsec_init.detach = ipsec_detached; #if IPSEC_NEXUS - result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp); - if (result != 0) { - printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result); - ipsec_free_pcb(pcb); - *unitinfo = NULL; - return result; - } - - result = ipsec_multistack_attach(pcb); - if (result != 0) { - printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result); - *unitinfo = NULL; - return result; - } + if (pcb->ipsec_use_netif) { + result = ipsec_nexus_ifattach(pcb, &ipsec_init, &pcb->ipsec_ifp); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_nexus_ifattach failed: %d\n", result); + ipsec_free_pcb(pcb, true); + *unitinfo = NULL; + return result; + } -#else // IPSEC_NEXUS - result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp); - if (result != 0) { - printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); - ipsec_free_pcb(pcb); - *unitinfo = NULL; - return result; - } - ipsec_ifnet_set_attrs(pcb->ipsec_ifp); + result = ipsec_multistack_attach(pcb); + if (result != 0) { + printf("ipsec_ctl_connect - ipsec_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } - /* Attach the interface */ - result = ifnet_attach(pcb->ipsec_ifp, NULL); - if (result != 0) { - printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result); - ifnet_release(pcb->ipsec_ifp); - ipsec_free_pcb(pcb); - *unitinfo = NULL; - return (result); - } + /* Attach to bpf */ + bpfattach(pcb->ipsec_ifp, DLT_RAW, 0); + } else #endif // IPSEC_NEXUS + { + result = ifnet_allocate_extended(&ipsec_init, &pcb->ipsec_ifp); + if (result != 0) { + printf("ipsec_ctl_connect - ifnet_allocate failed: %d\n", result); + ipsec_free_pcb(pcb, true); + *unitinfo = NULL; + return result; + } + ipsec_ifnet_set_attrs(pcb->ipsec_ifp); + + /* Attach the interface */ + result = ifnet_attach(pcb->ipsec_ifp, NULL); + if (result != 0) { + printf("ipsec_ctl_connect - ifnet_attach failed: %d\n", result); + ifnet_release(pcb->ipsec_ifp); + ipsec_free_pcb(pcb, true); + *unitinfo = NULL; + return (result); + } - /* Attach to bpf */ - bpfattach(pcb->ipsec_ifp, DLT_RAW, 0); + /* Attach to bpf */ + bpfattach(pcb->ipsec_ifp, DLT_NULL, 0); + } /* The interfaces resoures allocated, mark it as running */ ifnet_set_flags(pcb->ipsec_ifp, IFF_RUNNING, IFF_RUNNING); @@ -2388,46 +2427,84 @@ ipsec_ctl_disconnect(__unused kern_ctl_ref kctlref, pcb->ipsec_kpipe_enabled = FALSE; #endif // IPSEC_NEXUS - ifp = pcb->ipsec_ifp; - VERIFY(ifp != NULL); pcb->ipsec_ctlref = NULL; - /* - * Quiesce the interface and flush any pending outbound packets. - */ - if_down(ifp); + ifp = pcb->ipsec_ifp; + if (ifp != NULL) { +#if IPSEC_NEXUS + if (pcb->ipsec_netif_nexus != NULL) { + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + /* NOT REACHED */ + } - /* Increment refcnt, but detach interface */ - ifnet_incr_iorefcnt(ifp); - if ((result = ifnet_detach(ifp)) != 0) { - panic("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); - /* NOT REACHED */ - } + /* + * We want to do everything in our power to ensure that the interface + * really goes away when the socket is closed. We must remove IP/IPv6 + * addresses and detach the protocols. Finally, we can remove and + * release the interface. + */ + key_delsp_for_ipsec_if(ifp); - /* - * We want to do everything in our power to ensure that the interface - * really goes away when the socket is closed. We must remove IP/IPv6 - * addresses and detach the protocols. Finally, we can remove and - * release the interface. - */ - key_delsp_for_ipsec_if(ifp); - - ipsec_cleanup_family(ifp, AF_INET); - ipsec_cleanup_family(ifp, AF_INET6); + ipsec_cleanup_family(ifp, AF_INET); + ipsec_cleanup_family(ifp, AF_INET6); - lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { + ipsec_unregister_kernel_pipe_nexus(); + } + } + ipsec_nexus_detach(&pcb->ipsec_nx); + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); + } else +#endif // IPSEC_NEXUS + { + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); #if IPSEC_NEXUS - if (!uuid_is_null(kpipe_uuid)) { - if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { - ipsec_unregister_kernel_pipe_nexus(); - } - } - ipsec_nexus_detach(&pcb->ipsec_nx); + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(ipsec_ncd, kpipe_uuid) == 0) { + ipsec_unregister_kernel_pipe_nexus(); + } + } #endif // IPSEC_NEXUS - /* Decrement refcnt to finish detaching and freeing */ - ifnet_decr_iorefcnt(ifp); + /* + * We want to do everything in our power to ensure that the interface + * really goes away when the socket is closed. We must remove IP/IPv6 + * addresses and detach the protocols. Finally, we can remove and + * release the interface. + */ + key_delsp_for_ipsec_if(ifp); + + ipsec_cleanup_family(ifp, AF_INET); + ipsec_cleanup_family(ifp, AF_INET6); + + /* + * Detach now; ipsec_detach() will be called asynchronously once + * the I/O reference count drops to 0. There we will invoke + * ifnet_release(). + */ + if ((result = ifnet_detach(ifp)) != 0) { + printf("ipsec_ctl_disconnect - ifnet_detach failed: %d\n", result); + } + } + } else { + // Bound, but not connected + lck_rw_unlock_exclusive(&pcb->ipsec_pcb_lock); + ipsec_free_pcb(pcb, false); + } return 0; } @@ -2469,10 +2546,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, switch (opt) { case IPSEC_OPT_FLAGS: - if (len != sizeof(u_int32_t)) + if (len != sizeof(u_int32_t)) { result = EMSGSIZE; - else + } else { pcb->ipsec_flags = *(u_int32_t *)data; + } break; case IPSEC_OPT_EXT_IFDATA_STATS: @@ -2480,6 +2558,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } pcb->ipsec_ext_ifdata_stats = (*(int *)data) ? 1 : 0; break; @@ -2491,6 +2574,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EINVAL; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (!pcb->ipsec_ext_ifdata_stats) { result = EINVAL; break; @@ -2512,6 +2600,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */ bcopy(data, name, len); name[len] = 0; @@ -2534,6 +2627,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } mbuf_svc_class_t output_service_class = so_tc2msc(*(int *)data); if (output_service_class == MBUF_SC_UNSPEC) { pcb->ipsec_output_service_class = MBUF_SC_OAM; @@ -2552,6 +2650,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (*(int *)data) { result = ipsec_enable_channel(pcb, current_proc()); } else { @@ -2565,6 +2668,11 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->ipsec_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (!if_enable_netagent) { result = ENOTSUP; break; @@ -2597,6 +2705,92 @@ ipsec_ctl_setopt(__unused kern_ctl_ref kctlref, } break; } + case IPSEC_OPT_ENABLE_NETIF: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (pcb->ipsec_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + pcb->ipsec_use_netif = true; + break; + } + case IPSEC_OPT_SLOT_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->ipsec_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t slot_size = *(u_int32_t *)data; + if (slot_size < IPSEC_IF_MIN_SLOT_SIZE || + slot_size > IPSEC_IF_MAX_SLOT_SIZE) { + return (EINVAL); + } + pcb->ipsec_slot_size = slot_size; + break; + } + case IPSEC_OPT_NETIF_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->ipsec_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < IPSEC_IF_MIN_RING_SIZE || + ring_size > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->ipsec_netif_ring_size = ring_size; + break; + } + case IPSEC_OPT_TX_FSW_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->ipsec_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < IPSEC_IF_MIN_RING_SIZE || + ring_size > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->ipsec_tx_fsw_ring_size = ring_size; + break; + } + case IPSEC_OPT_RX_FSW_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->ipsec_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < IPSEC_IF_MIN_RING_SIZE || + ring_size > IPSEC_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->ipsec_rx_fsw_ring_size = ring_size; + break; + } + #endif // IPSEC_NEXUS default: @@ -2641,6 +2835,11 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, if (*len < MIN(strlen(pcb->ipsec_if_xname) + 1, sizeof(pcb->ipsec_if_xname))) { result = EMSGSIZE; } else { + if (pcb->ipsec_ifp == NULL) { + // Only can get after connecting + result = EINVAL; + break; + } *len = snprintf(data, *len, "%s", pcb->ipsec_if_xname) + 1; } break; @@ -2677,6 +2876,39 @@ ipsec_ctl_getopt(__unused kern_ctl_ref kctlref, } break; } + case IPSEC_OPT_SLOT_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_slot_size; + } + break; + } + case IPSEC_OPT_NETIF_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_netif_ring_size; + } + break; + } + case IPSEC_OPT_TX_FSW_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_tx_fsw_ring_size; + } + break; + } + case IPSEC_OPT_RX_FSW_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->ipsec_rx_fsw_ring_size; + } + break; + } + #endif // IPSEC_NEXUS default: { @@ -2721,6 +2953,14 @@ ipsec_output(ifnet_t interface, switch (ip_version) { case 4: { +#if IPSEC_NEXUS + if (!pcb->ipsec_use_netif) +#endif // IPSEC_NEXUS + { + int af = AF_INET; + bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); + } + /* Apply encryption */ memset(&ipsec_state, 0, sizeof(ipsec_state)); ipsec_state.m = data; @@ -2785,6 +3025,14 @@ ipsec_output(ifnet_t interface, goto done; } case 6: { +#if IPSEC_NEXUS + if (!pcb->ipsec_use_netif) +#endif // IPSEC_NEXUS + { + int af = AF_INET6; + bpf_tap_out(pcb->ipsec_ifp, DLT_NULL, data, &af, sizeof(af)); + } + data = ipsec6_splithdr(data); if (data == NULL) { printf("ipsec_output: ipsec6_splithdr returned NULL\n"); @@ -2863,7 +3111,6 @@ ipsec_output_err: goto done; } -#if !IPSEC_NEXUS static void ipsec_start(ifnet_t interface) { @@ -2878,7 +3125,6 @@ ipsec_start(ifnet_t interface) break; } } -#endif // !IPSEC_NEXUS /* Network Interface functions */ static errno_t @@ -2944,19 +3190,26 @@ ipsec_ioctl(ifnet_t interface, u_long command, void *data) { + struct ipsec_pcb *pcb = ifnet_softc(interface); errno_t result = 0; switch(command) { - case SIOCSIFMTU: + case SIOCSIFMTU: { #if IPSEC_NEXUS - // Make sure we can fit packets in the channel buffers - if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > IPSEC_IF_DEFAULT_SLOT_SIZE) { - ifnet_set_mtu(interface, IPSEC_IF_DEFAULT_SLOT_SIZE); - break; - } + if (pcb->ipsec_use_netif) { + // Make sure we can fit packets in the channel buffers + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) > pcb->ipsec_slot_size) { + result = EINVAL; + } else { + ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu); + } + } else #endif // IPSEC_NEXUS - ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); + { + ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); + } break; + } case SIOCSIFFLAGS: /* ifioctl() takes care of it */ @@ -2974,7 +3227,7 @@ ipsec_detached(ifnet_t interface) { struct ipsec_pcb *pcb = ifnet_softc(interface); (void)ifnet_release(interface); - ipsec_free_pcb(pcb); + ipsec_free_pcb(pcb, true); } /* Protocol Handlers */ @@ -2986,6 +3239,21 @@ ipsec_proto_input(ifnet_t interface, __unused char *frame_header) { mbuf_pkthdr_setrcvif(m, interface); + +#if IPSEC_NEXUS + struct ipsec_pcb *pcb = ifnet_softc(interface); + if (!pcb->ipsec_use_netif) +#endif // IPSEC_NEXUS + { + uint32_t af = 0; + struct ip *ip = mtod(m, struct ip *); + if (ip->ip_v == 4) { + af = AF_INET; + } else if (ip->ip_v == 6) { + af = AF_INET6; + } + bpf_tap_in(interface, DLT_NULL, m, &af, sizeof(af)); + } pktap_input(interface, protocol, m, NULL); if (proto_input(protocol, m) != 0) { @@ -3032,51 +3300,49 @@ ipsec_attach_proto(ifnet_t interface, return result; } -#if IPSEC_NEXUS errno_t ipsec_inject_inbound_packet(ifnet_t interface, mbuf_t packet) { struct ipsec_pcb *pcb = ifnet_softc(interface); - lck_rw_lock_shared(&pcb->ipsec_pcb_lock); +#if IPSEC_NEXUS + if (pcb->ipsec_use_netif) { + lck_rw_lock_shared(&pcb->ipsec_pcb_lock); - lck_mtx_lock(&pcb->ipsec_input_chain_lock); - if (pcb->ipsec_input_chain != NULL) { - pcb->ipsec_input_chain_last->m_nextpkt = packet; - } else { - pcb->ipsec_input_chain = packet; - } - while (packet->m_nextpkt) { - VERIFY(packet != packet->m_nextpkt); - packet = packet->m_nextpkt; - } - pcb->ipsec_input_chain_last = packet; - lck_mtx_unlock(&pcb->ipsec_input_chain_lock); + lck_mtx_lock(&pcb->ipsec_input_chain_lock); + if (pcb->ipsec_input_chain != NULL) { + pcb->ipsec_input_chain_last->m_nextpkt = packet; + } else { + pcb->ipsec_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->ipsec_input_chain_last = packet; + lck_mtx_unlock(&pcb->ipsec_input_chain_lock); - kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; - lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); + kern_channel_ring_t rx_ring = pcb->ipsec_netif_rxring; + lck_rw_unlock_shared(&pcb->ipsec_pcb_lock); - if (rx_ring != NULL) { - kern_channel_notify(rx_ring, 0); - } + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } - return (0); -} -#else // IPSEC_NEXUS -errno_t -ipsec_inject_inbound_packet(ifnet_t interface, - mbuf_t packet) -{ - errno_t error; - protocol_family_t protocol; - if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) { - return error; - } + return (0); + } else +#endif // IPSEC_NEXUS + { + errno_t error; + protocol_family_t protocol; + if ((error = ipsec_demux(interface, packet, NULL, &protocol)) != 0) { + return error; + } - return ipsec_proto_input(interface, protocol, packet, NULL); + return ipsec_proto_input(interface, protocol, packet, NULL); + } } -#endif // IPSEC_NEXUS void ipsec_set_pkthdr_for_interface(ifnet_t interface, mbuf_t packet, int family) diff --git a/bsd/net/if_ipsec.h b/bsd/net/if_ipsec.h index 159ef2036..93429c1ad 100644 --- a/bsd/net/if_ipsec.h +++ b/bsd/net/if_ipsec.h @@ -72,6 +72,12 @@ void ipsec_set_ip6oa_for_interface(ifnet_t interface, struct ip6_out_args *ip6oa #define IPSEC_OPT_ENABLE_FLOWSWITCH 10 /* enable a flowswitch nexus that clients can use */ #define IPSEC_OPT_INPUT_FRAG_SIZE 11 /* set the maximum size of input packets before fragmenting as a uint32_t */ +#define IPSEC_OPT_ENABLE_NETIF 12 /* Must be set before connecting */ +#define IPSEC_OPT_SLOT_SIZE 13 /* Must be set before connecting */ +#define IPSEC_OPT_NETIF_RING_SIZE 14 /* Must be set before connecting */ +#define IPSEC_OPT_TX_FSW_RING_SIZE 15 /* Must be set before connecting */ +#define IPSEC_OPT_RX_FSW_RING_SIZE 16 /* Must be set before connecting */ + /* * ipsec stats parameter structure */ diff --git a/bsd/net/if_loop.c b/bsd/net/if_loop.c index 00a8345e3..cc2089cb6 100644 --- a/bsd/net/if_loop.c +++ b/bsd/net/if_loop.c @@ -698,6 +698,13 @@ loopattach(void) __func__, result); /* NOTREACHED */ } + /* + * Disable ECN on loopback as ECN serves no purpose and otherwise + * TCP connections are subject to heuristics like SYN retransmits on RST + */ + lo_ifp->if_eflags &= ~IFEF_ECN_ENABLE; + lo_ifp->if_eflags |= IFEF_ECN_DISABLE; + bpfattach(lo_ifp, DLT_NULL, sizeof (u_int32_t)); } diff --git a/bsd/net/if_utun.c b/bsd/net/if_utun.c index 5e01509df..ffaa1124c 100644 --- a/bsd/net/if_utun.c +++ b/bsd/net/if_utun.c @@ -96,7 +96,6 @@ struct utun_pcb { // Input chain lock protects the list of input mbufs // The input chain lock must be taken AFTER the PCB lock if both are held lck_mtx_t utun_input_chain_lock; - bool utun_output_disabled; #if UTUN_NEXUS struct utun_nx utun_nx; @@ -109,10 +108,18 @@ struct utun_pcb { void * utun_netif_rxring; void * utun_netif_txring; uint64_t utun_netif_txring_size; + + u_int32_t utun_slot_size; + u_int32_t utun_netif_ring_size; + u_int32_t utun_tx_fsw_ring_size; + u_int32_t utun_rx_fsw_ring_size; + bool utun_use_netif; #endif // UTUN_NEXUS }; /* Kernel Control functions */ +static errno_t utun_ctl_bind(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, + void **unitinfo); static errno_t utun_ctl_connect(kern_ctl_ref kctlref, struct sockaddr_ctl *sac, void **unitinfo); static errno_t utun_ctl_disconnect(kern_ctl_ref kctlref, u_int32_t unit, @@ -127,12 +134,10 @@ static void utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags); /* Network Interface functions */ -#if !UTUN_NEXUS static void utun_start(ifnet_t interface); static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, const struct sockaddr *dest, const char *desk_linkaddr, const char *frame_type, u_int32_t *prepend_len, u_int32_t *postpend_len); -#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, mbuf_t data); static errno_t utun_demux(ifnet_t interface, mbuf_t data, char *frame_header, protocol_family_t *protocol); @@ -150,11 +155,11 @@ static errno_t utun_proto_input(ifnet_t interface, protocol_family_t protocol, static errno_t utun_proto_pre_output(ifnet_t interface, protocol_family_t protocol, mbuf_t *packet, const struct sockaddr *dest, void *route, char *frame_type, char *link_layer_dest); -static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m); +static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t m); #if UTUN_NEXUS -#define UTUN_IF_DEFAULT_SLOT_SIZE 4096 +#define UTUN_IF_DEFAULT_SLOT_SIZE 2048 #define UTUN_IF_DEFAULT_RING_SIZE 64 #define UTUN_IF_DEFAULT_TX_FSW_RING_SIZE 64 #define UTUN_IF_DEFAULT_RX_FSW_RING_SIZE 128 @@ -163,6 +168,9 @@ static errno_t utun_pkt_input (struct utun_pcb *pcb, mbuf_t m); #define UTUN_IF_MIN_RING_SIZE 16 #define UTUN_IF_MAX_RING_SIZE 1024 +#define UTUN_IF_MIN_SLOT_SIZE 1024 +#define UTUN_IF_MAX_SLOT_SIZE 4096 + static int sysctl_if_utun_ring_size SYSCTL_HANDLER_ARGS; static int sysctl_if_utun_tx_fsw_ring_size SYSCTL_HANDLER_ARGS; static int sysctl_if_utun_rx_fsw_ring_size SYSCTL_HANDLER_ARGS; @@ -427,7 +435,7 @@ utun_netif_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, tx_length += UTUN_HEADER_SIZE(pcb); tx_baddr += tx_offset; - length = MIN(tx_length, UTUN_IF_DEFAULT_SLOT_SIZE); + length = MIN(tx_length, pcb->utun_slot_size); // Copy in family memcpy(tx_baddr, &af, sizeof(af)); @@ -492,20 +500,23 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, { #pragma unused(nxprov) struct utun_pcb *pcb = kern_nexus_get_context(nexus); - - lck_rw_lock_shared(&pcb->utun_pcb_lock); - boolean_t more = false; errno_t rc = 0; - do { - /* Refill and sync the ring */ - rc = kern_channel_tx_refill(ring, UINT32_MAX, UINT32_MAX, true, &more); - if (rc != 0 && rc != EAGAIN && rc != EBUSY) { - printf("%s, tx refill failed %d\n", __func__, rc); - } - } while ((rc == 0) && more); - if (pcb->utun_kpipe_enabled && !pcb->utun_output_disabled) { + /* + * Refill and sync the ring; we may be racing against another thread doing + * an RX sync that also wants to do kr_enter(), and so use the blocking + * variant here. + */ + rc = kern_channel_tx_refill_canblock(ring, UINT32_MAX, UINT32_MAX, true, &more); + if (rc != 0 && rc != EAGAIN && rc != EBUSY) { + printf("%s, tx refill failed %d\n", __func__, rc); + } + + (void) kr_enter(ring, TRUE); + lck_rw_lock_shared(&pcb->utun_pcb_lock); + + if (pcb->utun_kpipe_enabled) { uint32_t tx_available = kern_channel_available_slot_count(ring); if (pcb->utun_netif_txring_size > 0 && tx_available >= pcb->utun_netif_txring_size - 1) { @@ -513,14 +524,11 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, errno_t error = ifnet_disable_output(pcb->utun_ifp); if (error != 0) { printf("utun_netif_tx_doorbell: ifnet_disable_output returned error %d\n", error); - } else { - pcb->utun_output_disabled = true; } } } - if (pcb->utun_kpipe_enabled && - (((rc != 0) && (rc != EAGAIN)) || pcb->utun_output_disabled)) { + if (pcb->utun_kpipe_enabled) { kern_channel_ring_t rx_ring = pcb->utun_kpipe_rxring; // Unlock while calling notify @@ -533,6 +541,8 @@ utun_netif_tx_doorbell(kern_nexus_provider_t nxprov, kern_nexus_t nexus, lck_rw_unlock_shared(&pcb->utun_pcb_lock); } + kr_exit(ring); + return (0); } @@ -581,8 +591,6 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, if (unlikely(error != 0)) { STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); STATS_INC(nifs, NETIF_STATS_DROPPED); - printf("utun_netif_sync_rx %s: failed to allocate packet\n", - pcb->utun_ifp->if_xname); lck_mtx_unlock(&pcb->utun_input_chain_lock); break; } @@ -706,8 +714,6 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, if (unlikely(error != 0)) { STATS_INC(nifs, NETIF_STATS_NOMEM_PKT); STATS_INC(nifs, NETIF_STATS_DROPPED); - printf("utun_netif_sync_rx %s: failed to allocate packet\n", - pcb->utun_ifp->if_xname); break; } @@ -731,7 +737,7 @@ utun_netif_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, } size_t length = MIN(tx_length - header_offset, - UTUN_IF_DEFAULT_SLOT_SIZE); + pcb->utun_slot_size); tx_ring_stats.kcrsi_slots_transferred++; tx_ring_stats.kcrsi_bytes_transferred += length; @@ -832,12 +838,12 @@ utun_nexus_ifattach(struct utun_pcb *pcb, goto failed; } - uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + uint64_t slot_buffer_size = pcb->utun_slot_size; err = kern_nexus_attr_set(nxa, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); VERIFY(err == 0); // Reset ring size for netif nexus to limit memory usage - uint64_t ring_size = if_utun_ring_size; + uint64_t ring_size = pcb->utun_netif_ring_size; err = kern_nexus_attr_set(nxa, NEXUS_ATTR_TX_SLOTS, ring_size); VERIFY(err == 0); err = kern_nexus_attr_set(nxa, NEXUS_ATTR_RX_SLOTS, ring_size); @@ -947,7 +953,8 @@ utun_nexus_detach(utun_nx_t nx) } static errno_t -utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, +utun_create_fs_provider_and_instance(struct utun_pcb *pcb, + uint32_t subtype, const char *type_name, const char *ifname, uuid_t *provider, uuid_t *instance) { @@ -976,15 +983,15 @@ utun_create_fs_provider_and_instance(uint32_t subtype, const char *type_name, err = kern_nexus_attr_set(attr, NEXUS_ATTR_EXTENSIONS, subtype); VERIFY(err == 0); - uint64_t slot_buffer_size = UTUN_IF_DEFAULT_SLOT_SIZE; + uint64_t slot_buffer_size = pcb->utun_slot_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_SLOT_BUF_SIZE, slot_buffer_size); VERIFY(err == 0); // Reset ring size for flowswitch nexus to limit memory usage. Larger RX than netif. - uint64_t tx_ring_size = if_utun_tx_fsw_ring_size; + uint64_t tx_ring_size = pcb->utun_tx_fsw_ring_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_TX_SLOTS, tx_ring_size); VERIFY(err == 0); - uint64_t rx_ring_size = if_utun_rx_fsw_ring_size; + uint64_t rx_ring_size = pcb->utun_rx_fsw_ring_size; err = kern_nexus_attr_set(attr, NEXUS_ATTR_RX_SLOTS, rx_ring_size); VERIFY(err == 0); @@ -1029,7 +1036,8 @@ utun_multistack_attach(struct utun_pcb *pcb) utun_nx_t nx = &pcb->utun_nx; // Allocate multistack flowswitch - err = utun_create_fs_provider_and_instance(NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, + err = utun_create_fs_provider_and_instance(pcb, + NEXUS_EXTENSION_FSW_TYPE_MULTISTACK, "multistack", pcb->utun_ifp->if_xname, &nx->ms_provider, @@ -1258,7 +1266,7 @@ utun_enable_channel(struct utun_pcb *pcb, struct proc *proc) * Make sure we can fit packets in the channel buffers and * Allow an extra 4 bytes for the protocol number header in the channel */ - if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > UTUN_IF_DEFAULT_SLOT_SIZE) { + if (pcb->utun_ifp->if_mtu + UTUN_HEADER_SIZE(pcb) > pcb->utun_slot_size) { result = EOPNOTSUPP; goto done; } @@ -1331,6 +1339,7 @@ utun_register_control(void) kern_ctl.ctl_flags = CTL_FLAG_PRIVILEGED | CTL_FLAG_REG_EXTENDED; /* Require root */ kern_ctl.ctl_sendsize = 512 * 1024; kern_ctl.ctl_recvsize = 512 * 1024; + kern_ctl.ctl_bind = utun_ctl_bind; kern_ctl.ctl_connect = utun_ctl_connect; kern_ctl.ctl_disconnect = utun_ctl_disconnect; kern_ctl.ctl_send = utun_ctl_send; @@ -1367,9 +1376,7 @@ utun_register_control(void) utun_lck_grp_attr = lck_grp_attr_alloc_init(); utun_lck_grp = lck_grp_alloc_init("utun", utun_lck_grp_attr); -#if UTUN_NEXUS lck_mtx_init(&utun_lock, utun_lck_grp, utun_lck_attr); -#endif // UTUN_NEXUS return 0; } @@ -1377,27 +1384,26 @@ utun_register_control(void) /* Kernel control functions */ static inline void -utun_free_pcb(struct utun_pcb *pcb) +utun_free_pcb(struct utun_pcb *pcb, bool in_list) { #ifdef UTUN_NEXUS mbuf_freem_list(pcb->utun_input_chain); lck_mtx_destroy(&pcb->utun_input_chain_lock, utun_lck_grp); #endif // UTUN_NEXUS lck_rw_destroy(&pcb->utun_pcb_lock, utun_lck_grp); - lck_mtx_lock(&utun_lock); - TAILQ_REMOVE(&utun_head, pcb, utun_chain); - lck_mtx_unlock(&utun_lock); + if (in_list) { + lck_mtx_lock(&utun_lock); + TAILQ_REMOVE(&utun_head, pcb, utun_chain); + lck_mtx_unlock(&utun_lock); + } zfree(utun_pcb_zone, pcb); } static errno_t -utun_ctl_connect(kern_ctl_ref kctlref, - struct sockaddr_ctl *sac, - void **unitinfo) +utun_ctl_bind(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) { - struct ifnet_init_eparams utun_init = {}; - errno_t result = 0; - struct utun_pcb *pcb = zalloc(utun_pcb_zone); memset(pcb, 0, sizeof(*pcb)); @@ -1406,9 +1412,34 @@ utun_ctl_connect(kern_ctl_ref kctlref, pcb->utun_unit = sac->sc_unit; pcb->utun_max_pending_packets = 1; +#if UTUN_NEXUS + pcb->utun_use_netif = false; + pcb->utun_slot_size = UTUN_IF_DEFAULT_SLOT_SIZE; + pcb->utun_netif_ring_size = UTUN_IF_DEFAULT_RING_SIZE; + pcb->utun_tx_fsw_ring_size = UTUN_IF_DEFAULT_TX_FSW_RING_SIZE; + pcb->utun_rx_fsw_ring_size = UTUN_IF_DEFAULT_RX_FSW_RING_SIZE; +#endif // UTUN_NEXUS + lck_mtx_init(&pcb->utun_input_chain_lock, utun_lck_grp, utun_lck_attr); lck_rw_init(&pcb->utun_pcb_lock, utun_lck_grp, utun_lck_attr); + return (0); +} + +static errno_t +utun_ctl_connect(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo) +{ + struct ifnet_init_eparams utun_init = {}; + errno_t result = 0; + + if (*unitinfo == NULL) { + (void)utun_ctl_bind(kctlref, sac, unitinfo); + } + + struct utun_pcb *pcb = *unitinfo; + lck_mtx_lock(&utun_lock); /* Find some open interface id */ @@ -1456,13 +1487,16 @@ utun_ctl_connect(kern_ctl_ref kctlref, utun_init.len = sizeof (utun_init); #if UTUN_NEXUS - utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); - utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE; -#else // UTUN_NEXUS - utun_init.flags = IFNET_INIT_NX_NOAUTO; - utun_init.start = utun_start; - utun_init.framer_extended = utun_framer; + if (pcb->utun_use_netif) { + utun_init.flags = (IFNET_INIT_SKYWALK_NATIVE | IFNET_INIT_NX_NOAUTO); + utun_init.tx_headroom = UTUN_IF_HEADROOM_SIZE; + } else #endif // UTUN_NEXUS + { + utun_init.flags = IFNET_INIT_NX_NOAUTO; + utun_init.start = utun_start; + utun_init.framer_extended = utun_framer; + } utun_init.name = "utun"; utun_init.unit = pcb->utun_unit - 1; utun_init.uniqueid = pcb->utun_unique_name; @@ -1478,62 +1512,68 @@ utun_ctl_connect(kern_ctl_ref kctlref, utun_init.detach = utun_detached; #if UTUN_NEXUS - result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp); - if (result != 0) { - printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result); - utun_free_pcb(pcb); - *unitinfo = NULL; - return result; - } + if (pcb->utun_use_netif) { + result = utun_nexus_ifattach(pcb, &utun_init, &pcb->utun_ifp); + if (result != 0) { + printf("utun_ctl_connect - utun_nexus_ifattach failed: %d\n", result); + utun_free_pcb(pcb, true); + *unitinfo = NULL; + return result; + } - result = utun_multistack_attach(pcb); - if (result != 0) { - printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result); - *unitinfo = NULL; - return result; - } + result = utun_multistack_attach(pcb); + if (result != 0) { + printf("utun_ctl_connect - utun_multistack_attach failed: %d\n", result); + *unitinfo = NULL; + return result; + } -#else // UTUN_NEXUS - /* - * Upon success, this holds an ifnet reference which we will - * release via ifnet_release() at final detach time. - */ - result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp); - if (result != 0) { - printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); - utun_free_pcb(pcb); - *unitinfo = NULL; - return result; - } - - /* Set flags and additional information. */ - ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU); - ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); + /* Attach to bpf */ + bpfattach(pcb->utun_ifp, DLT_RAW, 0); + } else +#endif // UTUN_NEXUS + { + /* + * Upon success, this holds an ifnet reference which we will + * release via ifnet_release() at final detach time. + */ + result = ifnet_allocate_extended(&utun_init, &pcb->utun_ifp); + if (result != 0) { + printf("utun_ctl_connect - ifnet_allocate failed: %d\n", result); + utun_free_pcb(pcb, true); + *unitinfo = NULL; + return result; + } - /* The interface must generate its own IPv6 LinkLocal address, - * if possible following the recommendation of RFC2472 to the 64bit interface ID - */ - ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); - - /* Reset the stats in case as the interface may have been recycled */ - struct ifnet_stats_param stats; - bzero(&stats, sizeof(struct ifnet_stats_param)); - ifnet_set_stat(pcb->utun_ifp, &stats); + /* Set flags and additional information. */ + ifnet_set_mtu(pcb->utun_ifp, UTUN_DEFAULT_MTU); + ifnet_set_flags(pcb->utun_ifp, IFF_UP | IFF_MULTICAST | IFF_POINTOPOINT, 0xffff); - /* Attach the interface */ - result = ifnet_attach(pcb->utun_ifp, NULL); - if (result != 0) { - printf("utun_ctl_connect - ifnet_attach failed: %d\n", result); - /* Release reference now since attach failed */ - ifnet_release(pcb->utun_ifp); - utun_free_pcb(pcb); - *unitinfo = NULL; - return (result); + /* The interface must generate its own IPv6 LinkLocal address, + * if possible following the recommendation of RFC2472 to the 64bit interface ID + */ + ifnet_set_eflags(pcb->utun_ifp, IFEF_NOAUTOIPV6LL, IFEF_NOAUTOIPV6LL); + + /* Reset the stats in case as the interface may have been recycled */ + struct ifnet_stats_param stats; + bzero(&stats, sizeof(struct ifnet_stats_param)); + ifnet_set_stat(pcb->utun_ifp, &stats); + + /* Attach the interface */ + result = ifnet_attach(pcb->utun_ifp, NULL); + if (result != 0) { + printf("utun_ctl_connect - ifnet_attach failed: %d\n", result); + /* Release reference now since attach failed */ + ifnet_release(pcb->utun_ifp); + utun_free_pcb(pcb, true); + *unitinfo = NULL; + return (result); + } + + /* Attach to bpf */ + bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); } -#endif // UTUN_NEXUS - /* Attach to bpf */ - bpfattach(pcb->utun_ifp, DLT_RAW, 0); /* The interfaces resoures allocated, mark it as running */ ifnet_set_flags(pcb->utun_ifp, IFF_RUNNING, IFF_RUNNING); @@ -1715,43 +1755,80 @@ utun_ctl_disconnect(__unused kern_ctl_ref kctlref, pcb->utun_kpipe_enabled = FALSE; #endif // UTUN_NEXUS - ifp = pcb->utun_ifp; - VERIFY(ifp != NULL); pcb->utun_ctlref = NULL; - /* - * Quiesce the interface and flush any pending outbound packets. - */ - if_down(ifp); + ifp = pcb->utun_ifp; + if (ifp != NULL) { +#if UTUN_NEXUS + // Tell the nexus to stop all rings + if (pcb->utun_netif_nexus != NULL) { + /* + * Quiesce the interface and flush any pending outbound packets. + */ + if_down(ifp); + + /* Increment refcnt, but detach interface */ + ifnet_incr_iorefcnt(ifp); + if ((result = ifnet_detach(ifp)) != 0) { + panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + } - /* Increment refcnt, but detach interface */ - ifnet_incr_iorefcnt(ifp); - if ((result = ifnet_detach(ifp)) != 0) { - panic("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); - } + /* + * We want to do everything in our power to ensure that the interface + * really goes away when the socket is closed. We must remove IP/IPv6 + * addresses and detach the protocols. Finally, we can remove and + * release the interface. + */ + utun_cleanup_family(ifp, AF_INET); + utun_cleanup_family(ifp, AF_INET6); - /* - * We want to do everything in our power to ensure that the interface - * really goes away when the socket is closed. We must remove IP/IPv6 - * addresses and detach the protocols. Finally, we can remove and - * release the interface. - */ - utun_cleanup_family(ifp, AF_INET); - utun_cleanup_family(ifp, AF_INET6); + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); - lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { + utun_unregister_kernel_pipe_nexus(); + } + } + utun_nexus_detach(&pcb->utun_nx); + + /* Decrement refcnt to finish detaching and freeing */ + ifnet_decr_iorefcnt(ifp); + } else +#endif // UTUN_NEXUS + { + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); #if UTUN_NEXUS - if (!uuid_is_null(kpipe_uuid)) { - if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { - utun_unregister_kernel_pipe_nexus(); - } - } - utun_nexus_detach(&pcb->utun_nx); + if (!uuid_is_null(kpipe_uuid)) { + if (kern_nexus_controller_free_provider_instance(utun_ncd, kpipe_uuid) == 0) { + utun_unregister_kernel_pipe_nexus(); + } + } #endif // UTUN_NEXUS - /* Decrement refcnt to finish detaching and freeing */ - ifnet_decr_iorefcnt(ifp); + /* + * We want to do everything in our power to ensure that the interface + * really goes away when the socket is closed. We must remove IP/IPv6 + * addresses and detach the protocols. Finally, we can remove and + * release the interface. + */ + utun_cleanup_family(ifp, AF_INET); + utun_cleanup_family(ifp, AF_INET6); + + /* + * Detach now; utun_detach() will be called asynchronously once + * the I/O reference count drops to 0. There we will invoke + * ifnet_release(). + */ + if ((result = ifnet_detach(ifp)) != 0) { + printf("utun_ctl_disconnect - ifnet_detach failed: %d\n", result); + } + } + } else { + // Bound, but not connected + lck_rw_unlock_exclusive(&pcb->utun_pcb_lock); + utun_free_pcb(pcb, false); + } return 0; } @@ -1802,7 +1879,25 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, if (len != sizeof(u_int32_t)) { result = EMSGSIZE; } else { - pcb->utun_flags = *(u_int32_t *)data; + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } +#if UTUN_NEXUS + if (pcb->utun_use_netif) { + pcb->utun_flags = *(u_int32_t *)data; + } else +#endif // UTUN_NEXUS + { + u_int32_t old_flags = pcb->utun_flags; + pcb->utun_flags = *(u_int32_t *)data; + if (((old_flags ^ pcb->utun_flags) & UTUN_FLAGS_ENABLE_PROC_UUID)) { + // If UTUN_FLAGS_ENABLE_PROC_UUID flag changed, update bpf + bpfdetach(pcb->utun_ifp); + bpfattach(pcb->utun_ifp, DLT_NULL, UTUN_HEADER_SIZE(pcb)); + } + } } break; @@ -1811,6 +1906,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } pcb->utun_ext_ifdata_stats = (*(int *)data) ? 1 : 0; break; @@ -1822,6 +1922,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, result = EINVAL; break; } + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (!pcb->utun_ext_ifdata_stats) { result = EINVAL; break; @@ -1842,6 +1947,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (len != 0) { /* if len==0, del_ifp will be NULL causing the delegate to be removed */ bcopy(data, name, len); name[len] = 0; @@ -1874,6 +1984,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (*(int *)data) { result = utun_enable_channel(pcb, current_proc()); } else { @@ -1886,6 +2001,11 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, result = EMSGSIZE; break; } + if (pcb->utun_ifp == NULL) { + // Only can set after connecting + result = EINVAL; + break; + } if (!if_enable_netagent) { result = ENOTSUP; break; @@ -1902,6 +2022,91 @@ utun_ctl_setopt(__unused kern_ctl_ref kctlref, } break; } + case UTUN_OPT_ENABLE_NETIF: { + if (len != sizeof(int)) { + result = EMSGSIZE; + break; + } + if (pcb->utun_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + pcb->utun_use_netif = true; + break; + } + case UTUN_OPT_SLOT_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->utun_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t slot_size = *(u_int32_t *)data; + if (slot_size < UTUN_IF_MIN_SLOT_SIZE || + slot_size > UTUN_IF_MAX_SLOT_SIZE) { + return (EINVAL); + } + pcb->utun_slot_size = slot_size; + break; + } + case UTUN_OPT_NETIF_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->utun_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < UTUN_IF_MIN_RING_SIZE || + ring_size > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->utun_netif_ring_size = ring_size; + break; + } + case UTUN_OPT_TX_FSW_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->utun_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < UTUN_IF_MIN_RING_SIZE || + ring_size > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->utun_tx_fsw_ring_size = ring_size; + break; + } + case UTUN_OPT_RX_FSW_RING_SIZE: { + if (len != sizeof(u_int32_t)) { + result = EMSGSIZE; + break; + } + if (pcb->utun_ifp != NULL) { + // Only can set before connecting + result = EINVAL; + break; + } + u_int32_t ring_size = *(u_int32_t *)data; + if (ring_size < UTUN_IF_MIN_RING_SIZE || + ring_size > UTUN_IF_MAX_RING_SIZE) { + return (EINVAL); + } + pcb->utun_rx_fsw_ring_size = ring_size; + break; + } #endif // UTUN_NEXUS default: { result = ENOPROTOOPT; @@ -1944,6 +2149,11 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref, if (*len < MIN(strlen(pcb->utun_if_xname) + 1, sizeof(pcb->utun_if_xname))) { result = EMSGSIZE; } else { + if (pcb->utun_ifp == NULL) { + // Only can get after connecting + result = EINVAL; + break; + } *len = snprintf(data, *len, "%s", pcb->utun_if_xname) + 1; } break; @@ -1958,7 +2168,7 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref, } #if UTUN_NEXUS - case UTUN_OPT_GET_CHANNEL_UUID: + case UTUN_OPT_GET_CHANNEL_UUID: { lck_rw_lock_shared(&pcb->utun_pcb_lock); if (uuid_is_null(pcb->utun_kpipe_uuid)) { result = ENXIO; @@ -1969,6 +2179,39 @@ utun_ctl_getopt(__unused kern_ctl_ref kctlref, } lck_rw_unlock_shared(&pcb->utun_pcb_lock); break; + } + case UTUN_OPT_SLOT_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->utun_slot_size; + } + break; + } + case UTUN_OPT_NETIF_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->utun_netif_ring_size; + } + break; + } + case UTUN_OPT_TX_FSW_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->utun_tx_fsw_ring_size; + } + break; + } + case UTUN_OPT_RX_FSW_RING_SIZE: { + if (*len != sizeof(u_int32_t)) { + result = EMSGSIZE; + } else { + *(u_int32_t *)data = pcb->utun_rx_fsw_ring_size; + } + break; + } #endif // UTUN_NEXUS default: @@ -2011,7 +2254,6 @@ utun_ctl_rcvd(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, int flags) } /* Network Interface functions */ -#if !UTUN_NEXUS static void utun_start(ifnet_t interface) { @@ -2074,7 +2316,6 @@ utun_start(ifnet_t interface) } } } -#endif // !UTUN_NEXUS static errno_t utun_output(ifnet_t interface, @@ -2085,6 +2326,15 @@ utun_output(ifnet_t interface, VERIFY(interface == pcb->utun_ifp); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) +#endif // UTUN_NEXUS + { + if (m_pktlen(data) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { + bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); + } + } + if (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT) { /* flush data */ mbuf_freem(data); @@ -2107,13 +2357,21 @@ utun_output(ifnet_t interface, if (result != 0) { mbuf_freem(data); printf("utun_output - ctl_enqueuembuf failed: %d\n", result); -#if !UTUN_NEXUS - ifnet_stat_increment_out(interface, 0, 0, 1); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) +#endif // UTUN_NEXUS + { + ifnet_stat_increment_out(interface, 0, 0, 1); + } } else { - if (!pcb->utun_ext_ifdata_stats) { - ifnet_stat_increment_out(interface, 1, length, 0); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) +#endif // UTUN_NEXUS + { + if (!pcb->utun_ext_ifdata_stats) { + ifnet_stat_increment_out(interface, 1, length, 0); + } } -#endif // !UTUN_NEXUS } } else { mbuf_freem(data); @@ -2128,7 +2386,7 @@ utun_demux(__unused ifnet_t interface, __unused char *frame_header, protocol_family_t *protocol) { - + struct utun_pcb *pcb = ifnet_softc(interface); struct ip *ip; u_int ip_version; @@ -2136,28 +2394,35 @@ utun_demux(__unused ifnet_t interface, data = mbuf_next(data); } - if (data == NULL) + if (data == NULL) { return ENOENT; + } - ip = mtod(data, struct ip *); - ip_version = ip->ip_v; - - switch(ip_version) { - case 4: - *protocol = PF_INET; - return 0; - case 6: - *protocol = PF_INET6; - return 0; - default: - *protocol = 0; - break; +#if UTUN_NEXUS + if (pcb->utun_use_netif) { + ip = mtod(data, struct ip *); + ip_version = ip->ip_v; + + switch(ip_version) { + case 4: + *protocol = PF_INET; + return 0; + case 6: + *protocol = PF_INET6; + return 0; + default: + *protocol = 0; + break; + } + } else +#endif // UTUN_NEXUS + { + *protocol = *(u_int32_t *)mbuf_data(data); } return 0; } -#if !UTUN_NEXUS static errno_t utun_framer(ifnet_t interface, mbuf_t *packet, @@ -2192,7 +2457,6 @@ utun_framer(ifnet_t interface, return 0; } -#endif // !UTUN_NEXUS static errno_t utun_add_proto(__unused ifnet_t interface, @@ -2224,22 +2488,27 @@ utun_ioctl(ifnet_t interface, u_long command, void *data) { + struct utun_pcb *pcb = ifnet_softc(interface); errno_t result = 0; switch(command) { - case SIOCSIFMTU: + case SIOCSIFMTU: { #if UTUN_NEXUS - { - // Make sure we can fit packets in the channel buffers - // Allow for the headroom in the slot - if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > UTUN_IF_DEFAULT_SLOT_SIZE) { - ifnet_set_mtu(interface, UTUN_IF_DEFAULT_SLOT_SIZE - UTUN_IF_HEADROOM_SIZE); - break; - } - } + if (pcb->utun_use_netif) { + // Make sure we can fit packets in the channel buffers + // Allow for the headroom in the slot + if (((uint64_t)((struct ifreq*)data)->ifr_mtu) + UTUN_IF_HEADROOM_SIZE > pcb->utun_slot_size) { + result = EINVAL; + } else { + ifnet_set_mtu(interface, (uint32_t)((struct ifreq*)data)->ifr_mtu); + } + } else #endif // UTUN_NEXUS - ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); + { + ifnet_set_mtu(interface, ((struct ifreq*)data)->ifr_mtu); + } break; + } case SIOCSIFFLAGS: /* ifioctl() takes care of it */ @@ -2257,7 +2526,7 @@ utun_detached(ifnet_t interface) { struct utun_pcb *pcb = ifnet_softc(interface); (void)ifnet_release(interface); - utun_free_pcb(pcb); + utun_free_pcb(pcb, true); } /* Protocol Handlers */ @@ -2268,13 +2537,28 @@ utun_proto_input(__unused ifnet_t interface, mbuf_t m, __unused char *frame_header) { + struct utun_pcb *pcb = ifnet_softc(interface); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) +#endif // UTUN_NEXUS + { + mbuf_adj(m, UTUN_HEADER_SIZE(pcb)); + } if (proto_input(protocol, m) != 0) { m_freem(m); -#if !UTUN_NEXUS - ifnet_stat_increment_in(interface, 0, 0, 1); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) +#endif // UTUN_NEXUS + { + ifnet_stat_increment_in(interface, 0, 0, 1); + } } else { - ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0); +#if UTUN_NEXUS + if (!pcb->utun_use_netif) #endif // UTUN_NEXUS + { + ifnet_stat_increment_in(interface, 1, m->m_pkthdr.len, 0); + } } return 0; @@ -2312,75 +2596,67 @@ utun_attach_proto(ifnet_t interface, return result; } -#if UTUN_NEXUS static errno_t utun_pkt_input(struct utun_pcb *pcb, mbuf_t packet) { - lck_rw_lock_shared(&pcb->utun_pcb_lock); - - lck_mtx_lock(&pcb->utun_input_chain_lock); - if (pcb->utun_input_chain != NULL) { - pcb->utun_input_chain_last->m_nextpkt = packet; - } else { - pcb->utun_input_chain = packet; - } - while (packet->m_nextpkt) { - VERIFY(packet != packet->m_nextpkt); - packet = packet->m_nextpkt; - } - pcb->utun_input_chain_last = packet; - lck_mtx_unlock(&pcb->utun_input_chain_lock); - - kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; - lck_rw_unlock_shared(&pcb->utun_pcb_lock); +#if UTUN_NEXUS + if (pcb->utun_use_netif) { + lck_rw_lock_shared(&pcb->utun_pcb_lock); - if (rx_ring != NULL) { - kern_channel_notify(rx_ring, 0); - } + lck_mtx_lock(&pcb->utun_input_chain_lock); + if (pcb->utun_input_chain != NULL) { + pcb->utun_input_chain_last->m_nextpkt = packet; + } else { + pcb->utun_input_chain = packet; + } + while (packet->m_nextpkt) { + VERIFY(packet != packet->m_nextpkt); + packet = packet->m_nextpkt; + } + pcb->utun_input_chain_last = packet; + lck_mtx_unlock(&pcb->utun_input_chain_lock); - return (0); -} -#else -static errno_t -utun_pkt_input (struct utun_pcb *pcb, mbuf_t m) -{ - errno_t result; - protocol_family_t protocol = 0; + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); - mbuf_pkthdr_setrcvif(m, pcb->utun_ifp); + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } - if (m_pktlen(m) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { - protocol = *(u_int32_t *)mbuf_data(m); + return (0); + } else +#endif // IPSEC_NEXUS + { + mbuf_pkthdr_setrcvif(packet, pcb->utun_ifp); - bpf_tap_in(pcb->utun_ifp, DLT_NULL, m, 0, 0); - } - if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) { - /* flush data */ - mbuf_freem(m); - return 0; - } + if (m_pktlen(packet) >= (int32_t)UTUN_HEADER_SIZE(pcb)) { + bpf_tap_in(pcb->utun_ifp, DLT_NULL, packet, 0, 0); + } + if (pcb->utun_flags & UTUN_FLAGS_NO_INPUT) { + /* flush data */ + mbuf_freem(packet); + return 0; + } - if (!pcb->utun_ext_ifdata_stats) { - struct ifnet_stat_increment_param incs; + errno_t result = 0; + if (!pcb->utun_ext_ifdata_stats) { + struct ifnet_stat_increment_param incs = {}; + incs.packets_in = 1; + incs.bytes_in = mbuf_pkthdr_len(packet); + result = ifnet_input(pcb->utun_ifp, packet, &incs); + } else { + result = ifnet_input(pcb->utun_ifp, packet, NULL); + } + if (result != 0) { + ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1); - bzero(&incs, sizeof(incs)); - incs.packets_in = 1; - incs.bytes_in = mbuf_pkthdr_len(m); - result = ifnet_input(pcb->utun_ifp, m, &incs); - } else { - result = ifnet_input(pcb->utun_ifp, m, NULL); - } - if (result != 0) { - ifnet_stat_increment_in(pcb->utun_ifp, 0, 0, 1); + printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result); + mbuf_freem(packet); + } - printf("%s - ifnet_input failed: %d\n", __FUNCTION__, result); - mbuf_freem(m); + return (0); } - - return 0; } -#endif // UTUN_NEXUS - #if UTUN_NEXUS @@ -2535,18 +2811,80 @@ utun_kpipe_sync_tx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, return 0; } - kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); - if (tx_slot == NULL) { - // Nothing to write, bail + if (pcb->utun_use_netif) { + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to write, bail + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } + + // Signal the netif ring to read + kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + if (rx_ring != NULL) { + kern_channel_notify(rx_ring, 0); + } + } else { lck_rw_unlock_shared(&pcb->utun_pcb_lock); - return 0; - } - // Signal the netif ring to read - kern_channel_ring_t rx_ring = pcb->utun_netif_rxring; - lck_rw_unlock_shared(&pcb->utun_pcb_lock); - if (rx_ring != NULL) { - kern_channel_notify(rx_ring, 0); + struct ifnet_stat_increment_param incs = {}; + struct kern_channel_ring_stat_increment tx_ring_stats = {}; + MBUFQ_HEAD(mbufq) mbq; + MBUFQ_INIT(&mbq); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + while (tx_slot != NULL) { + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); + + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); + + if (tx_ph == 0) { + continue; + } + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != 0); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + size_t length = MIN(kern_packet_get_data_length(tx_ph), + pcb->utun_slot_size); + + mbuf_t data = NULL; + if (length >= UTUN_HEADER_SIZE(pcb) && + !(pcb->utun_flags & UTUN_FLAGS_NO_INPUT)) { + errno_t error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_HEADER, &data); + VERIFY(0 == error); + error = mbuf_copyback(data, 0, length, tx_baddr, MBUF_WAITOK); + VERIFY(0 == error); + /* + * The userland ABI requires the first four bytes have + * the protocol family in network byte order: swap them + */ + *(uint32_t *)mbuf_data(data) = ntohl(*(uint32_t *)mbuf_data(data)); + mbuf_pkthdr_setrcvif(data, pcb->utun_ifp); + bpf_tap_in(pcb->utun_ifp, DLT_NULL, data, 0, 0); + incs.packets_in++; + incs.bytes_in += length; + MBUFQ_ENQUEUE(&mbq, data); + } + } + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + tx_ring_stats.kcrsi_slots_transferred = incs.packets_in; + tx_ring_stats.kcrsi_bytes_transferred = incs.bytes_in; + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void) kern_channel_reclaim(tx_ring); + } + if (!MBUFQ_EMPTY(&mbq)) { + (void) ifnet_input_extended(pcb->utun_ifp, MBUFQ_FIRST(&mbq), + MBUFQ_LAST(&mbq), &incs); + MBUFQ_INIT(&mbq); + } } return 0; @@ -2559,7 +2897,7 @@ utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, #pragma unused(nxprov) #pragma unused(flags) struct utun_pcb *pcb = kern_nexus_get_context(nexus); - struct kern_channel_ring_stat_increment rx_ring_stats; + struct kern_channel_ring_stat_increment rx_ring_stats = {}; lck_rw_lock_shared(&pcb->utun_pcb_lock); @@ -2578,185 +2916,276 @@ utun_kpipe_sync_rx(kern_nexus_provider_t nxprov, kern_nexus_t nexus, return 0; } - kern_channel_ring_t tx_ring = pcb->utun_netif_txring; - if (tx_ring == NULL || - pcb->utun_netif_nexus == NULL) { - // Net-If TX ring not set up yet, nothing to read - lck_rw_unlock_shared(&pcb->utun_pcb_lock); - return 0; - } - - struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats; - - // Unlock utun before entering ring - lck_rw_unlock_shared(&pcb->utun_pcb_lock); - - (void)kr_enter(tx_ring, TRUE); + if (pcb->utun_use_netif) { + kern_channel_ring_t tx_ring = pcb->utun_netif_txring; + if (tx_ring == NULL || + pcb->utun_netif_nexus == NULL) { + // Net-If TX ring not set up yet, nothing to read + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + return 0; + } - // Lock again after entering and validate - lck_rw_lock_shared(&pcb->utun_pcb_lock); - if (tx_ring != pcb->utun_netif_txring) { - // Ring no longer valid - // Unlock first, then exit ring - lck_rw_unlock_shared(&pcb->utun_pcb_lock); - kr_exit(tx_ring); - return 0; - } + struct netif_stats *nifs = &NX_NETIF_PRIVATE(pcb->utun_netif_nexus)->nif_stats; - struct kern_channel_ring_stat_increment tx_ring_stats; - bzero(&tx_ring_stats, sizeof(tx_ring_stats)); - kern_channel_slot_t tx_pslot = NULL; - kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); - if (tx_slot == NULL) { - // Nothing to read, don't bother signalling - // Unlock first, then exit ring + // Unlock utun before entering ring lck_rw_unlock_shared(&pcb->utun_pcb_lock); - kr_exit(tx_ring); - return 0; - } - - struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; - VERIFY(rx_pp != NULL); - bzero(&rx_ring_stats, sizeof(rx_ring_stats)); - kern_channel_slot_t rx_pslot = NULL; - kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); - while (rx_slot != NULL && tx_slot != NULL) { - size_t length; - kern_buflet_t rx_buf; - void *rx_baddr; + (void)kr_enter(tx_ring, TRUE); - kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); - - // Advance TX ring - tx_pslot = tx_slot; - tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); - - /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ - if (tx_ph == 0) { - continue; + // Lock again after entering and validate + lck_rw_lock_shared(&pcb->utun_pcb_lock); + if (tx_ring != pcb->utun_netif_txring) { + // Ring no longer valid + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; } - // Allocate rx packet - kern_packet_t rx_ph = 0; - errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); - if (unlikely(error != 0)) { - printf("utun_kpipe_sync_rx %s: failed to allocate packet\n", - pcb->utun_ifp->if_xname); - break; + struct kern_channel_ring_stat_increment tx_ring_stats; + bzero(&tx_ring_stats, sizeof(tx_ring_stats)); + kern_channel_slot_t tx_pslot = NULL; + kern_channel_slot_t tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL); + if (tx_slot == NULL) { + // Nothing to read, don't bother signalling + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + kr_exit(tx_ring); + return 0; } - kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); - VERIFY(tx_buf != NULL); - uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); - VERIFY(tx_baddr != NULL); - tx_baddr += kern_buflet_get_data_offset(tx_buf); + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); - bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); - - length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb), - UTUN_IF_DEFAULT_SLOT_SIZE); + while (rx_slot != NULL && tx_slot != NULL) { + size_t length; + kern_buflet_t rx_buf; + void *rx_baddr; - tx_ring_stats.kcrsi_slots_transferred++; - tx_ring_stats.kcrsi_bytes_transferred += length; + kern_packet_t tx_ph = kern_channel_slot_get_packet(tx_ring, tx_slot); - if (length < UTUN_HEADER_SIZE(pcb) || - length > UTUN_IF_DEFAULT_SLOT_SIZE || - length > rx_pp->pp_buflet_size || - (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { - /* flush data */ - kern_pbufpool_free(rx_pp, rx_ph); - printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n", - pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb)); - STATS_INC(nifs, NETIF_STATS_BADLEN); - STATS_INC(nifs, NETIF_STATS_DROPPED); - continue; - } + // Advance TX ring + tx_pslot = tx_slot; + tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL); - /* fillout packet */ - rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); - VERIFY(rx_buf != NULL); - rx_baddr = kern_buflet_get_object_address(rx_buf); - VERIFY(rx_baddr != NULL); + /* Skip slot if packet is zero-length or marked as dropped (QUMF_DROPPED) */ + if (tx_ph == 0) { + continue; + } - // Find family - uint32_t af = 0; - uint8_t vhl = *(uint8_t *)(tx_baddr); - u_int ip_version = (vhl >> 4); - switch (ip_version) { - case 4: { - af = AF_INET; + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("utun_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); break; } - case 6: { - af = AF_INET6; - break; + + kern_buflet_t tx_buf = kern_packet_get_next_buflet(tx_ph, NULL); + VERIFY(tx_buf != NULL); + uint8_t *tx_baddr = kern_buflet_get_object_address(tx_buf); + VERIFY(tx_baddr != NULL); + tx_baddr += kern_buflet_get_data_offset(tx_buf); + + bpf_tap_packet_out(pcb->utun_ifp, DLT_RAW, tx_ph, NULL, 0); + + length = MIN(kern_packet_get_data_length(tx_ph) + UTUN_HEADER_SIZE(pcb), + pcb->utun_slot_size); + + tx_ring_stats.kcrsi_slots_transferred++; + tx_ring_stats.kcrsi_bytes_transferred += length; + + if (length < UTUN_HEADER_SIZE(pcb) || + length > pcb->utun_slot_size || + length > rx_pp->pp_buflet_size || + (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { + /* flush data */ + kern_pbufpool_free(rx_pp, rx_ph); + printf("utun_kpipe_sync_rx %s: invalid length %zu header_size %zu\n", + pcb->utun_ifp->if_xname, length, UTUN_HEADER_SIZE(pcb)); + STATS_INC(nifs, NETIF_STATS_BADLEN); + STATS_INC(nifs, NETIF_STATS_DROPPED); + continue; } - default: { - printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n", - pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb)); - break; + + /* fillout packet */ + rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Find family + uint32_t af = 0; + uint8_t vhl = *(uint8_t *)(tx_baddr); + u_int ip_version = (vhl >> 4); + switch (ip_version) { + case 4: { + af = AF_INET; + break; + } + case 6: { + af = AF_INET6; + break; + } + default: { + printf("utun_kpipe_sync_rx %s: unknown ip version %u vhl %u header_size %zu\n", + pcb->utun_ifp->if_xname, ip_version, vhl, UTUN_HEADER_SIZE(pcb)); + break; + } } - } - // Copy header - af = htonl(af); - memcpy((void *)rx_baddr, &af, sizeof(af)); - if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { - kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af))); - } + // Copy header + af = htonl(af); + memcpy((void *)rx_baddr, &af, sizeof(af)); + if (pcb->utun_flags & UTUN_FLAGS_ENABLE_PROC_UUID) { + kern_packet_get_euuid(tx_ph, (void *)(rx_baddr + sizeof(af))); + } - // Copy data from tx to rx - memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb)); - kern_packet_clear_flow_uuid(rx_ph); // zero flow id + // Copy data from tx to rx + memcpy((void *)(rx_baddr + UTUN_HEADER_SIZE(pcb)), (void *)tx_baddr, length - UTUN_HEADER_SIZE(pcb)); + kern_packet_clear_flow_uuid(rx_ph); // zero flow id - /* finalize and attach the packet */ - error = kern_buflet_set_data_offset(rx_buf, 0); - VERIFY(error == 0); - error = kern_buflet_set_data_length(rx_buf, length); - VERIFY(error == 0); - error = kern_packet_finalize(rx_ph); - VERIFY(error == 0); - error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); - VERIFY(error == 0); + /* finalize and attach the packet */ + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); - STATS_INC(nifs, NETIF_STATS_TXPKTS); - STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); + STATS_INC(nifs, NETIF_STATS_TXPKTS); + STATS_INC(nifs, NETIF_STATS_TXCOPY_DIRECT); - rx_ring_stats.kcrsi_slots_transferred++; - rx_ring_stats.kcrsi_bytes_transferred += length; + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; - rx_pslot = rx_slot; - rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); - } + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } - if (rx_pslot) { - kern_channel_advance_slot(rx_ring, rx_pslot); - kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); - } + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_net_stats(rx_ring, pcb->utun_ifp, &rx_ring_stats); + } - if (tx_pslot) { - kern_channel_advance_slot(tx_ring, tx_pslot); - kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); - (void)kern_channel_reclaim(tx_ring); - } + if (tx_pslot) { + kern_channel_advance_slot(tx_ring, tx_pslot); + kern_channel_increment_ring_net_stats(tx_ring, pcb->utun_ifp, &tx_ring_stats); + (void)kern_channel_reclaim(tx_ring); + } - if (pcb->utun_output_disabled) { + /* just like utun_ctl_rcvd(), always reenable output */ errno_t error = ifnet_enable_output(pcb->utun_ifp); if (error != 0) { printf("utun_kpipe_sync_rx: ifnet_enable_output returned error %d\n", error); - } else { - pcb->utun_output_disabled = false; } - } - // Unlock first, then exit ring - lck_rw_unlock_shared(&pcb->utun_pcb_lock); + // Unlock first, then exit ring + lck_rw_unlock_shared(&pcb->utun_pcb_lock); - if (tx_pslot != NULL) { - kern_channel_notify(tx_ring, 0); + if (tx_pslot != NULL) { + kern_channel_notify(tx_ring, 0); + } + kr_exit(tx_ring); + } else { + lck_rw_unlock_shared(&pcb->utun_pcb_lock); + + uint32_t mb_cnt = 0; + uint32_t mb_len = 0; + struct mbuf *mb_head = NULL; + struct mbuf *mb_tail = NULL; + + if (ifnet_dequeue_multi(pcb->utun_ifp, avail, &mb_head, + &mb_tail, &mb_cnt, &mb_len) != 0) { + return 0; + } + VERIFY(mb_cnt <= avail); + + struct kern_pbufpool *rx_pp = rx_ring->ckr_pp; + VERIFY(rx_pp != NULL); + kern_channel_slot_t rx_pslot = NULL; + kern_channel_slot_t rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL); + while (rx_slot) { + size_t length = 0; + mbuf_t data = NULL; + if ((data = mb_head) == NULL) { + VERIFY(mb_cnt == 0); + break; + } + mb_head = mbuf_nextpkt(mb_head); + mbuf_setnextpkt(data, NULL); + VERIFY(mb_cnt != 0); + --mb_cnt; + length = mbuf_pkthdr_len(data); + if (length < UTUN_HEADER_SIZE(pcb) || + length > pcb->utun_slot_size || + (pcb->utun_flags & UTUN_FLAGS_NO_OUTPUT)) { + /* flush data */ + mbuf_freem(data); + continue; + } + bpf_tap_out(pcb->utun_ifp, DLT_NULL, data, 0, 0); + + // Allocate rx packet + kern_packet_t rx_ph = 0; + errno_t error = kern_pbufpool_alloc_nosleep(rx_pp, 1, &rx_ph); + if (unlikely(error != 0)) { + printf("utun_kpipe_sync_rx %s: failed to allocate packet\n", + pcb->utun_ifp->if_xname); + break; + } + + /* + * The ABI requires the protocol in network byte order + */ + *(u_int32_t *)mbuf_data(data) = htonl(*(u_int32_t *)mbuf_data(data)); + + // Fillout rx packet + kern_buflet_t rx_buf = kern_packet_get_next_buflet(rx_ph, NULL); + VERIFY(rx_buf != NULL); + void *rx_baddr = kern_buflet_get_object_address(rx_buf); + VERIFY(rx_baddr != NULL); + + // Copy-in data from mbuf to buflet + mbuf_copydata(data, 0, length, (void *)rx_baddr); + kern_packet_clear_flow_uuid(rx_ph); // Zero flow id + + // Finalize and attach the packet + error = kern_buflet_set_data_offset(rx_buf, 0); + VERIFY(error == 0); + error = kern_buflet_set_data_length(rx_buf, length); + VERIFY(error == 0); + error = kern_packet_finalize(rx_ph); + VERIFY(error == 0); + error = kern_channel_slot_attach_packet(rx_ring, rx_slot, rx_ph); + VERIFY(error == 0); + + rx_ring_stats.kcrsi_slots_transferred++; + rx_ring_stats.kcrsi_bytes_transferred += length; + + if (!pcb->utun_ext_ifdata_stats) { + ifnet_stat_increment_out(pcb->utun_ifp, 1, length, 0); + } + + mbuf_freem(data); + + rx_pslot = rx_slot; + rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL); + } + if (rx_pslot) { + kern_channel_advance_slot(rx_ring, rx_pslot); + kern_channel_increment_ring_stats(rx_ring, &rx_ring_stats); + } + if (mb_head != NULL) { + VERIFY(mb_cnt != 0); + mbuf_freem_list(mb_head); + } } - kr_exit(tx_ring); return 0; } diff --git a/bsd/net/if_utun.h b/bsd/net/if_utun.h index 008924154..4e9868a83 100644 --- a/bsd/net/if_utun.h +++ b/bsd/net/if_utun.h @@ -60,6 +60,13 @@ errno_t utun_register_control(void); #define UTUN_OPT_ENABLE_CHANNEL 17 #define UTUN_OPT_GET_CHANNEL_UUID 18 #define UTUN_OPT_ENABLE_FLOWSWITCH 19 + +#define UTUN_OPT_ENABLE_NETIF 20 /* Must be set before connecting */ +#define UTUN_OPT_SLOT_SIZE 21 /* Must be set before connecting */ +#define UTUN_OPT_NETIF_RING_SIZE 22 /* Must be set before connecting */ +#define UTUN_OPT_TX_FSW_RING_SIZE 23 /* Must be set before connecting */ +#define UTUN_OPT_RX_FSW_RING_SIZE 24 /* Must be set before connecting */ + /* * Flags for by UTUN_OPT_FLAGS */ diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h index 0541857f0..60aa09b8f 100644 --- a/bsd/net/if_var.h +++ b/bsd/net/if_var.h @@ -754,6 +754,8 @@ LIST_HEAD(ifmultihead, ifmultiaddr); TAILQ_HEAD(tailq_head, tqdummy); TAILQ_HEAD(ifnet_filter_head, ifnet_filter); TAILQ_HEAD(ddesc_head_name, dlil_demux_desc); + +extern boolean_t intcoproc_unrestricted; #endif /* BSD_KERNEL_PRIVATE */ #ifdef PRIVATE diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c index 737ce40ad..5c3535e68 100644 --- a/bsd/net/if_vlan.c +++ b/bsd/net/if_vlan.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003-2016 Apple Inc. All rights reserved. + * Copyright (c) 2003-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1835,6 +1835,11 @@ vlan_ioctl(ifnet_t ifp, u_long cmd, void * data) error = ENXIO; break; } + if (IFNET_IS_INTCOPROC(p)) { + error = EINVAL; + break; + } + /* can't do VLAN over anything but ethernet or ethernet aggregate */ if (ifnet_type(p) != IFT_ETHER && ifnet_type(p) != IFT_IEEE8023ADLAG) { diff --git a/bsd/net/necp_client.c b/bsd/net/necp_client.c index 0cb79a5b7..1588ac177 100644 --- a/bsd/net/necp_client.c +++ b/bsd/net/necp_client.c @@ -4706,8 +4706,18 @@ necp_get_socket_attributes(struct socket *so, struct sockopt *sopt) u_int8_t *buffer = NULL; u_int8_t *cursor = NULL; size_t valsize = 0; - struct inpcb *inp = sotoinpcb(so); + struct inpcb *inp = NULL; + if ((SOCK_DOM(so) != PF_INET +#if INET6 + && SOCK_DOM(so) != PF_INET6 +#endif + )) { + error = EINVAL; + goto done; + } + + inp = sotoinpcb(so); if (inp->inp_necp_attributes.inp_domain != NULL) { valsize += sizeof(struct necp_tlv_header) + strlen(inp->inp_necp_attributes.inp_domain); } diff --git a/bsd/net/packet_mangler.c b/bsd/net/packet_mangler.c index d6c8b9dbd..2efa88cc2 100644 --- a/bsd/net/packet_mangler.c +++ b/bsd/net/packet_mangler.c @@ -955,7 +955,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u } } - while (tcp_optlen) { + while (tcp_optlen > 0) { if (tcp_opt_buf[i] == 0x1) { PKT_MNGLR_LOG(LOG_INFO, "Skipping NOP\n"); tcp_optlen--; @@ -968,7 +968,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u continue; } else if (tcp_opt_buf[i] == TCP_OPT_MULTIPATH_TCP) { int j = 0; - int mptcpoptlen = tcp_opt_buf[i+1]; + unsigned char mptcpoptlen = tcp_opt_buf[i+1]; uint8_t sbtver = tcp_opt_buf[i+MPTCP_SBT_VER_OFFSET]; uint8_t subtype = sbtver >> 4; @@ -984,7 +984,7 @@ static errno_t pktmnglr_ipfilter_input(void *cookie, mbuf_t *data, int offset, u } PKT_MNGLR_LOG(LOG_INFO, "Got MPTCP option %x\n", tcp_opt_buf[i]); - for (; j < mptcpoptlen; j++) { + for (; j < mptcpoptlen && j < tcp_optlen; j++) { if (p_pkt_mnglr->proto_action_mask & PKT_MNGLR_TCP_ACT_NOP_MPTCP) { tcp_opt_buf[i+j] = 0x1; diff --git a/bsd/net/pf_ioctl.c b/bsd/net/pf_ioctl.c index 05bb21b56..62a475f9d 100644 --- a/bsd/net/pf_ioctl.c +++ b/bsd/net/pf_ioctl.c @@ -4341,7 +4341,7 @@ pf_af_hook(struct ifnet *ifp, struct mbuf **mppn, struct mbuf **mp, struct ifnet * pf_ifp = ifp; /* Always allow traffic on co-processor interfaces. */ - if (ifp && IFNET_IS_INTCOPROC(ifp)) + if (!intcoproc_unrestricted && ifp && IFNET_IS_INTCOPROC(ifp)) return (0); marks = net_thread_marks_push(NET_THREAD_HELD_PF); diff --git a/bsd/net/pktap.c b/bsd/net/pktap.c index 6608e9df3..66616a73f 100644 --- a/bsd/net/pktap.c +++ b/bsd/net/pktap.c @@ -994,6 +994,13 @@ pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m, void (*bpf_tap_func)(ifnet_t, u_int32_t, mbuf_t, void *, size_t) = outgoing ? bpf_tap_out : bpf_tap_in; + + /* + * Skip the coprocessor interface + */ + if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) + return; + lck_rw_lock_shared(pktap_lck_rw); /* diff --git a/bsd/netinet/flow_divert.c b/bsd/netinet/flow_divert.c index c5e974aab..b73a86172 100644 --- a/bsd/netinet/flow_divert.c +++ b/bsd/netinet/flow_divert.c @@ -1070,6 +1070,9 @@ flow_divert_create_connect_packet(struct flow_divert_pcb *fd_cb, struct sockaddr } } else { FDLOG0(LOG_WARNING, fd_cb, "Failed to get the code signing identity"); + if (fd_cb->group->flags & FLOW_DIVERT_GROUP_FLAG_NO_APP_MAP) { + error = 0; + } } if (src_proc != PROC_NULL) { @@ -3383,11 +3386,13 @@ flow_divert_token_set(struct socket *so, struct sockopt *sopt) error = soopt_getm(sopt, &token); if (error) { + token = NULL; goto done; } error = soopt_mcopyin(sopt, token); if (error) { + token = NULL; goto done; } diff --git a/bsd/netinet/in_pcb.c b/bsd/netinet/in_pcb.c index 3371c71b3..b3ff42cf9 100644 --- a/bsd/netinet/in_pcb.c +++ b/bsd/netinet/in_pcb.c @@ -131,7 +131,6 @@ static u_int16_t inpcb_timeout_run = 0; /* INPCB timer is scheduled to run */ static boolean_t inpcb_garbage_collecting = FALSE; /* gc timer is scheduled */ static boolean_t inpcb_ticking = FALSE; /* "slow" timer is scheduled */ static boolean_t inpcb_fast_timer_on = FALSE; -static boolean_t intcoproc_unrestricted = FALSE; extern char *proc_best_name(proc_t); @@ -308,9 +307,6 @@ in_pcbinit(void) RB_INIT(&inp_fc_tree); bzero(&key_inp, sizeof(key_inp)); lck_mtx_unlock(&inp_fc_lck); - - PE_parse_boot_argn("intcoproc_unrestricted", &intcoproc_unrestricted, - sizeof (intcoproc_unrestricted)); } #define INPCB_HAVE_TIMER_REQ(req) (((req).intimer_lazy > 0) || \ diff --git a/bsd/netinet/ip_icmp.c b/bsd/netinet/ip_icmp.c index 3d7ecafe3..ab1b0452e 100644 --- a/bsd/netinet/ip_icmp.c +++ b/bsd/netinet/ip_icmp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -538,9 +538,17 @@ icmp_input(struct mbuf *m, int hlen) * notification to TCP layer. */ ctlfunc = ip_protox[icp->icmp_ip.ip_p]->pr_ctlinput; - if (ctlfunc) + + if (ctlfunc) { + LCK_MTX_ASSERT(inet_domain_mutex, LCK_MTX_ASSERT_OWNED); + + lck_mtx_unlock(inet_domain_mutex); + (*ctlfunc)(code, (struct sockaddr *)&icmpsrc, (void *)&icp->icmp_ip, m->m_pkthdr.rcvif); + + lck_mtx_lock(inet_domain_mutex); + } break; badcode: diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c index 4e4c27040..1298ab28e 100644 --- a/bsd/netinet/ip_input.c +++ b/bsd/netinet/ip_input.c @@ -4117,7 +4117,7 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop) if (sav != NULL) { lck_mtx_lock(sadb_mutex); if (sav->sah != NULL) { - ro = &sav->sah->sa_route; + ro = (struct route *)&sav->sah->sa_route; if (ro->ro_rt != NULL) { RT_LOCK(ro->ro_rt); if (ro->ro_rt->rt_ifp != NULL) { diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c index 16aff3c65..f5b51ac52 100644 --- a/bsd/netinet/ip_output.c +++ b/bsd/netinet/ip_output.c @@ -1341,7 +1341,7 @@ sendit: if (flags & IP_ROUTETOIF) { bzero(&ipsec_state.ro, sizeof (ipsec_state.ro)); } else { - route_copyout(&ipsec_state.ro, ro, sizeof (ipsec_state.ro)); + route_copyout((struct route *)&ipsec_state.ro, ro, sizeof (struct route)); } ipsec_state.dst = SA(dst); @@ -1389,10 +1389,10 @@ sendit: */ if (ipsec_state.tunneled) { flags &= ~IP_ROUTETOIF; - ro = &ipsec_state.ro; + ro = (struct route *)&ipsec_state.ro; } } else { - ro = &ipsec_state.ro; + ro = (struct route *)&ipsec_state.ro; } dst = SIN(ipsec_state.dst); if (error) { diff --git a/bsd/netinet/mptcp.c b/bsd/netinet/mptcp.c index 5d901a9da..8cf437f64 100644 --- a/bsd/netinet/mptcp.c +++ b/bsd/netinet/mptcp.c @@ -173,7 +173,7 @@ SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED, * Static declarations */ static uint16_t mptcp_input_csum(struct tcpcb *, struct mbuf *, uint64_t, - uint32_t, uint16_t, uint16_t); + uint32_t, uint16_t, uint16_t, uint16_t); static int mptcp_reass_present(struct socket *mp_so) @@ -181,16 +181,17 @@ mptcp_reass_present(struct socket *mp_so) struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb; struct tseg_qent *q; int dowakeup = 0; + int flags = 0; /* * Present data to user, advancing rcv_nxt through * completed sequence space. */ if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) - return (0); + return (flags); q = LIST_FIRST(&mp_tp->mpt_segq); if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt) - return (0); + return (flags); /* * If there is already another thread doing reassembly for this @@ -198,7 +199,7 @@ mptcp_reass_present(struct socket *mp_so) * (radar 16316196) */ if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG) - return (0); + return (flags); mp_tp->mpt_flags |= MPTCPF_REASS_INPROG; @@ -208,7 +209,8 @@ mptcp_reass_present(struct socket *mp_so) if (mp_so->so_state & SS_CANTRCVMORE) { m_freem(q->tqe_m); } else { - if (sbappendstream(&mp_so->so_rcv, q->tqe_m)) + flags = !!(q->tqe_m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN); + if (sbappendstream_rcvdemux(mp_so, q->tqe_m, 0, 0)) dowakeup = 1; } zfree(tcp_reass_zone, q); @@ -219,7 +221,7 @@ mptcp_reass_present(struct socket *mp_so) if (dowakeup) sorwakeup(mp_so); /* done with socket lock held */ - return (0); + return (flags); } @@ -376,21 +378,40 @@ mptcp_input(struct mptses *mpte, struct mbuf *m) * In the degraded fallback case, data is accepted without DSS map */ if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) { + struct mbuf *iter; + int mb_dfin = 0; fallback: mptcp_sbrcv_grow(mp_tp); + for (iter = m; iter; iter = iter->m_next) { + if ((iter->m_flags & M_PKTHDR) && + (iter->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) { + mb_dfin = 1; + break; + } + } + /* * assume degraded flow as this may be the first packet * without DSS, and the subflow state is not updated yet. */ - if (sbappendstream(&mp_so->so_rcv, m)) + if (sbappendstream_rcvdemux(mp_so, m, 0, 0)) sorwakeup(mp_so); + DTRACE_MPTCP5(receive__degraded, struct mbuf *, m, struct socket *, mp_so, struct sockbuf *, &mp_so->so_rcv, struct sockbuf *, &mp_so->so_snd, struct mptses *, mpte); count = mp_so->so_rcv.sb_cc - count; + + mp_tp->mpt_rcvnxt += count; + + if (mb_dfin) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); + socantrcvmore(mp_so); + } + mptcplog((LOG_DEBUG, "%s: Fallback read %d bytes\n", __func__, count), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); return; @@ -400,6 +421,7 @@ fallback: u_int64_t mb_dsn; int32_t mb_datalen; int64_t todrop; + int mb_dfin = 0; /* If fallback occurs, mbufs will not have PKTF_MPTCP set */ if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) @@ -450,14 +472,21 @@ fallback: m_adj(m, -todrop); mb_datalen -= todrop; } + + /* + * We drop from the right edge of the mbuf, thus the + * DATA_FIN is dropped as well + */ + m->m_pkthdr.pkt_flags &= ~PKTF_MPTCP_DFIN; } if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) || !LIST_EMPTY(&mp_tp->mpt_segq)) { - mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m); + mb_dfin = mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m); goto next; } + mb_dfin = !!(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN); if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) { if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen), @@ -485,7 +514,7 @@ fallback: mptcp_sbrcv_grow(mp_tp); - if (sbappendstream(&mp_so->so_rcv, m)) + if (sbappendstream_rcvdemux(mp_so, m, 0, 0)) wakeup = 1; DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so, @@ -502,6 +531,10 @@ fallback: mp_tp->mpt_rcvnxt += count; next: + if (mb_dfin) { + mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); + socantrcvmore(mp_so); + } m = save; prev = save = NULL; count = mp_so->so_rcv.sb_cc; @@ -1076,19 +1109,13 @@ mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp, mptcp_notify_mpfail(tp->t_inpcb->inp_socket); return; } - mptcplog((LOG_DEBUG, - "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__, - seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); + mptcplog((LOG_DEBUG, + "%s: seqn = %x len = %x full = %llx rcvnxt = %llu \n", __func__, + seqn, mdss_data_len, full_dsn, mp_tp->mpt_rcvnxt), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_VERBOSE); - /* Process a Data FIN packet , handled in mptcp_do_fin_opt */ - if ((seqn == 0) && (mdss_data_len == 1)) { - mptcplog((LOG_INFO, "%s: Data FIN in %s state \n", __func__, - mptcp_state_to_str(mp_tp->mpt_state)), - MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); - return; - } mptcp_notify_mpready(tp->t_inpcb->inp_socket); + tp->t_rcv_map.mpt_dsn = full_dsn; tp->t_rcv_map.mpt_sseq = seqn; tp->t_rcv_map.mpt_len = mdss_data_len; @@ -1123,9 +1150,10 @@ mptcp_validate_dss_map(struct socket *so, struct tcpcb *tp, struct mbuf *m, } int -mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) +mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, + int drop_hdrlen) { - mptcp_insert_rmap(tp, m); + mptcp_insert_rmap(tp, m, th); if (mptcp_validate_dss_map(tp->t_inpcb->inp_socket, tp, m, drop_hdrlen) != 0) return -1; @@ -1142,11 +1170,11 @@ mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, int drop_hdrlen) int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, - uint32_t sseq, uint16_t dlen, uint16_t csum) + uint32_t sseq, uint16_t dlen, uint16_t csum, uint16_t dfin) { uint16_t mptcp_csum; - mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum); + mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum, dfin); if (mptcp_csum) { tp->t_mpflags |= TMPF_SND_MPFAIL; mptcp_notify_mpfail(tp->t_inpcb->inp_socket); @@ -1159,9 +1187,10 @@ mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, static uint16_t mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq, - uint16_t dlen, uint16_t csum) + uint16_t dlen, uint16_t csum, uint16_t dfin) { struct mptcb *mp_tp = tptomptp(tp); + uint16_t real_len = dlen - dfin; uint32_t sum = 0; if (mp_tp == NULL) @@ -1177,11 +1206,12 @@ mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq, * The remote side may send a packet with fewer bytes than the * claimed DSS checksum length. */ - if ((int)m_length2(m, NULL) < dlen) + if ((int)m_length2(m, NULL) < real_len) { return (0xffff); + } - if (dlen != 0) - sum = m_sum16(m, 0, dlen); + if (real_len != 0) + sum = m_sum16(m, 0, real_len); sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum); ADDCARRY(sum); diff --git a/bsd/netinet/mptcp_opt.c b/bsd/netinet/mptcp_opt.c index 8aa8a9a4e..40007851a 100644 --- a/bsd/netinet/mptcp_opt.c +++ b/bsd/netinet/mptcp_opt.c @@ -1366,33 +1366,6 @@ mptcp_do_dss_opt_meat(u_char *cp, struct tcpcb *tp, struct tcphdr *th) } } -static void -mptcp_do_fin_opt(struct tcpcb *tp) -{ - struct mptcb *mp_tp = tptomptp(tp); - - if (!(tp->t_mpflags & TMPF_RECV_DFIN)) { - if (mp_tp != NULL) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - - if (tp->t_inpcb->inp_socket != NULL) { - soevent(tp->t_inpcb->inp_socket, - SO_FILT_HINT_LOCKED | - SO_FILT_HINT_MPCANTRCVMORE); - } - - } - tp->t_mpflags |= TMPF_RECV_DFIN; - } - - tp->t_mpflags |= TMPF_MPTCP_ACKNOW; - /* - * Since this is a data level FIN, TCP needs to be explicitly told - * to send back an ACK on which the Data ACK is piggybacked. - */ - tp->t_flags |= TF_ACKNOW; -} - static void mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) { @@ -1408,9 +1381,8 @@ mptcp_do_dss_opt(struct tcpcb *tp, u_char *cp, struct tcphdr *th, int optlen) struct mptcp_dss_copt *dss_rsp = (struct mptcp_dss_copt *)cp; if (dss_rsp->mdss_subtype == MPO_DSS) { - if (dss_rsp->mdss_flags & MDSS_F) { - mptcp_do_fin_opt(tp); - } + if (dss_rsp->mdss_flags & MDSS_F) + tp->t_rcv_map.mpt_dfin = 1; mptcp_do_dss_opt_meat(cp, tp, th); } diff --git a/bsd/netinet/mptcp_subr.c b/bsd/netinet/mptcp_subr.c index 19e128687..590fa2c46 100644 --- a/bsd/netinet/mptcp_subr.c +++ b/bsd/netinet/mptcp_subr.c @@ -1733,7 +1733,7 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, SBLASTMBUFCHK(&so->so_rcv, "mptcp_subflow_soreceive 1"); while (m != NULL) { - int dlen = 0; + int dlen = 0, dfin = 0, error_out = 0; struct mbuf *start = m; uint64_t dsn; uint32_t sseq; @@ -1749,7 +1749,7 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, csum = m->m_pkthdr.mp_csum; } else { /* We did fallback */ - mptcp_adj_rmap(so, m, 0); + mptcp_adj_rmap(so, m, 0, 0, 0, 0); sbfree(&so->so_rcv, m); @@ -1770,10 +1770,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, continue; } + if (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN) + dfin = 1; + /* * Check if the full mapping is now present */ - if ((int)so->so_rcv.sb_cc < dlen) { + if ((int)so->so_rcv.sb_cc < dlen - dfin) { mptcplog((LOG_INFO, "%s not enough data (%u) need %u\n", __func__, so->so_rcv.sb_cc, dlen), MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_LOG); @@ -1785,7 +1788,13 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, /* Now, get the full mapping */ while (dlen > 0) { - mptcp_adj_rmap(so, m, orig_dlen - dlen); + if (mptcp_adj_rmap(so, m, orig_dlen - dlen, dsn, sseq, orig_dlen)) { + error_out = 1; + error = EIO; + dlen = 0; + soevent(so, SO_FILT_HINT_LOCKED | SO_FILT_HINT_MUSTRST); + break; + } dlen -= m->m_len; sbfree(&so->so_rcv, m); @@ -1797,6 +1806,9 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, *mp = NULL; } + if (dlen - dfin == 0) + dlen = 0; + VERIFY(dlen <= 0 || m); } @@ -1808,7 +1820,11 @@ mptcp_subflow_soreceive(struct socket *so, struct sockaddr **psa, SB_EMPTY_FIXUP(&so->so_rcv); } - if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum)) { + if (error_out) + goto release; + + + if (mptcp_validate_csum(sototcpcb(so), start, dsn, sseq, orig_dlen, csum, dfin)) { error = EIO; *mp0 = NULL; goto release; @@ -4118,16 +4134,8 @@ mptcp_subflow_workloop(struct mptses *mpte) } if (mpsofilt_hint_mask != SO_FILT_HINT_LOCKED) { - struct mptcb *mp_tp = mpte->mpte_mptcb; - VERIFY(mpsofilt_hint_mask & SO_FILT_HINT_LOCKED); - if (mpsofilt_hint_mask & SO_FILT_HINT_CANTRCVMORE) { - mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN); - socantrcvmore(mp_so); - mpsofilt_hint_mask &= ~SO_FILT_HINT_CANTRCVMORE; - } - soevent(mp_so, mpsofilt_hint_mask); } @@ -4778,31 +4786,49 @@ mptcp_output_getm_dsnmap64(struct socket *so, int off, uint64_t *dsn, * with mptcp_adj_rmap() */ void -mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m) +mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th) { + VERIFY(m->m_flags & M_PKTHDR); VERIFY(!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)); if (tp->t_mpflags & TMPF_EMBED_DSN) { - VERIFY(m->m_flags & M_PKTHDR); m->m_pkthdr.mp_dsn = tp->t_rcv_map.mpt_dsn; m->m_pkthdr.mp_rseq = tp->t_rcv_map.mpt_sseq; m->m_pkthdr.mp_rlen = tp->t_rcv_map.mpt_len; m->m_pkthdr.mp_csum = tp->t_rcv_map.mpt_csum; + if (tp->t_rcv_map.mpt_dfin) + m->m_pkthdr.pkt_flags |= PKTF_MPTCP_DFIN; + m->m_pkthdr.pkt_flags |= PKTF_MPTCP; + tp->t_mpflags &= ~TMPF_EMBED_DSN; tp->t_mpflags |= TMPF_MPTCP_ACKNOW; + } else if (tp->t_mpflags & TMPF_TCP_FALLBACK) { + if (th->th_flags & TH_FIN) + m->m_pkthdr.pkt_flags |= PKTF_MPTCP_DFIN; } } -void -mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off) +int +mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off, uint64_t dsn, + uint32_t rseq, uint16_t dlen) { struct mptsub *mpts = sototcpcb(so)->t_mpsub; if (m_pktlen(m) == 0) - return; + return (0); if ((m->m_flags & M_PKTHDR) && (m->m_pkthdr.pkt_flags & PKTF_MPTCP)) { + if (off && (dsn != m->m_pkthdr.mp_dsn || + rseq != m->m_pkthdr.mp_rseq || + dlen != m->m_pkthdr.mp_rlen)) { + mptcplog((LOG_ERR, "%s: Received incorrect second mapping: %llu - %llu , %u - %u, %u - %u\n", + __func__, dsn, m->m_pkthdr.mp_dsn, + rseq, m->m_pkthdr.mp_rseq, + dlen, m->m_pkthdr.mp_rlen), + MPTCP_RECEIVER_DBG, MPTCP_LOGLVL_ERR); + return (-1); + } m->m_pkthdr.mp_dsn += off; m->m_pkthdr.mp_rseq += off; m->m_pkthdr.mp_rlen = m->m_pkthdr.len; @@ -4817,7 +4843,7 @@ mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off) mpts->mpts_flags |= MPTSF_CONFIRMED; - return; + return (0); } /* diff --git a/bsd/netinet/mptcp_var.h b/bsd/netinet/mptcp_var.h index 2d1f99061..1ff155f54 100644 --- a/bsd/netinet/mptcp_var.h +++ b/bsd/netinet/mptcp_var.h @@ -223,7 +223,7 @@ static inline int mptcp_subflow_cwnd_space(struct socket *so) { struct tcpcb *tp = sototcpcb(so); - int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (tp->snd_nxt - tp->snd_una); + int cwnd = min(tp->snd_wnd, tp->snd_cwnd) - (so->so_snd.sb_cc); return (min(cwnd, sbspace(&so->so_snd))); } @@ -653,7 +653,8 @@ extern void mptcp_unset_cellicon(void); extern void mptcp_reset_rexmit_state(struct tcpcb *tp); extern void mptcp_reset_keepalive(struct tcpcb *tp); extern int mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, - uint32_t sseq, uint16_t dlen, uint16_t csum); + uint32_t sseq, uint16_t dlen, uint16_t csum, + uint16_t dfin); __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c index 741aa00d5..139984932 100644 --- a/bsd/netinet/tcp_input.c +++ b/bsd/netinet/tcp_input.c @@ -2741,18 +2741,19 @@ findpcb: */ if (tp->t_state != TCPS_LISTEN && optp) { tcp_dooptions(tp, optp, optlen, th, &to); + } #if MPTCP - if (mptcp_input_preproc(tp, m, drop_hdrlen) != 0) { - tp->t_flags |= TF_ACKNOW; - (void) tcp_output(tp); - tcp_check_timer_state(tp); - socket_unlock(so, 1); - KERNEL_DEBUG(DBG_FNC_TCP_INPUT | - DBG_FUNC_END,0,0,0,0,0); - return; - } -#endif /* MPTCP */ + if (tp->t_state != TCPS_LISTEN && (so->so_flags & SOF_MP_SUBFLOW) && + mptcp_input_preproc(tp, m, th, drop_hdrlen) != 0) { + tp->t_flags |= TF_ACKNOW; + (void) tcp_output(tp); + tcp_check_timer_state(tp); + socket_unlock(so, 1); + KERNEL_DEBUG(DBG_FNC_TCP_INPUT | + DBG_FUNC_END,0,0,0,0,0); + return; } +#endif /* MPTCP */ if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { if (!(thflags & TH_ACK) || (SEQ_GT(th->th_ack, tp->iss) && @@ -4940,7 +4941,14 @@ dodata: } } else { - m_freem(m); + if ((so->so_flags & SOF_MP_SUBFLOW) && tlen == 0 && + (m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) { + m_adj(m, drop_hdrlen); /* delayed header drop */ + mptcp_input(tptomptp(tp)->mpt_mpte, m); + tp->t_flags |= TF_ACKNOW; + } else { + m_freem(m); + } thflags &= ~TH_FIN; } diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c index 1be5b6a80..2d6d4d1bf 100644 --- a/bsd/netinet/tcp_subr.c +++ b/bsd/netinet/tcp_subr.c @@ -401,8 +401,6 @@ get_tcp_str_size(void) return (sizeof(struct tcpcb)); } -int tcp_freeq(struct tcpcb *tp); - static int scale_to_powerof2(int size); /* diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c index f410382ed..2c861120e 100644 --- a/bsd/netinet/tcp_timer.c +++ b/bsd/netinet/tcp_timer.c @@ -1040,7 +1040,7 @@ retransmit_packet: (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) && !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) && ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) || - tp->t_rxtshift > 2)) { + tp->t_rxtshift > 4)) { /* * For regular retransmissions, a first one is being * done for tail-loss probe. @@ -1062,7 +1062,7 @@ retransmit_packet: if (!(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) && (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) && - tp->t_rxtshift > 1) { + tp->t_rxtshift > 3) { if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) { tcp_heuristic_tfo_middlebox(tp); diff --git a/bsd/netinet/tcp_usrreq.c b/bsd/netinet/tcp_usrreq.c index 0f0dc353e..9d8fa7efb 100644 --- a/bsd/netinet/tcp_usrreq.c +++ b/bsd/netinet/tcp_usrreq.c @@ -2719,6 +2719,9 @@ tcp_disconnect(struct tcpcb *tp) { struct socket *so = tp->t_inpcb->inp_socket; + if (so->so_rcv.sb_cc != 0 || tp->t_reassqlen != 0) + return tcp_drop(tp, 0); + if (tp->t_state < TCPS_ESTABLISHED) tp = tcp_close(tp); else if ((so->so_options & SO_LINGER) && so->so_linger == 0) diff --git a/bsd/netinet/tcp_var.h b/bsd/netinet/tcp_var.h index 4a5e1d3b4..4fde35c90 100644 --- a/bsd/netinet/tcp_var.h +++ b/bsd/netinet/tcp_var.h @@ -216,6 +216,7 @@ struct mpt_dsn_map { uint32_t mpt_sseq; /* relative subflow # */ uint16_t mpt_len; /* length of mapping */ uint16_t mpt_csum; /* checksum value if on */ + uint8_t mpt_dfin; /* It's a DATA_FIN */ }; #define tcp6cb tcpcb /* for KAME src sync over BSD*'s */ @@ -1634,6 +1635,7 @@ extern boolean_t tfo_enabled(const struct tcpcb *tp); extern void tcp_disable_tfo(struct tcpcb *tp); extern void tcp_tfo_gen_cookie(struct inpcb *inp, u_char *out, size_t blk_size); #define TCP_FASTOPEN_KEYLEN 16 +extern int tcp_freeq(struct tcpcb *tp); extern errno_t tcp_notify_ack_id_valid(struct tcpcb *, struct socket *, u_int32_t); extern errno_t tcp_add_notify_ack_marker(struct tcpcb *, u_int32_t); extern void tcp_notify_ack_free(struct tcpcb *); @@ -1648,11 +1650,12 @@ extern int get_tcp_inp_list(struct inpcb **, int, inp_gen_t); extern bool tcp_notify_ack_active(struct socket *so); #if MPTCP -extern int mptcp_input_preproc(struct tcpcb *, struct mbuf *, int); +extern int mptcp_input_preproc(struct tcpcb *tp, struct mbuf *m, + struct tcphdr *th, int drop_hdrlen); extern uint32_t mptcp_output_csum(struct mbuf *m, uint64_t dss_val, uint32_t sseq, uint16_t dlen); extern int mptcp_adj_mss(struct tcpcb *, boolean_t); -extern void mptcp_insert_rmap(struct tcpcb *, struct mbuf *); +extern void mptcp_insert_rmap(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th); #endif __private_extern__ void tcp_update_stats_per_flow( diff --git a/bsd/netinet6/icmp6.c b/bsd/netinet6/icmp6.c index 1580cae9d..f80a4154a 100644 --- a/bsd/netinet6/icmp6.c +++ b/bsd/netinet6/icmp6.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -1133,8 +1133,14 @@ icmp6_notify_error(struct mbuf *m, int off, int icmp6len, int code) ctlfunc = ip6_protox[nxt]->pr_ctlinput; if (ctlfunc) { + LCK_MTX_ASSERT(inet6_domain_mutex, LCK_MTX_ASSERT_OWNED); + + lck_mtx_unlock(inet6_domain_mutex); + (void) (*ctlfunc)(code, (struct sockaddr *)&icmp6dst, &ip6cp, m->m_pkthdr.rcvif); + + lck_mtx_lock(inet6_domain_mutex); } } return(0); diff --git a/bsd/netinet6/in6_proto.c b/bsd/netinet6/in6_proto.c index 34f8f754a..e71714e69 100644 --- a/bsd/netinet6/in6_proto.c +++ b/bsd/netinet6/in6_proto.c @@ -477,7 +477,7 @@ u_int32_t rip6_sendspace = RIPV6SNDQ; u_int32_t rip6_recvspace = RIPV6RCVQ; /* ICMPV6 parameters */ -int icmp6_rediraccept = 0; /* accept and process redirects */ +int icmp6_rediraccept = 1; /* accept and process redirects */ int icmp6_redirtimeout = 10 * 60; /* 10 minutes */ int icmp6errppslim = 500; /* 500 packets per second */ int icmp6rappslim = 10; /* 10 packets per second */ diff --git a/bsd/netinet6/ip6_output.c b/bsd/netinet6/ip6_output.c index 5e541f14c..73a66159e 100644 --- a/bsd/netinet6/ip6_output.c +++ b/bsd/netinet6/ip6_output.c @@ -1055,8 +1055,8 @@ skip_ipsec: exthdrs.ip6e_ip6 = m; ipsec_state.m = m; - route_copyout(&ipsec_state.ro, (struct route *)ro, - sizeof (ipsec_state.ro)); + route_copyout((struct route *)&ipsec_state.ro, (struct route *)ro, + sizeof (struct route_in6)); ipsec_state.dst = SA(dst); /* So that we can see packets inside the tunnel */ diff --git a/bsd/netinet6/ipsec.c b/bsd/netinet6/ipsec.c index 895d63e77..1ba068011 100644 --- a/bsd/netinet6/ipsec.c +++ b/bsd/netinet6/ipsec.c @@ -2498,7 +2498,7 @@ ipsec6_update_routecache_and_output( struct secasvar *sav) { struct sockaddr_in6* dst6; - struct route *ro6; + struct route_in6 *ro6; struct ip6_hdr *ip6; errno_t error = 0; @@ -2530,7 +2530,7 @@ ipsec6_update_routecache_and_output( dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof(*dst6); dst6->sin6_addr = ip6->ip6_dst; - rtalloc_scoped(ro6, sav->sah->outgoing_if); + rtalloc_scoped((struct route *)ro6, sav->sah->outgoing_if); if (ro6->ro_rt) { RT_LOCK(ro6->ro_rt); } @@ -2557,7 +2557,7 @@ ipsec6_update_routecache_and_output( dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway; RT_UNLOCK(ro6->ro_rt); ROUTE_RELEASE(&state->ro); - route_copyout(&state->ro, ro6, sizeof(state->ro)); + route_copyout((struct route *)&state->ro, (struct route *)ro6, sizeof(struct route_in6)); state->dst = (struct sockaddr *)dst6; state->tunneled = 6; // release sadb_mutex, after updating sah's route cache @@ -3162,7 +3162,7 @@ ipsec4_output_internal(struct ipsec_output_state *state, struct secasvar *sav) // grab sadb_mutex, before updating sah's route cache lck_mtx_lock(sadb_mutex); - ro4= &sav->sah->sa_route; + ro4= (struct route *)&sav->sah->sa_route; dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst; if (ro4->ro_rt != NULL) { RT_LOCK(ro4->ro_rt); @@ -3201,7 +3201,7 @@ ipsec4_output_internal(struct ipsec_output_state *state, struct secasvar *sav) dst4 = (struct sockaddr_in *)(void *)ro4->ro_rt->rt_gateway; RT_UNLOCK(ro4->ro_rt); ROUTE_RELEASE(&state->ro); - route_copyout(&state->ro, ro4, sizeof(state->ro)); + route_copyout((struct route *)&state->ro, ro4, sizeof(struct route)); state->dst = (struct sockaddr *)dst4; state->tunneled = 4; // release sadb_mutex, after updating sah's route cache @@ -3644,7 +3644,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar int error = 0; int plen; struct sockaddr_in6* dst6; - struct route *ro6; + struct route_in6 *ro6; /* validity check */ if (sav == NULL || sav->sah == NULL || sav->sah->saidx.mode != IPSEC_MODE_TUNNEL) { @@ -3706,7 +3706,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar // grab sadb_mutex, to update sah's route cache and get a local copy of it lck_mtx_lock(sadb_mutex); - ro4 = &sav->sah->sa_route; + ro4 = (struct route *)&sav->sah->sa_route; dst4 = (struct sockaddr_in *)(void *)&ro4->ro_dst; if (ro4->ro_rt) { RT_LOCK(ro4->ro_rt); @@ -3724,7 +3724,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar } else { RT_UNLOCK(ro4->ro_rt); } - route_copyout(&ro4_copy, ro4, sizeof(ro4_copy)); + route_copyout(&ro4_copy, ro4, sizeof(struct route)); // release sadb_mutex, after updating sah's route cache and getting a local copy lck_mtx_unlock(sadb_mutex); state->m = ipsec4_splithdr(state->m); @@ -3789,7 +3789,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar state->m = NULL; // grab sadb_mutex, to synchronize the sah's route cache with the local copy lck_mtx_lock(sadb_mutex); - route_copyin(&ro4_copy, ro4, sizeof(ro4_copy)); + route_copyin(&ro4_copy, ro4, sizeof(struct route)); lck_mtx_unlock(sadb_mutex); if (error != 0) goto bad; @@ -3821,7 +3821,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar dst6->sin6_family = AF_INET6; dst6->sin6_len = sizeof(*dst6); dst6->sin6_addr = ip6->ip6_dst; - rtalloc_scoped(ro6, sav->sah->outgoing_if); + rtalloc_scoped((struct route *)ro6, sav->sah->outgoing_if); if (ro6->ro_rt) { RT_LOCK(ro6->ro_rt); } @@ -3848,7 +3848,7 @@ ipsec6_output_tunnel_internal(struct ipsec_output_state *state, struct secasvar dst6 = (struct sockaddr_in6 *)(void *)ro6->ro_rt->rt_gateway; RT_UNLOCK(ro6->ro_rt); ROUTE_RELEASE(&state->ro); - route_copyout(&state->ro, ro6, sizeof(state->ro)); + route_copyout((struct route *)&state->ro, (struct route *)ro6, sizeof(struct route_in6)); state->dst = (struct sockaddr *)dst6; state->tunneled = 6; // release sadb_mutex, after updating sah's route cache @@ -4743,7 +4743,7 @@ ipsec_send_natt_keepalive( rt_key(sav->sah->sa_route.ro_rt)->sa_family != AF_INET) ROUTE_RELEASE(&sav->sah->sa_route); - route_copyout(&ro, &sav->sah->sa_route, sizeof(ro)); + route_copyout(&ro, (struct route *)&sav->sah->sa_route, sizeof(struct route)); lck_mtx_unlock(sadb_mutex); necp_mark_packet_as_keepalive(m, TRUE); @@ -4752,7 +4752,7 @@ ipsec_send_natt_keepalive( // grab sadb_mutex, to synchronize the sah's route cache with the local copy lck_mtx_lock(sadb_mutex); - route_copyin(&ro, &sav->sah->sa_route, sizeof(ro)); + route_copyin(&ro, (struct route *)&sav->sah->sa_route, sizeof(struct route)); lck_mtx_unlock(sadb_mutex); if (error == 0) { sav->natt_last_activity = natt_now; diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h index 8d0e6dd51..e01608247 100644 --- a/bsd/netinet6/ipsec.h +++ b/bsd/netinet6/ipsec.h @@ -301,7 +301,7 @@ struct ipsecstat { struct ipsec_output_state { int tunneled; struct mbuf *m; - struct route ro; + struct route_in6 ro; struct sockaddr *dst; u_int outgoing_if; }; diff --git a/bsd/netinet6/nd6.c b/bsd/netinet6/nd6.c index 0a16cdb14..495a3b304 100644 --- a/bsd/netinet6/nd6.c +++ b/bsd/netinet6/nd6.c @@ -1310,9 +1310,13 @@ addrloop: /* * Only enqueue the Deprecated event when the address just - * becomes deprecated + * becomes deprecated. + * Keep it limited to the stable address it is common for + * older temporary addresses to get deprecated while we generate + * new ones. */ - if((oldflags & IN6_IFF_DEPRECATED) == 0) { + if((oldflags & IN6_IFF_DEPRECATED) == 0 && + (ia6->ia6_flags & IN6_IFF_TEMPORARY) == 0) { in6_event_enqueue_nwk_wq_entry(IN6_ADDR_MARKED_DEPRECATED, ia6->ia_ifa.ifa_ifp, &ia6->ia_addr.sin6_addr, 0); diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c index d8e8a1b39..a7935a5f2 100644 --- a/bsd/netkey/key.c +++ b/bsd/netkey/key.c @@ -9920,7 +9920,7 @@ key_sa_routechange( lck_mtx_lock(sadb_mutex); LIST_FOREACH(sah, &sahtree, chain) { - ro = &sah->sa_route; + ro = (struct route *)&sah->sa_route; if (ro->ro_rt && dst->sa_len == ro->ro_dst.sa_len && bcmp(dst, &ro->ro_dst, dst->sa_len) == 0) { ROUTE_RELEASE(ro); diff --git a/bsd/netkey/keydb.h b/bsd/netkey/keydb.h index c62dc964c..8e053fbff 100644 --- a/bsd/netkey/keydb.h +++ b/bsd/netkey/keydb.h @@ -67,7 +67,7 @@ struct secashead { /* SA chain */ /* The first of this list is newer SA */ - struct route sa_route; /* route cache */ + struct route_in6 sa_route; /* route cache */ }; /* Security Association */ diff --git a/bsd/nfs/krpc_subr.c b/bsd/nfs/krpc_subr.c index 53355a6e4..20fa08bd8 100644 --- a/bsd/nfs/krpc_subr.c +++ b/bsd/nfs/krpc_subr.c @@ -189,9 +189,12 @@ krpc_portmap( return error; rdata = mbuf_data(m); - *portp = rdata->port; - if (!rdata->port) + if (mbuf_len(m) >= sizeof(*rdata)) { + *portp = rdata->port; + } + + if (mbuf_len(m) < sizeof(*rdata) || !rdata->port) error = EPROGUNAVAIL; mbuf_freem(m); diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h index 9cacf96c5..2277221c4 100644 --- a/bsd/nfs/nfs.h +++ b/bsd/nfs/nfs.h @@ -1127,6 +1127,9 @@ struct nfsrv_descript { extern TAILQ_HEAD(nfsd_head, nfsd) nfsd_head, nfsd_queue; +typedef int (*nfsrv_proc_t)(struct nfsrv_descript *, struct nfsrv_sock *, + vfs_context_t, mbuf_t *); + /* mutex for nfs server */ extern lck_mtx_t *nfsd_mutex; extern int nfsd_thread_count, nfsd_thread_max; diff --git a/bsd/nfs/nfs_serv.c b/bsd/nfs/nfs_serv.c index 7a24596c8..027d7a5d8 100644 --- a/bsd/nfs/nfs_serv.c +++ b/bsd/nfs/nfs_serv.c @@ -4878,10 +4878,7 @@ nfsmout: return (error); } -int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd, - struct nfsrv_sock *slp, - vfs_context_t ctx, - mbuf_t *mrepp) = { +const nfsrv_proc_t nfsrv_procs[NFS_NPROCS] = { nfsrv_null, nfsrv_getattr, nfsrv_setattr, diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c index 2a20ee2ea..2e43de4b1 100644 --- a/bsd/nfs/nfs_syscalls.c +++ b/bsd/nfs/nfs_syscalls.c @@ -123,10 +123,8 @@ kern_return_t thread_terminate(thread_t); /* XXX */ #if NFSSERVER -extern int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd, - struct nfsrv_sock *slp, - vfs_context_t ctx, - mbuf_t *mrepp); +extern const nfsrv_proc_t nfsrv_procs[NFS_NPROCS]; + extern int nfsrv_wg_delay; extern int nfsrv_wg_delay_v3; diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c index 3c5f2831a..67b9befc0 100644 --- a/bsd/nfs/nfs_vfsops.c +++ b/bsd/nfs/nfs_vfsops.c @@ -204,7 +204,7 @@ int nfs_vfs_fhtovp(mount_t, int, unsigned char *, vnode_t *, vfs_context_t); int nfs_vfs_init(struct vfsconf *); int nfs_vfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t); -struct vfsops nfs_vfsops = { +const struct vfsops nfs_vfsops = { .vfs_mount = nfs_vfs_mount, .vfs_start = nfs_vfs_start, .vfs_unmount = nfs_vfs_unmount, @@ -237,7 +237,7 @@ int nfs3_getquota(struct nfsmount *, vfs_context_t, uid_t, int, struct dqblk *); int nfs4_getquota(struct nfsmount *, vfs_context_t, uid_t, int, struct dqblk *); #endif -struct nfs_funcs nfs3_funcs = { +const struct nfs_funcs nfs3_funcs = { nfs3_mount, nfs3_update_statfs, nfs3_getquota, @@ -258,7 +258,7 @@ struct nfs_funcs nfs3_funcs = { nfs3_unlock_rpc, nfs3_getlock_rpc }; -struct nfs_funcs nfs4_funcs = { +const struct nfs_funcs nfs4_funcs = { nfs4_mount, nfs4_update_statfs, nfs4_getquota, diff --git a/bsd/nfs/nfsm_subs.h b/bsd/nfs/nfsm_subs.h index 69d0f7865..50f72c5fc 100644 --- a/bsd/nfs/nfsm_subs.h +++ b/bsd/nfs/nfsm_subs.h @@ -144,6 +144,8 @@ int nfsm_chain_trim_data(struct nfsm_chain *, int, int *); VATTR_WANTED((VAP), va_change_time); \ if ((VERS) == NFS_VER2) \ VATTR_WANTED((VAP), va_iosize); \ + if ((VERS) == NFS_VER3) \ + VATTR_WANTED((VAP), va_filerev); \ } while (0) /* Initialize a vnode_attr to retrieve pre-operation attributes for the NFS server. */ diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h index 78aa778ad..90b75a548 100644 --- a/bsd/nfs/nfsmount.h +++ b/bsd/nfs/nfsmount.h @@ -267,7 +267,7 @@ struct nfsmount { uint32_t nm_minor_vers; /* minor version of above */ uint32_t nm_min_vers; /* minimum packed version to try */ uint32_t nm_max_vers; /* maximum packed version to try */ - struct nfs_funcs *nm_funcs; /* version-specific functions */ + const struct nfs_funcs *nm_funcs;/* version-specific functions */ kauth_cred_t nm_mcred; /* credential used for the mount */ mount_t nm_mountp; /* VFS structure for this filesystem */ nfsnode_t nm_dnp; /* root directory nfsnode pointer */ diff --git a/bsd/sys/event.h b/bsd/sys/event.h index f10e15777..04385bc6c 100644 --- a/bsd/sys/event.h +++ b/bsd/sys/event.h @@ -249,8 +249,9 @@ typedef uint64_t kqueue_id_t; #define EVFILTID_FSEVENT (EVFILT_SYSCOUNT + 13) #define EVFILTID_VN (EVFILT_SYSCOUNT + 14) #define EVFILTID_TTY (EVFILT_SYSCOUNT + 16) +#define EVFILTID_PTMX (EVFILT_SYSCOUNT + 17) -#define EVFILTID_MAX (EVFILT_SYSCOUNT + 17) +#define EVFILTID_MAX (EVFILT_SYSCOUNT + 18) #endif /* defined(XNU_KERNEL_PRIVATE) */ diff --git a/bsd/sys/kern_control.h b/bsd/sys/kern_control.h index 8e1b514bd..51324ea65 100644 --- a/bsd/sys/kern_control.h +++ b/bsd/sys/kern_control.h @@ -417,6 +417,31 @@ typedef void (*ctl_rcvd_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitin */ typedef errno_t (*ctl_send_list_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo, mbuf_t m, int flags); + +/*! + @typedef ctl_bind_func + @discussion The ctl_bind_func is an optional function that allows the client + to set up their unitinfo prior to connecting. + @param kctlref The control ref for the kernel control the client is + binding to. + @param sac The address used to connect to this control. The field sc_unit + contains the unit number of the kernel control instance the client is + binding to. If CTL_FLAG_REG_ID_UNIT was set when the kernel control + was registered, sc_unit is the ctl_unit of the kern_ctl_reg structure. + If CTL_FLAG_REG_ID_UNIT was not set when the kernel control was + registered, sc_unit is the dynamically allocated unit number of + the new kernel control instance that is used for this connection. + @param unitinfo A placeholder for a pointer to the optional user-defined + private data associated with this kernel control instance. This + opaque info will be provided to the user when the rest of the + callback routines are executed. For example, it can be used + to pass a pointer to an instance-specific data structure in + order for the user to keep track of the states related to this + kernel control instance. + */ +typedef errno_t (*ctl_bind_func)(kern_ctl_ref kctlref, + struct sockaddr_ctl *sac, + void **unitinfo); #endif /* KERNEL_PRIVATE */ /*! @@ -474,6 +499,7 @@ struct kern_ctl_reg #ifdef KERNEL_PRIVATE ctl_rcvd_func ctl_rcvd; /* Only valid if CTL_FLAG_REG_EXTENDED is set */ ctl_send_list_func ctl_send_list; /* Only valid if CTL_FLAG_REG_EXTENDED is set */ + ctl_bind_func ctl_bind; #endif /* KERNEL_PRIVATE */ }; diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h index 828f9f44d..3cac39def 100644 --- a/bsd/sys/mbuf.h +++ b/bsd/sys/mbuf.h @@ -504,6 +504,7 @@ struct pkthdr { #define PKTF_START_SEQ 0x8000000 /* valid start sequence */ #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ #define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ +#define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */ /* flags related to flow control/advisory and identification */ #define PKTF_FLOW_MASK \ diff --git a/bsd/sys/reason.h b/bsd/sys/reason.h index 2d81e3a41..a7dc43f69 100644 --- a/bsd/sys/reason.h +++ b/bsd/sys/reason.h @@ -42,7 +42,7 @@ __BEGIN_DECLS typedef struct os_reason { decl_lck_mtx_data(, osr_lock) - int osr_refcount; + unsigned int osr_refcount; uint32_t osr_namespace; uint64_t osr_code; uint64_t osr_flags; @@ -104,11 +104,12 @@ void os_reason_free(os_reason_t cur_reason); #define OS_REASON_FOUNDATION 19 #define OS_REASON_WATCHDOG 20 #define OS_REASON_METAL 21 +#define OS_REASON_WATCHKIT 22 /* * Update whenever new OS_REASON namespaces are added. */ -#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_METAL +#define OS_REASON_MAX_VALID_NAMESPACE OS_REASON_WATCHKIT #define OS_REASON_BUFFER_MAX_SIZE 5120 diff --git a/bsd/sys/reboot.h b/bsd/sys/reboot.h index 98db0e3d7..717ce89dc 100644 --- a/bsd/sys/reboot.h +++ b/bsd/sys/reboot.h @@ -92,6 +92,11 @@ __BEGIN_DECLS /* userspace reboot control */ int usrctl(uint32_t flags); +/* The normal reboot syscall. */ +int reboot(int howto); +/* Used with RB_PANIC to panic the kernel from userspace with a message. + * Requires an entitlement on Release. */ +int reboot_np(int howto, const char *message); __END_DECLS #endif diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h index a8faa3c64..17e1e2c35 100644 --- a/bsd/sys/socketvar.h +++ b/bsd/sys/socketvar.h @@ -980,7 +980,8 @@ extern void mptcp_fallback_sbdrop(struct socket *so, struct mbuf *m, int len); extern void mptcp_preproc_sbdrop(struct socket *, struct mbuf *, unsigned int); extern void mptcp_postproc_sbdrop(struct mbuf *, u_int64_t, u_int32_t, u_int32_t); -extern void mptcp_adj_rmap(struct socket *, struct mbuf *, int); +extern int mptcp_adj_rmap(struct socket *so, struct mbuf *m, int off, + uint64_t dsn, uint32_t rseq, uint16_t dlen); extern void netpolicy_post_msg(uint32_t, struct netpolicy_event_data *, uint32_t); diff --git a/bsd/sys/spawn_internal.h b/bsd/sys/spawn_internal.h index 9985b3cfc..29ea49d3b 100644 --- a/bsd/sys/spawn_internal.h +++ b/bsd/sys/spawn_internal.h @@ -48,6 +48,25 @@ #include #include #include /* COALITION_NUM_TYPES */ +#include + +/* + * Safely compute the size in bytes of a structure, '_type', whose last + * element, '_member', is a zero-sized array meant to hold 'x' bytes. + * + * If the size calculation overflows a size_t value, this macro returns 0. + */ +#define PS_ACTION_SIZE(x,_type,_member_type) ({ \ + size_t _ps_count = (size_t)x; \ + size_t _ps_size = 0; \ + /* (count * sizeof(_member_type)) + sizeof(_type) */ \ + if (os_mul_and_add_overflow(_ps_count, \ + sizeof(_member_type), \ + sizeof(_type), \ + &_ps_size)) { \ + _ps_size = 0; \ + } \ + _ps_size; }) /* * Allowable posix_spawn() port action types @@ -86,7 +105,7 @@ typedef struct _posix_spawn_port_actions { * Returns size in bytes of a _posix_spawn_port_actions holding x elements. */ #define PS_PORT_ACTIONS_SIZE(x) \ - __offsetof(struct _posix_spawn_port_actions, pspa_actions[(x)]) + PS_ACTION_SIZE(x, struct _posix_spawn_port_actions, _ps_port_action_t) #define NBINPREFS 4 @@ -115,7 +134,7 @@ typedef struct _posix_spawn_mac_policy_extensions { * Returns size in bytes of a _posix_spawn_mac_policy_extensions holding x elements. */ #define PS_MAC_EXTENSIONS_SIZE(x) \ - __offsetof(struct _posix_spawn_mac_policy_extensions, psmx_extensions[(x)]) + PS_ACTION_SIZE(x, struct _posix_spawn_mac_policy_extensions, _ps_mac_policy_extension_t) #define PS_MAC_EXTENSIONS_INIT_COUNT 2 @@ -334,7 +353,7 @@ typedef struct _posix_spawn_file_actions { * capable of containing. */ #define PSF_ACTIONS_SIZE(x) \ - __offsetof(struct _posix_spawn_file_actions, psfa_act_acts[(x)]) + PS_ACTION_SIZE(x, struct _posix_spawn_file_actions, _psfa_action_t) /* * Initial count of actions in a struct _posix_spawn_file_actions after it is diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c index 4aab3ac01..f5710e862 100644 --- a/bsd/vfs/vfs_attrlist.c +++ b/bsd/vfs/vfs_attrlist.c @@ -340,7 +340,7 @@ static struct getvolattrlist_attrtab getvolattrlist_common_tab[] = { static struct getvolattrlist_attrtab getvolattrlist_vol_tab[] = { {ATTR_VOL_FSTYPE, 0, sizeof(uint32_t)}, {ATTR_VOL_SIGNATURE, VFSATTR_BIT(f_signature), sizeof(uint32_t)}, - {ATTR_VOL_SIZE, VFSATTR_BIT(f_blocks), sizeof(off_t)}, + {ATTR_VOL_SIZE, VFSATTR_BIT(f_blocks) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, {ATTR_VOL_SPACEFREE, VFSATTR_BIT(f_bfree) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, {ATTR_VOL_SPACEAVAIL, VFSATTR_BIT(f_bavail) | VFSATTR_BIT(f_bsize), sizeof(off_t)}, {ATTR_VOL_MINALLOCATION, VFSATTR_BIT(f_bsize), sizeof(off_t)}, diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c index 788314616..2571c94d5 100644 --- a/bsd/vfs/vfs_syscalls.c +++ b/bsd/vfs/vfs_syscalls.c @@ -11993,9 +11993,11 @@ fs_snapshot(__unused proc_t p, struct fs_snapshot_args *uap, case SNAPSHOT_OP_REVERT: error = snapshot_revert(uap->dirfd, uap->name1, uap->flags, ctx); break; +#if !TARGET_OS_OSX case SNAPSHOT_OP_ROOT: error = snapshot_root(uap->dirfd, uap->name1, uap->flags, ctx); break; +#endif /* !TARGET_OS_OSX */ default: error = ENOSYS; } diff --git a/config/MasterVersion b/config/MasterVersion index 461cd787f..db6ac0645 100644 --- a/config/MasterVersion +++ b/config/MasterVersion @@ -1,4 +1,4 @@ -17.2.0 +17.3.0 # The first line of this file contains the master version number for the kernel. # All other instances of the kernel version in xnu are derived from this file. diff --git a/config/Private.arm64.exports b/config/Private.arm64.exports index ab9007317..3ac767939 100644 --- a/config/Private.arm64.exports +++ b/config/Private.arm64.exports @@ -4,6 +4,10 @@ _PE_consistent_debug_register _PE_get_default _PE_reboot_on_panic _PE_mark_hwaccess +_PE_smc_stashed_x86_system_state +_PE_smc_stashed_x86_power_state +_PE_smc_stashed_x86_efi_boot_state +_PE_pcie_stashed_link_state __ZN17IONVRAMController* __ZTV17IONVRAMController _cpu_broadcast_xcall diff --git a/config/Unsupported.arm.exports b/config/Unsupported.arm.exports index 6f33928e1..ee0e8db30 100644 --- a/config/Unsupported.arm.exports +++ b/config/Unsupported.arm.exports @@ -22,3 +22,5 @@ _ml_get_cpu_number _ml_get_max_cpu_number _ml_dbgwrap_halt_cpu_with_state _vm_map:_vm_map_external +__ZN5IORTC15getUTCTimeOfDayEPjS0_ +__ZN5IORTC15setUTCTimeOfDayEjj diff --git a/config/Unsupported.arm64.exports b/config/Unsupported.arm64.exports index 38e1a8c0e..883f24480 100644 --- a/config/Unsupported.arm64.exports +++ b/config/Unsupported.arm64.exports @@ -37,4 +37,5 @@ _ml_get_max_cpu_number _ml_lockdown_handler_register _ml_dbgwrap_halt_cpu_with_state _vm_map:_vm_map_external - +__ZN5IORTC15getUTCTimeOfDayEPmPj +__ZN5IORTC15setUTCTimeOfDayEmj diff --git a/config/Unsupported.exports b/config/Unsupported.exports index 9840aa96c..7938c5da9 100644 --- a/config/Unsupported.exports +++ b/config/Unsupported.exports @@ -188,4 +188,19 @@ _vnop_kqfilt_add_desc _vnop_kqfilt_remove_desc _vnop_makenamedstream_desc _vnop_removenamedstream_desc - +__ZN5IORTC10gMetaClassE +__ZN5IORTC10superClassE +__ZN5IORTC23getMonotonicClockOffsetEPx +__ZN5IORTC23setMonotonicClockOffsetEx +__ZN5IORTC29getMonotonicClockAndTimestampEPyS0_ +__ZN5IORTC9MetaClassC1Ev +__ZN5IORTC9MetaClassC2Ev +__ZN5IORTC9metaClassE +__ZN5IORTCC2EPK11OSMetaClass +__ZN5IORTCD0Ev +__ZN5IORTCD1Ev +__ZN5IORTCD2Ev +__ZNK5IORTC12getMetaClassEv +__ZNK5IORTC9MetaClass5allocEv +__ZTV5IORTC +__ZTVN5IORTC9MetaClassE diff --git a/config/Unsupported.x86_64.exports b/config/Unsupported.x86_64.exports index 57c5bb71e..c2af9c454 100644 --- a/config/Unsupported.x86_64.exports +++ b/config/Unsupported.x86_64.exports @@ -44,4 +44,13 @@ _tmrCvt _tsc_get_info _PE_state _vm_map - +__ZN5IORTC15_RESERVEDIORTC0Ev +__ZN5IORTC15_RESERVEDIORTC1Ev +__ZN5IORTC15_RESERVEDIORTC2Ev +__ZN5IORTC15_RESERVEDIORTC3Ev +__ZN5IORTC15_RESERVEDIORTC4Ev +__ZN5IORTC15_RESERVEDIORTC5Ev +__ZN5IORTC15_RESERVEDIORTC6Ev +__ZN5IORTC15_RESERVEDIORTC7Ev +__ZN5IORTC15getUTCTimeOfDayEPmPj +__ZN5IORTC15setUTCTimeOfDayEmj diff --git a/iokit/IOKit/IOCPU.h b/iokit/IOKit/IOCPU.h index 25d2398de..1ee373c3e 100644 --- a/iokit/IOKit/IOCPU.h +++ b/iokit/IOKit/IOCPU.h @@ -110,7 +110,7 @@ extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void); extern "C" kern_return_t IOCPURunPlatformActiveActions(void); extern "C" kern_return_t IOCPURunPlatformHaltRestartActions(uint32_t message); extern "C" kern_return_t IOCPURunPlatformPanicActions(uint32_t message); -extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, size_t len); +extern "C" kern_return_t IOCPURunPlatformPanicSyncAction(void *addr, uint32_t offset, uint32_t len); class IOCPUInterruptController : public IOInterruptController { diff --git a/iokit/IOKit/IOLocks.h b/iokit/IOKit/IOLocks.h index 7b42679f1..75d327b7f 100644 --- a/iokit/IOKit/IOLocks.h +++ b/iokit/IOKit/IOLocks.h @@ -140,6 +140,28 @@ int IOLockSleepDeadline( IOLock * lock, void *event, void IOLockWakeup(IOLock * lock, void *event, bool oneThread) __DARWIN14_ALIAS(IOLockWakeup); +#ifdef XNU_KERNEL_PRIVATE +/*! @enum IOLockAssertState + * @abstract Used with IOLockAssert to assert the state of a lock. + */ +typedef enum { + kIOLockAssertOwned = LCK_ASSERT_OWNED, + kIOLockAssertNotOwned = LCK_ASSERT_NOTOWNED +} IOLockAssertState; + +#ifdef IOLOCKS_INLINE +#define IOLockAssert(l, type) LCK_MTX_ASSERT(l, type) +#else +/*! @function IOLockAssert + * @abstract Assert that lock is either held or not held by current thread. + * @discussion Call with either kIOLockAssertOwned or kIOLockAssertNotOwned. + * Panics the kernel if the lock is not owned if called with kIOLockAssertOwned, + * and vice-versa. + */ +void IOLockAssert(IOLock * lock, IOLockAssertState type); +#endif /* !IOLOCKS_INLINE */ +#endif /* !XNU_KERNEL_PRIVATE */ + #ifdef __APPLE_API_OBSOLETE /* The following API is deprecated */ @@ -286,6 +308,30 @@ void IORWLockWrite( IORWLock * lock); void IORWLockUnlock( IORWLock * lock); #endif /* !IOLOCKS_INLINE */ +#ifdef XNU_KERNEL_PRIVATE +/*! @enum IORWLockAssertState + * @abstract Used with IORWLockAssert to assert the state of a lock. + */ +typedef enum { + kIORWLockAssertRead = LCK_RW_ASSERT_SHARED, + kIORWLockAssertWrite = LCK_RW_ASSERT_EXCLUSIVE, + kIORWLockAssertHeld = LCK_RW_ASSERT_HELD, + kIORWLockAssertNotHeld = LCK_RW_ASSERT_NOTHELD +} IORWLockAssertState; + +#ifdef IOLOCKS_INLINE +#define IORWLockAssert(l, type) LCK_RW_ASSERT(l, type) +#else +/*! @function IORWLockAssert + * @abstract Assert that a reader-writer lock is either held or not held + * by the current thread. + * @discussion Call with a value defined by the IORWLockAssertState type. + * If the specified lock is not in the state specified by the type argument, + * then the kernel will panic. + */ +void IORWLockAssert(IORWLock * lock, IORWLockAssertState type); +#endif /* !IOLOCKS_INLINE */ +#endif /* !XNU_KERNEL_PRIVATE */ #ifdef __APPLE_API_OBSOLETE @@ -371,6 +417,28 @@ boolean_t IOSimpleLockTryLock( IOSimpleLock * lock ); void IOSimpleLockUnlock( IOSimpleLock * lock ); #endif /* !IOLOCKS_INLINE */ +#ifdef XNU_KERNEL_PRIVATE +/*! @enum IOSimpleLockAssertState + * @abstract Used with IOSimpleLockAssert to assert the state of a lock. + */ +typedef enum { + kIOSimpleLockAssertOwned = LCK_ASSERT_OWNED, + kIOSimpleLockAssertNotOwned = LCK_ASSERT_NOTOWNED +} IOSimpleLockAssertState; + +#ifdef IOLOCKS_INLINE +#define IOSimpleLockAssert(l, type) LCK_SPIN_ASSERT(l, type) +#else +/*! @function IOSimpleLockAssert + * @abstract Assert that spinlock is either held or not held by current thread. + * @discussion Call with either kIOSimpleLockAssertOwned or kIOSimpleLockAssertNotOwned. + * Panics the kernel if the lock is not owned if called with + * kIOSimpleLockAssertOwned and vice-versa. + */ +void IOSimpleLockAssert(IOSimpleLock *lock, IOSimpleLockAssertState type); +#endif /* !IOLOCKS_INLINE */ +#endif /* !XNU_KERNEL_PRIVATE */ + #if __LP64__ typedef boolean_t IOInterruptState; #else diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h index f66f8da1f..63029bb1d 100644 --- a/iokit/IOKit/IOPlatformExpert.h +++ b/iokit/IOKit/IOPlatformExpert.h @@ -47,6 +47,13 @@ extern "C" { #include +typedef enum { + kCoprocessorVersionNone = 0x00000000, + kCoprocessorVersion1 = 0x00010000, + kCoprocessorVersion2 = 0x00020000, +} coprocessor_type_t; + + extern boolean_t PEGetMachineName( char * name, int maxLength ); extern boolean_t PEGetModelName( char * name, int maxLength ); extern int PEGetPlatformEpoch( void ); @@ -68,7 +75,13 @@ extern int PEHaltRestart(unsigned int type); // Save the Panic Info. Returns the number of bytes saved. extern UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length); -extern void PESavePanicInfoAction(void *buffer, size_t length); +extern void PESavePanicInfoAction(void *buffer, UInt32 offset, UInt32 length); + +/* + * SMC requires that all data is flushed in multiples of 16 bytes at 16 byte + * boundaries. + */ +#define PANIC_FLUSH_BOUNDARY 16 extern long PEGetGMTTimeOfDay( void ); extern void PESetGMTTimeOfDay( long secs ); @@ -84,12 +97,16 @@ extern boolean_t PEReadNVRAMProperty(const char *symbol, void *value, unsigned i extern boolean_t PERemoveNVRAMProperty(const char *symbol); +extern coprocessor_type_t PEGetCoprocessorVersion( void ); + #ifdef __cplusplus } /* extern "C" */ #define kIOPlatformMapperPresentKey "IOPlatformMapperPresent" + + extern OSSymbol * gPlatformInterruptControllerName; extern const OSSymbol * gIOPlatformSleepActionKey; diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h index 048020784..504d6f221 100644 --- a/iokit/IOKit/IOReturn.h +++ b/iokit/IOKit/IOReturn.h @@ -74,12 +74,12 @@ typedef kern_return_t IOReturn; #endif #define sub_iokit_platform err_sub(0x2A) #define sub_iokit_audio_video err_sub(0x45) +#define sub_iokit_cec err_sub(0x46) #define sub_iokit_baseband err_sub(0x80) -#define sub_iokit_HDA err_sub(254) +#define sub_iokit_HDA err_sub(0xFE) #define sub_iokit_hsic err_sub(0x147) #define sub_iokit_sdio err_sub(0x174) #define sub_iokit_wlan err_sub(0x208) - #define sub_iokit_appleembeddedsleepwakehandler err_sub(0x209) #define sub_iokit_vendor_specific err_sub(-2) diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h index b45162f87..89ce8909d 100644 --- a/iokit/IOKit/IOService.h +++ b/iokit/IOKit/IOService.h @@ -1399,7 +1399,9 @@ private: bool terminatePhase1( IOOptionBits options = 0 ); void scheduleTerminatePhase2( IOOptionBits options = 0 ); void scheduleStop( IOService * provider ); - static void terminateThread( void * arg, wait_result_t unused ); + + static void waitToBecomeTerminateThread( void ); + static void __attribute__((__noreturn__)) terminateThread( void * arg, wait_result_t unused ); static void terminateWorker( IOOptionBits options ); static void actionWillTerminate( IOService * victim, IOOptionBits options, OSArray * doPhase2List, void*, void * ); diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h index f9ca6ed8a..3fc00ebae 100644 --- a/iokit/IOKit/pwr_mgt/IOPM.h +++ b/iokit/IOKit/pwr_mgt/IOPM.h @@ -613,6 +613,8 @@ enum { #define kIOPMPSAdapterDetailsDescriptionKey "Description" #define kIOPMPSAdapterDetailsPMUConfigurationKey "PMUConfiguration" #define kIOPMPSAdapterDetailsVoltage "AdapterVoltage" +#define kIOPMPSAdapterDetailsSourceIDKey "SourceID" +#define kIOPMPSAdapterDetailsSharedSourceKey "SharedSource" // values for kIOPSPowerAdapterFamilyKey enum { diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h index 5a649dfdf..c5d2c17bc 100644 --- a/iokit/IOKit/pwr_mgt/RootDomain.h +++ b/iokit/IOKit/pwr_mgt/RootDomain.h @@ -557,6 +557,8 @@ public: void sleepWakeDebugSaveSpinDumpFile(); void swdDebugSetup(); void swdDebugTeardown(); + bool checkShutdownTimeout(); + void panicWithShutdownLog(uint32_t timeoutInMs); private: friend class PMSettingObject; diff --git a/iokit/IOKit/rtc/IORTCController.h b/iokit/IOKit/rtc/IORTCController.h index eaf681d12..2757c8a13 100644 --- a/iokit/IOKit/rtc/IORTCController.h +++ b/iokit/IOKit/rtc/IORTCController.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1998-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -28,6 +28,8 @@ /* * 24 Nov 1998 suurballe Created. */ +#ifndef _IORTCCONTROLLER_H +#define _IORTCCONTROLLER_H #include @@ -44,3 +46,40 @@ virtual IOReturn getRealTimeClock ( UInt8 * currentTime, IOByteCount * length ) virtual IOReturn setRealTimeClock ( UInt8 * newTime ) = 0; }; +class IORTC: public IOService +{ +OSDeclareAbstractStructors(IORTC); + +protected: + + /*! @var reserved + Reserved for future use. (Internal use only) */ + struct ExpansionData { }; + ExpansionData *reserved; + +public: + + virtual long getGMTTimeOfDay( void ) = 0; + virtual void setGMTTimeOfDay( long secs ) = 0; + + virtual void getUTCTimeOfDay( clock_sec_t * secs, clock_nsec_t * nsecs ); + virtual void setUTCTimeOfDay( clock_sec_t secs, clock_nsec_t nsecs ); + + virtual void setAlarmEnable( IOOptionBits message ) = 0; + + virtual IOReturn getMonotonicClockOffset( int64_t * usecs ); + virtual IOReturn setMonotonicClockOffset( int64_t usecs ); + virtual IOReturn getMonotonicClockAndTimestamp( uint64_t * usecs, uint64_t *mach_absolute_time ); + + + OSMetaClassDeclareReservedUnused(IORTC, 0); + OSMetaClassDeclareReservedUnused(IORTC, 1); + OSMetaClassDeclareReservedUnused(IORTC, 2); + OSMetaClassDeclareReservedUnused(IORTC, 3); + OSMetaClassDeclareReservedUnused(IORTC, 4); + OSMetaClassDeclareReservedUnused(IORTC, 5); + OSMetaClassDeclareReservedUnused(IORTC, 6); + OSMetaClassDeclareReservedUnused(IORTC, 7); +}; + +#endif /* !_IORTCCONTROLLER_H */ diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp index afbad6666..e9173c655 100644 --- a/iokit/Kernel/IOCPU.cpp +++ b/iokit/Kernel/IOCPU.cpp @@ -183,12 +183,18 @@ IOCPURunPlatformPanicActions(uint32_t message) extern "C" kern_return_t -IOCPURunPlatformPanicSyncAction(void *addr, size_t len) +IOCPURunPlatformPanicSyncAction(void *addr, uint32_t offset, uint32_t len) { + PE_panic_save_context_t context = { + .psc_buffer = addr, + .psc_offset = offset, + .psc_length = len + }; + // Don't allow nested calls of panic actions if (!gActionQueues[kQueuePanic].next) return (kIOReturnNotReady); return (iocpu_run_platform_actions(&gActionQueues[kQueuePanic], 0, 0U-1, - (void *)(uintptr_t)(kPEPanicSync), addr, (void *)(uintptr_t)len, FALSE)); + (void *)(uintptr_t)(kPEPanicSync), &context, NULL, FALSE)); } diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp index 66b566ca9..4ceb8cc0f 100644 --- a/iokit/Kernel/IOHibernateIO.cpp +++ b/iokit/Kernel/IOHibernateIO.cpp @@ -437,6 +437,7 @@ IOHibernateSystemSleep(void) gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock); } IOHibernateSystemPostWakeTrim(NULL, NULL); + thread_call_cancel(gIOHibernateTrimCalloutEntry); if (kFSIdle != gFSState) { HIBLOG("hibernate file busy\n"); @@ -677,14 +678,16 @@ IOHibernateSystemSleep(void) IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data); data->release(); } - if (gIOChosenEntry) + if (gIOChosenEntry && gIOOptionsEntry) { data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey)); if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy()); // set BootNext if (!gIOHibernateBoot0082Data) { + OSData * fileData = 0; data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path")); + if (data->getLength() >= 4) fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path")); if (data) { // AppleNVRAM_EFI_LOAD_OPTION @@ -696,11 +699,21 @@ IOHibernateSystemSleep(void) loadOptionHeader.Attributes = 1; loadOptionHeader.FilePathLength = data->getLength(); loadOptionHeader.Desc = 0; + if (fileData) + { + loadOptionHeader.FilePathLength -= 4; + loadOptionHeader.FilePathLength += fileData->getLength(); + } gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength); if (gIOHibernateBoot0082Data) { gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader)); - gIOHibernateBoot0082Data->appendBytes(data); + if (fileData) + { + gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4); + gIOHibernateBoot0082Data->appendBytes(fileData); + } + else gIOHibernateBoot0082Data->appendBytes(data); } } } @@ -1316,12 +1329,20 @@ IOHibernateSystemPostWake(bool now) { gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature; IOLockLock(gFSLock); - if (kFSTrimDelay == gFSState) IOHibernateSystemPostWakeTrim(NULL, NULL); + if (kFSTrimDelay == gFSState) + { + thread_call_cancel(gIOHibernateTrimCalloutEntry); + IOHibernateSystemPostWakeTrim(NULL, NULL); + } else if (kFSOpened != gFSState) gFSState = kFSIdle; else { gFSState = kFSTrimDelay; - if (now) IOHibernateSystemPostWakeTrim(NULL, NULL); + if (now) + { + thread_call_cancel(gIOHibernateTrimCalloutEntry); + IOHibernateSystemPostWakeTrim(NULL, NULL); + } else { AbsoluteTime deadline; @@ -1331,8 +1352,6 @@ IOHibernateSystemPostWake(bool now) } IOLockUnlock(gFSLock); - // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally - IOCloseDebugDataFile( ); return (kIOReturnSuccess); } diff --git a/iokit/Kernel/IOLocks.cpp b/iokit/Kernel/IOLocks.cpp index c2ece6b9c..2bbb712bb 100644 --- a/iokit/Kernel/IOLocks.cpp +++ b/iokit/Kernel/IOLocks.cpp @@ -80,6 +80,7 @@ void IOLockWakeup(IOLock * lock, void *event, bool oneThread) thread_wakeup_prim((event_t) event, oneThread, THREAD_AWAKENED); } + #if defined(__x86_64__) /* * For backwards compatibility, kexts built against pre-Darwin 14 headers will bind at runtime to this function, @@ -308,6 +309,30 @@ lck_spin_t * IOSimpleLockGetMachLock( IOSimpleLock * lock) return( (lck_spin_t *)lock); } +#ifndef IOLOCKS_INLINE +/* + * Lock assertions + */ + +void +IOLockAssert(IOLock * lock, IOLockAssertState type) +{ + LCK_MTX_ASSERT(lock, type); +} + +void +IORWLockAssert(IORWLock * lock, IORWLockAssertState type) +{ + LCK_RW_ASSERT(lock, type); +} + +void +IOSimpleLockAssert(IOSimpleLock *lock, IOSimpleLockAssertState type) +{ + LCK_SPIN_ASSERT(l, type); +} +#endif /* !IOLOCKS_INLINE */ + } /* extern "C" */ diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp index 3b778df40..bf8ecc94f 100644 --- a/iokit/Kernel/IOPMrootDomain.cpp +++ b/iokit/Kernel/IOPMrootDomain.cpp @@ -611,7 +611,7 @@ extern "C" IOReturn rootDomainShutdown ( void ) static void halt_log_putc(char c) { - if (gHaltLogPos >= (kHaltLogSize - 1)) return; + if (gHaltLogPos >= (kHaltLogSize - 2)) return; gHaltLog[gHaltLogPos++] = c; } @@ -645,12 +645,15 @@ halt_log_enter(const char * what, const void * pc, uint64_t time) IOLockLock(gHaltLogLock); if (pc) { - halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc)); - OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log, - OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse); - } else { - halt_log("%s: %qd ms\n", what, millis, VM_KERNEL_UNSLIDE(pc)); + halt_log("%s: %qd ms @ 0x%lx, ", what, millis, VM_KERNEL_UNSLIDE(pc)); + OSKext::printKextsInBacktrace((vm_offset_t *) &pc, 1, &halt_log, + OSKext::kPrintKextsLock | OSKext::kPrintKextsUnslide | OSKext::kPrintKextsTerse); + } + else { + halt_log("%s: %qd ms\n", what, millis); } + + gHaltLog[gHaltLogPos] = 0; IOLockUnlock(gHaltLogLock); } @@ -675,6 +678,7 @@ extern "C" void IOSystemShutdownNotification(void) #if HIBERNATION startTime = mach_absolute_time(); IOHibernateSystemPostWake(true); + gRootDomain->swdDebugTeardown(); halt_log_enter("IOHibernateSystemPostWake", 0, mach_absolute_time() - startTime); #endif if (OSCompareAndSwap(0, 1, &gPagingOff)) @@ -782,13 +786,15 @@ static void swdDebugSetupCallout( thread_call_param_t p0, thread_call_param_t p1 void IOPMrootDomain::swdDebugSetup( ) { #if HIBERNATION - static int32_t mem_only = -1; - if ((mem_only == -1) && - (PE_parse_boot_argn("swd_mem_only", &mem_only, sizeof(mem_only)) == false)) { - mem_only = 0; + static int32_t noDebugFile = -1; + if (noDebugFile == -1) { + if (PEGetCoprocessorVersion() >= kCoprocessorVersion2) + noDebugFile = 1; + else if (PE_parse_boot_argn("swd_mem_only", &noDebugFile, sizeof(noDebugFile)) == false) + noDebugFile = 0; } - if ((mem_only == 1) || (gRootDomain->sleepWakeDebugIsWdogEnabled() == false)) { + if ((noDebugFile == 1) || (gRootDomain->sleepWakeDebugIsWdogEnabled() == false)) { return; } DLOG("swdDebugSetup state:%d\n", swd_DebugImageSetup); @@ -797,6 +803,7 @@ void IOPMrootDomain::swdDebugSetup( ) if (CAP_GAIN(kIOPMSystemCapabilityGraphics) || (CAP_LOSS(kIOPMSystemCapabilityGraphics))) { IOHibernateSystemPostWake(true); + IOCloseDebugDataFile(); } IOOpenDebugDataFile(kSleepWakeStackBinFilename, SWD_BUF_SIZE); } @@ -844,6 +851,11 @@ static void disk_sync_callout( thread_call_param_t p0, thread_call_param_t p1 ) if (ON_STATE == powerState) { sync_internal(); + +#if HIBERNATION + // Block sleep until trim issued on previous wake path is completed. + IOHibernateSystemPostWake(true); +#endif swdDebugSetupCallout(p0, NULL); } #if HIBERNATION @@ -3083,9 +3095,6 @@ IOReturn IOPMrootDomain::sysPowerDownHandler( // We will ack within 20 seconds params->maxWaitForReply = 20 * 1000 * 1000; - // Remove EFI/BootRom's previous wake's failure data - PERemoveNVRAMProperty(kIOEFIBootRomFailureKey); - #if HIBERNATION gRootDomain->evaluateSystemSleepPolicyEarly(); @@ -4622,6 +4631,16 @@ IOReturn IOPMrootDomain::getSystemSleepType( uint32_t * sleepType, uint32_t * st // //****************************************************************************** +// Phases while performing shutdown/restart +typedef enum { + kNotifyDone = 0x00, + kNotifyPriorityClients = 0x10, + kNotifyPowerPlaneDrivers = 0x20, + kNotifyHaltRestartAction = 0x30, + kQuiescePM = 0x40, +} shutdownPhase_t; + + struct HaltRestartApplierContext { IOPMrootDomain * RootDomain; unsigned long PowerState; @@ -4629,7 +4648,29 @@ struct HaltRestartApplierContext { UInt32 MessageType; UInt32 Counter; const char * LogString; -}; + shutdownPhase_t phase; + + IOServiceInterestHandler handler; +} gHaltRestartCtx; + +const char *shutdownPhase2String(shutdownPhase_t phase) +{ + switch(phase) { + case kNotifyDone: + return "Notifications completed"; + case kNotifyPriorityClients: + return "Notifying priority clients"; + case kNotifyPowerPlaneDrivers: + return "Notifying power plane drivers"; + case kNotifyHaltRestartAction: + return "Notifying HaltRestart action handlers"; + case kQuiescePM: + return "Quiescing PM"; + default: + return "Unknown"; + } + +} static void platformHaltRestartApplier( OSObject * object, void * context ) @@ -4641,33 +4682,30 @@ platformHaltRestartApplier( OSObject * object, void * context ) ctx = (HaltRestartApplierContext *) context; + _IOServiceInterestNotifier * notifier; + notifier = OSDynamicCast(_IOServiceInterestNotifier, object); memset(¬ify, 0, sizeof(notify)); notify.powerRef = (void *)(uintptr_t)ctx->Counter; notify.returnValue = 0; notify.stateNumber = ctx->PowerState; notify.stateFlags = ctx->PowerFlags; + if (notifier) { + ctx->handler = notifier->handler; + } + clock_get_uptime(&startTime); ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)¬ify ); deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); - if ((deltaTime > kPMHaltTimeoutMS) || - (gIOKitDebug & kIOLogPMRootDomain)) - { - _IOServiceInterestNotifier * notifier; - notifier = OSDynamicCast(_IOServiceInterestNotifier, object); - - // IOService children of IOPMrootDomain are not instrumented. - // Only IORootParent currently falls under that group. + if ((deltaTime > kPMHaltTimeoutMS) && notifier) { - if (notifier) - { - LOG("%s handler %p took %u ms\n", + LOG("%s handler %p took %u ms\n", ctx->LogString, OBFUSCATE(notifier->handler), deltaTime); - halt_log_enter(ctx->LogString, (const void *) notifier->handler, elapsedTime); - } + halt_log_enter("PowerOff/Restart message to priority client", (const void *) notifier->handler, elapsedTime); } + ctx->handler = 0; ctx->Counter++; } @@ -4681,33 +4719,32 @@ static void quiescePowerTreeCallback( void * target, void * param ) void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) { - HaltRestartApplierContext ctx; AbsoluteTime startTime, elapsedTime; uint32_t deltaTime; - memset(&ctx, 0, sizeof(ctx)); - ctx.RootDomain = this; + memset(&gHaltRestartCtx, 0, sizeof(gHaltRestartCtx)); + gHaltRestartCtx.RootDomain = this; clock_get_uptime(&startTime); switch (pe_type) { case kPEHaltCPU: case kPEUPSDelayHaltCPU: - ctx.PowerState = OFF_STATE; - ctx.MessageType = kIOMessageSystemWillPowerOff; - ctx.LogString = "PowerOff"; + gHaltRestartCtx.PowerState = OFF_STATE; + gHaltRestartCtx.MessageType = kIOMessageSystemWillPowerOff; + gHaltRestartCtx.LogString = "PowerOff"; break; case kPERestartCPU: - ctx.PowerState = RESTART_STATE; - ctx.MessageType = kIOMessageSystemWillRestart; - ctx.LogString = "Restart"; + gHaltRestartCtx.PowerState = RESTART_STATE; + gHaltRestartCtx.MessageType = kIOMessageSystemWillRestart; + gHaltRestartCtx.LogString = "Restart"; break; case kPEPagingOff: - ctx.PowerState = ON_STATE; - ctx.MessageType = kIOMessageSystemPagingOff; - ctx.LogString = "PagingOff"; + gHaltRestartCtx.PowerState = ON_STATE; + gHaltRestartCtx.MessageType = kIOMessageSystemPagingOff; + gHaltRestartCtx.LogString = "PagingOff"; IOService::updateConsoleUsers(NULL, kIOMessageSystemPagingOff); #if HIBERNATION IOHibernateSystemRestart(); @@ -4718,8 +4755,9 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) return; } + gHaltRestartCtx.phase = kNotifyPriorityClients; // Notify legacy clients - applyToInterested(gIOPriorityPowerStateInterest, platformHaltRestartApplier, &ctx); + applyToInterested(gIOPriorityPowerStateInterest, platformHaltRestartApplier, &gHaltRestartCtx); // For normal shutdown, turn off File Server Mode. if (kPEHaltCPU == pe_type) @@ -4734,17 +4772,21 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) } } + if (kPEPagingOff != pe_type) { + gHaltRestartCtx.phase = kNotifyPowerPlaneDrivers; // Notify in power tree order - notifySystemShutdown(this, ctx.MessageType); + notifySystemShutdown(this, gHaltRestartCtx.MessageType); } + gHaltRestartCtx.phase = kNotifyHaltRestartAction; IOCPURunPlatformHaltRestartActions(pe_type); // Wait for PM to quiesce if ((kPEPagingOff != pe_type) && gPMHaltLock) { + gHaltRestartCtx.phase = kQuiescePM; AbsoluteTime quiesceTime = mach_absolute_time(); IOLockLock(gPMHaltLock); @@ -4762,23 +4804,47 @@ void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type ) DLOG("PM quiesce took %u ms\n", deltaTime); halt_log_enter("Quiesce", NULL, elapsedTime); } + gHaltRestartCtx.phase = kNotifyDone; deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); - LOG("%s all drivers took %u ms\n", ctx.LogString, deltaTime); + LOG("%s all drivers took %u ms\n", gHaltRestartCtx.LogString, deltaTime); - halt_log_enter(ctx.LogString, NULL, elapsedTime); - if (gHaltLog) gHaltLog[gHaltLogPos] = 0; + halt_log_enter(gHaltRestartCtx.LogString, NULL, elapsedTime); deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime); - LOG("%s total %u ms\n", ctx.LogString, deltaTime); + LOG("%s total %u ms\n", gHaltRestartCtx.LogString, deltaTime); if (gHaltLog && gHaltTimeMaxLog && (deltaTime >= gHaltTimeMaxLog)) { - printf("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + printf("%s total %d ms:%s\n", gHaltRestartCtx.LogString, deltaTime, gHaltLog); } - if (gHaltLog && gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic)) - { - panic("%s total %d ms:%s\n", ctx.LogString, deltaTime, gHaltLog); + + checkShutdownTimeout(); +} + +bool IOPMrootDomain::checkShutdownTimeout() +{ + AbsoluteTime elapsedTime; + uint32_t deltaTime = computeDeltaTimeMS(&gHaltStartTime, &elapsedTime); + + if (gHaltTimeMaxPanic && (deltaTime >= gHaltTimeMaxPanic)) { + return true; + } + return false; +} + +void IOPMrootDomain::panicWithShutdownLog(uint32_t timeoutInMs) +{ + if (gHaltLog) { + if ((gHaltRestartCtx.phase == kNotifyPriorityClients) && gHaltRestartCtx.handler) { + halt_log_enter("Blocked on priority client", (void *)gHaltRestartCtx.handler, mach_absolute_time() - gHaltStartTime); + } + panic("%s timed out in phase '%s'. Total %d ms:%s", + gHaltRestartCtx.LogString, shutdownPhase2String(gHaltRestartCtx.phase), timeoutInMs, gHaltLog); + } + else { + panic("%s timed out in phase \'%s\'. Total %d ms", + gHaltRestartCtx.LogString, shutdownPhase2String(gHaltRestartCtx.phase), timeoutInMs); } } @@ -7801,7 +7867,6 @@ IOReturn IOPMrootDomain::callPlatformFunction( void * param3, void * param4 ) { uint32_t bootFailureCode = 0xffffffff; - unsigned int len = sizeof(bootFailureCode); if (pmTracer && functionName && functionName->isEqualTo(kIOPMRegisterNVRAMTracePointHandlerKey) && !pmTracer->tracePointHandler && !pmTracer->tracePointTarget) @@ -7814,11 +7879,17 @@ IOReturn IOPMrootDomain::callPlatformFunction( tracePointPCI = (uint32_t)(uintptr_t) param3; tracePointPhases = (uint32_t)(uintptr_t) param4; if ((tracePointPhases & 0xff) == kIOPMTracePointSystemSleep) { - if (!PEReadNVRAMProperty(kIOEFIBootRomFailureKey, &bootFailureCode, &len)) { - MSG("Failed to read failure code from NVRam\n"); - } - // Failure code from EFI/BootRom is a four byte structure - tracePointPCI = OSSwapBigToHostInt32(bootFailureCode); + + IORegistryEntry *node = IORegistryEntry::fromPath( "/chosen", gIODTPlane ); + if ( node ) { + OSData *data = OSDynamicCast( OSData, node->getProperty(kIOEFIBootRomFailureKey) ); + if ( data && data->getLength() == sizeof(bootFailureCode) ) { + memcpy(&bootFailureCode, data->getBytesNoCopy(), sizeof(bootFailureCode)); + } + node->release(); + } + // Failure code from EFI/BootRom is a four byte structure + tracePointPCI = OSSwapBigToHostInt32(bootFailureCode); } statusCode = (((uint64_t)tracePointPCI) << 32) | tracePointPhases; if ((tracePointPhases & 0xff) != kIOPMTracePointSystemUp) { @@ -8454,16 +8525,14 @@ void PMHaltWorker::work( PMHaltWorker * me ) } deltaTime = computeDeltaTimeMS(&startTime, &elapsedTime); - if ((deltaTime > kPMHaltTimeoutMS) || timeout || - (gIOKitDebug & kIOLogPMRootDomain)) + if ((deltaTime > kPMHaltTimeoutMS) || timeout) { LOG("%s driver %s (0x%llx) took %u ms\n", (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart", service->getName(), service->getRegistryEntryID(), (uint32_t) deltaTime ); - halt_log_enter( - (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart", + halt_log_enter("PowerOff/Restart handler completed", OSMemberFunctionCast(const void *, service, &IOService::systemWillShutdown), elapsedTime); } @@ -8494,9 +8563,12 @@ void PMHaltWorker::checkTimeout( PMHaltWorker * me, AbsoluteTime * now ) if (nano > 3000000000ULL) { me->timeout = true; + + halt_log_enter("PowerOff/Restart still waiting on handler", + OSMemberFunctionCast(const void *, me->service, &IOService::systemWillShutdown), + endTime); MSG("%s still waiting on %s\n", - (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? - "PowerOff" : "Restart", + (gPMHaltMessageType == kIOMessageSystemWillPowerOff) ? "PowerOff" : "Restart", me->service->getName()); } } @@ -9700,7 +9772,7 @@ void IOPMrootDomain::takeStackshot(bool wdogTrigger, bool isOSXWatchdog, bool is if (wdogTrigger) { PE_parse_boot_argn("swd_panic", &wdog_panic, sizeof(wdog_panic)); PE_parse_boot_argn("stress-rack", &stress_rack, sizeof(stress_rack)); - if ((wdog_panic == 1) || (stress_rack == 1)) { + if ((wdog_panic == 1) || (stress_rack == 1) || (PEGetCoprocessorVersion() >= kCoprocessorVersion2)) { // If boot-arg specifies to panic then panic. panic("Sleep/Wake hang detected"); return; @@ -9886,6 +9958,9 @@ void IOPMrootDomain::sleepWakeDebugMemAlloc( ) if ( kIOSleepWakeWdogOff & gIOKitDebug ) return; + if (PEGetCoprocessorVersion() >= kCoprocessorVersion2) + return; + if (!OSCompareAndSwap(0, 1, &gRootDomain->swd_lock)) return; diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp index 274e69043..59c36062e 100644 --- a/iokit/Kernel/IOPlatformExpert.cpp +++ b/iokit/Kernel/IOPlatformExpert.cpp @@ -54,15 +54,9 @@ extern "C" { #include } -#if !CONFIG_EMBEDDED +#define kShutdownTimeout 30 //in secs -/* - * This will eventually be properly exported in - * ER: Expose coprocessor version (T208/T290) in a kernel/kext header - * although we'll always need to hardcode this here since we won't be able to include whatever - * header this ends up in. - */ -#define kCoprocessorMinVersion 0x00020000 +#if !CONFIG_EMBEDDED boolean_t coprocessor_cross_panic_enabled = TRUE; #endif /* !CONFIG_EMBEDDED */ @@ -114,11 +108,6 @@ bool IOPlatformExpert::start( IOService * provider ) OSData * busFrequency; uint32_t debugFlags; -#if !CONFIG_EMBEDDED - IORegistryEntry *platform_entry = NULL; - OSData *coprocessor_version_obj = NULL; - uint64_t coprocessor_version = 0; -#endif if (!super::start(provider)) return false; @@ -171,19 +160,11 @@ bool IOPlatformExpert::start( IOService * provider ) } #if !CONFIG_EMBEDDED - platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform"); - if (platform_entry != NULL) { - coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version")); - if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(coprocessor_version))) { - memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength()); - if (coprocessor_version >= kCoprocessorMinVersion) { - coprocessor_paniclog_flush = TRUE; - extended_debug_log_init(); - } - } - platform_entry->release(); + if (PEGetCoprocessorVersion() >= kCoprocessorVersion2) { + coprocessor_paniclog_flush = TRUE; + extended_debug_log_init(); } -#endif /* !CONFIG_EMBEDDED */ +#endif return( configure(provider) ); } @@ -788,6 +769,14 @@ static void IOShutdownNotificationsTimedOut( #else /* ! CONFIG_EMBEDDED */ int type = (int)(long)p0; + uint32_t timeout = (uint32_t)(uintptr_t)p1; + + IOPMrootDomain *pmRootDomain = IOService::getPMRootDomain(); + if (pmRootDomain) { + if ((PEGetCoprocessorVersion() >= kCoprocessorVersion2) || pmRootDomain->checkShutdownTimeout()) { + pmRootDomain->panicWithShutdownLog(timeout * 1000); + } + } /* 30 seconds has elapsed - resume shutdown */ if(gIOPlatform) gIOPlatform->haltRestart(type); @@ -832,7 +821,7 @@ int PEHaltRestart(unsigned int type) thread_call_t shutdown_hang; IORegistryEntry *node; OSData *data; - uint32_t timeout = 30; + uint32_t timeout = kShutdownTimeout; static boolean_t panic_begin_called = FALSE; if(type == kPEHaltCPU || type == kPERestartCPU || type == kPEUPSDelayHaltCPU) @@ -863,7 +852,7 @@ int PEHaltRestart(unsigned int type) shutdown_hang = thread_call_allocate( &IOShutdownNotificationsTimedOut, (thread_call_param_t)(uintptr_t) type); clock_interval_to_deadline( timeout, kSecondScale, &deadline ); - thread_call_enter1_delayed( shutdown_hang, 0, deadline ); + thread_call_enter1_delayed( shutdown_hang, (thread_call_param_t)(uintptr_t)timeout, deadline ); pmRootDomain->handlePlatformHaltRestart(type); /* This notification should have few clients who all do @@ -925,9 +914,9 @@ UInt32 PESavePanicInfo(UInt8 *buffer, UInt32 length) else return 0; } -void PESavePanicInfoAction(void *buffer, size_t length) +void PESavePanicInfoAction(void *buffer, UInt32 offset, UInt32 length) { - IOCPURunPlatformPanicSyncAction(buffer, length); + IOCPURunPlatformPanicSyncAction(buffer, offset, length); return; } @@ -1130,6 +1119,25 @@ void PESetUTCTimeOfDay(clock_sec_t secs, clock_usec_t usecs) gIOPlatform->setUTCTimeOfDay(secs, usecs * NSEC_PER_USEC); } +coprocessor_type_t PEGetCoprocessorVersion( void ) +{ + coprocessor_type_t coprocessor_version = kCoprocessorVersionNone; +#if !CONFIG_EMBEDDED + IORegistryEntry *platform_entry = NULL; + OSData *coprocessor_version_obj = NULL; + + platform_entry = IORegistryEntry::fromPath(kIODeviceTreePlane ":/efi/platform"); + if (platform_entry != NULL) { + coprocessor_version_obj = OSDynamicCast(OSData, platform_entry->getProperty("apple-coprocessor-version")); + if ((coprocessor_version_obj != NULL) && (coprocessor_version_obj->getLength() <= sizeof(uint64_t))) { + memcpy(&coprocessor_version, coprocessor_version_obj->getBytesNoCopy(), coprocessor_version_obj->getLength()); + } + platform_entry->release(); + } +#endif + return coprocessor_version; +} + } /* extern "C" */ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) @@ -1187,7 +1195,9 @@ void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller) data = OSDynamicCast( OSData, entry->getProperty( "EffectiveProductionStatus" ) ); if ( data && ( data->getLength( ) == sizeof( UInt8 ) ) ) { UInt8 *isProdFused = (UInt8 *) data->getBytesNoCopy( ); - if ( *isProdFused ) { + UInt32 debug_flags = 0; + if ( *isProdFused || ( PE_i_can_has_debugger(&debug_flags) && + ( debug_flags & DB_DISABLE_CROSS_PANIC ) ) ) { coprocessor_cross_panic_enabled = FALSE; } } diff --git a/iokit/Kernel/IOPolledInterface.cpp b/iokit/Kernel/IOPolledInterface.cpp index 2c9dfba68..821bbc81b 100644 --- a/iokit/Kernel/IOPolledInterface.cpp +++ b/iokit/Kernel/IOPolledInterface.cpp @@ -490,9 +490,11 @@ IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, uuid_t volUuid = {0}; size_t sizeOut = 0; err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut); - if (err == kIOReturnSuccess) { + if (err == kIOReturnSuccess) + { // No need to create uuid string if it's not requested - if (pKeyUUID) { + if (pKeyUUID) + { uuid_string_t volUuidStr; uuid_unparse(volUuid, volUuidStr); *pKeyUUID = OSString::withCString(volUuidStr); @@ -526,11 +528,15 @@ IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize); } bzero(&vek, sizeof(vek)); + + if (pKeyUUID) + { + // Create a copy because the caller would release it + *pKeyUUID = OSString::withString(keyUUID); + } } part->release(); - if (pKeyUUID) *pKeyUUID = keyUUID; - return (err); } @@ -538,11 +544,11 @@ IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, IOReturn IOPolledFileOpen(const char * filename, - uint64_t setFileSize, uint64_t fsFreeSize, - void * write_file_addr, size_t write_file_len, - IOPolledFileIOVars ** fileVars, - OSData ** imagePath, - uint8_t * volumeCryptKey, size_t keySize) + uint64_t setFileSize, uint64_t fsFreeSize, + void * write_file_addr, size_t write_file_len, + IOPolledFileIOVars ** fileVars, + OSData ** imagePath, + uint8_t * volumeCryptKey, size_t keySize) { IOReturn err = kIOReturnSuccess; IOPolledFileIOVars * vars; @@ -562,66 +568,66 @@ IOPolledFileOpen(const char * filename, do { - extentsData = OSData::withCapacity(32); - ctx.extents = extentsData; - ctx.size = 0; - clock_get_uptime(&startTime); + extentsData = OSData::withCapacity(32); + ctx.extents = extentsData; + ctx.size = 0; + clock_get_uptime(&startTime); - vars->fileRef = kern_open_file_for_direct_io(filename, + vars->fileRef = kern_open_file_for_direct_io(filename, (write_file_addr != NULL) || (0 != setFileSize), - &file_extent_callback, &ctx, - setFileSize, - fsFreeSize, - // write file: + &file_extent_callback, &ctx, + setFileSize, + fsFreeSize, + // write file: 0, write_file_addr, write_file_len, // results - &block_dev, - &image_dev, + &block_dev, + &image_dev, &vars->block0, &vars->maxiobytes, &vars->flags); #if 0 - uint32_t msDelay = (131071 & random()); - HIBLOG("sleep %d\n", msDelay); - IOSleep(msDelay); + uint32_t msDelay = (131071 & random()); + HIBLOG("sleep %d\n", msDelay); + IOSleep(msDelay); #endif clock_get_uptime(&endTime); SUB_ABSOLUTETIME(&endTime, &startTime); absolutetime_to_nanoseconds(endTime, &nsec); - if (!vars->fileRef) err = kIOReturnNoSpace; + if (!vars->fileRef) err = kIOReturnNoSpace; HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL); - if (kIOReturnSuccess != err) break; + if (kIOReturnSuccess != err) break; - HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size, - (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1, - vars->maxiobytes, kIOPolledFileSSD & vars->flags); - assert(!vars->block0); - if (extentsData->getLength() < sizeof(IOPolledFileExtent)) - { - err = kIOReturnNoSpace; - break; - } + HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size, + (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1, + vars->maxiobytes, kIOPolledFileSSD & vars->flags); + assert(!vars->block0); + if (extentsData->getLength() < sizeof(IOPolledFileExtent)) + { + err = kIOReturnNoSpace; + break; + } - vars->fileSize = ctx.size; - vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); + vars->fileSize = ctx.size; + vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); part = IOCopyMediaForDev(image_dev); if (!part) { err = kIOReturnNotFound; break; - } + } - if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break; + if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break; - if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) - vars->blockSize = num->unsigned32BitValue(); - if (vars->blockSize < 4096) vars->blockSize = 4096; + if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) + vars->blockSize = num->unsigned32BitValue(); + if (vars->blockSize < 4096) vars->blockSize = 4096; HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n", - major(image_dev), minor(image_dev), (long)vars->blockSize, + major(image_dev), minor(image_dev), (long)vars->blockSize, vars->pollers->pollers->getCount()); OSString * keyUUID = NULL; @@ -630,45 +636,52 @@ IOPolledFileOpen(const char * filename, err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize); } - *fileVars = vars; - vars->fileExtents = extentsData; - - // make imagePath - OSData * data; - if (imagePath) - { + *fileVars = vars; + vars->fileExtents = extentsData; + + // make imagePath + OSData * data; + if (imagePath) + { #if defined(__i386__) || defined(__x86_64__) - char str2[24 + sizeof(uuid_string_t) + 2]; + char str2[24 + sizeof(uuid_string_t) + 2]; if (keyUUID) - snprintf(str2, sizeof(str2), "%qx:%s", - vars->extentMap[0].start, keyUUID->getCStringNoCopy()); + snprintf(str2, sizeof(str2), "%qx:%s", + vars->extentMap[0].start, keyUUID->getCStringNoCopy()); else snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start); - err = IOService::getPlatform()->callPlatformFunction( - gIOCreateEFIDevicePathSymbol, false, - (void *) part, (void *) str2, - (void *) (uintptr_t) true, (void *) &data); + err = IOService::getPlatform()->callPlatformFunction( + gIOCreateEFIDevicePathSymbol, false, + (void *) part, (void *) str2, + (void *) (uintptr_t) true, (void *) &data); #else - data = 0; - err = kIOReturnSuccess; + data = 0; + err = kIOReturnSuccess; #endif - if (kIOReturnSuccess != err) - { - HIBLOG("error 0x%x getting path\n", err); - break; - } - *imagePath = data; - } + if (kIOReturnSuccess != err) + { + HIBLOG("error 0x%x getting path\n", err); + break; + } + *imagePath = data; + } + + // Release key UUID if we have one + if (keyUUID) + { + keyUUID->release(); + keyUUID = NULL; // Just in case + } } while (false); if (kIOReturnSuccess != err) { HIBLOG("error 0x%x opening polled file\n", err); - IOPolledFileClose(&vars, 0, 0, 0, 0, 0); - if (extentsData) extentsData->release(); + IOPolledFileClose(&vars, 0, 0, 0, 0, 0); + if (extentsData) extentsData->release(); } if (part) part->release(); @@ -816,7 +829,15 @@ IOPolledFileWrite(IOPolledFileIOVars * vars, if (bytes) { +#if KASAN + /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to + * avoid triggering global redzone sanitizer violations when accessing + * interstices between 'C' structures + */ + __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); +#else bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); +#endif bytes += copy; } else @@ -955,7 +976,11 @@ IOPolledFileRead(IOPolledFileIOVars * vars, if (bytes) { - bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); +#if KASAN + __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); +#else + bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); +#endif bytes += copy; } size -= copy; diff --git a/iokit/Kernel/IORTC.cpp b/iokit/Kernel/IORTC.cpp new file mode 100644 index 000000000..335bfc317 --- /dev/null +++ b/iokit/Kernel/IORTC.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#include + +#define super IOService + +OSDefineMetaClassAndAbstractStructors(IORTC, IOService); + +void IORTC::getUTCTimeOfDay( clock_sec_t * secs, clock_nsec_t * nsecs ) +{ + *nsecs = 0; + *secs = getGMTTimeOfDay(); +} + +void IORTC::setUTCTimeOfDay( clock_sec_t secs, clock_nsec_t nsecs ) +{ + setGMTTimeOfDay(secs); +} + +IOReturn IORTC::getMonotonicClockOffset( int64_t * usecs ) +{ + return kIOReturnUnsupported; +} + +IOReturn IORTC::setMonotonicClockOffset( int64_t usecs ) +{ + return kIOReturnUnsupported; +} + +IOReturn IORTC::getMonotonicClockAndTimestamp( uint64_t * usecs, uint64_t *mach_absolute_time ) +{ + return kIOReturnUnsupported; +} + +OSMetaClassDefineReservedUnused(IORTC, 0); +OSMetaClassDefineReservedUnused(IORTC, 1); +OSMetaClassDefineReservedUnused(IORTC, 2); +OSMetaClassDefineReservedUnused(IORTC, 3); +OSMetaClassDefineReservedUnused(IORTC, 4); +OSMetaClassDefineReservedUnused(IORTC, 5); +OSMetaClassDefineReservedUnused(IORTC, 6); +OSMetaClassDefineReservedUnused(IORTC, 7); diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp index 40055c5c9..207c8dc29 100644 --- a/iokit/Kernel/IOService.cpp +++ b/iokit/Kernel/IOService.cpp @@ -190,6 +190,7 @@ static IOLock * gIOServiceBusyLock; bool gCPUsRunning; static thread_t gIOTerminateThread; +static thread_t gIOTerminateWorkerThread; static UInt32 gIOTerminateWork; static OSArray * gIOTerminatePhase2List; static OSArray * gIOStopList; @@ -448,6 +449,11 @@ void IOService::initialize( void ) gIOStopProviderList = OSArray::withCapacity( 16 ); gIOFinalizeList = OSArray::withCapacity( 16 ); assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList ); + + // worker thread that is responsible for terminating / cleaning up threads + kernel_thread_start(&terminateThread, NULL, &gIOTerminateWorkerThread); + assert(gIOTerminateWorkerThread); + thread_set_thread_name(gIOTerminateWorkerThread, "IOServiceTerminateThread"); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -2221,13 +2227,27 @@ void IOService::setTerminateDefer(IOService * provider, bool defer) } } +// Must call this while holding gJobsLock +void IOService::waitToBecomeTerminateThread(void) +{ + IOLockAssert(gJobsLock, kIOLockAssertOwned); + bool wait; + do { + wait = (gIOTerminateThread != THREAD_NULL); + if (wait) { + IOLockSleep(gJobsLock, &gIOTerminateThread, THREAD_UNINT); + } + } while (wait); + gIOTerminateThread = current_thread(); +} + // call with lockForArbitration void IOService::scheduleTerminatePhase2( IOOptionBits options ) { AbsoluteTime deadline; uint64_t regID1; int waitResult = THREAD_AWAKENED; - bool wait, haveDeadline = false; + bool wait = false, haveDeadline = false; if (!(__state[0] & kIOServiceInactiveState)) return; @@ -2249,15 +2269,7 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) if( (options & kIOServiceSynchronous) && (current_thread() != gIOTerminateThread)) { - do { - wait = (gIOTerminateThread != 0); - if( wait) { - // wait to become the terminate thread - IOLockSleep( gJobsLock, &gIOTerminateThread, THREAD_UNINT); - } - } while( wait ); - - gIOTerminateThread = current_thread(); + waitToBecomeTerminateThread(); gIOTerminatePhase2List->setObject( this ); gIOTerminateWork++; @@ -2273,8 +2285,8 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) } waitResult = IOLockSleepDeadline( gJobsLock, &gIOTerminateWork, deadline, THREAD_UNINT ); - if( waitResult == THREAD_TIMED_OUT) { - IOLog("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID()); + if(__improbable(waitResult == THREAD_TIMED_OUT)) { + panic("%s[0x%qx]::terminate(kIOServiceSynchronous) timeout\n", getName(), getRegistryEntryID()); } } } while(gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT))); @@ -2287,11 +2299,9 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) gIOTerminatePhase2List->setObject( this ); if( 0 == gIOTerminateWork++) { - if( !gIOTerminateThread) - kernel_thread_start(&terminateThread, (void *)(uintptr_t) options, &gIOTerminateThread); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); - } + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); + } } IOLockUnlock( gJobsLock ); @@ -2299,18 +2309,23 @@ void IOService::scheduleTerminatePhase2( IOOptionBits options ) release(); } +__attribute__((__noreturn__)) void IOService::terminateThread( void * arg, wait_result_t waitResult ) { - IOLockLock( gJobsLock ); - - while (gIOTerminateWork) - terminateWorker( (uintptr_t) arg ); + // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once + IOLockLock(gJobsLock); + while (true) { + if (gIOTerminateThread != gIOTerminateWorkerThread) { + waitToBecomeTerminateThread(); + } - thread_deallocate(gIOTerminateThread); - gIOTerminateThread = 0; - IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); + while (gIOTerminateWork) + terminateWorker( (uintptr_t)arg ); - IOLockUnlock( gJobsLock ); + gIOTerminateThread = 0; + IOLockWakeup( gJobsLock, (event_t) &gIOTerminateThread, /* one-thread */ false); + IOLockSleep(gJobsLock, &gIOTerminateWork, THREAD_UNINT); + } } void IOService::scheduleStop( IOService * provider ) @@ -2331,10 +2346,8 @@ void IOService::scheduleStop( IOService * provider ) gIOStopProviderList->tailQ( provider ); if( 0 == gIOTerminateWork++) { - if( !gIOTerminateThread) - kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); } IOLockUnlock( gJobsLock ); @@ -2355,12 +2368,9 @@ void IOService::scheduleFinalize(bool now) { IOLockLock( gJobsLock ); gIOFinalizeList->tailQ(this); - if( 0 == gIOTerminateWork++) - { - if( !gIOTerminateThread) - kernel_thread_start(&terminateThread, (void *) 0, &gIOTerminateThread); - else - IOLockWakeup(gJobsLock, (event_t) &gIOTerminateWork, /* one-thread */ false ); + if( 0 == gIOTerminateWork++) { + assert(gIOTerminateWorkerThread); + IOLockWakeup(gJobsLock, (event_t)&gIOTerminateWork, /* one-thread */ false ); } IOLockUnlock( gJobsLock ); } @@ -5086,6 +5096,7 @@ bool IOResources::matchPropertyTable( OSDictionary * table ) OSString * str; OSSet * set; OSIterator * iter; + OSObject * obj; OSArray * keys; bool ok = true; @@ -5106,14 +5117,15 @@ bool IOResources::matchPropertyTable( OSDictionary * table ) } else if ((prop = table->getObject(gIOResourceMatchedKey))) { - keys = (OSArray *) copyProperty(gIOResourceMatchedKey); + obj = copyProperty(gIOResourceMatchedKey); + keys = OSDynamicCast(OSArray, obj); ok = false; if (keys) { // assuming OSSymbol ok = ((-1U) != keys->getNextIndexOfObject(prop, 0)); - keys->release(); } + OSSafeReleaseNULL(obj); } return( ok ); diff --git a/iokit/Tests/Tests.cpp b/iokit/Tests/Tests.cpp index 6336b67ac..173cad671 100644 --- a/iokit/Tests/Tests.cpp +++ b/iokit/Tests/Tests.cpp @@ -236,6 +236,7 @@ sysctl_iokittest(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused data->release(); } + if (changed && newValue) { error = IOWorkLoopTest(newValue); diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp index f68eebd1e..890b89946 100644 --- a/iokit/bsddev/IOKitBSDInit.cpp +++ b/iokit/bsddev/IOKitBSDInit.cpp @@ -60,10 +60,19 @@ extern void di_root_ramfile(IORegistryEntry * entry); #if CONFIG_EMBEDDED #define IOPOLLED_COREFILE (CONFIG_KDP_INTERACTIVE_DEBUGGING) -#define kIOCoreDumpPath "/private/var/vm/kernelcore" -#define kIOCoreDumpSize 350ULL*1024ULL*1024ULL + +#if defined(XNU_TARGET_OS_BRIDGE) +#define kIOCoreDumpSize 150ULL*1024ULL*1024ULL +// leave free space on volume: +#define kIOCoreDumpFreeSize 150ULL*1024ULL*1024ULL +#define kIOCoreDumpPath "/private/var/internal/kernelcore" +#else +#define kIOCoreDumpSize 350ULL*1024ULL*1024ULL // leave free space on volume: -#define kIOCoreDumpFreeSize 350ULL*1024ULL*1024ULL +#define kIOCoreDumpFreeSize 350ULL*1024ULL*1024ULL +#define kIOCoreDumpPath "/private/var/vm/kernelcore" +#endif + #elif DEVELOPMENT #define IOPOLLED_COREFILE 1 // no sizing @@ -909,7 +918,13 @@ IOBSDMountChange(struct mount * mp, uint32_t op) vnode_put(vn); if (0 != result) break; if (!pathLen) break; - if (0 != bcmp(path, kIOCoreDumpPath, pathLen - 1)) break; +#if defined(XNU_TARGET_OS_BRIDGE) + // on bridgeOS systems we put the core in /private/var/internal. We don't + // want to match with /private/var because /private/var/internal is often mounted + // over /private/var + if ((pathLen - 1) < (int) strlen("/private/var/internal")) break; +#endif + if (0 != strncmp(path, kIOCoreDumpPath, pathLen - 1)) break; IOOpenPolledCoreFile(kIOCoreDumpPath); break; diff --git a/iokit/conf/files b/iokit/conf/files index 8dd87dd65..bc356a362 100644 --- a/iokit/conf/files +++ b/iokit/conf/files @@ -85,6 +85,7 @@ iokit/Kernel/IOStateReporter.cpp optional iokitcpp iokit/Kernel/IOHistogramReporter.cpp optional iokitcpp iokit/Kernel/IOReportLegend.cpp optional iokitcpp +iokit/Kernel/IORTC.cpp optional iokitcpp iokit/Kernel/IOStringFuncs.c standard diff --git a/libkdd/kcdata.h b/libkdd/kcdata.h index 741c7f864..e358b4ec8 100644 --- a/libkdd/kcdata.h +++ b/libkdd/kcdata.h @@ -469,7 +469,7 @@ struct kcdata_type_definition { #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */ #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO 0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du /* struct stackshot_thread_waitinfo */ -#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */ +#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot or thread_group_snapshot_v2 */ #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu /* uint64_t */ #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */ #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u /* uint64_t */ @@ -659,6 +659,17 @@ struct thread_group_snapshot { char tgs_name[16]; } __attribute__((packed)); +enum thread_group_flags { + kThreadGroupEfficient = 0x1, + kThreadGroupUIApp = 0x2 +}; + +struct thread_group_snapshot_v2 { + uint64_t tgs_id; + char tgs_name[16]; + uint64_t tgs_flags; +} __attribute__((packed)); + enum coalition_flags { kCoalitionTermRequested = 0x1, kCoalitionTerminated = 0x2, diff --git a/libkdd/kcdtypes.c b/libkdd/kcdtypes.c index 8fe7aee23..73cbeb918 100644 --- a/libkdd/kcdtypes.c +++ b/libkdd/kcdtypes.c @@ -574,8 +574,9 @@ kcdata_get_typedescription(unsigned type_id, uint8_t * buffer, uint32_t buffer_s case STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT: { i = 0; - _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot, tgs_id); - _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot, tgs_name, 16); + _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot_v2, tgs_id); + _SUBTYPE_ARRAY(KC_ST_CHAR, struct thread_group_snapshot_v2, tgs_name, 16); + _SUBTYPE(KC_ST_UINT64, struct thread_group_snapshot_v2, tgs_flags); setup_type_definition(retval, type_id, i, "thread_group_snapshot"); break; } diff --git a/libkdd/kdd.xcodeproj/project.pbxproj b/libkdd/kdd.xcodeproj/project.pbxproj index 5a2b63714..e36cc086c 100644 --- a/libkdd/kdd.xcodeproj/project.pbxproj +++ b/libkdd/kdd.xcodeproj/project.pbxproj @@ -62,6 +62,8 @@ 13F3DA9E1C7C1C6600ACFFCC /* corpse-twr-sample-v2.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */; }; 1862B0341E7A083F0005ADF4 /* stackshot-sample-thread-groups in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */; }; 1862B0351E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */; }; + 18C577C31F96DB5200C67EB3 /* stackshot-sample-thread-groups-flags in Resources */ = {isa = PBXBuildFile; fileRef = 18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */; }; + 18C577C61F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */; }; 18E592981E9451A20018612A /* stackshot-sample-coalitions in Resources */ = {isa = PBXBuildFile; fileRef = 18E592961E9451A20018612A /* stackshot-sample-coalitions */; }; 18E592991E9451A20018612A /* stackshot-sample-coalitions.plist.gz in Resources */ = {isa = PBXBuildFile; fileRef = 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */; }; C91C93CB1ACB58B700119B60 /* kdd.h in Headers */ = {isa = PBXBuildFile; fileRef = C91C93CA1ACB58B700119B60 /* kdd.h */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -165,6 +167,8 @@ 13F3DA9D1C7C1C6000ACFFCC /* corpse-twr-sample-v2.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; name = "corpse-twr-sample-v2.plist.gz"; path = "tests/corpse-twr-sample-v2.plist.gz"; sourceTree = SOURCE_ROOT; }; 1862B0321E7A083F0005ADF4 /* stackshot-sample-thread-groups */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups"; sourceTree = ""; }; 1862B0331E7A083F0005ADF4 /* stackshot-sample-thread-groups.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups.plist.gz"; sourceTree = ""; }; + 18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-thread-groups-flags"; sourceTree = ""; }; + 18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-thread-groups-flags.plist.gz"; sourceTree = ""; }; 18E592961E9451A20018612A /* stackshot-sample-coalitions */ = {isa = PBXFileReference; lastKnownFileType = file; path = "stackshot-sample-coalitions"; sourceTree = ""; }; 18E592971E9451A20018612A /* stackshot-sample-coalitions.plist.gz */ = {isa = PBXFileReference; lastKnownFileType = archive.gzip; path = "stackshot-sample-coalitions.plist.gz"; sourceTree = ""; }; C91C93C71ACB58B700119B60 /* libkdd.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libkdd.a; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -214,6 +218,8 @@ 08603F351BF69EDE007D3784 /* tests */ = { isa = PBXGroup; children = ( + 18C577C51F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz */, + 18C577C11F96DB5100C67EB3 /* stackshot-sample-thread-groups-flags */, C9DCEF001F01C3790000BD02 /* stackshot-sample-instrs-cycles */, C9DCEEFF1F01C3790000BD02 /* stackshot-sample-instrs-cycles.plist.gz */, 088C36DF1EF323AE00ABB2E0 /* stackshot-sample-thread-policy */, @@ -437,6 +443,8 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + 18C577C61F96DB7100C67EB3 /* stackshot-sample-thread-groups-flags.plist.gz in Resources */, + 18C577C31F96DB5200C67EB3 /* stackshot-sample-thread-groups-flags in Resources */, C9DCEF011F01C3810000BD02 /* stackshot-sample-instrs-cycles in Resources */, C9DCEF021F01C3810000BD02 /* stackshot-sample-instrs-cycles.plist.gz in Resources */, 088C36E01EF323C300ABB2E0 /* stackshot-sample-thread-policy in Resources */, diff --git a/libkdd/tests/Tests.swift b/libkdd/tests/Tests.swift index cd7f46ea0..0886e0251 100644 --- a/libkdd/tests/Tests.swift +++ b/libkdd/tests/Tests.swift @@ -1338,6 +1338,10 @@ class Tests: XCTestCase { self.testSampleStackshot("stackshot-sample-thread-groups") } + func testSampleThreadGroupsFlags() { + self.testSampleStackshot("stackshot-sample-thread-groups-flags") + } + func testSampleCoalitions() { self.testSampleStackshot("stackshot-sample-coalitions") } diff --git a/libkdd/tests/stackshot-sample-thread-groups-flags b/libkdd/tests/stackshot-sample-thread-groups-flags new file mode 100644 index 0000000000000000000000000000000000000000..7cd85bafae3f658bc5968ef733e983e4bd7ad9dd GIT binary patch literal 8240 zcmdU!dypJO8NmDU4gvBYhL_}4ULtaryW~P35Y6Q-kR#mX$XyahjA!pm?`~&jriYo? z+dCiO0tg5JML{J*j1Uzq5!A%DCZfTl3ak9XvM8`pR4M+YR6 zX8ZB$*K~jV_4m!rzV*r52!tv517R9GUGVIc83`Z)WZ_&w z7C_$2U%|xB#Th;$!DMnlhLg2~oW6sQ$L?Q5UOhdZ%m?m6z58Wl2>mo@a<8j#a!k^V>S^FQH zG<~aTJGSQ@NT-wUqhz=Z(hqwlj|fNtmJ;$VOhuw$)j}3Q|1pz^i!lc~U$;fmvpcyL zu^_H<38tzniYDXIR^n1j_WHiV2Ksy9h{zKjSGWvQaK2dnbP{qo@}`xZm+9c$m>BT` z?p4`!KOrm}hZl;$WtOw^iqM*P6mwM0>mg@UWwTJmTJpYPUV@>`qvN@~mFWG*-N}w2Rdymza0~!FcakL3jHR@HPfLhs0a2PopE+_PPQgtYD3vwDd`s8Is z*o@rm9~#-rLLZb*Y7!&J1M2%C$XYjqjGF!}OWu&#*CSg@|LS}xEuLD)t;j=WHG+)o zAkjADUf2kn5H9eHj=I%w>Vbt-1t|(rQmC9Ws57X>zY&BS?2rwu^SK?{(_-~Ob|Bw0 z+B*t58dPlB0Q~^+b#T5$g>Wz%M>{dqC-u)pHfhyB3l6t7;e*JPeoC#324l#*#QQn4 zfu-7)kr|5yb)MfIVk{lFSqc?+8GAzez!D;`xo28IzsMu2_<}W4mipfY6=U+)jF^=M zs{|~ZYCt2=dU>c+!HOyQOYDzP#Wu}}mTGA(wK5kj0~6iKtO7UvYx%RlN0B!amJ+hd zaA_Yqyd2Q9{M37A zE)maSz3~v^m@?y=``T1!X1KYp4WGzxb6*>7Y)T(gOVRp&7%QTEhUTqrnv;)U$^zpU zVXB{xn)XWn!0W>)N{%D_c^SM)@%h-TF&XLGP^jO*#aCfG@4Bh-DUC^M|2M-eZf-q7 z_8_O01CQ}kZ&y_sof{!GsycKM!kkV3`zjJNMltH@(6PM{sY>JO!%jd zBCmpINgF=^J(sjmK8w7Zv%EF<9P-Ud`SbuHe&w(}sA8}X zqAy~yUQTa}nco5AA;?eB_n|Ktp7Q^V)+XdhEDmgwl4hJ$JCPTEX<97)!A7CD?6FkzEKgXau~zW{`C z0^p8JRQnck^?kB{f7@^u>_)g-Dcr7!=HxrbYr=ysP>f~2zZ^2{!NBz!VeSj#VdOS^ z9~q}s%669MyU2wnwR4C3gxQhw-@k|45^g7)5r=i&-T)&EO9}ZtrsDgfL-{!|{{g1@ zs8wVAN=W;`m5KQebM56PmKc*C#T*Wp`u1WhP7QvHJb<0c`VJEPB<9L{$6PD=DJHf4 zTc@-v%r&lX1XI$!EmE*Yk@taaguf0(GN0GT&yf3dZGNSkFJR9j4|%{lq>~)*awBTJ zL5$+l|)x9zzcP&G5#7UipLZa@QF(tLftQHki)=$WFl{!yFy83QQj3m}ko4>{Oh3TZyzy$B{p$a8z2JuJsQHSf$; zA*804bav-62Zk#!&g ze6_RTSps#>eC>(5WZ%xO|20jGvGCt2zI2|B|NinnRsK->8!CR}4W0kp2_5&m@()$s z^|FpfU()gZD_&RSwPO#e{$U6eO7J@ckDTLT82`nqmd!o|?|AT@3swAwNXJL*3sin$ zOGU+dx9fQEn>Cf6vxI9tu};NgpjX8Q=4ifnmW~e{7+2*_AM4cho?onE7+;2VTrYDz zj_cz1_y0{^<)N3s$YK8f&$-hj-*@+G)khy&*}Lz7bN6?7i{OLFT(wW8z0ngFWaMd7 wDgX%cFVnGTKfweE37I;16~Ae?Ith@~_*2```fm`PPU3icMAYAH?sVD*ylh literal 0 HcmV?d00001 diff --git a/libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz b/libkdd/tests/stackshot-sample-thread-groups-flags.plist.gz new file mode 100644 index 0000000000000000000000000000000000000000..7ca2eb553745b88b1b98eac3df3744f6830be0b4 GIT binary patch literal 4175 zcmV-V5U}qbiwFoIn&w#m19Nm?V{3D0Z*(nlVQp}1Wi51Qa%Ev;EoX9Xb#QYnW^7?+ zb1rafX>)V{?Ooe)+c*+^p7{!G_SqJ%5|wo_<=B~xYjSbL&P;7>ZLuH{k}xI-1^_K< zzkUHqvPJQxcI4@o^^08=8#n?QeY(*=^X!+0G#T7eAsNr!ys}TNR|A?wJZ9Pa&8u&> zzn%oIetG-z+21a|T-^Ta`u!kJm{f!7Z|^={T?}5G42S1=p3vd&^7e9Y{qgGNcA(oB z4&Q%%HF&j9Dt|p3E|<$wVxBsUcxoP$!)w8FD%9$uZtO%0oW?4CrMp?TUV)?k8?#8g z{rRV}JGy#%7sW)8P%0AL$pu$uL-Sw#;pU_W8_Rp=rrmLLHX(-u$X!*6To_&7Iv%KvE6!nhDbRaU{ag|Tl1^)+0V47oN4qAox2f^nBT%feYg zcFpGFDtAXtU^iKiigeE6ChO(dU2+=iGrC<#MN>lr+Vr(ZJ9RNd0OZauqVz74xjgs8y2bZzw2< zWkGKhOJUXU(@@ zt$ovGsEq^v7r{-bB0b7HZkv`z+143#g3;J%6VJ3k)6S7*=I3bij`Z_1y0`0Ab?oc5 zMjFFCOLK?=(qMPmosgZ*eQV;_UdyO=`y)H8jJ%2Kjvdc#wnq)lNvn3-zSpj$wp#aI{SL{l&qjV%qn+Qf1CG=6J^b4E@32LO&9Lnb(_nWB)|dOP;i&0y zg?YQQR~h11hBHzms#)oR<`t(5eZ-+!2(DBDp$;wC(VbJDvaM92y^kzfh2MFzXlr*V zdC)W~S!|-vcC)CX5~r^D+eUIdP-$9`+(a;Pu^Y@(M6~N7beA-0c1W~`>5M0CP*b+Y z7W9f{S<6V()b~+p$1$w*x9=P7$Y={%b?mBwZctlhkCKJVX%>TRwh>+Xl_n%M4y|^h zQWQyT={WYRjbMaI4t^jV`;NGh7)3*eGi68X58nsM|JnCVm#R^8!l zSodCF`4hL&&(0(3XQI3)zgtBos?SxzgJMt5J6)o+WI28va;_6h8tnV9hP5qBXvW(X zDs%-4J>f!6z;IZ|(CH~35;pYuuZM*W2h3bg_;6SYogu`&@6tUuo&7!k!>WINSU}MM zyk%1UV({(ZMOT1vK#0*7WE>V|9JUHNLXE>$!2u9D)VII;27j+^yEa?Wk~&fsw&ytZ#H+_A$L+#(UbqW)GP3I-!cn{MEXTE9Qlo4;7~3Q3B{fP%a1(d( z!W!l3pO>C)*BRA4>0@@`YnN~QvKnRifi-Ff)Q@ODf?({A$4zeFk95&HDtnB-8y3t3 zFY+p{JVm|@Jk=0U^!e3lA;U7Yxpj@&NV(0qHVs5PQ!Fd0Ntym(*ow#B*9qjTNZTA6 z*V6S&V^p-;ng7B{9iOIDvt6pX~dv{{9IA<^JkEKvNSwN16P6pRf<= zH|bSO%(@3Zvb zoypQ?mndNOj}G*jFY{Xxk>-L=DGKD;*_nRKlIiFrQ#f#;d=_!-E);5kgBHqXa>-PL zgcZtX+DT9>L(*C(t)==mB`FSAD6y3`SJ0Fzq|t@)+C66x5>zmsF&t{BokKjtSGcDM zkCwU4Jk^6Ds9b5;Oh@ z#SwR)6lM$pjuwFV%bfcOvzbIe3nef8k10Vi1yJ_F3e6pG7)DZ-tTJxWFA+fr&T9v` zd!28ZpbV}H-c95)WwsECJSIpkVl9W-hnZzMXiSg>7)oR7#GzW5eGl!~qF{znnledG zNtIR#6ueM6)1K{J%$HewYQzKzW+<`EsWPVu5RgJitt@GAb$MQ>h1nfN!3-rgBDMPB zL}xBII*X2jv(+-S;DmD8O>Ta@;{+#w@W8u^OsFEGI9gD5b>)$wF zp@cRidv{p>Dxgp{)4o`FFc>H83+0INLOxEy*MDaEiKmF;3*`uNRGhF(+c1*pkw(D` z<%mezg@oZ4vD(%8LmXX!rw?#;$Z`C!T0w|jEl79~=h}zFfeX$>@hMB_l9-e)pQAvA zGDWU%wikS8;4{-&kSAmnpMG#oV-zDc0tY|yxuA2bExi%=s9=HPk&nPZ54pzy3T3fW zrBuwR4kvSz+{p;aWYeUeigTo_?+D5drlx|TB0gPPUKGqwlDOb$dO@+aNNczJd)Jyt z`QRK_7(qEAn&pDdaD1CFlpJCvRfcsUXbk0s1Zh;E)R5od;Dj=R$(BqxO3XWkl7gnM z8B3596&l>+GaQ^y>LN=VXKM`3;$$RQnQ~D$aKU^<7c|Ln$aM@pVMnsL5+uSpix1^2 zMOY$phO^HRKsk%ZWWD2sq6`6)y3)(fH(#mDc@`rf2A>!wdCmpS#eo34Nf(vnt8mO% ztt{kWTS;XG9S1Ntkwc?GU>$7^prn?`LL`#nrSxj`4*K^zGA`&`2Mq#gjG^TAsZ`;A zQgcpBeLxA$C*U1Bq@D2zl-s`Cp5GFAr~gE|ELp2>)ra|AQqWt@6BMveZo4q4hu`Qb zS`e09qww(rN^xbn1V?|N{I+Y(5>lXG24|y)xMMhUJbA{oD72*d1j=ojv&0eSB4!!N z?TrbP-Bfg>6NBLUz3{71g2n}sA8A`qW-}FYQ#KbwaTKd-TR#-I;W#?=7fSRGGt#}u zT78<{oD}0J)qh4<#@Du<8NHdxh~}IS4rnO7mA8}=HWdUZ!muqU%grfALJFn01xGrO zX@K?T@Q}JEQ*${_f$x1z!Z-!1Zz9 zLb*+-NKmtf%8<@L+u&jrth~=5mGLTZ5JQRW>eJ=5c{{5#??s~LIG~|~u~pa+T(ar$|%*t|UR?`16{TuQH@#$2Pc}gi(sS zj4u-!V-Xytz za3P8HX_A|@!}|$mOC4NKVwU2ZMz+CqBo-8@qKFMHA7S3NQl?hW3Tc8%NXS(d#dQ5T zZzRZ2E;B_suyO$4DiS(=U%Tl@c)@qT5=ycnzZNtnf?nf*hLYOa0$;7qYT%Rrv^^-n z&Go-c{R6?11nHE$4K9c=J=TTyR=#RO3MII&W%A-eAQeWbzh zMOg6=xH5(;=5s2M3|Z~#;r1;1x8 zRuBZPkFg~8Sg)Xg+THVV?}6#?n5osbD;&_^pdy`>WgOs}?d71MjXcN2jV zLPSW4bm#$Gks@Dk#mj0B66>8x;NleJnRR5$P+Od)#5|vo3{8*0bt-s9S5rz6Yx~>a zVihULWnM_6hyz@$BF+W7ClOLg16-|wXC~8qVqV2~f>c!21{bV|czRl%k2(Ebu;>m4 zG(35%M0UJ0w5j18`Fp+s@4-D7B0A6(l4Ijir0OI*Z)=Tk~hY?}kGaM6wSY)FJ4 zrOdjeg`gx3b8}>272cbhj68d6_7O+b!j_VSq>&6I%OV=;DUB9&DciRKC-8j7cHD7L ZwRl@dPrvg-TjaMt{|_5R6Tzaf008mj5hVZs literal 0 HcmV?d00001 diff --git a/libsyscall/Libsyscall.xcodeproj/project.pbxproj b/libsyscall/Libsyscall.xcodeproj/project.pbxproj index 8175500a0..d0053b2da 100644 --- a/libsyscall/Libsyscall.xcodeproj/project.pbxproj +++ b/libsyscall/Libsyscall.xcodeproj/project.pbxproj @@ -107,6 +107,7 @@ 29A59AE2183B0DE000E8B896 /* renameat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE1183B0DE000E8B896 /* renameat.c */; }; 29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A59AE5183B110C00E8B896 /* unlinkat.c */; }; 2BA88DCC1810A3CE00EB63F6 /* coalition.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BA88DCB1810A3CE00EB63F6 /* coalition.c */; }; + 3695DCC91F3D2C5F0072C0B3 /* reboot.c in Sources */ = {isa = PBXBuildFile; fileRef = 3695DCC81F3D2C5A0072C0B3 /* reboot.c */; }; 374A36E314748F1300AAF39D /* varargs_wrappers.s in Sources */ = {isa = PBXBuildFile; fileRef = 374A36E214748EE400AAF39D /* varargs_wrappers.s */; }; 3F538F891A659C5600B37EFD /* persona.c in Sources */ = {isa = PBXBuildFile; fileRef = 3F538F881A659C5600B37EFD /* persona.c */; }; 401BB71A1BCAE57B005080D3 /* os_channel.c in Sources */ = {isa = PBXBuildFile; fileRef = 401BB7161BCAE539005080D3 /* os_channel.c */; settings = {COMPILER_FLAGS = "-fno-builtin"; }; }; @@ -491,6 +492,7 @@ 29A59AE1183B0DE000E8B896 /* renameat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = renameat.c; sourceTree = ""; }; 29A59AE5183B110C00E8B896 /* unlinkat.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = unlinkat.c; sourceTree = ""; }; 2BA88DCB1810A3CE00EB63F6 /* coalition.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = coalition.c; sourceTree = ""; }; + 3695DCC81F3D2C5A0072C0B3 /* reboot.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = reboot.c; sourceTree = ""; }; 374A36E214748EE400AAF39D /* varargs_wrappers.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = varargs_wrappers.s; sourceTree = ""; }; 37DDFB7614748713009D3355 /* syscall.map */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = syscall.map; sourceTree = ""; }; 3F538F881A659C5600B37EFD /* persona.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = persona.c; sourceTree = ""; }; @@ -805,6 +807,7 @@ BA9973461C3B4C8A00B14D8C /* quota_obsolete.c */, 24B8C2611237F53900D36CC3 /* remove-counter.c */, 248AA966122C7CDA0085F5B1 /* rename.c */, + 3695DCC81F3D2C5A0072C0B3 /* reboot.c */, 29A59AE1183B0DE000E8B896 /* renameat.c */, 906AA2D018F74CD1001C681A /* renamex.c */, 248AA964122C7C330085F5B1 /* rmdir.c */, @@ -1319,6 +1322,7 @@ C9D9BD4D114B00600000D8B9 /* processor.defs in Sources */, C9D9BD53114B00600000D8B9 /* netname.defs in Sources */, C9D9BD57114B00600000D8B9 /* task.defs in Sources */, + 3695DCC91F3D2C5F0072C0B3 /* reboot.c in Sources */, C9D9BD58114B00600000D8B9 /* thread_act.defs in Sources */, 72E09E941B444B19006F11A4 /* mach_continuous_time.c in Sources */, 29A59AE6183B110C00E8B896 /* unlinkat.c in Sources */, diff --git a/libsyscall/wrappers/libproc/libproc.h b/libsyscall/wrappers/libproc/libproc.h index 3cfcfdc70..ccf413fc3 100644 --- a/libsyscall/wrappers/libproc/libproc.h +++ b/libsyscall/wrappers/libproc/libproc.h @@ -127,6 +127,7 @@ int proc_clear_dirty(pid_t pid, uint32_t flags); int proc_terminate(pid_t pid, int *sig); #ifdef PRIVATE +#include /* * Enumerate potential userspace pointers embedded in kernel data structures. * Currently inspects kqueues only. diff --git a/libsyscall/wrappers/mach_absolute_time.s b/libsyscall/wrappers/mach_absolute_time.s index cf93161b9..2c637bfc9 100644 --- a/libsyscall/wrappers/mach_absolute_time.s +++ b/libsyscall/wrappers/mach_absolute_time.s @@ -232,16 +232,13 @@ _mach_continuous_time_kernel: * read the register. If the offset changes, we have gone to sleep in the midst of * doing a read. This case should be exceedingly rare, but could result in a terribly * inaccurate result, so we need to get a fresh timebase value. - * - * Note that the commpage address construction expects our top 2 bytes to be 0xFFFF. - * If this changes (i.e, we significantly relocate the commpage), this logic will need - * to change as well (use 4 movk instructions rather than cheating with the movn). */ .text .align 2 .globl _mach_absolute_time _mach_absolute_time: - movn x3, #(~((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32 + movk x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 48) & 0x000000000000FFFF), lsl #48 + movk x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 32) & 0x000000000000FFFF), lsl #32 movk x3, #(((_COMM_PAGE_TIMEBASE_OFFSET) >> 16) & 0x000000000000FFFF), lsl #16 movk x3, #((_COMM_PAGE_TIMEBASE_OFFSET) & 0x000000000000FFFF) ldrb w2, [x3, #((_COMM_PAGE_USER_TIMEBASE) - (_COMM_PAGE_TIMEBASE_OFFSET))] diff --git a/libsyscall/wrappers/reboot.c b/libsyscall/wrappers/reboot.c new file mode 100644 index 000000000..c703ed4c3 --- /dev/null +++ b/libsyscall/wrappers/reboot.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include + +extern int __reboot(int, const char *); + +int +reboot(int howto) +{ + int res = __reboot(howto, NULL); + return res; +} + +int +reboot_np(int howto, const char *msg) +{ + int res = __reboot(howto, msg); + return res; +} diff --git a/libsyscall/wrappers/spawn/posix_spawn.c b/libsyscall/wrappers/spawn/posix_spawn.c index 69002dc0b..1334f4c3d 100644 --- a/libsyscall/wrappers/spawn/posix_spawn.c +++ b/libsyscall/wrappers/spawn/posix_spawn.c @@ -713,7 +713,6 @@ posix_spawn_growportactions_np(posix_spawnattr_t *attr) { _posix_spawnattr_t psattr; _posix_spawn_port_actions_t acts; - int newnum; if (attr == NULL || *attr == NULL) return EINVAL; @@ -724,8 +723,14 @@ posix_spawn_growportactions_np(posix_spawnattr_t *attr) return EINVAL; /* Double number of port actions allocated for */ - newnum = 2 * acts->pspa_alloc; - acts = realloc(acts, PS_PORT_ACTIONS_SIZE(newnum)); + int newnum = 0; + if (os_mul_overflow(acts->pspa_alloc, 2, &newnum)) + return ENOMEM; + size_t newsize = PS_PORT_ACTIONS_SIZE(newnum); + if (newsize == 0) + return ENOMEM; + + acts = realloc(acts, newsize); if (acts == NULL) return ENOMEM; @@ -1031,8 +1036,13 @@ posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *file_actions) static int _posix_spawn_file_actions_grow(_posix_spawn_file_actions_t *psactsp) { - int new_alloc = (*psactsp)->psfa_act_alloc * 2; - _posix_spawn_file_actions_t new_psacts; + int newnum = 0; + if (os_mul_overflow((*psactsp)->psfa_act_alloc, 2, &newnum)) + return ENOMEM; + + size_t newsize = PSF_ACTIONS_SIZE(newnum); + if (newsize == 0) + return ENOMEM; /* * XXX may want to impose an administrative limit here; POSIX does @@ -1040,13 +1050,14 @@ _posix_spawn_file_actions_grow(_posix_spawn_file_actions_t *psactsp) * XXX so it's probably acceptable to just fail catastrophically * XXX instead of implementing one. */ - if ((new_psacts = (_posix_spawn_file_actions_t)realloc((*psactsp), PSF_ACTIONS_SIZE(new_alloc))) == NULL) { - return (ENOMEM); + _posix_spawn_file_actions_t new_psacts; + if ((new_psacts = (_posix_spawn_file_actions_t)realloc((*psactsp), newsize)) == NULL) { + return ENOMEM; } - new_psacts->psfa_act_alloc = new_alloc; + new_psacts->psfa_act_alloc = newnum; *psactsp = new_psacts; - return (0); + return 0; } @@ -1515,10 +1526,16 @@ posix_spawnattr_setmacpolicyinfo_np(posix_spawnattr_t * __restrict attr, psmx->psmx_count = 0; } else if (psmx->psmx_count == psmx->psmx_alloc) { - psmx = psattr->psa_mac_extensions = reallocf(psmx, PS_MAC_EXTENSIONS_SIZE(psmx->psmx_alloc * 2)); + int newnum = 0; + if (os_mul_overflow(psmx->psmx_alloc, 2, &newnum)) + return ENOMEM; + size_t extsize = PS_MAC_EXTENSIONS_SIZE(newnum); + if (extsize == 0) + return ENOMEM; + psmx = psattr->psa_mac_extensions = reallocf(psmx, extsize); if (psmx == NULL) return ENOMEM; - psmx->psmx_alloc *= 2; + psmx->psmx_alloc = newnum; } extension = &psmx->psmx_extensions[psmx->psmx_count]; strlcpy(extension->policyname, policyname, sizeof(extension->policyname)); @@ -1801,14 +1818,24 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path, ad.attrp = psattr; if (psattr->psa_ports != NULL) { + size_t psact_size = PS_PORT_ACTIONS_SIZE(psattr->psa_ports->pspa_count); + if (psact_size == 0 && psattr->psa_ports->pspa_count != 0) { + errno = EINVAL; + ret = -1; + goto out; + } ad.port_actions = psattr->psa_ports; - ad.port_actions_size = PS_PORT_ACTIONS_SIZE( - ad.port_actions->pspa_count); + ad.port_actions_size = psact_size; } if (psattr->psa_mac_extensions != NULL) { + size_t macext_size = PS_MAC_EXTENSIONS_SIZE(psattr->psa_mac_extensions->psmx_count); + if (macext_size == 0 && psattr->psa_mac_extensions->psmx_count != 0) { + errno = EINVAL; + ret = -1; + goto out; + } ad.mac_extensions = psattr->psa_mac_extensions; - ad.mac_extensions_size = PS_MAC_EXTENSIONS_SIZE( - ad.mac_extensions->psmx_count); + ad.mac_extensions_size = macext_size; } if (psattr->psa_coalition_info != NULL) { ad.coal_info_size = sizeof(struct _posix_spawn_coalition_info); @@ -1824,7 +1851,13 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path, *(_posix_spawn_file_actions_t *)file_actions; if (psactsp->psfa_act_count > 0) { - ad.file_actions_size = PSF_ACTIONS_SIZE(psactsp->psfa_act_count); + size_t fa_size = PSF_ACTIONS_SIZE(psactsp->psfa_act_count); + if (fa_size == 0 && psactsp->psfa_act_count != 0) { + errno = EINVAL; + ret = -1; + goto out; + } + ad.file_actions_size = fa_size; ad.file_actions = psactsp; } } @@ -1833,6 +1866,7 @@ posix_spawn(pid_t * __restrict pid, const char * __restrict path, } else ret = __posix_spawn(pid, path, NULL, argv, envp); +out: if (ret < 0) ret = errno; errno = saveerrno; diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def index 4eca357c9..8c49bd74b 100644 --- a/makedefs/MakeInc.def +++ b/makedefs/MakeInc.def @@ -60,7 +60,7 @@ else ifeq ($(PLATFORM),tvOS) else ifeq ($(PLATFORM),AppleTVOS) DEPLOYMENT_TARGET_FLAGS = -mtvos-version-min=$(SDKVERSION) else ifeq ($(PLATFORM),BridgeOS) - DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) + DEPLOYMENT_TARGET_FLAGS = -mbridgeos-version-min=$(SDKVERSION) -DXNU_TARGET_OS_BRIDGE else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) DEPLOYMENT_TARGET_FLAGS = -miphoneos-version-min=$(SDKVERSION) else ifneq ($(filter $(SUPPORTED_SIMULATOR_PLATFORMS),$(PLATFORM)),) @@ -694,14 +694,6 @@ INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) INSTALL_KERNEL_SYM_TO_KDK = 1 INSTALL_XNU_DEBUG_FILES = 1 -else ifeq ($(RC_ProjectName),xnu_kasan) -ifeq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) -# MacOS -INSTALL_KERNEL_DIR := $(DEVELOPER_EXTRAS_DIR) -INSTALL_KERNEL_SYM_DIR := $(DEVELOPER_EXTRAS_DIR) -endif -INSTALL_KERNEL_SYM_TO_KDK = 1 -INSTALL_KASAN_ONLY = 1 else ifneq ($(filter $(SUPPORTED_EMBEDDED_PLATFORMS),$(PLATFORM)),) INSTALL_KERNEL_SYM_TO_KDK = 1 USE_BINARY_PLIST = 1 @@ -713,4 +705,8 @@ INSTALL_KERNEL_SYM_DIR := $(SYSTEM_LIBRARY_KERNELS_DIR) INSTALL_KERNEL_SYM_TO_KDK = $(if $(filter YES,$(DWARF_DSYM_FILE_SHOULD_ACCOMPANY_PRODUCT)),1,0) endif +ifeq ($(RC_ProjectName),xnu_kasan) +INSTALL_KASAN_ONLY = 1 +endif + # vim: set ft=make: diff --git a/osfmk/arm/commpage/commpage.c b/osfmk/arm/commpage/commpage.c index 520a140d0..430a149b7 100644 --- a/osfmk/arm/commpage/commpage.c +++ b/osfmk/arm/commpage/commpage.c @@ -263,22 +263,14 @@ commpage_init_cpu_capabilities( void ) bits |= (cpus << kNumCPUsShift); bits |= kFastThreadLocalStorage; // TPIDRURO for TLS + #if __ARM_VFP__ bits |= kHasVfp; -#if (__ARM_VFP__ >= 3) - bits |= kHasNeon; - -#if defined(__arm64__) - bits |= kHasNeonHPFP; -#else - boolean_t intr = ml_set_interrupts_enabled(FALSE); - unsigned int mvfr1 = get_mvfr1(); - - if (mvfr1 & MVFR_ASIMD_HPFP) + arm_mvfp_info_t *mvfp_info = arm_mvfp_info(); + if (mvfp_info->neon) + bits |= kHasNeon; + if (mvfp_info->neon_hpfp) bits |= kHasNeonHPFP; - (void) ml_set_interrupts_enabled(intr); -#endif -#endif #endif #if defined(__arm64__) bits |= kHasFMA; diff --git a/osfmk/arm/cpu_capabilities.h b/osfmk/arm/cpu_capabilities.h index f1d16dde6..71050291c 100644 --- a/osfmk/arm/cpu_capabilities.h +++ b/osfmk/arm/cpu_capabilities.h @@ -100,8 +100,9 @@ typedef struct { #if defined(__arm64__) -#define _COMM_PAGE64_BASE_ADDRESS (0xfffffff0001fc000ULL) /* Just below the kernel, safely in TTBR1 */ -#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_PAGE64_BASE_ADDRESS - (PAGE_SIZE)) /* Privileged RO in kernel mode */ +#define _COMM_PAGE64_BASE_ADDRESS (0x0000000FFFFFC000ULL) /* In TTBR0 */ +#define _COMM_HIGH_PAGE64_BASE_ADDRESS (0xFFFFFFF0001FC000ULL) /* Just below the kernel, safely in TTBR1; only used for testing */ +#define _COMM_PRIV_PAGE64_BASE_ADDRESS (_COMM_HIGH_PAGE64_BASE_ADDRESS - (PAGE_SIZE)) /* Privileged RO in kernel mode */ #define _COMM_PAGE64_AREA_LENGTH (_COMM_PAGE32_AREA_LENGTH) #define _COMM_PAGE64_AREA_USED (-1) @@ -117,12 +118,12 @@ extern vm_address_t sharedpage_rw_addr; #define _COMM_PAGE_BASE_ADDRESS (sharedpage_rw_addr) #define _COMM_PAGE_START_ADDRESS (sharedpage_rw_addr) -#else +#else /* KERNEL_PRIVATE */ #define _COMM_PAGE_AREA_LENGTH (4096) #define _COMM_PAGE_BASE_ADDRESS _COMM_PAGE64_BASE_ADDRESS #define _COMM_PAGE_START_ADDRESS _COMM_PAGE64_BASE_ADDRESS -#endif +#endif /* KERNEL_PRIVATE */ #elif defined(__arm__) diff --git a/osfmk/arm/cpu_data_internal.h b/osfmk/arm/cpu_data_internal.h index 264e7ed96..ecef9f939 100644 --- a/osfmk/arm/cpu_data_internal.h +++ b/osfmk/arm/cpu_data_internal.h @@ -188,9 +188,9 @@ typedef struct cpu_data uint64_t cpu_idle_latency; uint64_t cpu_idle_pop; -#if __arm__ +#if __arm__ || __ARM_KERNEL_PROTECT__ vm_offset_t cpu_exc_vectors; -#endif +#endif /* __ARM_KERNEL_PROTECT__ */ vm_offset_t cpu_reset_handler; uint32_t cpu_reset_type; uintptr_t cpu_reset_assist; diff --git a/osfmk/arm/machine_cpuid.c b/osfmk/arm/machine_cpuid.c index 232c6e64a..3943f47d4 100644 --- a/osfmk/arm/machine_cpuid.c +++ b/osfmk/arm/machine_cpuid.c @@ -135,24 +135,20 @@ machine_arm_debug_info(void) void machine_do_mvfpid() { +#if __arm__ arm_mvfr0_info_t arm_mvfr0_info; arm_mvfr1_info_t arm_mvfr1_info; - uint64_t tmp; -#if __arm__ - (void)tmp; __asm__ volatile("vmrs %0, mvfr0":"=r"(arm_mvfr0_info.value)); __asm__ volatile("vmrs %0, mvfr1":"=r"(arm_mvfr1_info.value)); -#else - __asm__ volatile("mrs %0, MVFR0_EL1":"=r"(tmp)); - arm_mvfr0_info.value = (uint32_t)tmp; - - __asm__ volatile("mrs %0, MVFR1_EL1":"=r"(tmp)); - arm_mvfr1_info.value = (uint32_t)tmp; -#endif cpuid_mvfp_info.neon = arm_mvfr1_info.bits.SP; cpuid_mvfp_info.neon_hpfp = arm_mvfr1_info.bits.HPFP; +#else + cpuid_mvfp_info.neon = 1; + cpuid_mvfp_info.neon_hpfp = 1; +#endif + } arm_mvfp_info_t * diff --git a/osfmk/arm/machine_routines.h b/osfmk/arm/machine_routines.h index 3510649b4..3e46c61e1 100644 --- a/osfmk/arm/machine_routines.h +++ b/osfmk/arm/machine_routines.h @@ -529,10 +529,6 @@ void set_fpscr(uint32_t); extern void init_vfp(void); extern boolean_t get_vfp_enabled(void); -#if (__ARM_VFP__ >= 3) -extern unsigned int get_mvfr0(void); -extern unsigned int get_mvfr1(void); -#endif extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); extern void fiq_context_init(boolean_t enable_fiq); @@ -551,6 +547,9 @@ void rorgn_stash_range(void); void rorgn_lockdown(void); #endif /* defined(KERNEL_INTEGRITY_KTRR)*/ +#if __ARM_KERNEL_PROTECT__ +extern void set_vbar_el1(uint64_t); +#endif /* __ARM_KERNEL_PROTECT__ */ #endif /* MACH_KERNEL_PRIVATE */ extern uint32_t arm_debug_read_dscr(void); diff --git a/osfmk/arm/machine_routines_asm.s b/osfmk/arm/machine_routines_asm.s index b64d28373..d0b0a6b96 100644 --- a/osfmk/arm/machine_routines_asm.s +++ b/osfmk/arm/machine_routines_asm.s @@ -177,18 +177,6 @@ LEXT(set_fpscr) #endif bx lr -#if (__ARM_VFP__ >= 3) - .align 2 - .globl EXT(get_mvfr0) -LEXT(get_mvfr0) - vmrs r0, mvfr0 - bx lr - .globl EXT(get_mvfr1) -LEXT(get_mvfr1) - vmrs r0, mvfr1 - bx lr -#endif - /* * void OSSynchronizeIO(void) */ diff --git a/osfmk/arm/model_dep.c b/osfmk/arm/model_dep.c index ab930d549..ad9fd23ea 100644 --- a/osfmk/arm/model_dep.c +++ b/osfmk/arm/model_dep.c @@ -92,7 +92,7 @@ extern int kdp_stack_snapshot_bytes_traced(void); * Increment the PANICLOG_VERSION if you change the format of the panic * log in any way. */ -#define PANICLOG_VERSION 8 +#define PANICLOG_VERSION 9 static struct kcdata_descriptor kc_panic_data; extern char firmware_version[]; @@ -147,6 +147,14 @@ unsigned int debug_ack_timeout_count = 0; volatile unsigned int debugger_sync = 0; volatile unsigned int mp_kdp_trap = 0; /* CPUs signalled by the debug CPU will spin on this */ unsigned int DebugContextCount = 0; + +#if defined(__arm64__) +uint8_t PE_smc_stashed_x86_system_state = 0xFF; +uint8_t PE_smc_stashed_x86_power_state = 0xFF; +uint8_t PE_smc_stashed_x86_efi_boot_state = 0xFF; +uint32_t PE_pcie_stashed_link_state = UINT32_MAX; +#endif + // Convenient macros to easily validate one or more pointers if // they have defined types @@ -319,6 +327,14 @@ do_print_all_backtraces( if (last_hwaccess_thread) { paniclog_append_noflush("AppleHWAccess Thread: 0x%llx\n", last_hwaccess_thread); } +#if defined(XNU_TARGET_OS_BRIDGE) + paniclog_append_noflush("PCIeUp link state: "); + if (PE_pcie_stashed_link_state != UINT32_MAX) { + paniclog_append_noflush("0x%x", PE_pcie_stashed_link_state); + } else { + paniclog_append_noflush("not available\n"); + } +#endif } paniclog_append_noflush("Memory ID: 0x%x\n", gPlatformMemoryID); paniclog_append_noflush("OS version: %.256s\n", @@ -332,6 +348,26 @@ do_print_all_backtraces( paniclog_append_noflush("iBoot version: %.128s\n", firmware_version); paniclog_append_noflush("secure boot?: %s\n", debug_enabled ? "NO": "YES"); +#if defined(XNU_TARGET_OS_BRIDGE) + paniclog_append_noflush("x86 EFI Boot State: "); + if (PE_smc_stashed_x86_efi_boot_state != 0xFF) { + paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_efi_boot_state); + } else { + paniclog_append_noflush("not available\n"); + } + paniclog_append_noflush("x86 System State: "); + if (PE_smc_stashed_x86_system_state != 0xFF) { + paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_system_state); + } else { + paniclog_append_noflush("not available\n"); + } + paniclog_append_noflush("x86 Power State: "); + if (PE_smc_stashed_x86_power_state != 0xFF) { + paniclog_append_noflush("0x%x\n", PE_smc_stashed_x86_power_state); + } else { + paniclog_append_noflush("not available\n"); + } +#endif paniclog_append_noflush("Paniclog version: %d\n", logversion); panic_display_kernel_aslr(); @@ -574,6 +610,12 @@ SavePanicInfo( panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC; } +#if defined(XNU_TARGET_OS_BRIDGE) + panic_info->eph_x86_power_state = PE_smc_stashed_x86_power_state; + panic_info->eph_x86_efi_boot_state = PE_smc_stashed_x86_efi_boot_state; + panic_info->eph_x86_system_state = PE_smc_stashed_x86_system_state; +#endif + /* * On newer targets, panic data is stored directly into the iBoot panic region. * If we re-enter SavePanicInfo (e.g. on a double panic) on such a target, update the diff --git a/osfmk/arm/pmap.c b/osfmk/arm/pmap.c index 327ac9b9f..990ab667d 100644 --- a/osfmk/arm/pmap.c +++ b/osfmk/arm/pmap.c @@ -88,6 +88,17 @@ #include +#if MACH_ASSERT +int pmap_stats_assert = 1; +#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...) \ + MACRO_BEGIN \ + if (pmap_stats_assert && (pmap)->pmap_stats_assert) \ + assertf(cond, fmt, ##__VA_ARGS__); \ + MACRO_END +#else /* MACH_ASSERT */ +#define PMAP_STATS_ASSERTF(cond, pmap, fmt, ...) +#endif /* MACH_ASSERT */ + #if DEVELOPMENT || DEBUG #define PMAP_FOOTPRINT_SUSPENDED(pmap) ((pmap)->footprint_suspended) #else /* DEVELOPMENT || DEBUG */ @@ -366,7 +377,7 @@ SECURITY_READ_ONLY_LATE(vm_map_offset_t) arm64_pmap_max_offset_default = 0x0; static uint32_t asid_bitmap[MAX_ASID / (sizeof(uint32_t) * NBBY)] MARK_AS_PMAP_DATA; #if (__ARM_VMSA__ > 7) -SECURITY_READ_ONLY_LATE(pmap_t) u32_sharedpage_pmap; +SECURITY_READ_ONLY_LATE(pmap_t) sharedpage_pmap; #endif @@ -975,7 +986,7 @@ static inline tt_entry_t *pmap_tt2e( static inline pt_entry_t *pmap_tt3e( pmap_t, vm_map_address_t); -static void pmap_unmap_sharedpage32( +static void pmap_unmap_sharedpage( pmap_t pmap); static void pmap_sharedpage_flush_32_to_64( @@ -1605,11 +1616,33 @@ alloc_asid( if (temp > 0) { temp -= 1; asid_bitmap[asid_bitmap_index] &= ~(1 << temp); +#if __ARM_KERNEL_PROTECT__ + /* + * We need two ASIDs: n and (n | 1). n is used for EL0, + * (n | 1) for EL1. + */ + unsigned int temp2 = temp | 1; + assert(temp2 < MAX_ASID); + assert(temp2 < 32); + assert(temp2 != temp); + assert(asid_bitmap[asid_bitmap_index] & (1 << temp2)); + + /* Grab the second ASID. */ + asid_bitmap[asid_bitmap_index] &= ~(1 << temp2); +#endif /* __ARM_KERNEL_PROTECT__ */ simple_unlock(&pmaps_lock); - /* We should never vend out physical ASID 0 through this method. */ + /* + * We should never vend out physical ASID 0 through this + * method, as it belongs to the kernel. + */ assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 0); +#if __ARM_KERNEL_PROTECT__ + /* Or the kernel EL1 ASID. */ + assert(((asid_bitmap_index * sizeof(uint32_t) * NBBY + temp) % ARM_MAX_ASID) != 1); +#endif /* __ARM_KERNEL_PROTECT__ */ + return (asid_bitmap_index * sizeof(uint32_t) * NBBY + temp); } } @@ -1631,6 +1664,13 @@ free_asid( simple_lock(&pmaps_lock); setbit(asid, (int *) asid_bitmap); + +#if __ARM_KERNEL_PROTECT__ + assert((asid | 1) < MAX_ASID); + assert((asid | 1) != asid); + setbit(asid | 1, (int *) asid_bitmap); +#endif /* __ARM_KERNEL_PROTECT__ */ + simple_unlock(&pmaps_lock); } @@ -2510,6 +2550,9 @@ pmap_map_bd_with_options( tmplate = pa_to_pte(start) | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) | mem_attr | ARM_PTE_TYPE | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF; +#if __ARM_KERNEL_PROTECT__ + tmplate |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ vaddr = virt; paddr = start; @@ -2554,8 +2597,11 @@ pmap_map_bd( /* not cacheable and not buffered */ tmplate = pa_to_pte(start) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX - | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) + | ARM_PTE_AP((prot & VM_PROT_WRITE) ? AP_RWNA : AP_RONA) | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DISABLE); +#if __ARM_KERNEL_PROTECT__ + tmplate |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ vaddr = virt; paddr = start; @@ -2644,6 +2690,9 @@ scan: #else pte |= ARM_PTE_SH; #endif +#if __ARM_KERNEL_PROTECT__ + pte |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ WRITE_PTE(ptep, pte); } PMAP_UPDATE_TLBS(kernel_pmap, va_start, va_start + len); @@ -2898,6 +2947,11 @@ pmap_bootstrap( */ for (i = 0; i < MAX_ASID; i += ARM_MAX_ASID) { asid_bitmap[i / (sizeof(uint32_t) * NBBY)] &= ~(1 << (i % (sizeof(uint32_t) * NBBY))); + +#if __ARM_KERNEL_PROTECT__ + assert((i + 1) < MAX_ASID); + asid_bitmap[(i + 1) / (sizeof(uint32_t) * NBBY)] &= ~(1 << ((i + 1) % (sizeof(uint32_t) * NBBY))); +#endif /* __ARM_KERNEL_PROTECT__ */ } kernel_pmap->asid = 0; @@ -2928,6 +2982,12 @@ pmap_bootstrap( pmap_nesting_size_max = ARM_NESTING_SIZE_MAX; simple_lock_init(&phys_backup_lock, 0); + +#if MACH_ASSERT + PE_parse_boot_argn("pmap_stats_assert", + &pmap_stats_assert, + sizeof (pmap_stats_assert)); +#endif /* MACH_ASSERT */ } @@ -3231,6 +3291,7 @@ pmap_create_internal( p->nested_region_asid_bitmap_size = 0x0UL; #if MACH_ASSERT + p->pmap_stats_assert = TRUE; p->pmap_pid = 0; strlcpy(p->pmap_procname, "", sizeof (p->pmap_procname)); #endif /* MACH_ASSERT */ @@ -3278,9 +3339,31 @@ pmap_set_process_internal( pmap->pmap_pid = pid; strlcpy(pmap->pmap_procname, procname, sizeof (pmap->pmap_procname)); -#endif + if (!strncmp(procname, "corecaptured", sizeof (pmap->pmap_procname))) { + /* + * XXX FBDP + * "corecaptured" somehow triggers some issues that make + * the pmap stats and ledgers to go off track, causing + * some assertion failures and ledger panics. + * Turn that off if the terminating process is "corecaptured". + */ + pmap->pmap_stats_assert = FALSE; + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.phys_footprint); + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.internal); + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.internal_compressed); + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.iokit_mapped); + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.alternate_accounting); + ledger_disable_panic_on_negative(pmap->ledger, + task_ledgers.alternate_accounting_compressed); + } +#endif /* MACH_ASSERT */ } -#endif +#endif /* MACH_ASSERT*/ #if MACH_ASSERT void @@ -3513,8 +3596,7 @@ pmap_destroy_internal( return; } - if (!pmap->is_64bit) - pmap_unmap_sharedpage32(pmap); + pmap_unmap_sharedpage(pmap); if (hw_atomic_sub(&pmap->ref_count, 1) == 0) { @@ -3560,7 +3642,6 @@ pmap_destroy_internal( assert((tt_free_entry_t*)pmap->tt_entry_free == NULL); - flush_mmu_tlb_asid((uint64_t)(pmap->asid) << TLBI_ASID_SHIFT); free_asid(pmap->vasid); @@ -4238,7 +4319,8 @@ pmap_remove_range_options( /* sanity checks... */ #if MACH_ASSERT if (pmap->stats.internal < num_internal) { - if ((pmap->stats.internal + pmap->stats.reusable) == + if ((! pmap_stats_assert) || + (pmap->stats.internal + pmap->stats.reusable) == (num_internal + num_reusable)) { num_reusable_mismatch++; printf("pmap_remove_range_options(%p,0x%llx,%p,%p,0x%x): num_internal=%d num_removed=%d num_unwired=%d num_external=%d num_reusable=%d num_compressed=%lld num_alt_internal=%d num_alt_compressed=%lld num_pte_changed=%d stats.internal=%d stats.reusable=%d\n", @@ -4282,23 +4364,27 @@ pmap_remove_range_options( } } #endif /* MACH_ASSERT */ - assertf(pmap->stats.external >= num_external, - "pmap=%p num_external=%d stats.external=%d", - pmap, num_external, pmap->stats.external); - assertf(pmap->stats.internal >= num_internal, - "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", - pmap, - num_internal, pmap->stats.internal, - num_reusable, pmap->stats.reusable); - assertf(pmap->stats.reusable >= num_reusable, - "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", - pmap, - num_internal, pmap->stats.internal, - num_reusable, pmap->stats.reusable); - assertf(pmap->stats.compressed >= num_compressed, - "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld", - pmap, num_compressed, num_alt_compressed, - pmap->stats.compressed); + PMAP_STATS_ASSERTF(pmap->stats.external >= num_external, + pmap, + "pmap=%p num_external=%d stats.external=%d", + pmap, num_external, pmap->stats.external); + PMAP_STATS_ASSERTF(pmap->stats.internal >= num_internal, + pmap, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + PMAP_STATS_ASSERTF(pmap->stats.reusable >= num_reusable, + pmap, + "pmap=%p num_internal=%d stats.internal=%d num_reusable=%d stats.reusable=%d", + pmap, + num_internal, pmap->stats.internal, + num_reusable, pmap->stats.reusable); + PMAP_STATS_ASSERTF(pmap->stats.compressed >= num_compressed, + pmap, + "pmap=%p num_compressed=%lld num_alt_compressed=%lld stats.compressed=%lld", + pmap, num_compressed, num_alt_compressed, + pmap->stats.compressed); /* update pmap stats... */ OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count); @@ -4493,6 +4579,9 @@ pmap_flush_core_tlb_asid(pmap_t pmap) flush_core_tlb_asid(pmap->asid); #else flush_core_tlb_asid(((uint64_t) pmap->asid) << TLBI_ASID_SHIFT); +#if __ARM_KERNEL_PROTECT__ + flush_core_tlb_asid(((uint64_t) pmap->asid + 1) << TLBI_ASID_SHIFT); +#endif /* __ARM_KERNEL_PROTECT__ */ #endif } @@ -4758,13 +4847,13 @@ pmap_page_protect_options_internal( if (IS_REUSABLE_PAGE(pai) && IS_INTERNAL_PAGE(pai) && !is_altacct) { - assert(pmap->stats.reusable > 0); + PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable); OSAddAtomic(-1, &pmap->stats.reusable); } else if (IS_INTERNAL_PAGE(pai)) { - assert(pmap->stats.internal > 0); + PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal); OSAddAtomic(-1, &pmap->stats.internal); } else { - assert(pmap->stats.external > 0); + PMAP_STATS_ASSERTF(pmap->stats.external > 0, pmap, "stats.external %d", pmap->stats.external); OSAddAtomic(-1, &pmap->stats.external); } if ((options & PMAP_OPTIONS_COMPRESSOR) && @@ -5579,6 +5668,9 @@ Pmap_enter_retry: pte |= wimg_to_pte(wimg_bits); if (pmap == kernel_pmap) { +#if __ARM_KERNEL_PROTECT__ + pte |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ if (prot & VM_PROT_WRITE) { pte |= ARM_PTE_AP(AP_RWNA); pa_set_bits(pa, PP_ATTR_MODIFIED | PP_ATTR_REFERENCED); @@ -5960,7 +6052,7 @@ pmap_change_wiring_internal( OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count); pmap_ledger_credit(pmap, task_ledgers.wired_mem, PAGE_SIZE); } else if (!wired && pte_is_wired(*pte_p)) { - assert(pmap->stats.wired_count >= 1); + PMAP_STATS_ASSERTF(pmap->stats.wired_count >= 1, pmap, "stats.wired_count %d", pmap->stats.wired_count); pte_set_wired(pte_p, wired); OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count); pmap_ledger_debit(pmap, task_ledgers.wired_mem, PAGE_SIZE); @@ -7294,12 +7386,12 @@ arm_force_fast_fault_internal( is_internal && pmap != kernel_pmap) { /* one less "reusable" */ - assert(pmap->stats.reusable > 0); + PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable); OSAddAtomic(-1, &pmap->stats.reusable); /* one more "internal" */ OSAddAtomic(+1, &pmap->stats.internal); PMAP_STATS_PEAK(pmap->stats.internal); - assert(pmap->stats.internal > 0); + PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal); pmap_ledger_credit(pmap, task_ledgers.internal, machine_ptob(1)); @@ -7325,9 +7417,9 @@ arm_force_fast_fault_internal( /* one more "reusable" */ OSAddAtomic(+1, &pmap->stats.reusable); PMAP_STATS_PEAK(pmap->stats.reusable); - assert(pmap->stats.reusable > 0); + PMAP_STATS_ASSERTF(pmap->stats.reusable > 0, pmap, "stats.reusable %d", pmap->stats.reusable); /* one less "internal" */ - assert(pmap->stats.internal > 0); + PMAP_STATS_ASSERTF(pmap->stats.internal > 0, pmap, "stats.internal %d", pmap->stats.internal); OSAddAtomic(-1, &pmap->stats.internal); pmap_ledger_debit(pmap, task_ledgers.internal, @@ -7696,6 +7788,9 @@ pmap_map_globals( assert(*ptep == ARM_PTE_EMPTY); pte = pa_to_pte(ml_static_vtop((vm_offset_t)&lowGlo)) | AP_RONA | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_AF | ARM_PTE_TYPE; +#if __ARM_KERNEL_PROTECT__ + pte |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ pte |= ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); #if (__ARM_VMSA__ > 7) pte |= ARM_PTE_SH(SH_OUTER_MEMORY); @@ -7738,6 +7833,9 @@ pmap_map_cpu_windows_copy_internal( } pte = pa_to_pte(ptoa(pn)) | ARM_PTE_TYPE | ARM_PTE_AF | ARM_PTE_NX | ARM_PTE_PNX; +#if __ARM_KERNEL_PROTECT__ + pte |= ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ pte |= wimg_to_pte(wimg_bits); @@ -8606,7 +8704,7 @@ pmap_update_cache_attributes_locked( va = ptep_get_va(pte_p); tmplate = *pte_p; - tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_NX | ARM_PTE_PNX | ARM_PTE_SHMASK); + tmplate &= ~(ARM_PTE_ATTRINDXMASK | ARM_PTE_SHMASK); tmplate |= wimg_to_pte(attributes); WRITE_PTE(pte_p, tmplate); @@ -8670,20 +8768,24 @@ pmap_create_sharedpage( pmap_paddr_t pa = 0; - kr = pmap_expand(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL); - assert(kr == KERN_SUCCESS); - (void) pmap_pages_alloc(&pa, PAGE_SIZE, 0); memset((char *) phystokv(pa), 0, PAGE_SIZE); /* - * This is the mapping which U64 will refer to. - * Create with common path, update to be non-global and user-readable. + * The kernel pmap maintains a user accessible mapping of the commpage + * to test PAN. */ - kr = pmap_enter(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + kr = pmap_expand(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, 0, PMAP_TT_L3_LEVEL); + assert(kr == KERN_SUCCESS); + kr = pmap_enter(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); assert(kr == KERN_SUCCESS); - pmap_update_tt3e(kernel_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); + + /* + * This mapping should not be global (as we only expect to reference it + * during testing). + */ + pmap_update_tt3e(kernel_pmap, _COMM_HIGH_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE | ARM_PTE_NG); /* * With PAN enabled kernel drivers can no longer use the previous mapping which is user readable @@ -8695,31 +8797,50 @@ pmap_create_sharedpage( assert(kr == KERN_SUCCESS); /* - * Preferrable to just use a single entry,but we consume - * the full entry 0 of the L1 page table for U32 (i.e. - * for the 1GB of user address space), so we have no choice - * but to burn another L1 entry for the shared page... unless - * something clever comes to us. For the short term (i.e. - * bringup) do a kind of forced nesting (we don't have a - * notion of nesting more than one pmap, and the shared cache - * wins). In effect, just allocate a pmap, fill it out - * to include the one entry we care about, and then - * use its L1 pointer in U32 TTBR0 page tables. + * In order to avoid burning extra pages on mapping the shared page, we + * create a dedicated pmap for the shared page. We forcibly nest the + * translation tables from this pmap into other pmaps. The level we + * will nest at depends on the MMU configuration (page size, TTBR range, + * etc). + * + * Note that this is NOT "the nested pmap" (which is used to nest the + * shared cache). * - * Note that we update parameters of entry for our unique - * needs (NG entry, etc.). + * Note that we update parameters of the entry for our unique needs (NG + * entry, etc.). */ - u32_sharedpage_pmap = pmap_create(NULL, 0x0, FALSE); - assert(u32_sharedpage_pmap != NULL); - kr = pmap_enter(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + sharedpage_pmap = pmap_create(NULL, 0x0, FALSE); + assert(sharedpage_pmap != NULL); + + /* The user 64-bit mapping... */ + kr = pmap_enter(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); + assert(kr == KERN_SUCCESS); + pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE64_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); + + /* ...and the user 32-bit mapping. */ + kr = pmap_enter(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, (ppnum_t)atop(pa), VM_PROT_READ, VM_PROT_NONE, VM_WIMG_USE_DEFAULT, TRUE); assert(kr == KERN_SUCCESS); - pmap_update_tt3e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); + pmap_update_tt3e(sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS, PMAP_COMM_PAGE_PTE_TEMPLATE); /* For manipulation in kernel, go straight to physical page */ sharedpage_rw_addr = phystokv(pa); return((vm_map_address_t)sharedpage_rw_addr); } +/* + * Asserts to ensure that the TTEs we nest to map the shared page do not overlap + * with user controlled TTEs. + */ +#if (ARM_PGSHIFT == 14) || __ARM64_TWO_LEVEL_PMAP__ +static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= MACH_VM_MAX_ADDRESS); +static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) >= VM_MAX_ADDRESS); +#elif (ARM_PGSHIFT == 12) +static_assert((_COMM_PAGE64_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= MACH_VM_MAX_ADDRESS); +static_assert((_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) >= VM_MAX_ADDRESS); +#else +#error Nested shared page mapping is unsupported on this config +#endif + static void pmap_insert_sharedpage_internal( pmap_t pmap) @@ -8727,14 +8848,16 @@ pmap_insert_sharedpage_internal( #if (ARM_PGSHIFT == 14) && !__ARM64_TWO_LEVEL_PMAP__ kern_return_t kr; #endif + vm_offset_t sharedpage_vaddr; pt_entry_t *ttep, *src_ttep; #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE #error We assume a single page. #endif if (pmap_is_64bit(pmap)) { - /* Already in kernel pmap */ - return; + sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS; + } else { + sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS; } PMAP_LOCK(pmap); @@ -8751,12 +8874,13 @@ pmap_insert_sharedpage_internal( #error A two level page table with a page shift of 12 is not currently supported #endif /* Just slam in the L1 entry. */ - ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS); + ttep = pmap_tt1e(pmap, sharedpage_vaddr); + if (*ttep != ARM_PTE_EMPTY) { panic("%s: Found something mapped at the commpage address?!", __FUNCTION__); } - src_ttep = pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS); + src_ttep = pmap_tt1e(sharedpage_pmap, sharedpage_vaddr); #elif (ARM_PGSHIFT == 14) #if !__ARM64_TWO_LEVEL_PMAP__ /* Allocate for the L2 entry if necessary, and slam it into place. */ @@ -8764,7 +8888,7 @@ pmap_insert_sharedpage_internal( * As long as we are use a three level page table, the first level * should always exist, so we don't need to check for it. */ - while (*pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS) == ARM_PTE_EMPTY) { + while (*pmap_tt1e(pmap, sharedpage_vaddr) == ARM_PTE_EMPTY) { PMAP_UNLOCK(pmap); kr = pmap_expand(pmap, _COMM_PAGE32_BASE_ADDRESS, 0, PMAP_TT_L2_LEVEL); @@ -8777,23 +8901,26 @@ pmap_insert_sharedpage_internal( } #endif - ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS); + ttep = pmap_tt2e(pmap, sharedpage_vaddr); + if (*ttep != ARM_PTE_EMPTY) { panic("%s: Found something mapped at the commpage address?!", __FUNCTION__); } - src_ttep = pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS); + src_ttep = pmap_tt2e(sharedpage_pmap, sharedpage_vaddr); #endif *ttep = *src_ttep; #ifndef __ARM_L1_PTW__ CleanPoU_DcacheRegion((vm_offset_t) ttep, sizeof(tt_entry_t)); #endif - flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); + /* TODO: Should we flush in the 64-bit case? */ + flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE); + #if (ARM_PGSHIFT == 12) && !__ARM64_TWO_LEVEL_PMAP__ - flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); + flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); #elif (ARM_PGSHIFT == 14) - flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); + flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); #endif PMAP_UNLOCK(pmap); @@ -8807,50 +8934,59 @@ pmap_sharedpage_flush_32_to_64( } static void -pmap_unmap_sharedpage32( +pmap_unmap_sharedpage( pmap_t pmap) { pt_entry_t *ttep; + vm_offset_t sharedpage_vaddr; #if _COMM_PAGE_AREA_LENGTH != PAGE_SIZE #error We assume a single page. #endif + if (pmap_is_64bit(pmap)) { + sharedpage_vaddr = _COMM_PAGE64_BASE_ADDRESS; + } else { + sharedpage_vaddr = _COMM_PAGE32_BASE_ADDRESS; + } + #if (ARM_PGSHIFT == 12) #if __ARM64_TWO_LEVEL_PMAP__ #error A two level page table with a page shift of 12 is not currently supported #endif - ttep = pmap_tt1e(pmap, _COMM_PAGE32_BASE_ADDRESS); + ttep = pmap_tt1e(pmap, sharedpage_vaddr); + if (ttep == NULL) { return; } /* It had better be mapped to the shared page */ - if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt1e(sharedpage_pmap, sharedpage_vaddr)) { panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); } #elif (ARM_PGSHIFT == 14) - ttep = pmap_tt2e(pmap, _COMM_PAGE32_BASE_ADDRESS); + ttep = pmap_tt2e(pmap, sharedpage_vaddr); + if (ttep == NULL) { return; } /* It had better be mapped to the shared page */ - if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(u32_sharedpage_pmap, _COMM_PAGE32_BASE_ADDRESS)) { + if (*ttep != ARM_TTE_EMPTY && *ttep != *pmap_tt2e(sharedpage_pmap, sharedpage_vaddr)) { panic("%s: Something other than commpage mapped in shared page slot?", __FUNCTION__); } #endif *ttep = ARM_TTE_EMPTY; - flush_mmu_tlb_region(_COMM_PAGE32_BASE_ADDRESS, PAGE_SIZE); + flush_mmu_tlb_region(sharedpage_vaddr, PAGE_SIZE); #if (ARM_PGSHIFT == 12) #if __ARM64_TWO_LEVEL_PMAP__ #error A two level page table with a page shift of 12 is not currently supported #endif - flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); + flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L1_OFFMASK) | tlbi_asid(pmap->asid)); #elif (ARM_PGSHIFT == 14) - flush_mmu_tlb_entry(tlbi_addr(_COMM_PAGE32_BASE_ADDRESS & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); + flush_mmu_tlb_entry(tlbi_addr(sharedpage_vaddr & ~ARM_TT_L2_OFFMASK) | tlbi_asid(pmap->asid)); #endif } @@ -9478,15 +9614,15 @@ pmap_check_ledgers( } } - assert(pmap->stats.resident_count == 0); + PMAP_STATS_ASSERTF(pmap->stats.resident_count == 0, pmap, "stats.resident_count %d", pmap->stats.resident_count); #if 00 - assert(pmap->stats.wired_count == 0); + PMAP_STATS_ASSERTF(pmap->stats.wired_count == 0, pmap, "stats.wired_count %d", pmap->stats.wired_count); #endif - assert(pmap->stats.device == 0); - assert(pmap->stats.internal == 0); - assert(pmap->stats.external == 0); - assert(pmap->stats.reusable == 0); - assert(pmap->stats.compressed == 0); + PMAP_STATS_ASSERTF(pmap->stats.device == 0, pmap, "stats.device %d", pmap->stats.device); + PMAP_STATS_ASSERTF(pmap->stats.internal == 0, pmap, "stats.internal %d", pmap->stats.internal); + PMAP_STATS_ASSERTF(pmap->stats.external == 0, pmap, "stats.external %d", pmap->stats.external); + PMAP_STATS_ASSERTF(pmap->stats.reusable == 0, pmap, "stats.reusable %d", pmap->stats.reusable); + PMAP_STATS_ASSERTF(pmap->stats.compressed == 0, pmap, "stats.compressed %lld", pmap->stats.compressed); } #endif /* MACH_ASSERT */ diff --git a/osfmk/arm/pmap.h b/osfmk/arm/pmap.h index 75ed29d26..7653401cf 100644 --- a/osfmk/arm/pmap.h +++ b/osfmk/arm/pmap.h @@ -44,6 +44,42 @@ #include #include + +#if __ARM_KERNEL_PROTECT__ +/* + * For __ARM_KERNEL_PROTECT__, we need twice as many ASIDs to support having + * unique EL0 and EL1 ASIDs for each pmap. + */ +#define ASID_SHIFT (12) /* Shift for the maximum virtual ASID value (2048)*/ +#else /* __ARM_KERNEL_PROTECT__ */ +#define ASID_SHIFT (11) /* Shift for the maximum virtual ASID value (2048) */ +#endif /* __ARM_KERNEL_PROTECT__ */ +#define MAX_ASID (1 << ASID_SHIFT) /* Max supported ASIDs (can be virtual) */ +#define ARM_ASID_SHIFT (8) /* Shift for the maximum ARM ASID value (256) */ +#define ARM_MAX_ASID (1 << ARM_ASID_SHIFT) /* Max ASIDs supported by the hardware */ +#define ASID_VIRT_BITS (ASID_SHIFT - ARM_ASID_SHIFT) /* The number of virtual bits in a virtaul ASID */ +#define NBBY 8 + +struct pmap_cpu_data { + pmap_t cpu_user_pmap; + unsigned int cpu_number; + unsigned int cpu_user_pmap_stamp; + + /* + * This supports overloading of ARM ASIDs by the pmap. The field needs + * to be wide enough to cover all the virtual bits in a virtual ASID. + * With 256 physical ASIDs, 8-bit fields let us support up to 65536 + * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global + * ASID). + * + * If we were to use bitfield shenanigans here, we could save a bit of + * memory by only having enough bits to support MAX_ASID. However, such + * an implementation would be more error prone. + */ + uint8_t cpu_asid_high_bits[ARM_MAX_ASID]; +}; +typedef struct pmap_cpu_data pmap_cpu_data_t; + #include #include #include @@ -136,18 +172,8 @@ extern void flush_mmu_tlb_entry(uint64_t); extern void flush_mmu_tlb_entries(uint64_t, uint64_t); extern void flush_mmu_tlb_asid(uint64_t); extern void flush_core_tlb_asid(uint64_t); -/* - * TLBI appers to only deal in 4KB page addresses, so give - * it an explicit shift of 12. - */ -#define TLBI_ADDR_SIZE 44 -#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) -#define TLBI_ADDR_SHIFT (12) -#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK) -#define TLBI_ASID_SHIFT 48 -#define TLBI_ASID_SIZE 16 -#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT) +#define tlbi_addr(x) (((x) >> TLBI_ADDR_SHIFT) & TLBI_ADDR_MASK) #define tlbi_asid(x) (((uint64_t)x << TLBI_ASID_SHIFT) & TLBI_ASID_MASK) #else extern void flush_mmu_tlb_entry(uint32_t); @@ -268,8 +294,9 @@ struct pmap { decl_simple_lock_data(,tt1_lock) /* lock on tt1 */ #endif #if MACH_ASSERT - int pmap_pid; - char pmap_procname[17]; + boolean_t pmap_stats_assert; + int pmap_pid; + char pmap_procname[17]; #endif /* MACH_ASSERT */ #if DEVELOPMENT || DEBUG boolean_t footprint_suspended; @@ -423,12 +450,6 @@ extern boolean_t pmap_is_empty(pmap_t pmap, vm_map_offset_t start, vm_map_offset #define ARM_PMAP_MAX_OFFSET_DEVICE 0x08 #define ARM_PMAP_MAX_OFFSET_JUMBO 0x10 -#define ASID_SHIFT (11) /* Shift for the maximum virtual ASID value (2048) */ -#define MAX_ASID (1 << ASID_SHIFT) /* Max supported ASIDs (can be virtual) */ -#define ARM_ASID_SHIFT (8) /* Shift for the maximum ARM ASID value (256) */ -#define ARM_MAX_ASID (1 << ARM_ASID_SHIFT) /* Max ASIDs supported by the hardware */ -#define ASID_VIRT_BITS (ASID_SHIFT - ARM_ASID_SHIFT) /* The number of virtual bits in a virtaul ASID */ -#define NBBY 8 extern vm_map_offset_t pmap_max_offset(boolean_t is64, unsigned int option); @@ -478,27 +499,6 @@ boolean_t pmap_enforces_execute_only(pmap_t pmap); #define PMAP_INVALID_CPU_NUM (~0U) -struct pmap_cpu_data { - pmap_t cpu_user_pmap; - unsigned int cpu_number; - unsigned int cpu_user_pmap_stamp; - - /* - * This supports overloading of ARM ASIDs by the pmap. The field needs - * to be wide enough to cover all the virtual bits in a virtual ASID. - * With 256 physical ASIDs, 8-bit fields let us support up to 65536 - * Virtual ASIDs, minus all that would map on to 0 (as 0 is a global - * ASID). - * - * If we were to use bitfield shenanigans here, we could save a bit of - * memory by only having enough bits to support MAX_ASID. However, such - * an implementation would be more error prone. - */ - uint8_t cpu_asid_high_bits[ARM_MAX_ASID]; -}; - -typedef struct pmap_cpu_data pmap_cpu_data_t; - /* Initialize the pmap per-CPU data for the current CPU. */ extern void pmap_cpu_data_init(void); @@ -512,4 +512,12 @@ extern kern_return_t pmap_return(boolean_t do_panic, boolean_t do_recurse); #endif /* #ifndef ASSEMBLER */ +#if __ARM_KERNEL_PROTECT__ +/* + * The exception vector mappings start at the middle of the kernel page table + * range (so that the EL0 mapping can be located at the base of the range). + */ +#define ARM_KERNEL_PROTECT_EXCEPTION_START ((~((ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK) / 2ULL)) + 1ULL) +#endif /* __ARM_KERNEL_PROTECT__ */ + #endif /* #ifndef _ARM_PMAP_H_ */ diff --git a/osfmk/arm/proc_reg.h b/osfmk/arm/proc_reg.h index ca58e18ae..9dc5e2ec8 100644 --- a/osfmk/arm/proc_reg.h +++ b/osfmk/arm/proc_reg.h @@ -64,6 +64,17 @@ #ifndef _ARM_PROC_REG_H_ #define _ARM_PROC_REG_H_ +#if __ARM_KERNEL_PROTECT__ +/* + * This feature is not currently implemented for 32-bit ARM CPU architectures. + * A discussion of this feature for 64-bit ARM CPU architectures can be found + * in the ARM64 version of this file. + */ +#if __arm__ +#error __ARM_KERNEL_PROTECT__ is not supported on ARM32 +#endif +#endif /* __ARM_KERNEL_PROTECT__ */ + #if defined (__arm64__) #include #elif defined (__arm__) @@ -110,6 +121,7 @@ #define __ARM_ENABLE_SWAP__ 1 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1 #define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_KERNEL_PROTECT__ 1 #elif defined (APPLETYPHOON) #define __ARM_ARCH__ 8 @@ -124,6 +136,7 @@ #define __ARM_ENABLE_SWAP__ 1 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1 #define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_KERNEL_PROTECT__ 1 #elif defined (APPLETWISTER) #define __ARM_ARCH__ 8 @@ -138,7 +151,8 @@ #define __ARM_ENABLE_SWAP__ 1 #define __ARM_V8_CRYPTO_EXTENSIONS__ 1 #define __ARM_16K_PG__ 1 -#define __ARM64_TWO_LEVEL_PMAP__ 1 +#define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_KERNEL_PROTECT__ 1 #elif defined (APPLEHURRICANE) #define __ARM_ARCH__ 8 @@ -154,6 +168,7 @@ #define __ARM_V8_CRYPTO_EXTENSIONS__ 1 #define __ARM_16K_PG__ 1 #define __ARM64_PMAP_SUBPAGE_L1__ 1 +#define __ARM_KERNEL_PROTECT__ 1 #define __ARM_GLOBAL_SLEEP_BIT__ 1 #define __ARM_PAN_AVAILABLE__ 1 @@ -678,6 +693,7 @@ * Convenience definitions for: * ARM_TT_LEAF: The last level of the configured page table format. * ARM_TT_TWIG: The second to last level of the configured page table format. + * ARM_TT_ROOT: The first level of the configured page table format. * * My apologies to any botanists who may be reading this. */ @@ -691,6 +707,11 @@ #define ARM_TT_TWIG_SHIFT ARM_TT_L1_SHIFT #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L1_INDEX_MASK +#define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE +#define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK +#define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT +#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK + /* * Level 1 Translation Table Entry * diff --git a/osfmk/arm64/arm_vm_init.c b/osfmk/arm64/arm_vm_init.c index dc0cb7430..9b06f9504 100644 --- a/osfmk/arm64/arm_vm_init.c +++ b/osfmk/arm64/arm_vm_init.c @@ -47,6 +47,28 @@ #include #include +#if __ARM_KERNEL_PROTECT__ +#include +#endif /* __ARM_KERNEL_PROTECT__ */ + +#if __ARM_KERNEL_PROTECT__ +/* + * If we want to support __ARM_KERNEL_PROTECT__, we need a sufficient amount of + * mappable space preceeding the kernel (as we unmap the kernel by cutting the + * range covered by TTBR1 in half). This must also cover the exception vectors. + */ +static_assert(KERNEL_PMAP_HEAP_RANGE_START > ARM_KERNEL_PROTECT_EXCEPTION_START); + +/* The exception vectors and the kernel cannot share root TTEs. */ +static_assert((KERNEL_PMAP_HEAP_RANGE_START & ~ARM_TT_ROOT_OFFMASK) > ARM_KERNEL_PROTECT_EXCEPTION_START); + +/* + * We must have enough space in the TTBR1_EL1 range to create the EL0 mapping of + * the exception vectors. + */ +static_assert((((~ARM_KERNEL_PROTECT_EXCEPTION_START) + 1) * 2ULL) <= (ARM_TT_ROOT_SIZE + ARM_TT_ROOT_INDEX_MASK)); +#endif /* __ARM_KERNEL_PROTECT__ */ + #if KASAN extern vm_offset_t shadow_pbase; extern vm_offset_t shadow_ptop; @@ -178,6 +200,11 @@ SECURITY_READ_ONLY_LATE(vm_offset_t) static_memory_end; SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_start; SECURITY_READ_ONLY_LATE(pmap_paddr_t) avail_end; +#if __ARM_KERNEL_PROTECT__ +extern void ExceptionVectorsBase; +extern void ExceptionVectorsEnd; +#endif /* __ARM_KERNEL_PROTECT__ */ + #define MEM_SIZE_MAX 0x100000000ULL #if defined(KERNEL_INTEGRITY_KTRR) @@ -344,6 +371,240 @@ void dump_kva_space() { #endif /* DEBUG */ +#if __ARM_KERNEL_PROTECT__ +/* + * arm_vm_map: + * root_ttp: The kernel virtual address for the root of the target page tables + * vaddr: The target virtual address + * pte: A page table entry value (may be ARM_PTE_EMPTY) + * + * This function installs pte at vaddr in root_ttp. Any page table pages needed + * to install pte will be allocated by this function. + */ +static void +arm_vm_map(tt_entry_t * root_ttp, vm_offset_t vaddr, pt_entry_t pte) +{ + vm_offset_t ptpage = 0; + tt_entry_t * ttp = root_ttp; + +#if !__ARM64_TWO_LEVEL_PMAP__ + tt_entry_t * l1_ttep = NULL; + tt_entry_t l1_tte = 0; +#endif + + tt_entry_t * l2_ttep = NULL; + tt_entry_t l2_tte = 0; + pt_entry_t * ptep = NULL; + pt_entry_t cpte = 0; + + /* + * Walk the target page table to find the PTE for the given virtual + * address. Allocate any page table pages needed to do this. + */ +#if !__ARM64_TWO_LEVEL_PMAP__ + l1_ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + l1_tte = *l1_ttep; + + if (l1_tte == ARM_TTE_EMPTY) { + ptpage = alloc_ptpage(TRUE); + bzero((void *)ptpage, ARM_PGBYTES); + l1_tte = kvtophys(ptpage); + l1_tte &= ARM_TTE_TABLE_MASK; + l1_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + *l1_ttep = l1_tte; + ptpage = 0; + } + + ttp = (tt_entry_t *)phystokv(l1_tte & ARM_TTE_TABLE_MASK); +#endif + + l2_ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + l2_tte = *l2_ttep; + + if (l2_tte == ARM_TTE_EMPTY) { + ptpage = alloc_ptpage(TRUE); + bzero((void *)ptpage, ARM_PGBYTES); + l2_tte = kvtophys(ptpage); + l2_tte &= ARM_TTE_TABLE_MASK; + l2_tte |= ARM_TTE_VALID | ARM_TTE_TYPE_TABLE; + *l2_ttep = l2_tte; + ptpage = 0; + } + + ttp = (tt_entry_t *)phystokv(l2_tte & ARM_TTE_TABLE_MASK); + + ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + cpte = *ptep; + + /* + * If the existing PTE is not empty, then we are replacing a valid + * mapping. + */ + if (cpte != ARM_PTE_EMPTY) { + panic("%s: cpte=%#llx is not empty, " + "vaddr=%#lx, pte=%#llx", + __FUNCTION__, cpte, + vaddr, pte); + } + + *ptep = pte; +} + +/* + * arm_vm_kernel_el0_map: + * vaddr: The target virtual address + * pte: A page table entry value (may be ARM_PTE_EMPTY) + * + * This function installs pte at vaddr for the EL0 kernel mappings. + */ +static void +arm_vm_kernel_el0_map(vm_offset_t vaddr, pt_entry_t pte) +{ + /* Calculate where vaddr will be in the EL1 kernel page tables. */ + vm_offset_t kernel_pmap_vaddr = vaddr - ((ARM_TT_ROOT_INDEX_MASK + ARM_TT_ROOT_SIZE) / 2ULL); + arm_vm_map(cpu_tte, kernel_pmap_vaddr, pte); +} + +/* + * arm_vm_kernel_el1_map: + * vaddr: The target virtual address + * pte: A page table entry value (may be ARM_PTE_EMPTY) + * + * This function installs pte at vaddr for the EL1 kernel mappings. + */ +static void +arm_vm_kernel_el1_map(vm_offset_t vaddr, pt_entry_t pte) { + arm_vm_map(cpu_tte, vaddr, pte); +} + +/* + * arm_vm_kernel_pte: + * vaddr: The target virtual address + * + * This function returns the PTE value for the given vaddr from the kernel page + * tables. If the region has been been block mapped, we return what an + * equivalent PTE value would be (as regards permissions and flags). We also + * remove the HINT bit (as we are not necessarily creating contiguous mappings. + */ +static pt_entry_t +arm_vm_kernel_pte(vm_offset_t vaddr) +{ + tt_entry_t * ttp = cpu_tte; + tt_entry_t * ttep = NULL; + tt_entry_t tte = 0; + pt_entry_t * ptep = NULL; + pt_entry_t pte = 0; + +#if !__ARM64_TWO_LEVEL_PMAP__ + ttep = ttp + ((vaddr & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); + tte = *ttep; + + assert(tte & ARM_TTE_VALID); + + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) { + /* This is a block mapping; return the equivalent PTE value. */ + pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK); + pte |= ARM_PTE_TYPE_VALID; + pte |= vaddr & ((ARM_TT_L1_SIZE - 1) & ARM_PTE_PAGE_MASK); + pte &= ~ARM_PTE_HINT_MASK; + return pte; + } + + ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK); +#endif + ttep = ttp + ((vaddr & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT); + tte = *ttep; + + assert(tte & ARM_TTE_VALID); + + if ((tte & ARM_TTE_TYPE_MASK) == ARM_TTE_TYPE_BLOCK) { + /* This is a block mapping; return the equivalent PTE value. */ + pte = (pt_entry_t)(tte & ~ARM_TTE_TYPE_MASK); + pte |= ARM_PTE_TYPE_VALID; + pte |= vaddr & ((ARM_TT_L2_SIZE - 1) & ARM_PTE_PAGE_MASK); + pte &= ~ARM_PTE_HINT_MASK; + return pte; + } + + ttp = (tt_entry_t *)phystokv(tte & ARM_TTE_TABLE_MASK); + + ptep = ttp + ((vaddr & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT); + pte = *ptep; + pte &= ~ARM_PTE_HINT_MASK; + return pte; +} + +/* + * arm_vm_prepare_kernel_el0_mappings: + * alloc_only: Indicates if PTE values should be copied from the EL1 kernel + * mappings. + * + * This function expands the kernel page tables to support the EL0 kernel + * mappings, and conditionally installs the PTE values for the EL0 kernel + * mappings (if alloc_only is false). + */ +static void +arm_vm_prepare_kernel_el0_mappings(bool alloc_only) +{ + pt_entry_t pte = 0; + vm_offset_t start = ((vm_offset_t)&ExceptionVectorsBase) & ~PAGE_MASK; + vm_offset_t end = (((vm_offset_t)&ExceptionVectorsEnd) + PAGE_MASK) & ~PAGE_MASK; + vm_offset_t cur = 0; + vm_offset_t cur_fixed = 0; + + /* Expand for/map the exceptions vectors in the EL0 kernel mappings. */ + for (cur = start, cur_fixed = ARM_KERNEL_PROTECT_EXCEPTION_START; cur < end; cur += ARM_PGBYTES, cur_fixed += ARM_PGBYTES) { + /* + * We map the exception vectors at a different address than that + * of the kernelcache to avoid sharing page table pages with the + * kernelcache (as this may cause issues with TLB caching of + * page table pages. + */ + if (!alloc_only) { + pte = arm_vm_kernel_pte(cur); + } + + arm_vm_kernel_el1_map(cur_fixed, pte); + arm_vm_kernel_el0_map(cur_fixed, pte); + } + + __builtin_arm_dmb(DMB_ISH); + __builtin_arm_isb(ISB_SY); + + if (!alloc_only) { + /* + * If we have created the alternate exception vector mappings, + * the boot CPU may now switch over to them. + */ + set_vbar_el1(ARM_KERNEL_PROTECT_EXCEPTION_START); + __builtin_arm_isb(ISB_SY); + } +} + +/* + * arm_vm_populate_kernel_el0_mappings: + * + * This function adds all required mappings to the EL0 kernel mappings. + */ +static void +arm_vm_populate_kernel_el0_mappings(void) +{ + arm_vm_prepare_kernel_el0_mappings(FALSE); +} + +/* + * arm_vm_expand_kernel_el0_mappings: + * + * This function expands the kernel page tables to accomodate the EL0 kernel + * mappings. + */ +static void +arm_vm_expand_kernel_el0_mappings(void) +{ + arm_vm_prepare_kernel_el0_mappings(TRUE); +} +#endif /* __ARM_KERNEL_PROTECT__ */ + #if defined(KERNEL_INTEGRITY_KTRR) extern void bootstrap_instructions; @@ -474,6 +735,9 @@ arm_vm_page_granular_helper(vm_offset_t start, vm_offset_t _end, vm_offset_t va, ptmp = ptmp | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_DEFAULT); ptmp = ptmp | ARM_PTE_AP(pte_prot_APX); ptmp = ptmp | ARM_PTE_NX; +#if __ARM_KERNEL_PROTECT__ + ptmp = ptmp | ARM_PTE_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ if (pte_prot_XN) { ptmp = ptmp | ARM_PTE_PNX; @@ -566,6 +830,9 @@ arm_vm_page_granular_prot(vm_offset_t start, unsigned long size, tmplate = (tmplate & ~ARM_TTE_BLOCK_APMASK) | ARM_TTE_BLOCK_AP(pte_prot_APX); tmplate = tmplate | ARM_TTE_BLOCK_NX; +#if __ARM_KERNEL_PROTECT__ + tmplate = tmplate | ARM_TTE_BLOCK_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ if (tte_prot_XN) tmplate = tmplate | ARM_TTE_BLOCK_PNX; @@ -742,6 +1009,10 @@ arm_vm_prot_finalize(boot_args * args) arm_vm_page_granular_RNX(segPLKDATACONSTB, segSizePLKDATACONST, FALSE); } +#if __ARM_KERNEL_PROTECT__ + arm_vm_populate_kernel_el0_mappings(); +#endif /* __ARM_KERNEL_PROTECT__ */ + #if defined(KERNEL_INTEGRITY_KTRR) /* * __LAST,__pinst should no longer be executable. @@ -754,6 +1025,7 @@ arm_vm_prot_finalize(boot_args * args) * and will become immutable. */ #endif + arm_vm_page_granular_RNX(segDATACONSTB, segSizeDATACONST, FALSE); #ifndef __ARM_L1_PTW__ @@ -776,6 +1048,10 @@ boolean_t user_tbi = TRUE; static void set_tbi(void) { +#if !__ARM_KERNEL_PROTECT__ + /* If we are not built with __ARM_KERNEL_PROTECT__, TBI can be turned + * off with a boot-arg. + */ uint64_t old_tcr, new_tcr; int tbi = 0; @@ -789,6 +1065,7 @@ set_tbi(void) set_tcr(new_tcr); sysreg_restore.tcr_el1 = new_tcr; } +#endif /* !__ARM_KERNEL_PROTECT__ */ } void @@ -863,7 +1140,6 @@ arm_vm_init(uint64_t memory_size, boot_args * args) cpu_tte = (tt_entry_t *)alloc_ptpage(TRUE); cpu_ttep = kvtophys((vm_offset_t)cpu_tte); bzero(cpu_tte, ARM_PGBYTES); - avail_end = gPhysBase + mem_size; /* @@ -873,7 +1149,6 @@ arm_vm_init(uint64_t memory_size, boot_args * args) * * the so called physical aperture should be statically mapped */ - #if !__ARM64_TWO_LEVEL_PMAP__ pa_l1 = gPhysBase; va_l1 = gVirtBase; @@ -920,6 +1195,9 @@ arm_vm_init(uint64_t memory_size, boot_args * args) | ARM_TTE_VALID | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_AP(AP_RWNA) | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK); +#if __ARM_KERNEL_PROTECT__ + *cpu_l2_tte |= ARM_TTE_BLOCK_NG; +#endif /* __ARM_KERNEL_PROTECT__ */ va_l2 += ARM_TT_L2_SIZE; pa_l2 += ARM_TT_L2_SIZE; cpu_l2_tte++; @@ -931,6 +1209,11 @@ arm_vm_init(uint64_t memory_size, boot_args * args) } #endif +#if __ARM_KERNEL_PROTECT__ + /* Expand the page tables to prepare for the EL0 mappings. */ + arm_vm_expand_kernel_el0_mappings(); +#endif /* __ARM_KERNEL_PROTECT__ */ + /* * Now retrieve addresses for end, edata, and etext from MACH-O headers */ @@ -1010,7 +1293,7 @@ arm_vm_init(uint64_t memory_size, boot_args * args) */ #if !__ARM64_TWO_LEVEL_PMAP__ va_l1 = (gVirtBase+MEM_SIZE_MAX+ ~0xFFFFFFFFFF800000ULL) & 0xFFFFFFFFFF800000ULL; - va_l1_end = VM_MAX_KERNEL_ADDRESS; + va_l1_end = VM_MAX_KERNEL_ADDRESS; cpu_l1_tte = cpu_tte + ((va_l1 & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT); while (va_l1 < va_l1_end) { diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c index 6e0d5ed52..d4712a612 100644 --- a/osfmk/arm64/cpu.c +++ b/osfmk/arm64/cpu.c @@ -78,6 +78,10 @@ void sleep_token_buffer_init(void); extern uintptr_t resume_idle_cpu; extern uintptr_t start_cpu; +#if __ARM_KERNEL_PROTECT__ +extern void exc_vectors_table; +#endif /* __ARM_KERNEL_PROTECT__ */ + extern void __attribute__((noreturn)) arm64_prepare_for_sleep(void); extern void arm64_force_wfi_clock_gate(void); #if (defined(APPLECYCLONE) || defined(APPLETYPHOON)) @@ -537,6 +541,9 @@ cpu_data_init(cpu_data_t *cpu_data_ptr) pmap_cpu_data_ptr->cpu_asid_high_bits[i] = 0; } cpu_data_ptr->halt_status = CPU_NOT_HALTED; +#if __ARM_KERNEL_PROTECT__ + cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table; +#endif /* __ARM_KERNEL_PROTECT__ */ } kern_return_t diff --git a/osfmk/arm64/genassym.c b/osfmk/arm64/genassym.c index 5c06aabce..6e47758b1 100644 --- a/osfmk/arm64/genassym.c +++ b/osfmk/arm64/genassym.c @@ -296,6 +296,10 @@ main( offsetof(cpu_data_t, fiqstackptr)); DECLARE("CPU_FIQSTACK_TOP", offsetof(cpu_data_t, fiqstack_top)); +#if __ARM_KERNEL_PROTECT__ + DECLARE("CPU_EXC_VECTORS", + offsetof(cpu_data_t, cpu_exc_vectors)); +#endif /* __ARM_KERNEL_PROTECT__ */ DECLARE("CPU_NUMBER_GS", offsetof(cpu_data_t,cpu_number)); DECLARE("CPU_IDENT", diff --git a/osfmk/arm64/locore.s b/osfmk/arm64/locore.s index 4f8b9e9ad..ea33d6b42 100644 --- a/osfmk/arm64/locore.s +++ b/osfmk/arm64/locore.s @@ -34,6 +34,10 @@ #include #include "assym.s" +#if __ARM_KERNEL_PROTECT__ +#include +#endif + /* * INIT_SAVED_STATE_FLAVORS @@ -53,16 +57,6 @@ str $1, [$0, NS_COUNT] .endmacro -.macro EL1_SP0_VECTOR - msr SPSel, #0 // Switch to SP0 - sub sp, sp, ARM_CONTEXT_SIZE // Create exception frame - stp x0, x1, [sp, SS64_X0] // Save x0, x1 to exception frame - add x0, sp, ARM_CONTEXT_SIZE // Calculate the original stack pointer - str x0, [sp, SS64_SP] // Save stack pointer to exception frame - stp fp, lr, [sp, SS64_FP] // Save fp and lr to exception frame - INIT_SAVED_STATE_FLAVORS sp, w0, w1 - mov x0, sp // Copy saved state pointer to x0 -.endmacro /* * SPILL_REGISTERS @@ -142,11 +136,183 @@ #endif .endmacro +/* + * MAP_KERNEL + * + * Restores the kernel EL1 mappings, if necessary. + * + * This may mutate x18. + */ +.macro MAP_KERNEL +#if __ARM_KERNEL_PROTECT__ + /* Switch to the kernel ASID (low bit set) for the task. */ + mrs x18, TTBR0_EL1 + orr x18, x18, #(1 << TTBR_ASID_SHIFT) + msr TTBR0_EL1, x18 + + /* + * We eschew some barriers on Apple CPUs, as relative ordering of writes + * to the TTBRs and writes to the TCR should be ensured by the + * microarchitecture. + */ +#if !defined(APPLE_ARM64_ARCH_FAMILY) + isb sy +#endif + + /* + * Update the TCR to map the kernel now that we are using the kernel + * ASID. + */ + MOV64 x18, TCR_EL1_BOOT + msr TCR_EL1, x18 + isb sy +#endif /* __ARM_KERNEL_PROTECT__ */ +.endmacro + +/* + * BRANCH_TO_KVA_VECTOR + * + * Branches to the requested long exception vector in the kernelcache. + * arg0 - The label to branch to + * arg1 - The index of the label in exc_vectors_tables + * + * This may mutate x18. + */ +.macro BRANCH_TO_KVA_VECTOR +#if __ARM_KERNEL_PROTECT__ + /* + * Find the kernelcache table for the exception vectors by accessing + * the per-CPU data. + */ + mrs x18, TPIDR_EL1 + ldr x18, [x18, ACT_CPUDATAP] + ldr x18, [x18, CPU_EXC_VECTORS] + + /* + * Get the handler for this exception and jump to it. + */ + ldr x18, [x18, #($1 << 3)] + br x18 +#else + b $0 +#endif /* __ARM_KERNEL_PROTECT__ */ +.endmacro + +#if __ARM_KERNEL_PROTECT__ + .data + .align 3 + .globl EXT(exc_vectors_table) +LEXT(exc_vectors_table) + /* Table of exception handlers. */ + .quad Lel1_sp0_synchronous_vector_long + .quad Lel1_sp0_irq_vector_long + .quad Lel1_sp0_fiq_vector_long + .quad Lel1_sp0_serror_vector_long + .quad Lel1_sp1_synchronous_vector_long + .quad Lel1_sp1_irq_vector_long + .quad Lel1_sp1_fiq_vector_long + .quad Lel1_sp1_serror_vector_long + .quad Lel0_synchronous_vector_64_long + .quad Lel0_irq_vector_64_long + .quad Lel0_fiq_vector_64_long + .quad Lel0_serror_vector_64_long +#endif /* __ARM_KERNEL_PROTECT__ */ + .text +#if __ARM_KERNEL_PROTECT__ + /* + * We need this to be on a page boundary so that we may avoiding mapping + * other text along with it. As this must be on the VM page boundary + * (due to how the coredumping code currently works), this will be a + * 16KB page boundary. + */ + .align 14 +#else .align 12 +#endif /* __ARM_KERNEL_PROTECT__ */ .globl EXT(ExceptionVectorsBase) LEXT(ExceptionVectorsBase) Lel1_sp0_synchronous_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp0_synchronous_vector_long, 0 + + .text + .align 7 +Lel1_sp0_irq_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp0_irq_vector_long, 1 + + .text + .align 7 +Lel1_sp0_fiq_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp0_fiq_vector_long, 2 + + .text + .align 7 +Lel1_sp0_serror_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp0_serror_vector_long, 3 + + .text + .align 7 +Lel1_sp1_synchronous_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp1_synchronous_vector_long, 4 + + .text + .align 7 +Lel1_sp1_irq_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp1_irq_vector_long, 5 + + .text + .align 7 +Lel1_sp1_fiq_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp1_fiq_vector_long, 6 + + .text + .align 7 +Lel1_sp1_serror_vector: + BRANCH_TO_KVA_VECTOR Lel1_sp1_serror_vector, 7 + + .text + .align 7 +Lel0_synchronous_vector_64: + MAP_KERNEL + BRANCH_TO_KVA_VECTOR Lel0_synchronous_vector_64_long, 8 + + .text + .align 7 +Lel0_irq_vector_64: + MAP_KERNEL + BRANCH_TO_KVA_VECTOR Lel0_irq_vector_64_long, 9 + + .text + .align 7 +Lel0_fiq_vector_64: + MAP_KERNEL + BRANCH_TO_KVA_VECTOR Lel0_fiq_vector_64_long, 10 + + .text + .align 7 +Lel0_serror_vector_64: + MAP_KERNEL + BRANCH_TO_KVA_VECTOR Lel0_serror_vector_64_long, 11 + + /* Fill out the rest of the page */ + .align 12 + +/********************************* + * END OF EXCEPTION VECTORS PAGE * + *********************************/ + +.macro EL1_SP0_VECTOR + msr SPSel, #0 // Switch to SP0 + sub sp, sp, ARM_CONTEXT_SIZE // Create exception frame + stp x0, x1, [sp, SS64_X0] // Save x0, x1 to exception frame + add x0, sp, ARM_CONTEXT_SIZE // Calculate the original stack pointer + str x0, [sp, SS64_SP] // Save stack pointer to exception frame + stp fp, lr, [sp, SS64_FP] // Save fp and lr to exception frame + INIT_SAVED_STATE_FLAVORS sp, w0, w1 + mov x0, sp // Copy saved state pointer to x0 +.endmacro + +Lel1_sp0_synchronous_vector_long: sub sp, sp, ARM_CONTEXT_SIZE // Make space on the exception stack stp x0, x1, [sp, SS64_X0] // Save x0, x1 to the stack mrs x1, ESR_EL1 // Get the exception syndrome @@ -168,9 +334,7 @@ Lkernel_stack_valid: add x1, x1, fleh_synchronous@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp0_irq_vector: +Lel1_sp0_irq_vector_long: EL1_SP0_VECTOR mrs x1, TPIDR_EL1 ldr x1, [x1, ACT_CPUDATAP] @@ -180,9 +344,7 @@ Lel1_sp0_irq_vector: add x1, x1, fleh_irq@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp0_fiq_vector: +Lel1_sp0_fiq_vector_long: // ARM64_TODO write optimized decrementer EL1_SP0_VECTOR mrs x1, TPIDR_EL1 @@ -193,9 +355,7 @@ Lel1_sp0_fiq_vector: add x1, x1, fleh_fiq@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp0_serror_vector: +Lel1_sp0_serror_vector_long: EL1_SP0_VECTOR adrp x1, fleh_serror@page // Load address for fleh add x1, x1, fleh_serror@pageoff @@ -211,9 +371,7 @@ Lel1_sp0_serror_vector: mov x0, sp // Copy saved state pointer to x0 .endmacro - .text - .align 7 -Lel1_sp1_synchronous_vector: +Lel1_sp1_synchronous_vector_long: #if defined(KERNEL_INTEGRITY_KTRR) b check_ktrr_sctlr_trap Lel1_sp1_synchronous_vector_continue: @@ -223,25 +381,19 @@ Lel1_sp1_synchronous_vector_continue: add x1, x1, fleh_synchronous_sp1@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp1_irq_vector: +Lel1_sp1_irq_vector_long: EL1_SP1_VECTOR adrp x1, fleh_irq_sp1@page add x1, x1, fleh_irq_sp1@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp1_fiq_vector: +Lel1_sp1_fiq_vector_long: EL1_SP1_VECTOR adrp x1, fleh_fiq_sp1@page add x1, x1, fleh_fiq_sp1@pageoff b fleh_dispatch64 - .text - .align 7 -Lel1_sp1_serror_vector: +Lel1_sp1_serror_vector_long: EL1_SP1_VECTOR adrp x1, fleh_serror_sp1@page add x1, x1, fleh_serror_sp1@pageoff @@ -264,9 +416,8 @@ Lel1_sp1_serror_vector: mov x0, sp // Copy the user PCB pointer to x0 .endmacro - .text - .align 7 -Lel0_synchronous_vector_64: + +Lel0_synchronous_vector_64_long: EL0_64_VECTOR mrs x1, TPIDR_EL1 // Load the thread register ldr x1, [x1, TH_KSTACKPTR] // Load the top of the kernel stack to x1 @@ -275,9 +426,7 @@ Lel0_synchronous_vector_64: add x1, x1, fleh_synchronous@pageoff b fleh_dispatch64 - .text - .align 7 -Lel0_irq_vector_64: +Lel0_irq_vector_64_long: EL0_64_VECTOR mrs x1, TPIDR_EL1 ldr x1, [x1, ACT_CPUDATAP] @@ -287,9 +436,7 @@ Lel0_irq_vector_64: add x1, x1, fleh_irq@pageoff b fleh_dispatch64 - .text - .align 7 -Lel0_fiq_vector_64: +Lel0_fiq_vector_64_long: EL0_64_VECTOR mrs x1, TPIDR_EL1 ldr x1, [x1, ACT_CPUDATAP] @@ -299,9 +446,7 @@ Lel0_fiq_vector_64: add x1, x1, fleh_fiq@pageoff b fleh_dispatch64 - .text - .align 7 -Lel0_serror_vector_64: +Lel0_serror_vector_64_long: EL0_64_VECTOR mrs x1, TPIDR_EL1 // Load the thread register ldr x1, [x1, TH_KSTACKPTR] // Load the top of the kernel stack to x1 @@ -310,13 +455,6 @@ Lel0_serror_vector_64: add x1, x1, fleh_serror@pageoff b fleh_dispatch64 - /* Fill out the rest of the page */ - .align 12 - -/********************************* - * END OF EXCEPTION VECTORS PAGE * - *********************************/ - /* * check_kernel_stack @@ -755,6 +893,26 @@ exception_return: ldr x0, [x3, TH_CTH_DATA] // Load cthread data pointer str x0, [sp, SS64_X18] // and use it to trash x18 +#if __ARM_KERNEL_PROTECT__ + /* + * If we are going to eret to userspace, we must return through the EL0 + * eret mapping. + */ + ldr w1, [sp, SS64_CPSR] // Load CPSR + tbnz w1, PSR64_MODE_EL_SHIFT, Lskip_el0_eret_mapping // Skip if returning to EL1 + + /* We need to switch to the EL0 mapping of this code to eret to EL0. */ + adrp x0, EXT(ExceptionVectorsBase)@page // Load vector base + adrp x1, Lexception_return_restore_registers@page // Load target PC + add x1, x1, Lexception_return_restore_registers@pageoff + MOV64 x2, ARM_KERNEL_PROTECT_EXCEPTION_START // Load EL0 vector address + sub x1, x1, x0 // Calculate delta + add x0, x2, x1 // Convert KVA to EL0 vector address + br x0 + +Lskip_el0_eret_mapping: +#endif /* __ARM_KERNEL_PROTECT__ */ + Lexception_return_restore_registers: /* Restore special register state */ ldr x0, [sp, SS64_PC] // Get the return address @@ -809,8 +967,39 @@ Lexception_return_restore_registers: // Restore stack pointer and our last two GPRs ldr x1, [x0, SS64_SP] mov sp, x1 + +#if __ARM_KERNEL_PROTECT__ + ldr w18, [x0, SS64_CPSR] // Stash CPSR +#endif /* __ARM_KERNEL_PROTECT__ */ + ldp x0, x1, [x0, SS64_X0] // Restore the GPRs +#if __ARM_KERNEL_PROTECT__ + /* If we are going to eret to userspace, we must unmap the kernel. */ + tbnz w18, PSR64_MODE_EL_SHIFT, Lskip_ttbr1_switch + + /* Update TCR to unmap the kernel. */ + MOV64 x18, TCR_EL1_USER + msr TCR_EL1, x18 + + /* + * On Apple CPUs, TCR writes and TTBR writes should be ordered relative to + * each other due to the microarchitecture. + */ +#if !defined(APPLE_ARM64_ARCH_FAMILY) + isb sy +#endif + + /* Switch to the user ASID (low bit clear) for the task. */ + mrs x18, TTBR0_EL1 + bic x18, x18, #(1 << TTBR_ASID_SHIFT) + msr TTBR0_EL1, x18 + mov x18, xzr + + /* We don't need an ISB here, as the eret is synchronizing. */ +Lskip_ttbr1_switch: +#endif /* __ARM_KERNEL_PROTECT__ */ + eret user_take_ast: @@ -856,6 +1045,18 @@ L_rwlock_count_notzero_str: .asciz "RW lock count not 0 on thread %p (%u)" .align 2 +#if __ARM_KERNEL_PROTECT__ + /* + * This symbol denotes the end of the exception vector/eret range; we page + * align it so that we can avoid mapping other text in the EL0 exception + * vector mapping. + */ + .text + .align 14 + .globl EXT(ExceptionVectorsEnd) +LEXT(ExceptionVectorsEnd) +#endif /* __ARM_KERNEL_PROTECT__ */ + .text .align 2 .globl EXT(ml_panic_trap_to_debugger) diff --git a/osfmk/arm64/machine_routines.c b/osfmk/arm64/machine_routines.c index fefbaa51b..defb3f837 100644 --- a/osfmk/arm64/machine_routines.c +++ b/osfmk/arm64/machine_routines.c @@ -186,6 +186,11 @@ pmap_paddr_t get_mmu_ttb(void) MARK_AS_PMAP_TEXT void set_mmu_ttb(pmap_paddr_t value) { +#if __ARM_KERNEL_PROTECT__ + /* All EL1-mode ASIDs are odd. */ + value |= (1ULL << TTBR_ASID_SHIFT); +#endif /* __ARM_KERNEL_PROTECT__ */ + __builtin_arm_dsb(DSB_ISH); MSR("TTBR0_EL1", value); __builtin_arm_isb(ISB_SY); diff --git a/osfmk/arm64/machine_routines_asm.s b/osfmk/arm64/machine_routines_asm.s index c23ec81da..042246006 100644 --- a/osfmk/arm64/machine_routines_asm.s +++ b/osfmk/arm64/machine_routines_asm.s @@ -82,19 +82,19 @@ LEXT(set_fpscr) #endif ret -#if (__ARM_VFP__ >= 3) - .align 2 - .globl EXT(get_mvfr0) -LEXT(get_mvfr0) - mrs x0, MVFR0_EL1 - ret - - .globl EXT(get_mvfr1) -LEXT(get_mvfr1) - mrs x0, MVFR1_EL1 - ret - -#endif +#if __ARM_KERNEL_PROTECT__ +/* + * __ARM_KERNEL_PROTECT__ adds two complications to TLB management: + * + * 1. As each pmap has two ASIDs, every TLB operation that targets an ASID must + * target both ASIDs for the pmap that owns the target ASID. + * + * 2. Any TLB operation targeting the kernel_pmap ASID (ASID 0) must target all + * ASIDs (as kernel_pmap mappings may be referenced while using an ASID that + * belongs to another pmap). We expect these routines to be called with the + * EL0 ASID for the target; not the EL1 ASID. + */ +#endif /* __ARM_KERNEL_PROTECT__ */ /* * void flush_mmu_tlb(void) @@ -154,12 +154,11 @@ LEXT(flush_mmu_tlb_allentries) add x1, x1, #0x3 and x1, x1, #~0x3 #endif - -1: +Lflush_mmu_tlb_allentries_loop: tlbi vaae1is, x0 add x0, x0, #(ARM_PGBYTES / 4096) // Units are 4KB pages, as defined by the ISA cmp x0, x1 - b.lt 1b + b.lt Lflush_mmu_tlb_allentries_loop dsb ish isb sy ret @@ -173,10 +172,30 @@ LEXT(flush_mmu_tlb_allentries) .align 2 .globl EXT(flush_mmu_tlb_entry) LEXT(flush_mmu_tlb_entry) +#if __ARM_KERNEL_PROTECT__ + /* + * If we are flushing ASID 0, this is a kernel operation. With this + * ASID scheme, this means we should flush all ASIDs. + */ + lsr x2, x0, #TLBI_ASID_SHIFT + cmp x2, #0 + b.eq Lflush_mmu_tlb_entry_globally + + bic x0, x0, #(1 << TLBI_ASID_SHIFT) tlbi vae1is, x0 + orr x0, x0, #(1 << TLBI_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ + tlbi vae1is, x0 + dsb ish + isb sy + ret +#if __ARM_KERNEL_PROTECT__ +Lflush_mmu_tlb_entry_globally: + tlbi vaae1is, x0 dsb ish isb sy ret +#endif /* __ARM_KERNEL_PROTECT__ */ /* * void flush_mmu_tlb_entries(uint64_t, uint64_t) @@ -207,16 +226,41 @@ LEXT(flush_mmu_tlb_entries) */ add x1, x1, #0x3 and x1, x1, #~0x3 -#endif +#endif /* __ARM_KERNEL_PROTECT__ */ +#if __ARM_KERNEL_PROTECT__ + /* + * If we are flushing ASID 0, this is a kernel operation. With this + * ASID scheme, this means we should flush all ASIDs. + */ + lsr x2, x0, #TLBI_ASID_SHIFT + cmp x2, #0 + b.eq Lflush_mmu_tlb_entries_globally_loop -1: + bic x0, x0, #(1 << TLBI_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ +Lflush_mmu_tlb_entries_loop: + tlbi vae1is, x0 +#if __ARM_KERNEL_PROTECT__ + orr x0, x0, #(1 << TLBI_ASID_SHIFT) tlbi vae1is, x0 + bic x0, x0, #(1 << TLBI_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ + add x0, x0, #(ARM_PGBYTES / 4096) // Units are pages + cmp x0, x1 + b.lt Lflush_mmu_tlb_entries_loop + dsb ish + isb sy + ret +#if __ARM_KERNEL_PROTECT__ +Lflush_mmu_tlb_entries_globally_loop: + tlbi vaae1is, x0 add x0, x0, #(ARM_PGBYTES / 4096) // Units are pages cmp x0, x1 - b.lt 1b + b.lt Lflush_mmu_tlb_entries_globally_loop dsb ish isb sy ret +#endif /* __ARM_KERNEL_PROTECT__ */ /* * void flush_mmu_tlb_asid(uint64_t) @@ -227,10 +271,30 @@ LEXT(flush_mmu_tlb_entries) .align 2 .globl EXT(flush_mmu_tlb_asid) LEXT(flush_mmu_tlb_asid) +#if __ARM_KERNEL_PROTECT__ + /* + * If we are flushing ASID 0, this is a kernel operation. With this + * ASID scheme, this means we should flush all ASIDs. + */ + lsr x1, x0, #TLBI_ASID_SHIFT + cmp x1, #0 + b.eq Lflush_mmu_tlb_globally + + bic x0, x0, #(1 << TLBI_ASID_SHIFT) tlbi aside1is, x0 + orr x0, x0, #(1 << TLBI_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ + tlbi aside1is, x0 + dsb ish + isb sy + ret +#if __ARM_KERNEL_PROTECT__ +Lflush_mmu_tlb_globally: + tlbi vmalle1is dsb ish isb sy ret +#endif /* __ARM_KERNEL_PROTECT__ */ /* * void flush_core_tlb_asid(uint64_t) @@ -241,10 +305,30 @@ LEXT(flush_mmu_tlb_asid) .align 2 .globl EXT(flush_core_tlb_asid) LEXT(flush_core_tlb_asid) +#if __ARM_KERNEL_PROTECT__ + /* + * If we are flushing ASID 0, this is a kernel operation. With this + * ASID scheme, this means we should flush all ASIDs. + */ + lsr x1, x0, #TLBI_ASID_SHIFT + cmp x1, #0 + b.eq Lflush_core_tlb_asid_globally + + bic x0, x0, #(1 << TLBI_ASID_SHIFT) + tlbi aside1, x0 + orr x0, x0, #(1 << TLBI_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ tlbi aside1, x0 dsb ish isb sy ret +#if __ARM_KERNEL_PROTECT__ +Lflush_core_tlb_asid_globally: + tlbi vmalle1 + dsb ish + isb sy + ret +#endif /* __ARM_KERNEL_PROTECT__ */ /* * Set MMU Translation Table Base Alternate @@ -277,6 +361,19 @@ LEXT(set_aux_control) isb sy ret +#if __ARM_KERNEL_PROTECT__ + .text + .align 2 + .globl EXT(set_vbar_el1) +LEXT(set_vbar_el1) +#if defined(KERNEL_INTEGRITY_KTRR) + b EXT(pinst_set_vbar) +#else + msr VBAR_EL1, x0 + ret +#endif +#endif /* __ARM_KERNEL_PROTECT__ */ + /* * set translation control register diff --git a/osfmk/arm64/platform_tests.c b/osfmk/arm64/platform_tests.c index 689250acf..d5391b327 100644 --- a/osfmk/arm64/platform_tests.c +++ b/osfmk/arm64/platform_tests.c @@ -1049,10 +1049,10 @@ arm64_pan_test() pan_exception_level = 0; pan_fault_value = 0xDE; // convert priv_addr to one that is accessible from user mode - pan_test_addr = priv_addr + _COMM_PAGE64_BASE_ADDRESS - + pan_test_addr = priv_addr + _COMM_HIGH_PAGE64_BASE_ADDRESS - _COMM_PAGE_START_ADDRESS; - // Below should trigger a PAN exception as pan_test_addr is accessible + // Below should trigger a PAN exception as pan_test_addr is accessible // in user mode // The exception handler, upon recognizing the fault address is pan_test_addr, // will disable PAN and rerun this instruction successfully diff --git a/osfmk/arm64/proc_reg.h b/osfmk/arm64/proc_reg.h index 55c370635..ce62e62bd 100644 --- a/osfmk/arm64/proc_reg.h +++ b/osfmk/arm64/proc_reg.h @@ -33,6 +33,61 @@ #include +#if __ARM_KERNEL_PROTECT__ +/* + * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential + * architectural or microarchitectural vulnerabilities that could allow cores to + * read/access EL1-only mappings while in EL0 mode. This is achieved by + * removing as many mappings as possible when the core transitions to EL0 mode + * from EL1 mode, and restoring those mappings when the core transitions to EL1 + * mode from EL0 mode. + * + * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is + * used to map and unmap the ordinary kernel mappings, by contracting and + * expanding translation zone size for TTBR1 when exiting and entering EL1, + * respectively: + * + * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is + * is in EL0. + * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in + * EL1. + * + * T1SZ_USER: T1SZ_BOOT + 1 + * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) + * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) + * TTBR1_EL1_MAX: (2^64) - 1 + * + * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the + * the following TTBR1 layout: + * + * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX + * +---------------------------------------------------------+ + * | Kernel EL0 Mappings | Kernel EL1 Mappings | + * +---------------------------------------------------------+ + * + * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: + * + * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX + * +---------------------------------------------------------+ + * | Kernel EL0 Mappings | + * +---------------------------------------------------------+ + * + * With the current implementation, both the EL0 and EL1 mappings for the kernel + * use otherwise empty translation tables for mapping the exception vectors (so + * that we do not need to TLB flush the exception vector address when switching + * between EL0 and EL1). The rationale here is that the TLBI would require a + * DSB, and DSBs can be extremely expensive. + * + * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 + * ASID. The core switches between ASIDs on EL transitions, so that the TLB + * does not need to be fully invalidated on an EL transition. + * + * Most kernel mappings will be marked non-global in this configuration, as + * global mappings would be visible to userspace unless we invalidate them on + * eret. + */ +#endif /* __ARM_KERNEL_PROTECT */ + /* * 64-bit Program Status Register (PSR64) * @@ -484,22 +539,30 @@ * we support the following: * * 4KB pages, full page L1: 39 bit range. - * 4KB pages, sub-page L1: 36 bit range. + * 4KB pages, sub-page L1: 38 bit range. * 16KB pages, full page L1: 47 bit range. - * 16KB pages, sub-page L1: 37 bit range. + * 16KB pages, sub-page L1: 39 bit range. * 16KB pages, two level page tables: 36 bit range. */ +#if __ARM_KERNEL_PROTECT__ +/* + * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the + * address space is used for the mappings that will remain in place when in EL0. + * As a result, 1 bit less of address space is available to the rest of the + * the kernel. + */ +#endif /* __ARM_KERNEL_PROTECT__ */ #ifdef __ARM_16K_PG__ #if __ARM64_TWO_LEVEL_PMAP__ #define T0SZ_BOOT 28ULL #elif __ARM64_PMAP_SUBPAGE_L1__ -#define T0SZ_BOOT 27ULL +#define T0SZ_BOOT 25ULL #else /* __ARM64_TWO_LEVEL_PMAP__ */ #define T0SZ_BOOT 17ULL #endif /* __ARM64_TWO_LEVEL_PMAP__ */ #else /* __ARM_16K_PG__ */ #if __ARM64_PMAP_SUBPAGE_L1__ -#define T0SZ_BOOT 28ULL +#define T0SZ_BOOT 26ULL #else /* __ARM64_PMAP_SUBPAGE_L1__ */ #define T0SZ_BOOT 25ULL #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ @@ -513,22 +576,32 @@ #if __ARM64_TWO_LEVEL_PMAP__ #define T1SZ_BOOT 28ULL #elif __ARM64_PMAP_SUBPAGE_L1__ -#define T1SZ_BOOT 27ULL +#define T1SZ_BOOT 25ULL #else /* __ARM64_TWO_LEVEL_PMAP__ */ #define T1SZ_BOOT 17ULL #endif /* __ARM64_TWO_LEVEL_PMAP__ */ #else /* __ARM_16K_PG__ */ #if __ARM64_PMAP_SUBPAGE_L1__ -#define T1SZ_BOOT 28ULL +#define T1SZ_BOOT 26ULL #else /* __ARM64_PMAP_SUBPAGE_L1__ */ #define T1SZ_BOOT 25ULL #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ #endif /* __ARM_16K_PG__ */ #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ -#define TCR_EL1_BOOT (TCR_IPS_40BITS | \ +#define TCR_EL1_BASE (TCR_IPS_40BITS | \ TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\ - TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | TCR_IRGN1_WRITEBACK | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG1_GRANULE_SIZE)) + TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE)) + +#if __ARM_KERNEL_PROTECT__ +#define TCR_EL1_BOOT (TCR_EL1_BASE | \ + (T1SZ_BOOT << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED) +#define T1SZ_USER (T1SZ_BOOT + 1) +#define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED) +#else +#define TCR_EL1_BOOT (TCR_EL1_BASE | \ + (T1SZ_BOOT << TCR_T1SZ_SHIFT)) +#endif /* __ARM_KERNEL_PROTECT__ */ /* * Translation Table Base Register (TTBR) @@ -686,21 +759,21 @@ #define ARM_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ #define ARM_TT_L1_SHIFT 36 /* page descriptor shift */ #ifdef __ARM64_PMAP_SUBPAGE_L1__ -/* This config supports 128GB per TTBR. */ -#define ARM_TT_L1_INDEX_MASK 0x0000001000000000ULL /* mask for getting index into L1 table from virtual address */ -#else +/* This config supports 512GB per TTBR. */ +#define ARM_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ +#else /* __ARM64_PMAP_SUBPAGE_L1__ */ #define ARM_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ -#endif -#else +#endif /* __ARM64_PMAP_SUBPAGE_L1__ */ +#else /* __ARM_16K_PG__ */ #define ARM_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ #define ARM_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ #define ARM_TT_L1_SHIFT 30 /* page descriptor shift */ #ifdef __ARM64_PMAP_SUBPAGE_L1__ -/* This config supports 64GB per TTBR. */ -#define ARM_TT_L1_INDEX_MASK 0x0000000fc0000000ULL /* mask for getting index into L1 table from virtual address */ -#else +/* This config supports 256GB per TTBR. */ +#define ARM_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ +#else /* __ARM64_PMAP_SUBPAGE_L1__ */ #define ARM_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ -#endif +#endif /* __ARM64_PMAP_SUBPAGE_L1__ */ #endif /* @@ -759,6 +832,7 @@ * Convenience definitions for: * ARM_TT_LEAF: The last level of the configured page table format. * ARM_TT_TWIG: The second to last level of the configured page table format. + * ARM_TT_ROOT: The first level of the configured page table format. * * My apologies to any botanists who may be reading this. */ @@ -772,6 +846,18 @@ #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK +#if __ARM64_TWO_LEVEL_PMAP__ +#define ARM_TT_ROOT_SIZE ARM_TT_L2_SIZE +#define ARM_TT_ROOT_OFFMASK ARM_TT_L2_OFFMASK +#define ARM_TT_ROOT_SHIFT ARM_TT_L2_SHIFT +#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L2_INDEX_MASK +#else +#define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE +#define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK +#define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT +#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK +#endif + /* * 4KB granule size: * @@ -941,8 +1027,14 @@ #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ +#if __ARM_KERNEL_PROTECT__ +#define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \ + | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF \ + | ARM_TTE_BLOCK_NG) +#else /* __ARM_KERNEL_PROTECT__ */ #define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \ | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) +#endif /* __ARM_KERNEL_PROTECT__ */ #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) /* @@ -1083,9 +1175,26 @@ #define ARM_PTE_PGTRACE_MASK 0x0200000000000000ULL /* software trace mask */ #endif -#define ARM_PTE_BOOT_PAGE (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) \ +#define ARM_PTE_BOOT_PAGE_BASE (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) \ | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) +#if __ARM_KERNEL_PROTECT__ +#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) +#else /* __ARM_KERNEL_PROTECT__ */ +#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) +#endif /* __ARM_KERNEL_PROTECT__ */ + +/* + * TLBI appers to only deal in 4KB page addresses, so give + * it an explicit shift of 12. + */ +#define TLBI_ADDR_SIZE (44) +#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) +#define TLBI_ADDR_SHIFT (12) +#define TLBI_ASID_SHIFT (48) +#define TLBI_ASID_SIZE (16) +#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT) + /* * Exception Syndrome Register * diff --git a/osfmk/arm64/sleh.c b/osfmk/arm64/sleh.c index f77de678b..6e65c7598 100644 --- a/osfmk/arm64/sleh.c +++ b/osfmk/arm64/sleh.c @@ -942,7 +942,7 @@ handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */ if (is_vm_fault(fault_code)) { - kern_return_t result; + kern_return_t result = KERN_FAILURE; vm_map_t map = thread->map; vm_offset_t vm_fault_addr = fault_addr; @@ -982,7 +982,10 @@ handle_user_abort(arm_saved_state_t *state, uint32_t esr, vm_offset_t fault_addr #endif /* check to see if it is just a pmap ref/modify fault */ - result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE); + + if (result != KERN_SUCCESS) { + result = arm_fast_fault(map->pmap, trunc_page(vm_fault_addr), fault_type, TRUE); + } if (result != KERN_SUCCESS) { { diff --git a/osfmk/arm64/start.s b/osfmk/arm64/start.s index 2776c8f46..ad181f705 100644 --- a/osfmk/arm64/start.s +++ b/osfmk/arm64/start.s @@ -35,6 +35,10 @@ #include #include "assym.s" +#if __ARM_KERNEL_PROTECT__ +#include +#endif /* __ARM_KERNEL_PROTECT__ */ + .macro MSR_VBAR_EL1_X0 #if defined(KERNEL_INTEGRITY_KTRR) @@ -180,6 +184,15 @@ Lfound_cpu_data_entry: +#if __ARM_KERNEL_PROTECT__ && defined(KERNEL_INTEGRITY_KTRR) + /* + * Populate TPIDR_EL1 (in case the CPU takes an exception while + * turning on the MMU). + */ + ldr x13, [x21, CPU_ACTIVE_THREAD] + msr TPIDR_EL1, x13 +#endif /* __ARM_KERNEL_PROTECT__ */ + blr x0 Lskip_cpu_reset_handler: b . // Hang if the handler is NULL or returns @@ -586,6 +599,10 @@ common_start: */ #endif and x0, x25, #(TTBR_BADDR_MASK) +#if __ARM_KERNEL_PROTECT__ + /* We start out with a kernel ASID. */ + orr x0, x0, #(1 << TTBR_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ msr TTBR0_EL1, x0 #if __ARM64_TWO_LEVEL_PMAP__ /* @@ -656,10 +673,20 @@ common_start: 1: // Set up the exception vectors +#if __ARM_KERNEL_PROTECT__ + /* If this is not the first reset of the boot CPU, the alternate mapping + * for the exception vectors will be set up, so use it. Otherwise, we + * should use the mapping located in the kernelcache mapping. + */ + MOV64 x0, ARM_KERNEL_PROTECT_EXCEPTION_START + + cbnz x21, 1f +#endif /* __ARM_KERNEL_PROTECT__ */ adrp x0, EXT(ExceptionVectorsBase)@page // Load exception vectors base address add x0, x0, EXT(ExceptionVectorsBase)@pageoff add x0, x0, x22 // Convert exception vector address to KVA sub x0, x0, x23 +1: MSR_VBAR_EL1_X0 @@ -874,6 +901,10 @@ arm_init_tramp: adrp x0, EXT(invalid_ttep)@page add x0, x0, EXT(invalid_ttep)@pageoff ldr x0, [x0] +#if __ARM_KERNEL_PROTECT__ + /* We start out with a kernel ASID. */ + orr x0, x0, #(1 << TTBR_ASID_SHIFT) +#endif /* __ARM_KERNEL_PROTECT__ */ msr TTBR0_EL1, x0 diff --git a/osfmk/bank/bank.c b/osfmk/bank/bank.c index 66a4798f5..b7d5d11d0 100644 --- a/osfmk/bank/bank.c +++ b/osfmk/bank/bank.c @@ -44,6 +44,7 @@ #include #include #include +#include static zone_t bank_task_zone, bank_account_zone; #define MAX_BANK_TASK (CONFIG_TASK_MAX) @@ -69,13 +70,15 @@ struct _bank_ledger_indices bank_ledgers = { -1, -1 }; static bank_task_t bank_task_alloc_init(task_t task); static bank_account_t bank_account_alloc_init(bank_task_t bank_holder, bank_task_t bank_merchant, - bank_task_t bank_secureoriginator, bank_task_t bank_proximateprocess); + bank_task_t bank_secureoriginator, bank_task_t bank_proximateprocess, struct thread_group* banktg); static bank_task_t get_bank_task_context(task_t task, boolean_t initialize); static void bank_task_dealloc(bank_task_t bank_task, mach_voucher_attr_value_reference_t sync); static kern_return_t bank_account_dealloc_with_sync(bank_account_t bank_account, mach_voucher_attr_value_reference_t sync); static void bank_rollup_chit_to_tasks(ledger_t bill, bank_task_t bank_holder, bank_task_t bank_merchant); static void init_bank_ledgers(void); static boolean_t bank_task_is_propagate_entitled(task_t t); +static struct thread_group *bank_get_bank_task_thread_group(bank_task_t bank_task __unused); +static struct thread_group *bank_get_bank_account_thread_group(bank_account_t bank_account __unused); static lck_spin_t g_bank_task_lock_data; /* lock to protect task->bank_context transition */ @@ -322,6 +325,8 @@ bank_get_value( task_t task; kern_return_t kr = KERN_SUCCESS; mach_msg_type_number_t i; + struct thread_group *thread_group = NULL; + struct thread_group *cur_thread_group = NULL; assert(MACH_VOUCHER_ATTR_KEY_BANK == key); assert(manager == &bank_manager); @@ -354,11 +359,14 @@ bank_get_value( bank_holder = CAST_TO_BANK_TASK(bank_element); bank_secureoriginator = bank_holder; bank_proximateprocess = bank_holder; + thread_group = bank_get_bank_task_thread_group(bank_holder); } else if (bank_element->be_type == BANK_ACCOUNT) { old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element); bank_holder = old_bank_account->ba_holder; bank_secureoriginator = old_bank_account->ba_secureoriginator; bank_proximateprocess = old_bank_account->ba_proximateprocess; + thread_group = bank_get_bank_account_thread_group(old_bank_account); + } else { panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type); } @@ -367,17 +375,26 @@ bank_get_value( if (bank_merchant == BANK_TASK_NULL) return KERN_RESOURCE_SHORTAGE; + cur_thread_group = bank_get_bank_task_thread_group(bank_merchant); + + /* Change voucher thread group to current thread group for Apps */ + if (task_is_app(task)) { + thread_group = cur_thread_group; + } + /* Check if trying to redeem for self task, return the default bank task */ if (bank_holder == bank_merchant && bank_holder == bank_secureoriginator && - bank_holder == bank_proximateprocess) { + bank_holder == bank_proximateprocess && + thread_group == cur_thread_group) { *out_value = BANK_ELEMENT_TO_HANDLE(BANK_DEFAULT_TASK_VALUE); *out_flags = MACH_VOUCHER_ATTR_VALUE_FLAGS_PERSIST; return kr; } bank_account = bank_account_alloc_init(bank_holder, bank_merchant, - bank_secureoriginator, bank_proximateprocess); + bank_secureoriginator, bank_proximateprocess, + thread_group); if (bank_account == BANK_ACCOUNT_NULL) return KERN_RESOURCE_SHORTAGE; @@ -405,10 +422,12 @@ bank_get_value( if (bank_element->be_type == BANK_TASK) { bank_holder = CAST_TO_BANK_TASK(bank_element); bank_secureoriginator = bank_holder; + thread_group = bank_get_bank_task_thread_group(bank_holder); } else if (bank_element->be_type == BANK_ACCOUNT) { old_bank_account = CAST_TO_BANK_ACCOUNT(bank_element); bank_holder = old_bank_account->ba_holder; bank_secureoriginator = old_bank_account->ba_secureoriginator; + thread_group = bank_get_bank_account_thread_group(old_bank_account); } else { panic("Bogus bank type: %d passed in get_value\n", bank_element->be_type); } @@ -417,6 +436,8 @@ bank_get_value( if (bank_merchant == BANK_TASK_NULL) return KERN_RESOURCE_SHORTAGE; + cur_thread_group = bank_get_bank_task_thread_group(bank_merchant); + /* * If the process doesn't have secure persona entitlement, * then replace the secure originator to current task. @@ -433,7 +454,8 @@ bank_get_value( /* Check if trying to redeem for self task, return the bank task */ if (bank_holder == bank_merchant && bank_holder == bank_secureoriginator && - bank_holder == bank_proximateprocess) { + bank_holder == bank_proximateprocess && + thread_group == cur_thread_group) { lck_mtx_lock(&bank_holder->bt_acc_to_pay_lock); bank_task_made_reference(bank_holder); @@ -448,7 +470,8 @@ bank_get_value( return kr; } bank_account = bank_account_alloc_init(bank_holder, bank_merchant, - bank_secureoriginator, bank_proximateprocess); + bank_secureoriginator, bank_proximateprocess, + thread_group); if (bank_account == BANK_ACCOUNT_NULL) return KERN_RESOURCE_SHORTAGE; @@ -782,7 +805,8 @@ bank_account_alloc_init( bank_task_t bank_holder, bank_task_t bank_merchant, bank_task_t bank_secureoriginator, - bank_task_t bank_proximateprocess) + bank_task_t bank_proximateprocess, + struct thread_group *thread_group) { bank_account_t new_bank_account; bank_account_t bank_account; @@ -815,7 +839,8 @@ bank_account_alloc_init( queue_iterate(&bank_holder->bt_accounts_to_pay, bank_account, bank_account_t, ba_next_acc_to_pay) { if (bank_account->ba_merchant != bank_merchant || bank_account->ba_secureoriginator != bank_secureoriginator || - bank_account->ba_proximateprocess != bank_proximateprocess) + bank_account->ba_proximateprocess != bank_proximateprocess || + bank_get_bank_account_thread_group(bank_account) != thread_group) continue; entry_found = TRUE; @@ -1322,7 +1347,8 @@ bank_serviced_balance(bank_task_t bank_task, uint64_t *cpu_time, uint64_t *energ * Routine: bank_get_voucher_bank_account * Purpose: Get the bank account from the voucher. * Returns: bank_account if bank_account attribute present in voucher. - * NULL on no attribute, no bank_element, or if holder and merchant bank accounts are the same. + * NULL on no attribute, no bank_element, or if holder and merchant bank accounts + * and voucher thread group and current thread group are the same. */ static bank_account_t bank_get_voucher_bank_account(ipc_voucher_t voucher) @@ -1352,12 +1378,20 @@ bank_get_voucher_bank_account(ipc_voucher_t voucher) return BANK_ACCOUNT_NULL; } else if (bank_element->be_type == BANK_ACCOUNT) { bank_account = CAST_TO_BANK_ACCOUNT(bank_element); - if (bank_account->ba_holder != bank_account->ba_merchant) { - bank_task_t bank_merchant = get_bank_task_context(current_task(), FALSE); - if (bank_account->ba_merchant == bank_merchant) - return bank_account; - else - return BANK_ACCOUNT_NULL; + /* + * Return BANK_ACCOUNT_NULL if the ba_holder is same as ba_merchant + * and bank account thread group is same as current thread group + * i.e. ba_merchant's thread group. + * + * The bank account might have ba_holder same as ba_merchant but different + * thread group if daemon sends a voucher to an App and then App sends the + * same voucher back to the daemon (IPC code will replace thread group in the + * voucher to App's thread group when it gets auto redeemed by the App). + */ + if (bank_account->ba_holder != bank_account->ba_merchant || + bank_get_bank_account_thread_group(bank_account) != + bank_get_bank_task_thread_group(bank_account->ba_merchant)) { + return bank_account; } else { return BANK_ACCOUNT_NULL; } @@ -1369,19 +1403,45 @@ bank_get_voucher_bank_account(ipc_voucher_t voucher) /* * Routine: bank_get_bank_account_ledger - * Purpose: Get the bankledger from the bank account + * Purpose: Get the bankledger from the bank account if ba_merchant different than ba_holder */ static ledger_t bank_get_bank_account_ledger(bank_account_t bank_account) { ledger_t bankledger = NULL; - if (bank_account != BANK_ACCOUNT_NULL) + if (bank_account != BANK_ACCOUNT_NULL && + bank_account->ba_holder != bank_account->ba_merchant) bankledger = bank_account->ba_bill; return (bankledger); } +/* + * Routine: bank_get_bank_task_thread_group + * Purpose: Get the bank task's thread group from the bank task + */ +static struct thread_group * +bank_get_bank_task_thread_group(bank_task_t bank_task __unused) +{ + struct thread_group *banktg = NULL; + + + return (banktg); +} + +/* + * Routine: bank_get_bank_account_thread_group + * Purpose: Get the bank account's thread group from the bank account + */ +static struct thread_group * +bank_get_bank_account_thread_group(bank_account_t bank_account __unused) +{ + thread_group_t banktg = NULL; + + + return (banktg); +} /* * Routine: bank_get_bank_ledger_and_thread_group @@ -1393,12 +1453,21 @@ kern_return_t bank_get_bank_ledger_and_thread_group( ipc_voucher_t voucher, ledger_t *bankledger, - thread_group_t *banktg __unused) + thread_group_t *banktg) { bank_account_t bank_account; + struct thread_group *thread_group = NULL; bank_account = bank_get_voucher_bank_account(voucher); *bankledger = bank_get_bank_account_ledger(bank_account); + thread_group = bank_get_bank_account_thread_group(bank_account); + + /* Return NULL thread group if voucher has current task's thread group */ + if (thread_group == bank_get_bank_task_thread_group( + get_bank_task_context(current_task(), FALSE))) { + thread_group = NULL; + } + *banktg = thread_group; return KERN_SUCCESS; } diff --git a/osfmk/i386/AT386/model_dep.c b/osfmk/i386/AT386/model_dep.c index 2d29bf968..10b4c4c3f 100644 --- a/osfmk/i386/AT386/model_dep.c +++ b/osfmk/i386/AT386/model_dep.c @@ -134,10 +134,26 @@ #define DPRINTF(x...) #endif +#ifndef ROUNDUP +#define ROUNDUP(a, b) (((a) + ((b) - 1)) & (~((b) - 1))) +#endif + +#ifndef ROUNDDOWN +#define ROUNDDOWN(x,y) (((x)/(y))*(y)) +#endif + static void machine_conf(void); void panic_print_symbol_name(vm_address_t search); void RecordPanicStackshot(void); +typedef enum paniclog_flush_type { + kPaniclogFlushBase = 1, /* Flush the initial log and paniclog header */ + kPaniclogFlushStackshot = 2, /* Flush only the stackshot data, then flush the header */ + kPaniclogFlushOtherLog = 3 /* Flush the other log, then flush the header */ +} paniclog_flush_type_t; + +void paniclog_flush_internal(paniclog_flush_type_t variant); + extern const char version[]; extern char osversion[]; extern int max_unsafe_quanta; @@ -919,8 +935,6 @@ RecordPanicStackshot() } } } - - paniclog_flush(); #if DEVELOPMENT || DEBUG if (panic_stackshot_buf != 0) { // We're going to try to take another stackshot, reset the state. @@ -1008,7 +1022,7 @@ SavePanicInfo( } /* Flush the panic log */ - paniclog_flush(); + paniclog_flush_internal(kPaniclogFlushBase); /* Try to take a panic stackshot */ RecordPanicStackshot(); @@ -1018,17 +1032,12 @@ SavePanicInfo( * from when we tried to capture it. */ if (extended_debug_log_enabled) { - paniclog_flush(); + paniclog_flush_internal(kPaniclogFlushStackshot); } } -void -paniclog_flush() +void paniclog_flush_internal(paniclog_flush_type_t variant) { - unsigned long pi_size = 0; - - assert(panic_info != NULL); - /* Update the other log offset if we've opened the other log */ if (panic_info->mph_other_log_offset != 0) { panic_info->mph_other_log_len = PE_get_offset_into_panic_region(debug_buf_ptr) - panic_info->mph_other_log_offset; @@ -1039,7 +1048,8 @@ paniclog_flush() * panic callbacks, otherwise we flush via nvram (unless that has been disabled). */ if (coprocessor_paniclog_flush) { - unsigned int size_to_flush = debug_buf_size; + uint32_t overall_buffer_size = debug_buf_size; + uint32_t size_to_flush = 0, offset_to_flush = 0; if (extended_debug_log_enabled) { /* * debug_buf_size for the extended log does not include the length of the header. @@ -1047,16 +1057,47 @@ paniclog_flush() * for the non-extended case (this is a concession we make to not shrink the paniclog data * for non-coprocessor systems that only use the basic log). */ - size_to_flush = debug_buf_size + sizeof(struct macos_panic_header); + overall_buffer_size = debug_buf_size + sizeof(struct macos_panic_header); } - /* We need to calculate the CRC for co-processor platforms */ - panic_info->mph_crc = crc32(0L, &panic_info->mph_version, (size_to_flush - offsetof(struct macos_panic_header, mph_version))); + /* Update the CRC */ + panic_info->mph_crc = crc32(0L, &panic_info->mph_version, (overall_buffer_size - offsetof(struct macos_panic_header, mph_version))); + + if (variant == kPaniclogFlushBase) { + /* Flush the header and base panic log. */ + kprintf("Flushing base panic log\n"); + size_to_flush = ROUNDUP((panic_info->mph_panic_log_offset + panic_info->mph_panic_log_len), PANIC_FLUSH_BOUNDARY); + offset_to_flush = 0; + PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush); + } else if ((variant == kPaniclogFlushStackshot) || (variant == kPaniclogFlushOtherLog)) { + if (variant == kPaniclogFlushStackshot) { + /* + * We flush the stackshot before flushing the updated header because the stackshot + * can take a while to flush. We want the paniclog header to be as consistent as possible even + * if the stackshot isn't flushed completely. Flush starting from the end of the panic log. + */ + kprintf("Flushing panic log stackshot\n"); + offset_to_flush = ROUNDDOWN((panic_info->mph_panic_log_offset + panic_info->mph_panic_log_len), PANIC_FLUSH_BOUNDARY); + size_to_flush = ROUNDUP((panic_info->mph_stackshot_len + (panic_info->mph_stackshot_offset - offset_to_flush)), PANIC_FLUSH_BOUNDARY); + PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush); + } - PESavePanicInfoAction(panic_info, size_to_flush); + /* Flush the other log -- everything after the stackshot */ + kprintf("Flushing panic 'other' log\n"); + offset_to_flush = ROUNDDOWN((panic_info->mph_stackshot_offset + panic_info->mph_stackshot_len), PANIC_FLUSH_BOUNDARY); + size_to_flush = ROUNDUP((panic_info->mph_other_log_len + (panic_info->mph_other_log_offset - offset_to_flush)), PANIC_FLUSH_BOUNDARY); + PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush); + + /* Flush the header -- everything before the paniclog */ + kprintf("Flushing panic log header\n"); + size_to_flush = ROUNDUP(panic_info->mph_panic_log_offset, PANIC_FLUSH_BOUNDARY); + offset_to_flush = 0; + PESavePanicInfoAction(panic_info, offset_to_flush, size_to_flush); + } } else if (commit_paniclog_to_nvram) { assert(debug_buf_size != 0); unsigned int bufpos; + unsigned long pi_size = 0; uintptr_t cr0; debug_putc(0); @@ -1109,6 +1150,14 @@ paniclog_flush() } } +void +paniclog_flush() +{ + /* Called outside of this file to update logging appended to the "other" log */ + paniclog_flush_internal(kPaniclogFlushOtherLog); + return; +} + char * machine_boot_info(char *buf, __unused vm_size_t size) { diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h index 27c493c24..618389fd9 100644 --- a/osfmk/i386/cpu_data.h +++ b/osfmk/i386/cpu_data.h @@ -78,11 +78,17 @@ typedef struct rtclock_timer { } rtclock_timer_t; typedef struct { - struct x86_64_tss *cdi_ktss; - x86_64_desc_register_t cdi_gdt; - x86_64_desc_register_t cdi_idt; - struct fake_descriptor *cdi_ldt; - vm_offset_t cdi_sstk; + /* The 'u' suffixed fields store the double-mapped descriptor addresses */ + struct x86_64_tss *cdi_ktssu; + struct x86_64_tss *cdi_ktssb; + x86_64_desc_register_t cdi_gdtu; + x86_64_desc_register_t cdi_gdtb; + x86_64_desc_register_t cdi_idtu; + x86_64_desc_register_t cdi_idtb; + struct fake_descriptor *cdi_ldtu; + struct fake_descriptor *cdi_ldtb; + vm_offset_t cdi_sstku; + vm_offset_t cdi_sstkb; } cpu_desc_index_t; typedef enum { @@ -109,7 +115,7 @@ typedef uint8_t pcid_ref_t; #define CPU_RTIME_BINS (12) #define CPU_ITIME_BINS (CPU_RTIME_BINS) -#define MAXPLFRAMES (32) +#define MAXPLFRAMES (16) typedef struct { boolean_t pltype; int plevel; @@ -129,6 +135,13 @@ typedef struct { * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu * pointers. */ +typedef struct { + pcid_t cpu_pcid_free_hint; +#define PMAP_PCID_MAX_PCID (0x800) + pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; + pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; +} pcid_cdata_t; + typedef struct cpu_data { struct pal_cpu_data cpu_pal_data; /* PAL-specific data */ @@ -143,8 +156,6 @@ typedef struct cpu_data vm_offset_t cpu_kernel_stack; /* kernel stack top */ vm_offset_t cpu_int_stack_top; int cpu_interrupt_level; - int cpu_phys_number; /* Physical CPU */ - cpu_id_t cpu_id; /* Platform Expert */ volatile int cpu_signals; /* IPI events */ volatile int cpu_prior_signals; /* Last set of events, * debugging @@ -166,18 +177,22 @@ typedef struct cpu_data }; volatile task_map_t cpu_task_map; volatile addr64_t cpu_task_cr3; + volatile addr64_t cpu_ucr3; addr64_t cpu_kernel_cr3; boolean_t cpu_pagezero_mapped; cpu_uber_t cpu_uber; - void *cpu_chud; - void *cpu_console_buf; - struct x86_lcpu lcpu; + /* Double-mapped per-CPU exception stack address */ + uintptr_t cd_estack; + /* Address of shadowed, partially mirrored CPU data structures located + * in the double mapped PML4 + */ + void *cd_shadow; struct processor *cpu_processor; #if NCOPY_WINDOWS > 0 struct cpu_pmap *cpu_pmap; #endif struct cpu_desc_table *cpu_desc_tablep; - struct fake_descriptor *cpu_ldtp; + struct real_descriptor *cpu_ldtp; cpu_desc_index_t cpu_desc_index; int cpu_ldt; #if NCOPY_WINDOWS > 0 @@ -191,7 +206,6 @@ typedef struct cpu_data #define HWINTCNT_SIZE 256 uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */ uint64_t cpu_hwIntpexits[HWINTCNT_SIZE]; - uint64_t cpu_hwIntcexits[HWINTCNT_SIZE]; uint64_t cpu_dr7; /* debug control register */ uint64_t cpu_int_event_time; /* intr entry/exit time */ pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */ @@ -211,10 +225,7 @@ typedef struct cpu_data pcid_t cpu_kernel_pcid; volatile pcid_ref_t *cpu_pmap_pcid_coherentp; volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel; -#define PMAP_PCID_MAX_PCID (0x1000) - pcid_t cpu_pcid_free_hint; - pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID]; - pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID]; + pcid_cdata_t *cpu_pcid_data; #ifdef PCID_STATS uint64_t cpu_pmap_pcid_flushes; uint64_t cpu_pmap_pcid_preserves; @@ -263,10 +274,20 @@ typedef struct cpu_data boolean_t cpu_iflag; boolean_t cpu_boot_complete; int cpu_hibernate; -#define MAX_PREEMPTION_RECORDS (128) +#define MAX_PREEMPTION_RECORDS (8) #if DEVELOPMENT || DEBUG int cpu_plri; plrecord_t plrecords[MAX_PREEMPTION_RECORDS]; +#endif + void *cpu_chud; + void *cpu_console_buf; + struct x86_lcpu lcpu; + int cpu_phys_number; /* Physical CPU */ + cpu_id_t cpu_id; /* Platform Expert */ +#if DEBUG + uint64_t cpu_entry_cr3; + uint64_t cpu_exit_cr3; + uint64_t cpu_pcid_last_cr3; #endif } cpu_data_t; @@ -585,12 +606,18 @@ _mp_enable_preemption_no_check(void) { #define MACHINE_PREEMPTION_MACROS (1) #endif - static inline cpu_data_t * cpu_datap(int cpu) { return cpu_data_ptr[cpu]; } +#ifdef MACH_KERNEL_PRIVATE +static inline cpu_data_t * +cpu_shadowp(int cpu) { + return cpu_data_ptr[cpu]->cd_shadow; +} + +#endif extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu); extern void cpu_data_realloc(void); diff --git a/osfmk/i386/cpu_topology.c b/osfmk/i386/cpu_topology.c index a8517d148..e079e6e38 100644 --- a/osfmk/i386/cpu_topology.c +++ b/osfmk/i386/cpu_topology.c @@ -50,6 +50,19 @@ static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep); x86_affinity_set_t *x86_affinities = NULL; static int x86_affinity_count = 0; +extern cpu_data_t cpshadows[]; +/* Re-sort double-mapped CPU data shadows after topology discovery sorts the + * primary CPU data structures by physical/APIC CPU ID. + */ +static void cpu_shadow_sort(int ncpus) { + for (int i = 0; i < ncpus; i++) { + cpu_data_t *cpup = cpu_datap(i); + ptrdiff_t coff = cpup - cpu_datap(0); + + cpup->cd_shadow = &cpshadows[coff]; + } +} + /* * cpu_topology_sort() is called after all processors have been registered * but before any non-boot processor id started. @@ -122,6 +135,7 @@ cpu_topology_sort(int ncpus) x86_set_logical_topology(&cpup->lcpu, cpup->cpu_phys_number, i); } + cpu_shadow_sort(ncpus); x86_validate_topology(); ml_set_interrupts_enabled(istate); diff --git a/osfmk/i386/genassym.c b/osfmk/i386/genassym.c index 510bdf89f..f6812c6f0 100644 --- a/osfmk/i386/genassym.c +++ b/osfmk/i386/genassym.c @@ -311,11 +311,11 @@ main( DECLARE("KERNEL_PML4_INDEX", KERNEL_PML4_INDEX); DECLARE("IDTSZ", IDTSZ); DECLARE("GDTSZ", GDTSZ); - DECLARE("LDTSZ", LDTSZ); DECLARE("KERNEL_DS", KERNEL_DS); DECLARE("USER_CS", USER_CS); DECLARE("USER_DS", USER_DS); + DECLARE("USER_CTHREAD", USER_CTHREAD); DECLARE("KERNEL32_CS", KERNEL32_CS); DECLARE("KERNEL64_CS", KERNEL64_CS); DECLARE("USER64_CS", USER64_CS); @@ -350,14 +350,6 @@ main( offsetof(cpu_data_t,cpu_running)); DECLARE("CPU_PENDING_AST", offsetof(cpu_data_t,cpu_pending_ast)); - DECLARE("CPU_DESC_TABLEP", - offsetof(cpu_data_t,cpu_desc_tablep)); - DECLARE("CPU_DESC_INDEX", - offsetof(cpu_data_t,cpu_desc_index)); - DECLARE("CDI_GDT", - offsetof(cpu_desc_index_t,cdi_gdt)); - DECLARE("CDI_IDT", - offsetof(cpu_desc_index_t,cdi_idt)); DECLARE("CPU_PROCESSOR", offsetof(cpu_data_t,cpu_processor)); DECLARE("CPU_INT_STATE", @@ -371,6 +363,14 @@ main( offsetof(cpu_data_t, cpu_active_cr3)); DECLARE("CPU_KERNEL_CR3", offsetof(cpu_data_t, cpu_kernel_cr3)); + DECLARE("CPU_UCR3", + offsetof(cpu_data_t, cpu_ucr3)); +#if DEBUG + DECLARE("CPU_ENTRY_CR3", + offsetof(cpu_data_t, cpu_entry_cr3)); + DECLARE("CPU_EXIT_CR3", + offsetof(cpu_data_t, cpu_exit_cr3)); +#endif DECLARE("CPU_TLB_INVALID", offsetof(cpu_data_t, cpu_tlb_invalid)); DECLARE("CPU_PAGEZERO_MAPPED", @@ -418,6 +418,11 @@ main( offsetof(cpu_data_t, cpu_tlb_invalid_local)); DECLARE("CPU_TLB_INVALID_GLOBAL", offsetof(cpu_data_t, cpu_tlb_invalid_global)); + DECLARE("CPU_ESTACK", + offsetof(cpu_data_t, cd_estack)); + DECLARE("CPU_DSHADOW", + offsetof(cpu_data_t, cd_shadow)); + DECLARE("enaExpTrace", enaExpTrace); DECLARE("enaUsrFCall", enaUsrFCall); DECLARE("enaUsrPhyMp", enaUsrPhyMp); diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c index 73a30a22b..5a7655de9 100644 --- a/osfmk/i386/i386_init.c +++ b/osfmk/i386/i386_init.c @@ -137,6 +137,7 @@ pdpt_entry_t *IdlePDPT; pml4_entry_t *IdlePML4; char *physfree; +void idt64_remap(void); /* * Note: ALLOCPAGES() can only be used safely within Idle_PTs_init() @@ -249,32 +250,7 @@ physmap_init(void) KERNEL_PHYSMAP_PML4_INDEX, IdlePML4[KERNEL_PHYSMAP_PML4_INDEX]); } -static void -descriptor_alias_init() -{ - vm_offset_t master_gdt_phys; - vm_offset_t master_gdt_alias_phys; - vm_offset_t master_idt_phys; - vm_offset_t master_idt_alias_phys; - - assert(((vm_offset_t)master_gdt & PAGE_MASK) == 0); - assert(((vm_offset_t)master_idt64 & PAGE_MASK) == 0); - - master_gdt_phys = (vm_offset_t) ID_MAP_VTOP(master_gdt); - master_idt_phys = (vm_offset_t) ID_MAP_VTOP(master_idt64); - master_gdt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_GDT_ALIAS); - master_idt_alias_phys = (vm_offset_t) ID_MAP_VTOP(MASTER_IDT_ALIAS); - - DBG("master_gdt_phys: %p\n", (void *) master_gdt_phys); - DBG("master_idt_phys: %p\n", (void *) master_idt_phys); - DBG("master_gdt_alias_phys: %p\n", (void *) master_gdt_alias_phys); - DBG("master_idt_alias_phys: %p\n", (void *) master_idt_alias_phys); - - KPTphys[atop_kernel(master_gdt_alias_phys)] = master_gdt_phys | - INTEL_PTE_VALID | INTEL_PTE_NX | INTEL_PTE_WRITE; - KPTphys[atop_kernel(master_idt_alias_phys)] = master_idt_phys | - INTEL_PTE_VALID | INTEL_PTE_NX; /* read-only */ -} +void doublemap_init(void); static void Idle_PTs_init(void) @@ -304,10 +280,8 @@ Idle_PTs_init(void) postcode(VSTART_PHYSMAP_INIT); physmap_init(); - - postcode(VSTART_DESC_ALIAS_INIT); - - descriptor_alias_init(); + doublemap_init(); + idt64_remap(); postcode(VSTART_SET_CR3); @@ -685,4 +659,115 @@ i386_init_slave_fast(void) do_init_slave(TRUE); } +#include + +/* TODO: Evaluate global PTEs for the double-mapped translations */ + +uint64_t dblmap_base, dblmap_max; +kernel_segment_command_t *hdescseg; +pt_entry_t *dblmapL3; +unsigned int dblallocs; +uint64_t dblmap_dist; +extern uint64_t idt64_hndl_table0[]; + + +void doublemap_init(void) { + dblmapL3 = ALLOCPAGES(1); // for 512 1GiB entries + dblallocs++; + + struct { + pt_entry_t entries[PTE_PER_PAGE]; + } * dblmapL2 = ALLOCPAGES(1); // for 512 2MiB entries + dblallocs++; + + dblmapL3[0] = ((uintptr_t)ID_MAP_VTOP(&dblmapL2[0])) + | INTEL_PTE_VALID + | INTEL_PTE_WRITE; + + hdescseg = getsegbynamefromheader(&_mh_execute_header, "__HIB"); + + vm_offset_t hdescb = hdescseg->vmaddr; + unsigned long hdescsz = hdescseg->vmsize; + unsigned long hdescszr = round_page_64(hdescsz); + vm_offset_t hdescc = hdescb, hdesce = hdescb + hdescszr; + + kernel_section_t *thdescsect = getsectbynamefromheader(&_mh_execute_header, "__HIB", "__text"); + vm_offset_t thdescb = thdescsect->addr; + unsigned long thdescsz = thdescsect->size; + unsigned long thdescszr = round_page_64(thdescsz); + vm_offset_t thdesce = thdescb + thdescszr; + + assert((hdescb & 0xFFF) == 0); + /* Mirror HIB translations into the double-mapped pagetable subtree*/ + for(int i = 0; hdescc < hdesce; i++) { + struct { + pt_entry_t entries[PTE_PER_PAGE]; + } * dblmapL1 = ALLOCPAGES(1); + dblallocs++; + dblmapL2[0].entries[i] = ((uintptr_t)ID_MAP_VTOP(&dblmapL1[0])) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF; + int hdescn = (int) ((hdesce - hdescc) / PAGE_SIZE); + for (int j = 0; j < MIN(PTE_PER_PAGE, hdescn); j++) { + uint64_t template = INTEL_PTE_VALID; + if ((hdescc >= thdescb) && (hdescc < thdesce)) { + /* executable */ + } else { + template |= INTEL_PTE_WRITE | INTEL_PTE_NX ; /* Writeable, NX */ + } + dblmapL1[0].entries[j] = ((uintptr_t)ID_MAP_VTOP(hdescc)) | template; + hdescc += PAGE_SIZE; + } + } + + IdlePML4[KERNEL_DBLMAP_PML4_INDEX] = ((uintptr_t)ID_MAP_VTOP(dblmapL3)) | INTEL_PTE_VALID | INTEL_PTE_WRITE | INTEL_PTE_REF; + + dblmap_base = KVADDR(KERNEL_DBLMAP_PML4_INDEX, dblmapL3, 0, 0); + dblmap_max = dblmap_base + hdescszr; + /* Calculate the double-map distance, which accounts for the current + * KASLR slide + */ + + dblmap_dist = dblmap_base - hdescb; + idt64_hndl_table0[1] = DBLMAP(idt64_hndl_table0[1]); + + extern cpu_data_t cpshadows[], scdatas[]; + uintptr_t cd1 = (uintptr_t) &cpshadows[0]; + uintptr_t cd2 = (uintptr_t) &scdatas[0]; +/* Record the displacement from the kernel's per-CPU data pointer, eventually + * programmed into GSBASE, to the "shadows" in the doublemapped + * region. These are not aliases, but separate physical allocations + * containing data required in the doublemapped trampolines. +*/ + idt64_hndl_table0[2] = dblmap_dist + cd1 - cd2; + + DBG("Double map base: 0x%qx\n", dblmap_base); + DBG("double map idlepml4[%d]: 0x%llx\n", KERNEL_DBLMAP_PML4_INDEX, IdlePML4[KERNEL_DBLMAP_PML4_INDEX]); + assert(LDTSZ > LDTSZ_MIN); +} + +vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t); + +#include + +/* Use of this routine is expected to be synchronized by callers + * Creates non-executable aliases. + */ +vm_offset_t dyn_dblmap(vm_offset_t cva, vm_offset_t sz) { + vm_offset_t ava = dblmap_max; + + assert((sz & PAGE_MASK) == 0); + assert(cva != 0); + + pmap_alias(ava, cva, cva + sz, VM_PROT_READ | VM_PROT_WRITE, PMAP_EXPAND_OPTIONS_ALIASMAP); + dblmap_max += sz; + return (ava - cva); +} +/* Adjust offsets interior to the bootstrap interrupt descriptor table to redirect + * control to the double-mapped interrupt vectors. The IDTR proper will be + * programmed via cpu_desc_load() + */ +void idt64_remap(void) { + for (int i = 0; i < IDTSZ; i++) { + master_idt64[i].offset64 = DBLMAP(master_idt64[i].offset64); + } +} diff --git a/osfmk/i386/ktss.c b/osfmk/i386/ktss.c index f5d3118f7..9d49d2571 100644 --- a/osfmk/i386/ktss.c +++ b/osfmk/i386/ktss.c @@ -73,10 +73,10 @@ * due to a single-step trace trap at system call entry. */ struct sysenter_stack master_sstk - __attribute__ ((section ("__DESC, master_sstk"))) + __attribute__ ((section ("__HIB, __desc"))) __attribute__ ((aligned (16))) = { {0}, 0 }; -struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) = { +struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) __attribute__ ((section ("__HIB, __desc"))) = { .io_bit_map_offset = 0x0FFF, }; diff --git a/osfmk/i386/ldt.c b/osfmk/i386/ldt.c index 4027c05f2..6e1a1e115 100644 --- a/osfmk/i386/ldt.c +++ b/osfmk/i386/ldt.c @@ -62,8 +62,9 @@ */ #include - -struct real_descriptor master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = { +struct real_descriptor master_ldt[LDTSZ] __attribute__ ((aligned (4096))) + __attribute__ ((section ("__HIB, __desc"))) + = { [SEL_TO_INDEX(USER_CTHREAD)] = MAKE_REAL_DESCRIPTOR( /* user cthread segment */ 0, 0xfffff, @@ -71,3 +72,5 @@ struct real_descriptor master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = { ACC_P|ACC_PL_U|ACC_DATA_W ), }; + +unsigned mldtsz = sizeof(master_ldt); diff --git a/osfmk/i386/mp.c b/osfmk/i386/mp.c index 0c971b1e0..1c11d9601 100644 --- a/osfmk/i386/mp.c +++ b/osfmk/i386/mp.c @@ -189,7 +189,7 @@ boolean_t i386_smp_init(int nmi_vector, i386_intr_func_t nmi_handler, int ipi_vector, i386_intr_func_t ipi_handler); void i386_start_cpu(int lapic_id, int cpu_num); void i386_send_NMI(int cpu); - +void NMIPI_enable(boolean_t); #if GPROF /* * Initialize dummy structs for profiling. These aren't used but @@ -1034,6 +1034,7 @@ NMIPI_panic(cpumask_t cpu_mask, NMI_reason_t why) { unsigned int cpu, cpu_bit; uint64_t deadline; + NMIPI_enable(TRUE); NMI_panic_reason = why; for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) { @@ -1647,8 +1648,12 @@ mp_kdp_enter(boolean_t proceed_on_failure) cpu_pause(); } /* If we've timed out, and some processor(s) are still unresponsive, - * interrupt them with an NMI via the local APIC. + * interrupt them with an NMI via the local APIC, iff a panic is + * in progress. */ + if (panic_active()) { + NMIPI_enable(TRUE); + } if (mp_kdp_ncpus != ncpus) { DBG("mp_kdp_enter() timed-out on cpu %d, NMI-ing\n", my_cpu); for (cpu = 0; cpu < real_ncpus; cpu++) { diff --git a/osfmk/i386/mp_desc.c b/osfmk/i386/mp_desc.c index 5f95beb4d..788e71663 100644 --- a/osfmk/i386/mp_desc.c +++ b/osfmk/i386/mp_desc.c @@ -176,15 +176,17 @@ extern uint32_t low_eintstack[]; /* top */ /* * Per-cpu data area pointers. - * The master cpu (cpu 0) has its data area statically allocated; - * others are allocated dynamically and this array is updated at runtime. */ -static cpu_data_t cpu_data_master = { - .cpu_this = &cpu_data_master, - .cpu_nanotime = &pal_rtc_nanotime_info, - .cpu_int_stack_top = (vm_offset_t) low_eintstack, +cpu_data_t cpshadows[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc"))); +cpu_data_t scdatas[MAX_CPUS] __attribute__((aligned(64))) = { + [0].cpu_this = &scdatas[0], + [0].cpu_nanotime = &pal_rtc_nanotime_info, + [0].cpu_int_stack_top = (vm_offset_t) low_eintstack, + [0].cd_shadow = &cpshadows[0] }; -cpu_data_t *cpu_data_ptr[MAX_CPUS] = { [0] = &cpu_data_master }; +cpu_data_t *cpu_data_master = &scdatas[0]; + +cpu_data_t *cpu_data_ptr[MAX_CPUS] = { [0] = &scdatas[0] }; decl_simple_lock_data(,ncpus_lock); /* protects real_ncpus */ unsigned int real_ncpus = 1; @@ -193,6 +195,15 @@ unsigned int max_ncpus = MAX_CPUS; extern void hi64_sysenter(void); extern void hi64_syscall(void); +typedef struct { + struct real_descriptor pcldts[LDTSZ]; +} cldt_t; + +cpu_desc_table64_t scdtables[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc"))); +cpu_fault_stack_t scfstks[MAX_CPUS] __attribute__((aligned(64))) __attribute__((section("__HIB, __desc"))); + +cldt_t *dyn_ldts; + /* * Multiprocessor i386/i486 systems use a separate copy of the * GDT, IDT, LDT, and kernel TSS per processor. The first three @@ -356,51 +367,36 @@ fix_desc64(void *descp, int count) } } -static void -cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias) -{ - pt_entry_t *pte = NULL; - - /* Require page alignment */ - assert(page_aligned(gdt)); - assert(page_aligned(alias)); - - pte = pmap_pte(kernel_pmap, alias); - pmap_store_pte(pte, kvtophys(gdt) | INTEL_PTE_REF - | INTEL_PTE_MOD - | INTEL_PTE_WIRED - | INTEL_PTE_VALID - | INTEL_PTE_WRITE - | INTEL_PTE_NX); -#if KASAN - kasan_notify_address(alias, PAGE_SIZE); -#endif -} - - +extern unsigned mldtsz; void cpu_desc_init(cpu_data_t *cdp) { cpu_desc_index_t *cdi = &cdp->cpu_desc_index; - if (cdp == &cpu_data_master) { + if (cdp == cpu_data_master) { /* - * Master CPU uses the tables built at boot time. - * Just set the index pointers to the low memory space. + * Populate the double-mapped 'u' and base 'b' fields in the + * KTSS with I/G/LDT and sysenter stack data. */ - cdi->cdi_ktss = (void *)&master_ktss64; - cdi->cdi_sstk = (vm_offset_t) &master_sstk.top; - cdi->cdi_gdt.ptr = (void *)MASTER_GDT_ALIAS; - cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS; - cdi->cdi_ldt = (struct fake_descriptor *) master_ldt; + cdi->cdi_ktssu = (void *)DBLMAP(&master_ktss64); + cdi->cdi_ktssb = (void *)&master_ktss64; + cdi->cdi_sstku = (vm_offset_t) DBLMAP(&master_sstk.top); + cdi->cdi_sstkb = (vm_offset_t) &master_sstk.top; + + cdi->cdi_gdtu.ptr = (void *)DBLMAP((uintptr_t) &master_gdt); + cdi->cdi_gdtb.ptr = (void *)&master_gdt; + cdi->cdi_idtu.ptr = (void *)DBLMAP((uintptr_t) &master_idt64); + cdi->cdi_idtb.ptr = (void *)((uintptr_t) &master_idt64); + cdi->cdi_ldtu = (struct fake_descriptor *) (void *) DBLMAP((uintptr_t)&master_ldt[0]); + cdi->cdi_ldtb = (struct fake_descriptor *) (void *) &master_ldt[0]; /* Replace the expanded LDTs and TSS slots in the GDT */ - kernel_ldt_desc64.offset64 = (uintptr_t) &master_ldt; + kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu; *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] = kernel_ldt_desc64; *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] = kernel_ldt_desc64; - kernel_tss_desc64.offset64 = (uintptr_t) &master_ktss64; + kernel_tss_desc64.offset64 = (uintptr_t) DBLMAP(&master_ktss64); *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] = kernel_tss_desc64; @@ -412,69 +408,62 @@ cpu_desc_init(cpu_data_t *cdp) /* * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS - * Note: this will be dynamically re-allocated in VM later. */ master_ktss64.ist2 = (uintptr_t) low_eintstack; - master_ktss64.ist1 = (uintptr_t) low_eintstack - - sizeof(x86_64_intr_stack_frame_t); - } else if (cdi->cdi_ktss == NULL) { /* Skipping re-init on wake */ + master_ktss64.ist1 = (uintptr_t) low_eintstack - sizeof(x86_64_intr_stack_frame_t); + } else if (cdi->cdi_ktssu == NULL) { /* Skipping re-init on wake */ cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep; - /* - * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel - * heap (cpu_desc_table). - * LDT descriptors are mapped into a separate area. - * GDT descriptors are addressed by alias to avoid sgdt leaks to user-space. - */ - cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS; - cdi->cdi_gdt.ptr = (void *)CPU_GDT_ALIAS(cdp->cpu_number); - cdi->cdi_ktss = (void *)&cdt->ktss; - cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top; - cdi->cdi_ldt = cdp->cpu_ldtp; + cdi->cdi_idtu.ptr = (void *)DBLMAP((uintptr_t) &master_idt64); - /* Make the virtual alias address for the GDT */ - cpu_gdt_alias((vm_map_offset_t) &cdt->gdt, - (vm_map_offset_t) cdi->cdi_gdt.ptr); + cdi->cdi_ktssu = (void *)DBLMAP(&cdt->ktss); + cdi->cdi_ktssb = (void *)(&cdt->ktss); + cdi->cdi_sstku = (vm_offset_t)DBLMAP(&cdt->sstk.top); + cdi->cdi_sstkb = (vm_offset_t)(&cdt->sstk.top); + cdi->cdi_ldtu = (void *)LDTALIAS(cdp->cpu_ldtp); + cdi->cdi_ldtb = (void *)(cdp->cpu_ldtp); /* * Copy the tables */ bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt)); - bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, sizeof(master_ldt)); + bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, mldtsz); bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss)); - + cdi->cdi_gdtu.ptr = (void *)DBLMAP(cdt->gdt); + cdi->cdi_gdtb.ptr = (void *)(cdt->gdt); /* * Fix up the entries in the GDT to point to * this LDT and this TSS. + * Note reuse of global 'kernel_ldt_desc64, which is not + * concurrency-safe. Higher level synchronization is expected */ - kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt; + kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu; *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] = kernel_ldt_desc64; fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1); - kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt; + kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldtu; *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] = kernel_ldt_desc64; fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1); - kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktss; + kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktssu; *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] = kernel_tss_desc64; fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1); /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */ - bzero((void *) cdt->fstk, sizeof(cdt->fstk)); - cdt->ktss.ist2 = (unsigned long)cdt->fstk + sizeof(cdt->fstk); - cdt->ktss.ist1 = cdt->ktss.ist2 - - sizeof(x86_64_intr_stack_frame_t); + uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0]; + cdt->fstkp = cfstk; + bzero((void *) cfstk, FSTK_SZ); + cdt->ktss.ist2 = DBLMAP((uint64_t)cdt->fstkp + FSTK_SZ); + cdt->ktss.ist1 = cdt->ktss.ist2 - sizeof(x86_64_intr_stack_frame_t); } /* Require that the top of the sysenter stack is 16-byte aligned */ - if ((cdi->cdi_sstk % 16) != 0) + if ((cdi->cdi_sstku % 16) != 0) panic("cpu_desc_init() sysenter stack not 16-byte aligned"); } - - void cpu_desc_load(cpu_data_t *cdp) { @@ -496,13 +485,15 @@ cpu_desc_load(cpu_data_t *cdp) gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY; /* Load the GDT, LDT, IDT and TSS */ - cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1; - cdi->cdi_idt.size = 0x1000 + cdp->cpu_number; - + cdi->cdi_gdtb.size = sizeof(struct real_descriptor)*GDTSZ - 1; + cdi->cdi_gdtu.size = cdi->cdi_gdtb.size; + cdi->cdi_idtb.size = 0x1000 + cdp->cpu_number; + cdi->cdi_idtu.size = cdi->cdi_idtb.size; + postcode(CPU_DESC_LOAD_GDT); - lgdt((uintptr_t *) &cdi->cdi_gdt); + lgdt((uintptr_t *) &cdi->cdi_gdtu); postcode(CPU_DESC_LOAD_IDT); - lidt((uintptr_t *) &cdi->cdi_idt); + lidt((uintptr_t *) &cdi->cdi_idtu); postcode(CPU_DESC_LOAD_LDT); lldt(KERNEL_LDT); postcode(CPU_DESC_LOAD_TSS); @@ -514,7 +505,6 @@ cpu_desc_load(cpu_data_t *cdp) postcode(CPU_DESC_LOAD_EXIT); } - /* * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit. */ @@ -527,8 +517,8 @@ cpu_syscall_init(cpu_data_t *cdp) #pragma unused(cdp) #endif /* !MONOTONIC */ wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); - wrmsr64(MSR_IA32_SYSENTER_EIP, (uintptr_t) hi64_sysenter); - wrmsr64(MSR_IA32_SYSENTER_ESP, current_sstk()); + wrmsr64(MSR_IA32_SYSENTER_EIP, DBLMAP((uintptr_t) hi64_sysenter)); + wrmsr64(MSR_IA32_SYSENTER_ESP, current_cpu_datap()->cpu_desc_index.cdi_sstku); /* Enable syscall/sysret */ wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE); @@ -537,9 +527,8 @@ cpu_syscall_init(cpu_data_t *cdp) * Note USER_CS because sysret uses this + 16 when returning to * 64-bit code. */ - wrmsr64(MSR_IA32_LSTAR, (uintptr_t) hi64_syscall); - wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) | - (((uint64_t)KERNEL64_CS) << 32)); + wrmsr64(MSR_IA32_LSTAR, DBLMAP((uintptr_t) hi64_syscall)); + wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) | (((uint64_t)KERNEL64_CS) << 32)); /* * Emulate eflags cleared by sysenter but note that * we also clear the trace trap to avoid the complications @@ -550,7 +539,8 @@ cpu_syscall_init(cpu_data_t *cdp) wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT); } - +extern vm_offset_t dyn_dblmap(vm_offset_t, vm_offset_t); +uint64_t ldt_alias_offset; cpu_data_t * cpu_data_alloc(boolean_t is_boot_cpu) @@ -571,17 +561,24 @@ cpu_data_alloc(boolean_t is_boot_cpu) return cdp; } + boolean_t do_ldt_alloc = FALSE; + simple_lock(&ncpus_lock); + int cnum = real_ncpus; + real_ncpus++; + if (dyn_ldts == NULL) { + do_ldt_alloc = TRUE; + } + simple_unlock(&ncpus_lock); + /* * Allocate per-cpu data: */ - ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU); - if (ret != KERN_SUCCESS) { - printf("cpu_data_alloc() failed, ret=%d\n", ret); - goto abort; - } + + cdp = &scdatas[cnum]; bzero((void*) cdp, sizeof(cpu_data_t)); cdp->cpu_this = cdp; - + cdp->cpu_number = cnum; + cdp->cd_shadow = &cpshadows[cnum]; /* * Allocate interrupt stack: */ @@ -589,8 +586,7 @@ cpu_data_alloc(boolean_t is_boot_cpu) (vm_offset_t *) &cdp->cpu_int_stack_top, INTSTACK_SIZE, VM_KERN_MEMORY_CPU); if (ret != KERN_SUCCESS) { - printf("cpu_data_alloc() int stack failed, ret=%d\n", ret); - goto abort; + panic("cpu_data_alloc() int stack failed, ret=%d\n", ret); } bzero((void*) cdp->cpu_int_stack_top, INTSTACK_SIZE); cdp->cpu_int_stack_top += INTSTACK_SIZE; @@ -598,39 +594,49 @@ cpu_data_alloc(boolean_t is_boot_cpu) /* * Allocate descriptor table: */ - ret = kmem_alloc(kernel_map, - (vm_offset_t *) &cdp->cpu_desc_tablep, - sizeof(cpu_desc_table64_t), - VM_KERN_MEMORY_CPU); - if (ret != KERN_SUCCESS) { - printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret); - goto abort; - } + cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[cnum]; /* * Allocate LDT */ - ret = kmem_alloc(kernel_map, - (vm_offset_t *) &cdp->cpu_ldtp, - sizeof(struct real_descriptor) * LDTSZ, - VM_KERN_MEMORY_CPU); - if (ret != KERN_SUCCESS) { - printf("cpu_data_alloc() ldt failed, ret=%d\n", ret); - goto abort; + if (do_ldt_alloc) { + boolean_t do_ldt_free = FALSE; + vm_offset_t sldtoffset = 0; + /* + * Allocate LDT + */ + vm_offset_t ldtalloc = 0, ldtallocsz = round_page_64(MAX_CPUS * sizeof(struct real_descriptor) * LDTSZ); + ret = kmem_alloc(kernel_map, (vm_offset_t *) &ldtalloc, ldtallocsz, VM_KERN_MEMORY_CPU); + if (ret != KERN_SUCCESS) { + panic("cpu_data_alloc() ldt failed, kmem_alloc=%d\n", ret); + } + + simple_lock(&ncpus_lock); + if (dyn_ldts == NULL) { + dyn_ldts = (cldt_t *)ldtalloc; + } else { + do_ldt_free = TRUE; + } + simple_unlock(&ncpus_lock); + + if (do_ldt_free) { + kmem_free(kernel_map, ldtalloc, ldtallocsz); + } else { + /* CPU registration and startup are expected to execute + * serially, as invoked by the platform driver. + * Create trampoline alias of LDT region. + */ + sldtoffset = dyn_dblmap(ldtalloc, ldtallocsz); + ldt_alias_offset = sldtoffset; + } } + cdp->cpu_ldtp = &dyn_ldts[cnum].pcldts[0]; #if CONFIG_MCA /* Machine-check shadow register allocation. */ mca_cpu_alloc(cdp); #endif - simple_lock(&ncpus_lock); - - cpu_data_ptr[real_ncpus] = cdp; - cdp->cpu_number = real_ncpus; - real_ncpus++; - simple_unlock(&ncpus_lock); - /* * Before this cpu has been assigned a real thread context, * we give it a fake, unique, non-zero thread id which the locking @@ -648,20 +654,10 @@ cpu_data_alloc(boolean_t is_boot_cpu) "int_stack: 0x%lx-0x%lx\n", cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp, (long)(cdp->cpu_int_stack_top - INTSTACK_SIZE), (long)(cdp->cpu_int_stack_top)); + cpu_data_ptr[cnum] = cdp; return cdp; -abort: - if (cdp) { - if (cdp->cpu_desc_tablep) - kfree((void *) cdp->cpu_desc_tablep, - sizeof(cpu_desc_table64_t)); - if (cdp->cpu_int_stack_top) - kfree((void *) (cdp->cpu_int_stack_top - INTSTACK_SIZE), - INTSTACK_SIZE); - kfree((void *) cdp, sizeof(*cdp)); - } - return NULL; } boolean_t @@ -678,7 +674,6 @@ valid_user_data_selector(uint16_t selector) if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U) return (TRUE); } - return (FALSE); } @@ -841,7 +836,6 @@ cpu_data_realloc(void) { int ret; vm_offset_t istk; - vm_offset_t fstk; cpu_data_t *cdp; boolean_t istate; @@ -852,10 +846,7 @@ cpu_data_realloc(void) bzero((void*) istk, INTSTACK_SIZE); istk += INTSTACK_SIZE; - ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU); - if (ret != KERN_SUCCESS) { - panic("cpu_data_realloc() cpu data alloc, ret=%d\n", ret); - } + cdp = &scdatas[0]; /* Copy old contents into new area and make fix-ups */ assert(cpu_number() == 0); @@ -863,28 +854,25 @@ cpu_data_realloc(void) cdp->cpu_this = cdp; cdp->cpu_int_stack_top = istk; timer_call_queue_init(&cdp->rtclock_timer.queue); + cdp->cpu_desc_tablep = (struct cpu_desc_table *) &scdtables[0]; + cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep; - /* Allocate the separate fault stack */ - ret = kmem_alloc(kernel_map, &fstk, PAGE_SIZE, VM_KERN_MEMORY_CPU); - if (ret != KERN_SUCCESS) { - panic("cpu_data_realloc() fault stack alloc, ret=%d\n", ret); - } - bzero((void*) fstk, PAGE_SIZE); - fstk += PAGE_SIZE; + uint8_t *cfstk = &scfstks[cdp->cpu_number].fstk[0]; + cdt->fstkp = cfstk; + cfstk += FSTK_SZ; /* * With interrupts disabled commmit the new areas. */ istate = ml_set_interrupts_enabled(FALSE); cpu_data_ptr[0] = cdp; - master_ktss64.ist2 = (uintptr_t) fstk; - master_ktss64.ist1 = (uintptr_t) fstk - - sizeof(x86_64_intr_stack_frame_t); + master_ktss64.ist2 = DBLMAP((uintptr_t) cfstk); + master_ktss64.ist1 = DBLMAP((uintptr_t) cfstk - sizeof(x86_64_intr_stack_frame_t)); wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp); wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp); (void) ml_set_interrupts_enabled(istate); kprintf("Reallocated master cpu data: %p," " interrupt stack: %p, fault stack: %p\n", - (void *) cdp, (void *) istk, (void *) fstk); + (void *) cdp, (void *) istk, (void *) cfstk); } diff --git a/osfmk/i386/mp_desc.h b/osfmk/i386/mp_desc.h index ebc75cfb0..50f6c62a6 100644 --- a/osfmk/i386/mp_desc.h +++ b/osfmk/i386/mp_desc.h @@ -80,18 +80,23 @@ __BEGIN_DECLS * The descriptor tables are together in a structure * allocated one per processor (except for the boot processor). */ +#define FSTK_SZ (PAGE_SIZE/2) typedef struct cpu_desc_table64 { struct fake_descriptor gdt[GDTSZ] __attribute__ ((aligned (16))); struct x86_64_tss ktss __attribute__ ((aligned (16))); struct sysenter_stack sstk __attribute__ ((aligned (16))); - uint8_t fstk[PAGE_SIZE] __attribute__ ((aligned (16))); + uint8_t *fstkp; } cpu_desc_table64_t; -#define current_gdt() (current_cpu_datap()->cpu_desc_index.cdi_gdt.ptr) -#define current_idt() (current_cpu_datap()->cpu_desc_index.cdi_idt.ptr) -#define current_ldt() (current_cpu_datap()->cpu_desc_index.cdi_ldt) -#define current_ktss() (current_cpu_datap()->cpu_desc_index.cdi_ktss) -#define current_sstk() (current_cpu_datap()->cpu_desc_index.cdi_sstk) +typedef struct { + uint8_t fstk[FSTK_SZ] __attribute__ ((aligned (16))); +} cpu_fault_stack_t; + +#define current_gdt() (current_cpu_datap()->cpu_desc_index.cdi_gdtb.ptr) +#define current_idt() (current_cpu_datap()->cpu_desc_index.cdi_idtb.ptr) +#define current_ldt() (current_cpu_datap()->cpu_desc_index.cdi_ldtb) +#define current_ktss() (current_cpu_datap()->cpu_desc_index.cdi_ktssb) +#define current_sstk() (current_cpu_datap()->cpu_desc_index.cdi_sstkb) #define current_ktss64() ((struct x86_64_tss *) current_ktss()) #define current_sstk64() ((addr64_t *) current_sstk()) diff --git a/osfmk/i386/mp_native.c b/osfmk/i386/mp_native.c index f517f173a..e64344db7 100644 --- a/osfmk/i386/mp_native.c +++ b/osfmk/i386/mp_native.c @@ -42,6 +42,7 @@ boolean_t i386_smp_init(int nmi_vector, i386_intr_func_t nmi_handler, void i386_start_cpu(int lapic_id, int cpu_num); void i386_send_NMI(int cpu); void handle_pending_TLB_flushes(void); +void NMIPI_enable(boolean_t); extern void slave_pstart(void); @@ -75,16 +76,27 @@ i386_start_cpu(int lapic_id, __unused int cpu_num ) LAPIC_ICR_DM_STARTUP|(REAL_MODE_BOOTSTRAP_OFFSET>>12)); } +static boolean_t NMIPIs_enabled = FALSE; + +void NMIPI_enable(boolean_t enable) { + NMIPIs_enabled = enable; +} + void i386_send_NMI(int cpu) { boolean_t state = ml_set_interrupts_enabled(FALSE); + + if (NMIPIs_enabled == FALSE) { + i386_cpu_IPI(cpu); + } else { /* Program the interrupt command register */ /* The vector is ignored in this case--the target CPU will enter on the * NMI vector. */ LAPIC_WRITE_ICR(cpu_to_lapic[cpu], LAPIC_VECTOR(INTERPROCESSOR)|LAPIC_ICR_DM_NMI); + } (void) ml_set_interrupts_enabled(state); } diff --git a/osfmk/i386/pcb_native.c b/osfmk/i386/pcb_native.c index c260ec7cd..34df7c119 100644 --- a/osfmk/i386/pcb_native.c +++ b/osfmk/i386/pcb_native.c @@ -137,6 +137,7 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new) set_ds(NULL_SEG); set_es(NULL_SEG); set_fs(NULL_SEG); + if (get_gs() != NULL_SEG) { swapgs(); /* switch to user's GS context */ set_gs(NULL_SEG); @@ -158,9 +159,7 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new) /* require 16-byte alignment */ assert((pcb_stack_top & 0xF) == 0); - /* Interrupt stack is pcb */ - current_ktss64()->rsp0 = pcb_stack_top; - + current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku; /* * Top of temporary sysenter stack points to pcb stack. * Although this is not normally used by 64-bit users, @@ -168,6 +167,8 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new) */ *current_sstk64() = pcb_stack_top; + cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku; + if (is_saved_state64(pcb->iss)) { cdp->cpu_task_map = new->map->pmap->pm_task_map; @@ -195,7 +196,6 @@ act_machine_switch_pcb(__unused thread_t old, thread_t new) wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self); } } - } else { cdp->cpu_task_map = TASK_MAP_32BIT; @@ -403,9 +403,7 @@ machine_thread_create( * segment. */ if ((pcb->cthread_desc.access & ACC_P) == 0) { - struct real_descriptor *ldtp; - ldtp = (struct real_descriptor *)current_ldt(); - pcb->cthread_desc = ldtp[sel_idx(USER_DS)]; + pcb->cthread_desc = *gdt_desc_p(USER_DS); } return(KERN_SUCCESS); diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h index 0c26eee60..3458ce7cf 100644 --- a/osfmk/i386/pmap.h +++ b/osfmk/i386/pmap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2012 Apple Inc. All rights reserved. + * Copyright (c) 2000-2017 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -70,7 +70,6 @@ #ifndef ASSEMBLER - #include #include #include @@ -110,10 +109,7 @@ #define PTESHIFT 12ULL - -#ifdef __x86_64__ #define LOW_4GB_MASK ((vm_offset_t)0x00000000FFFFFFFFUL) -#endif #define PDESIZE sizeof(pd_entry_t) /* for assembly files */ #define PTESIZE sizeof(pt_entry_t) /* for assembly files */ @@ -194,11 +190,7 @@ extern int pmap_asserts_traced; #endif /* superpages */ -#ifdef __x86_64__ #define SUPERPAGE_NBASEPAGES 512 -#else -#define SUPERPAGE_NBASEPAGES 1 /* we don't support superpages on i386 */ -#endif /* * Atomic 64-bit store of a page table entry. @@ -220,10 +212,11 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) #define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t)))) #define KERNEL_PML4_INDEX 511 -#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ +#define KERNEL_KEXTS_INDEX 510 /* Home of KEXTs - the basement */ #define KERNEL_PHYSMAP_PML4_INDEX 509 /* virtual to physical map */ #define KERNEL_KASAN_PML4_INDEX0 508 #define KERNEL_KASAN_PML4_INDEX1 507 +#define KERNEL_DBLMAP_PML4_INDEX (506) #define KERNEL_BASE (0ULL - NBPML4) #define KERNEL_BASEMENT (KERNEL_BASE - NBPML4) @@ -460,9 +453,6 @@ extern uint64_t pmap_pv_hashlist_cnts; extern uint32_t pmap_pv_hashlist_max; extern uint32_t pmap_kernel_text_ps; - - -#ifdef __x86_64__ #define ID_MAP_VTOP(x) ((void *)(((uint64_t)(x)) & LOW_4GB_MASK)) extern uint64_t physmap_base, physmap_max; @@ -484,6 +474,30 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { } #define PHYSMAP_PTOV(x) (PHYSMAP_PTOV_check((void*) (x))) +#if MACH_KERNEL_PRIVATE +extern uint64_t dblmap_base, dblmap_max, dblmap_dist; + +static inline uint64_t DBLMAP_CHECK(uintptr_t x) { + uint64_t dbladdr = (uint64_t)x + dblmap_dist; + if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) { + panic("DBLMAP bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx", + (uint64_t)x, dbladdr, dblmap_base, dblmap_max); + } + return dbladdr; + +} +#define DBLMAP(x) (DBLMAP_CHECK((uint64_t) x)) +extern uint64_t ldt_alias_offset; +static inline uint64_t LDTALIAS_CHECK(uintptr_t x) { + uint64_t dbladdr = (uint64_t)x + ldt_alias_offset; + if (__improbable((dbladdr >= dblmap_max) || (dbladdr < dblmap_base))) { + panic("LDTALIAS: bounds exceeded, 0x%qx, 0x%qx 0x%qx, 0x%qx", + (uint64_t)x, dbladdr, dblmap_base, dblmap_max); + } + return dbladdr; +} +#define LDTALIAS(x) (LDTALIAS_CHECK((uint64_t) x)) +#endif /* * For KASLR, we alias the master processor's IDT and GDT at fixed @@ -491,10 +505,7 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { * And non-boot processor's GDT aliases likewise (skipping LOWGLOBAL_ALIAS) * The low global vector page is mapped at a fixed alias also. */ -#define MASTER_IDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x0000) -#define MASTER_GDT_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x1000) #define LOWGLOBAL_ALIAS (VM_MIN_KERNEL_ADDRESS + 0x2000) -#define CPU_GDT_ALIAS(_cpu) (LOWGLOBAL_ALIAS + (0x1000*(_cpu))) /* * This indicates (roughly) where there is free space for the VM @@ -502,8 +513,6 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { */ #define KERNEL_PMAP_HEAP_RANGE_START VM_MIN_KERNEL_AND_KEXT_ADDRESS -#endif /*__x86_64__ */ - #include /* @@ -514,7 +523,8 @@ static inline void * PHYSMAP_PTOV_check(void *paddr) { struct pmap { decl_simple_lock_data(,lock) /* lock on map */ - pmap_paddr_t pm_cr3; /* physical addr */ + pmap_paddr_t pm_cr3; /* Kernel+user shared PML4 physical*/ + pmap_paddr_t pm_ucr3; /* Mirrored user PML4 physical */ task_map_t pm_task_map; boolean_t pm_shared; boolean_t pagezero_accessible; @@ -524,13 +534,12 @@ struct pmap { struct pmap_statistics stats; /* map statistics */ int ref_count; /* reference count */ int nx_enabled; - pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */ pml4_entry_t *pm_pml4; /* VKA of top level */ + pml4_entry_t *pm_upml4; /* Shadow VKA of top level */ vm_object_t pm_obj; /* object to hold pde's */ vm_object_t pm_obj_pdpt; /* holds pdpt pages */ vm_object_t pm_obj_pml4; /* holds pml4 pages */ pmap_paddr_t pm_eptp; /* EPTP */ - pd_entry_t *dirbase; /* page directory pointer */ ledger_t ledger; /* ledger tracking phys mappings */ #if MACH_ASSERT int pmap_pid; @@ -602,7 +611,13 @@ extern pmap_memory_region_t pmap_memory_regions[]; static inline void set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) { int ccpu = my_cpu; - cpu_datap(ccpu)->cpu_task_cr3 = tpmap->pm_cr3; + uint64_t pcr3 = tpmap->pm_cr3, ucr3 = tpmap->pm_ucr3; + cpu_datap(ccpu)->cpu_task_cr3 = pcr3; + cpu_shadowp(ccpu)->cpu_task_cr3 = pcr3; + + cpu_datap(ccpu)->cpu_ucr3 = ucr3; + cpu_shadowp(ccpu)->cpu_ucr3 = ucr3; + cpu_datap(ccpu)->cpu_task_map = tpmap->pm_task_map; assert((get_preemption_level() > 0) || (ml_get_interrupts_enabled() == FALSE)); @@ -650,10 +665,6 @@ set_dirbase(pmap_t tpmap, thread_t thread, int my_cpu) { extern void process_pmap_updates(void); extern void pmap_update_interrupt(void); -/* - * Machine dependent routines that are used only for i386/i486/i860. - */ - extern addr64_t (kvtophys)( vm_offset_t addr); @@ -661,23 +672,6 @@ extern kern_return_t pmap_expand( pmap_t pmap, vm_map_offset_t addr, unsigned int options); -#if !defined(__x86_64__) -extern pt_entry_t *pmap_pte( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pd_entry_t *pmap_pde( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pd_entry_t *pmap64_pde( - struct pmap *pmap, - vm_map_offset_t addr); - -extern pdpt_entry_t *pmap64_pdpt( - struct pmap *pmap, - vm_map_offset_t addr); -#endif extern vm_offset_t pmap_map( vm_offset_t virt, vm_map_offset_t start, @@ -691,7 +685,6 @@ extern vm_offset_t pmap_map_bd( vm_map_offset_t end, vm_prot_t prot, unsigned int flags); - extern void pmap_bootstrap( vm_offset_t load_start, boolean_t IA32e); @@ -870,11 +863,6 @@ extern int pmap_stats_assert; #else /* MACH_ASSERT */ #define PMAP_STATS_ASSERTF(args) #endif /* MACH_ASSERT */ - #endif /* ASSEMBLER */ - - #endif /* _PMAP_MACHINE_ */ - - #endif /* KERNEL_PRIVATE */ diff --git a/osfmk/i386/pmap_common.c b/osfmk/i386/pmap_common.c index d948f34a3..9436719d4 100644 --- a/osfmk/i386/pmap_common.c +++ b/osfmk/i386/pmap_common.c @@ -61,7 +61,7 @@ event_t mapping_replenish_event, pmap_user_pv_throttle_event; uint64_t pmap_pv_throttle_stat, pmap_pv_throttled_waiters; -int pmap_asserts_enabled = DEBUG; +int pmap_asserts_enabled = (DEBUG); int pmap_asserts_traced = 0; unsigned int pmap_cache_attributes(ppnum_t pn) { diff --git a/osfmk/i386/pmap_internal.h b/osfmk/i386/pmap_internal.h index 3523d8a14..4ddabaa20 100644 --- a/osfmk/i386/pmap_internal.h +++ b/osfmk/i386/pmap_internal.h @@ -407,7 +407,7 @@ extern boolean_t pmap_disable_kstack_nx; #define PMAP_EXPAND_OPTIONS_NONE (0x0) #define PMAP_EXPAND_OPTIONS_NOWAIT (PMAP_OPTIONS_NOWAIT) #define PMAP_EXPAND_OPTIONS_NOENTER (PMAP_OPTIONS_NOENTER) - +#define PMAP_EXPAND_OPTIONS_ALIASMAP (0x40000000U) /* * Amount of virtual memory mapped by one * page-directory entry. @@ -473,11 +473,14 @@ extern ppnum_t highest_hi; #define MAX_PREEMPTION_LATENCY_NS 20000 extern uint64_t max_preemption_latency_tsc; -/* #define DEBUGINTERRUPTS 1 uncomment to ensure pmap callers have interrupts enabled */ -#ifdef DEBUGINTERRUPTS +#if DEBUG +#define PMAP_INTR_DEBUG (1) +#endif + +#if PMAP_INTR_DEBUG #define pmap_intr_assert() { \ if (processor_avail_count > 1 && !ml_get_interrupts_enabled()) \ - panic("pmap interrupt assert %s, %d",__FILE__, __LINE__); \ + panic("pmap interrupt assert %d %s, %d", processor_avail_count, __FILE__, __LINE__); \ } #else #define pmap_intr_assert() @@ -495,7 +498,6 @@ pvhashidx(pmap_t pmap, vm_map_offset_t va) return hashidx; } - /* * unlinks the pv_hashed_entry_t pvh from the singly linked hash chain. * properly deals with the anchor. @@ -964,11 +966,6 @@ PMAP_ZINFO_SFREE(pmap_t pmap, vm_size_t bytes) extern boolean_t pmap_initialized;/* Has pmap_init completed? */ #define valid_page(x) (pmap_initialized && pmap_valid_page(x)) -// XXX -#define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */ -// XXX - - int phys_attribute_test( ppnum_t phys, int bits); @@ -1033,7 +1030,6 @@ static inline void pmap_update_pte(pt_entry_t *mptep, uint64_t pclear_bits, uint } while (!pmap_cmpx_pte(mptep, opte, npte)); } -#if defined(__x86_64__) /* * The single pml4 page per pmap is allocated at pmap create time and exists * for the duration of the pmap. we allocate this page in kernel vm. @@ -1055,6 +1051,21 @@ pmap64_pml4(pmap_t pmap, vm_map_offset_t vaddr) #endif } +static inline pml4_entry_t * +pmap64_user_pml4(pmap_t pmap, vm_map_offset_t vaddr) +{ + if (__improbable((vaddr > 0x00007FFFFFFFFFFFULL) && + (vaddr < 0xFFFF800000000000ULL))) { + return (NULL); + } + +#if DEBUG + return PHYSMAP_PTOV(&((pml4_entry_t *)pmap->pm_ucr3)[(vaddr >> PML4SHIFT) & (NPML4PG-1)]); +#else + return &pmap->pm_upml4[(vaddr >> PML4SHIFT) & (NPML4PG-1)]; +#endif +} + /* * Returns address of requested PDPT entry in the physmap. */ @@ -1134,7 +1145,13 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr) } return (NULL); } -#endif +extern void pmap_alias( + vm_offset_t ava, + vm_map_offset_t start, + vm_map_offset_t end, + vm_prot_t prot, + unsigned int options); + #if DEBUG #define DPRINTF(x...) kprintf(x) #else diff --git a/osfmk/i386/pmap_pcid.h b/osfmk/i386/pmap_pcid.h index fc0854d7e..372f0a4ed 100644 --- a/osfmk/i386/pmap_pcid.h +++ b/osfmk/i386/pmap_pcid.h @@ -61,8 +61,12 @@ tlb_flush_global(void) { * count of kernel invalidations, but that scheme * has its disadvantages as well. */ - set_cr4(cr4 & ~CR4_PGE); - set_cr4(cr4 | CR4_PGE); + if (cr4 & CR4_PGE) { + set_cr4(cr4 & ~CR4_PGE); + set_cr4(cr4 | CR4_PGE); + } else { + set_cr3_raw(get_cr3_raw()); + } return; } @@ -85,7 +89,6 @@ static inline void pmap_pcid_validate_current(void) { if (cptr) { *cptr = 0; } - } static inline void pmap_pcid_invalidate_cpu(pmap_t tpmap, int ccpu) { diff --git a/osfmk/i386/pmap_x86_common.c b/osfmk/i386/pmap_x86_common.c index 72fd27e76..b66630233 100644 --- a/osfmk/i386/pmap_x86_common.c +++ b/osfmk/i386/pmap_x86_common.c @@ -2234,9 +2234,11 @@ pmap_map_bd( unsigned int flags) { pt_entry_t template; - pt_entry_t *pte; - spl_t spl; + pt_entry_t *ptep; + vm_offset_t base = virt; + boolean_t doflush = FALSE; + template = pa_to_pte(start_addr) | INTEL_PTE_REF | INTEL_PTE_MOD @@ -2249,31 +2251,80 @@ pmap_map_bd( template |= INTEL_PTE_PTA; } -#if defined(__x86_64__) if ((prot & VM_PROT_EXECUTE) == 0) template |= INTEL_PTE_NX; -#endif if (prot & VM_PROT_WRITE) template |= INTEL_PTE_WRITE; while (start_addr < end_addr) { - spl = splhigh(); - pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt); - if (pte == PT_ENTRY_NULL) { - panic("pmap_map_bd: Invalid kernel address\n"); + ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)virt); + if (ptep == PT_ENTRY_NULL) { + panic("pmap_map_bd: Invalid kernel address"); + } + if (pte_to_pa(*ptep)) { + doflush = TRUE; } - pmap_store_pte(pte, template); - splx(spl); + pmap_store_pte(ptep, template); pte_increment_pa(template); virt += PAGE_SIZE; start_addr += PAGE_SIZE; } - flush_tlb_raw(); - PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr); + if (doflush) { + flush_tlb_raw(); + PMAP_UPDATE_TLBS(kernel_pmap, base, base + end_addr - start_addr); + } return(virt); } +/* Create a virtual alias beginning at 'ava' of the specified kernel virtual + * range. The aliased pagetable range is expanded if + * PMAP_EXPAND_OPTIONS_ALIASMAP is specified. Performs no synchronization, + * assumes caller has stabilized the source and destination ranges. Currently + * used to populate sections of the trampoline "doublemap" at CPU startup. + */ + +void +pmap_alias( + vm_offset_t ava, + vm_map_offset_t start_addr, + vm_map_offset_t end_addr, + vm_prot_t prot, + unsigned int eoptions) +{ + pt_entry_t prot_template, template; + pt_entry_t *aptep, *sptep; + + prot_template = INTEL_PTE_REF | INTEL_PTE_MOD | INTEL_PTE_WIRED | INTEL_PTE_VALID; + if ((prot & VM_PROT_EXECUTE) == 0) + prot_template |= INTEL_PTE_NX; + + if (prot & VM_PROT_WRITE) + prot_template |= INTEL_PTE_WRITE; + assert(((start_addr | end_addr) & PAGE_MASK) == 0); + while (start_addr < end_addr) { + aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava); + if (aptep == PT_ENTRY_NULL) { + if (eoptions & PMAP_EXPAND_OPTIONS_ALIASMAP) { + pmap_expand(kernel_pmap, ava, PMAP_EXPAND_OPTIONS_ALIASMAP); + aptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ava); + } else { + panic("pmap_alias: Invalid alias address"); + } + } + /* The aliased range should not have any active mappings */ + assert(pte_to_pa(*aptep) == 0); + + sptep = pmap_pte(kernel_pmap, start_addr); + assert(sptep != PT_ENTRY_NULL && (pte_to_pa(*sptep) != 0)); + template = pa_to_pte(pte_to_pa(*sptep)) | prot_template; + pmap_store_pte(aptep, template); + + ava += PAGE_SIZE; + start_addr += PAGE_SIZE; + } +} + mach_vm_size_t pmap_query_resident( pmap_t pmap, diff --git a/osfmk/i386/proc_reg.h b/osfmk/i386/proc_reg.h index 9e1e744d1..530f9e61c 100644 --- a/osfmk/i386/proc_reg.h +++ b/osfmk/i386/proc_reg.h @@ -306,9 +306,9 @@ static inline uintptr_t get_cr3_base(void) return(cr3 & ~(0xFFFULL)); } -static inline void set_cr3_composed(uintptr_t base, uint16_t pcid, uint32_t preserve) +static inline void set_cr3_composed(uintptr_t base, uint16_t pcid, uint64_t preserve) { - __asm__ volatile("mov %0, %%cr3" : : "r" (base | pcid | ( ( (uint64_t)preserve) << 63) ) ); + __asm__ volatile("mov %0, %%cr3" : : "r" (base | pcid | ( (preserve) << 63) ) ); } static inline uintptr_t get_cr4(void) @@ -383,7 +383,13 @@ static inline void hlt(void) static inline void flush_tlb_raw(void) { - set_cr3_raw(get_cr3_raw()); + uintptr_t cr4 = get_cr4(); + if (cr4 & CR4_PGE) { + set_cr4(cr4 & ~CR4_PGE); + set_cr4(cr4 | CR4_PGE); + } else { + set_cr3_raw(get_cr3_raw()); + } } extern int rdmsr64_carefully(uint32_t msr, uint64_t *val); extern int wrmsr64_carefully(uint32_t msr, uint64_t val); diff --git a/osfmk/i386/rtclock_asm.h b/osfmk/i386/rtclock_asm.h index dba69a6e7..9a380be1e 100644 --- a/osfmk/i386/rtclock_asm.h +++ b/osfmk/i386/rtclock_asm.h @@ -54,7 +54,7 @@ #define NANOTIME \ movq %gs:CPU_NANOTIME,%rdi ; \ PAL_RTC_NANOTIME_READ_FAST() - +/* TODO nobarrier */ /* * Add 64-bit delta in register reg to timer pointed to by register treg. */ diff --git a/osfmk/i386/rtclock_asm_native.h b/osfmk/i386/rtclock_asm_native.h index ab16668e3..50919f1fd 100644 --- a/osfmk/i386/rtclock_asm_native.h +++ b/osfmk/i386/rtclock_asm_native.h @@ -69,7 +69,23 @@ jz 0b /* - wait if so */ ; \ lfence ; \ rdtsc ; \ - lfence ; \ + shlq $32,%rdx ; \ + movl RNT_SHIFT(%rdi),%ecx ; \ + orq %rdx,%rax /* %rax := tsc */ ; \ + subq RNT_TSC_BASE(%rdi),%rax /* tsc - tsc_base */ ; \ + shlq %cl,%rax ; \ + movl RNT_SCALE(%rdi),%ecx ; \ + mulq %rcx /* delta * scale */ ; \ + shrdq $32,%rdx,%rax /* %rdx:%rax >>= 32 */ ; \ + addq RNT_NS_BASE(%rdi),%rax /* add ns_base */ ; \ + cmpl RNT_GENERATION(%rdi),%esi /* repeat if changed */ ; \ + jne 0b + +#define PAL_RTC_NANOTIME_READ_NOBARRIER() \ +0: movl RNT_GENERATION(%rdi),%esi ; \ + test %esi,%esi /* info updating? */ ; \ + jz 0b /* - wait if so */ ; \ + rdtsc ; \ shlq $32,%rdx ; \ movl RNT_SHIFT(%rdi),%ecx ; \ orq %rdx,%rax /* %rax := tsc */ ; \ diff --git a/osfmk/i386/seg.h b/osfmk/i386/seg.h index 75f5182e0..d5760f7aa 100644 --- a/osfmk/i386/seg.h +++ b/osfmk/i386/seg.h @@ -90,7 +90,9 @@ selector_to_sel(uint16_t selector) return (tconv.sel); } -#define LDTSZ 8192 /* size of the kernel ldt in entries */ +#define LDTSZ_MAX 8192 /* maximal size of the kernel ldt in entries */ +#define LDTSZ_DFL (128) +#define LDTSZ (LDTSZ_MAX) #define LDTSZ_MIN SEL_TO_INDEX(USER_SETTABLE) /* kernel ldt entries */ @@ -182,11 +184,13 @@ typedef struct __attribute__((packed)) { void *ptr; } x86_64_desc_register_t; + + /* * Boot-time data for master (or only) CPU */ extern struct real_descriptor master_gdt[GDTSZ]; -extern struct real_descriptor master_ldt[LDTSZ]; +extern struct real_descriptor master_ldt[]; extern struct i386_tss master_ktss; extern struct sysenter_stack master_sstk; @@ -285,6 +289,7 @@ __END_DECLS #define SYSENTER_TF_CS (USER_CS|0x10000) #define SYSENTER_DS KERNEL64_SS /* sysenter kernel data segment */ +#endif /* _I386_SEG_H_ */ #ifdef __x86_64__ /* * 64-bit kernel LDT descriptors @@ -293,5 +298,3 @@ __END_DECLS #define USER_CTHREAD 0x0f /* user cthread area */ #define USER_SETTABLE 0x1f /* start of user settable ldt entries */ #endif - -#endif /* _I386_SEG_H_ */ diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c index ad07c8887..bb1f9a17e 100644 --- a/osfmk/i386/trap.c +++ b/osfmk/i386/trap.c @@ -790,7 +790,6 @@ debugger_entry: */ } - static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip) { @@ -814,8 +813,8 @@ panic_trap(x86_saved_state64_t *regs, uint32_t pl, kern_return_t fault_result) */ panic_io_port_read(); - kprintf("panic trap number 0x%x, rip 0x%016llx\n", - regs->isf.trapno, regs->isf.rip); + kprintf("CPU %d panic trap number 0x%x, rip 0x%016llx\n", + cpu_number(), regs->isf.trapno, regs->isf.rip); kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n", cr0, cr2, cr3, cr4); @@ -940,11 +939,6 @@ user_trap( subcode = 0; exc = 0; -#if DEBUG_TRACE - kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n", - saved_state, type, vaddr); -#endif - perfASTCallback astfn = perfASTHook; if (__improbable(astfn != NULL)) { myast = ast_pending(); @@ -1245,6 +1239,10 @@ sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) { } #if DEBUG +#define TERI 1 +#endif + +#if TERI extern void thread_exception_return_internal(void) __dead2; void thread_exception_return(void) { diff --git a/osfmk/i386/tss.h b/osfmk/i386/tss.h index 866d2056b..eab037c24 100644 --- a/osfmk/i386/tss.h +++ b/osfmk/i386/tss.h @@ -105,8 +105,9 @@ struct i386_tss { * Although this is defined as a 64-bit stack, the space is also used in * 32-bit legacy mode. For 64-bit the stack is 16-byte aligned. */ + struct sysenter_stack { - uint64_t stack[16]; /* Space for a 64-bit frame and some */ + uint64_t sysestack[64]; /* Space for a 64-bit frame and some */ uint64_t top; /* Top and pointer to ISS in PCS */ }; diff --git a/osfmk/i386/user_ldt.c b/osfmk/i386/user_ldt.c index 8b5791455..35dd2cef7 100644 --- a/osfmk/i386/user_ldt.c +++ b/osfmk/i386/user_ldt.c @@ -224,7 +224,6 @@ i386_set_ldt( } else { bzero(&new_ldt->ldt[start_sel - begin_sel], num_sels * sizeof(struct real_descriptor)); } - /* * Validate descriptors. * Only allow descriptors with user privileges. @@ -253,6 +252,12 @@ i386_set_ldt( user_ldt_free(new_ldt); return EACCES; } + /* Reject attempts to create segments with 64-bit granules */ + if (dp->granularity & SZ_64) { + task_unlock(task); + user_ldt_free(new_ldt); + return EACCES; + } } } @@ -292,9 +297,9 @@ i386_get_ldt( unsigned int ldt_count; kern_return_t err; - if (start_sel >= 8192) + if (start_sel >= LDTSZ) return EINVAL; - if ((uint64_t)start_sel + (uint64_t)num_sels > 8192) + if ((uint64_t)start_sel + (uint64_t)num_sels > LDTSZ) return EINVAL; if (descs == 0) return EINVAL; diff --git a/osfmk/ipc/ipc_entry.c b/osfmk/ipc/ipc_entry.c index 677e054ea..288ea6a58 100644 --- a/osfmk/ipc/ipc_entry.c +++ b/osfmk/ipc/ipc_entry.c @@ -103,8 +103,9 @@ ipc_entry_lookup( if (index < space->is_table_size) { entry = &space->is_table[index]; if (IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name) || - IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE) + IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE) { entry = IE_NULL; + } } else { entry = IE_NULL; @@ -189,10 +190,14 @@ ipc_entry_claim( assert(table->ie_next < space->is_table_size); /* - * Initialize the new entry. We need only - * increment the generation number and clear ie_request. + * Initialize the new entry: increment gencount and reset + * rollover point if it rolled over, and clear ie_request. */ - gen = IE_BITS_NEW_GEN(entry->ie_bits); + gen = ipc_entry_new_gen(entry->ie_bits); + if (__improbable(ipc_entry_gen_rolled(entry->ie_bits, gen))) { + ipc_entry_bits_t roll = ipc_space_get_rollpoint(space); + gen = ipc_entry_new_rollpoint(roll); + } entry->ie_bits = gen; entry->ie_request = IE_REQ_NONE; @@ -434,7 +439,7 @@ ipc_entry_dealloc( if ((index < size) && (entry == &table[index])) { assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name)); - entry->ie_bits &= IE_BITS_GEN_MASK; + entry->ie_bits &= (IE_BITS_GEN_MASK | IE_BITS_ROLL_MASK); entry->ie_next = table->ie_next; table->ie_next = index; space->is_table_free++; @@ -611,14 +616,7 @@ ipc_entry_grow_table( return KERN_RESOURCE_SHORTAGE; } - /* initialize new entries (free chain in backwards order) */ - for (i = osize; i < size; i++) { - table[i].ie_object = IO_NULL; - table[i].ie_bits = IE_BITS_GEN_MASK; - table[i].ie_index = 0; - table[i].ie_next = i + 1; - } - table[size-1].ie_next = 0; + ipc_space_rand_freelist(space, table, osize, size); /* clear out old entries in new table */ memset((void *)table, 0, osize * sizeof(*table)); @@ -771,10 +769,10 @@ mach_port_name_t ipc_entry_name_mask(mach_port_name_t name) { #ifndef NO_PORT_GEN - static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_NEW_GEN(IE_BITS_GEN_MASK)); + static mach_port_name_t null_name = MACH_PORT_MAKE(0, IE_BITS_GEN_MASK + IE_BITS_GEN_ONE); return name | null_name; #else - static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_NEW_GEN(IE_BITS_GEN_MASK))); + static mach_port_name_t null_name = MACH_PORT_MAKE(0, ~(IE_BITS_GEN_MASK + IE_BITS_GEN_ONE)); return name & ~null_name; #endif } diff --git a/osfmk/ipc/ipc_entry.h b/osfmk/ipc/ipc_entry.h index 531434edf..285a6adf2 100644 --- a/osfmk/ipc/ipc_entry.h +++ b/osfmk/ipc/ipc_entry.h @@ -119,17 +119,79 @@ struct ipc_entry { #define IE_BITS_GEN_MASK 0xff000000 /* 8 bits for generation */ #define IE_BITS_GEN(bits) ((bits) & IE_BITS_GEN_MASK) #define IE_BITS_GEN_ONE 0x04000000 /* low bit of generation */ -#define IE_BITS_NEW_GEN(old) (((old) + IE_BITS_GEN_ONE) & IE_BITS_GEN_MASK) +#define IE_BITS_ROLL_POS 22 /* LSB pos of generation rollover */ +#define IE_BITS_ROLL_BITS 2 /* number of generation rollover bits */ +#define IE_BITS_ROLL_MASK (((1 << IE_BITS_ROLL_BITS) - 1) << IE_BITS_ROLL_POS) +#define IE_BITS_ROLL(bits) ((((bits) & IE_BITS_ROLL_MASK) << 8) ^ IE_BITS_GEN_MASK) + +/* + * Restart a generation counter with the specified bits for the rollover point. + * There are 4 different rollover points: + * bits rollover period + * 0 0 64 + * 0 1 48 + * 1 0 32 + * 1 1 16 + */ +static inline ipc_entry_bits_t ipc_entry_new_rollpoint( + ipc_entry_bits_t rollbits) +{ + rollbits = (rollbits << IE_BITS_ROLL_POS) & IE_BITS_ROLL_MASK; + ipc_entry_bits_t newgen = IE_BITS_GEN_MASK + IE_BITS_GEN_ONE; + return (newgen | rollbits); +} + +/* + * Get the next gencount, modulo the entry's rollover point. If the sum rolls over, + * the caller should re-start the generation counter with a different rollpoint. + */ +static inline ipc_entry_bits_t ipc_entry_new_gen( + ipc_entry_bits_t oldgen) +{ + ipc_entry_bits_t sum = (oldgen + IE_BITS_GEN_ONE) & IE_BITS_GEN_MASK; + ipc_entry_bits_t roll = oldgen & IE_BITS_ROLL_MASK; + ipc_entry_bits_t newgen = (sum % IE_BITS_ROLL(oldgen)) | roll; + return newgen; +} + +/* Determine if a gencount has rolled over or not. */ +static inline boolean_t ipc_entry_gen_rolled( + ipc_entry_bits_t oldgen, + ipc_entry_bits_t newgen) +{ + return (oldgen & IE_BITS_GEN_MASK) > (newgen & IE_BITS_GEN_MASK); +} + #else #define IE_BITS_GEN_MASK 0 #define IE_BITS_GEN(bits) 0 #define IE_BITS_GEN_ONE 0 -#define IE_BITS_NEW_GEN(old) (old) -#endif /* !USE_PORT_GEN */ +#define IE_BITS_ROLL_POS 0 +#define IE_BITS_ROLL_MASK 0 +#define IE_BITS_ROLL(bits) (bits) + +static inline ipc_entry_bits_t ipc_entry_new_rollpoint( + ipc_entry_bits_t rollbits) +{ + return 0; +} + +static inline ipc_entry_bits_t ipc_entry_new_gen( + ipc_entry_bits_t oldgen) +{ + return 0; +} + +static inline boolean_t ipc_entry_gen_rolled( + ipc_entry_bits_t oldgen, + ipc_entry_bits_t newgen) +{ + return FALSE; +} +#endif /* !USE_PORT_GEN */ #define IE_BITS_RIGHT_MASK 0x007fffff /* relevant to the right */ - /* * Exported interfaces */ diff --git a/osfmk/ipc/ipc_init.c b/osfmk/ipc/ipc_init.c index 49221fa52..8d5ea071f 100644 --- a/osfmk/ipc/ipc_init.c +++ b/osfmk/ipc/ipc_init.c @@ -275,7 +275,6 @@ ipc_init(void) msg_ool_size_small -= cpy_kdata_hdr_sz; ipc_host_init(); - } diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c index 4ec900c7b..e5445a290 100644 --- a/osfmk/ipc/ipc_kmsg.c +++ b/osfmk/ipc/ipc_kmsg.c @@ -1160,6 +1160,12 @@ ipc_kmsg_rmqueue( queue->ikmq_base = IKM_NULL; } else { + if (__improbable(next->ikm_prev != kmsg || prev->ikm_next != kmsg)) { + panic("ipc_kmsg_rmqueue: inconsistent prev/next pointers. " + "(prev->next: %p, next->prev: %p, kmsg: %p)", + prev->ikm_next, next->ikm_prev, kmsg); + } + if (queue->ikmq_base == kmsg) queue->ikmq_base = next; diff --git a/osfmk/ipc/ipc_pset.c b/osfmk/ipc/ipc_pset.c index 2fa71904d..fe989b283 100644 --- a/osfmk/ipc/ipc_pset.c +++ b/osfmk/ipc/ipc_pset.c @@ -192,8 +192,10 @@ ipc_pset_alloc_special( reserved_link = waitq_link_reserve(NULL); __IGNORE_WCASTALIGN(pset = (ipc_pset_t)io_alloc(IOT_PORT_SET)); - if (pset == IPS_NULL) + if (pset == IPS_NULL) { + waitq_link_release(reserved_link); return IPS_NULL; + } bzero((char *)pset, sizeof(*pset)); diff --git a/osfmk/ipc/ipc_space.c b/osfmk/ipc/ipc_space.c index 4c38dbab7..8596ad530 100644 --- a/osfmk/ipc/ipc_space.c +++ b/osfmk/ipc/ipc_space.c @@ -83,8 +83,15 @@ #include #include #include +#include #include +/* Remove this in the future so port names are less predictable. */ +#define CONFIG_SEMI_RANDOM_ENTRIES +#ifdef CONFIG_SEMI_RANDOM_ENTRIES +#define NUM_SEQ_ENTRIES 8 +#endif + zone_t ipc_space_zone; ipc_space_t ipc_space_kernel; ipc_space_t ipc_space_reply; @@ -110,6 +117,102 @@ ipc_space_release( is_release(space); } +/* Routine: ipc_space_get_rollpoint + * Purpose: + * Generate a new gencount rollover point from a space's entropy pool + */ +ipc_entry_bits_t +ipc_space_get_rollpoint( + ipc_space_t space) +{ + return random_bool_gen_bits( + &space->bool_gen, + &space->is_entropy[0], + IS_ENTROPY_CNT, + IE_BITS_ROLL_BITS); +} + +/* + * Routine: ipc_entry_rand_freelist + * Purpose: + * Pseudo-randomly permute the order of entries in an IPC space + * Arguments: + * space: the ipc space to initialize. + * table: the corresponding ipc table to initialize. + * bottom: the start of the range to initialize (inclusive). + * top: the end of the range to initialize (noninclusive). + */ +void +ipc_space_rand_freelist( + ipc_space_t space, + ipc_entry_t table, + mach_port_index_t bottom, + mach_port_index_t top) +{ +#ifdef CONFIG_SEMI_RANDOM_ENTRIES + /* + * Only make sequential entries at the start of the table, and not when + * we're growing the space. + */ + int at_start = (bottom == 0); + ipc_entry_num_t total = 0; +#endif + + /* First entry in the free list is always free, and is the start of the free list. */ + mach_port_index_t curr = bottom; + bottom++; + top--; + + /* + * Initialize the free list in the table. + * Add the entries in pseudo-random order and randomly set the generation + * number, in order to frustrate attacks involving port name reuse. + */ + while (bottom <= top) { + ipc_entry_t entry = &table[curr]; + int which; +#ifdef CONFIG_SEMI_RANDOM_ENTRIES + /* + * XXX: This is a horrible hack to make sure that randomizing the port + * doesn't break programs that might have (sad) hard-coded values for + * certain port names. + */ + if (at_start && total++ < NUM_SEQ_ENTRIES) + which = 0; + else +#endif + which = random_bool_gen_bits( + &space->bool_gen, + &space->is_entropy[0], + IS_ENTROPY_CNT, + 1); + + mach_port_index_t next; + if (which) { + next = top; + top--; + } else { + next = bottom; + bottom++; + } + + /* + * The entry's gencount will roll over on its first allocation, at which + * point a random rollover will be set for the entry. + */ + entry->ie_bits = IE_BITS_GEN_MASK; + entry->ie_next = next; + entry->ie_object = IO_NULL; + entry->ie_index = 0; + curr = next; + } + table[curr].ie_next = 0; + table[curr].ie_object = IO_NULL; + table[curr].ie_index = 0; + table[curr].ie_bits = IE_BITS_GEN_MASK; +} + + /* * Routine: ipc_space_create * Purpose: @@ -132,7 +235,6 @@ ipc_space_create( ipc_space_t space; ipc_entry_t table; ipc_entry_num_t new_size; - mach_port_index_t index; space = is_alloc(); if (space == IS_NULL) @@ -147,19 +249,11 @@ ipc_space_create( new_size = initial->its_size; memset((void *) table, 0, new_size * sizeof(struct ipc_entry)); - /* - * Initialize the free list in the table. - * Add the entries in reverse order, and - * set the generation number to -1, so that - * initial allocations produce "natural" names. - */ - for (index = 0; index < new_size; index++) { - ipc_entry_t entry = &table[index]; + /* Set to 0 so entropy pool refills */ + memset((void *) space->is_entropy, 0, sizeof(space->is_entropy)); - entry->ie_bits = IE_BITS_GEN_MASK; - entry->ie_next = index+1; - } - table[new_size-1].ie_next = 0; + random_bool_init(&space->bool_gen); + ipc_space_rand_freelist(space, table, 0, new_size); is_lock_init(space); space->is_bits = 2; /* 2 refs, active, not growing */ diff --git a/osfmk/ipc/ipc_space.h b/osfmk/ipc/ipc_space.h index 60662c8ee..06fdf3e79 100644 --- a/osfmk/ipc/ipc_space.h +++ b/osfmk/ipc/ipc_space.h @@ -73,6 +73,7 @@ #define _IPC_IPC_SPACE_H_ +#include #include #include #include @@ -109,6 +110,7 @@ typedef natural_t ipc_space_refs_t; #define IS_REFS_MAX 0x0fffffff #define IS_INACTIVE 0x40000000 /* space is inactive */ #define IS_GROWING 0x20000000 /* space is growing */ +#define IS_ENTROPY_CNT 1 /* per-space entropy pool size */ struct ipc_space { lck_spin_t is_lock_data; @@ -120,6 +122,8 @@ struct ipc_space { struct ipc_table_size *is_table_next; /* info for larger table */ ipc_entry_num_t is_low_mod; /* lowest modified entry during growth */ ipc_entry_num_t is_high_mod; /* highest modified entry during growth */ + struct bool_gen bool_gen; /* state for boolean RNG */ + unsigned int is_entropy[IS_ENTROPY_CNT]; /* pool of entropy taken from RNG */ int is_node_id; /* HOST_LOCAL_NODE, or remote node if proxy space */ }; @@ -215,7 +219,7 @@ is_release(ipc_space_t is) { extern kern_return_t ipc_space_create_special( ipc_space_t *spacep); -/* Create new IPC space */ +/* Create a new IPC space */ extern kern_return_t ipc_space_create( ipc_table_size_t initial, ipc_space_t *spacep); @@ -228,6 +232,15 @@ extern void ipc_space_terminate( extern void ipc_space_clean( ipc_space_t space); +/* Permute the order of a range within an IPC space */ +extern void ipc_space_rand_freelist( + ipc_space_t space, + ipc_entry_t table, + mach_port_index_t bottom, + mach_port_index_t top); + +/* Generate a new gencount rollover point from a space's entropy pool */ +extern ipc_entry_bits_t ipc_space_get_rollpoint(ipc_space_t space); #endif /* MACH_KERNEL_PRIVATE */ #endif /* __APPLE_API_PRIVATE */ diff --git a/osfmk/kdp/kdp_core.c b/osfmk/kdp/kdp_core.c index 5cf505770..ac71c8061 100644 --- a/osfmk/kdp/kdp_core.c +++ b/osfmk/kdp/kdp_core.c @@ -709,7 +709,7 @@ kernel_pmap_present_mapping(uint64_t vaddr, uint64_t * pvincr, uintptr_t * pvphy } else #if defined(__arm64__) - if (vaddr == _COMM_PAGE64_BASE_ADDRESS) + if (vaddr == _COMM_HIGH_PAGE64_BASE_ADDRESS) { /* not readable */ ppn = 0; @@ -1148,8 +1148,10 @@ do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant) existing_log_size = (panic_info->eph_panic_log_offset - sizeof(struct embedded_panic_header)) + panic_info->eph_panic_log_len + panic_info->eph_other_log_len; #else /* CONFIG_EMBEDDED */ - existing_log_size = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) + + if (panic_info->mph_panic_log_offset != 0) { + existing_log_size = (panic_info->mph_panic_log_offset - sizeof(struct macos_panic_header)) + panic_info->mph_panic_log_len + panic_info->mph_other_log_len; + } #endif /* CONFIG_EMBEDDED */ assert (existing_log_size <= debug_buf_size); @@ -1261,7 +1263,9 @@ do_kern_dump(kern_dump_output_proc outproc, enum kern_dump_type kd_variant) #if CONFIG_EMBEDDED existing_log_size -= panic_info->eph_other_log_len; #else - existing_log_size -= panic_info->mph_other_log_len; + if (existing_log_size) { + existing_log_size -= panic_info->mph_other_log_len; + } #endif /* @@ -1308,15 +1312,18 @@ exit: dump_succeeded = FALSE; } + /* If applicable, update the panic header and flush it so we update the CRC */ #if CONFIG_EMBEDDED panic_info->eph_panic_flags |= (dump_succeeded ? EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE : EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED); + paniclog_flush(); #else - panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE : + if (panic_info->mph_panic_log_offset != 0) { + panic_info->mph_panic_flags |= (dump_succeeded ? MACOS_PANIC_HEADER_FLAG_COREDUMP_COMPLETE : MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED); + paniclog_flush(); + } #endif - /* We touched the panic header, flush it so we update the CRC */ - paniclog_flush(); return (dump_succeeded ? 0 : -1); } diff --git a/osfmk/kdp/processor_core.c b/osfmk/kdp/processor_core.c index e6ebc85a2..f33d9915c 100644 --- a/osfmk/kdp/processor_core.c +++ b/osfmk/kdp/processor_core.c @@ -678,8 +678,8 @@ kern_do_coredump(void *core_outvars, boolean_t kernel_only, uint64_t first_file_ current_core = kern_coredump_core_list; while (current_core) { /* Seek to the beginning of the next file */ - ret = kern_dump_seek_to_next_file(core_outvars, *last_file_offset); - if (ret != KERN_SUCCESS) { + cur_ret = kern_dump_seek_to_next_file(core_outvars, *last_file_offset); + if (cur_ret != KERN_SUCCESS) { kern_coredump_log(NULL, "Failed to seek to beginning of next core\n"); return KERN_FAILURE; } diff --git a/osfmk/kern/debug.c b/osfmk/kern/debug.c index e330be0d6..a84f86d75 100644 --- a/osfmk/kern/debug.c +++ b/osfmk/kern/debug.c @@ -181,6 +181,7 @@ int mach_assert = 1; * EXTENDED_/DEBUG_BUF_SIZE can't grow without updates to SMC and iBoot to store larger panic logs on co-processor systems */ #define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data)) #define EXTENDED_DEBUG_BUF_SIZE 0x0013ff80 +static_assert(((EXTENDED_DEBUG_BUF_SIZE % PANIC_FLUSH_BOUNDARY) == 0), "Extended debug buf size must match SMC alignment requirements"); #define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace" #endif @@ -315,7 +316,12 @@ extended_debug_log_init(void) * to point at this new buffer. */ char *new_debug_buf = kalloc(EXTENDED_DEBUG_BUF_SIZE); - bzero(new_debug_buf, EXTENDED_DEBUG_BUF_SIZE); + /* + * iBoot pre-initializes the panic region with the NULL character. We set this here + * so we can accurately calculate the CRC for the region without needing to flush the + * full region over SMC. + */ + memset(new_debug_buf, '\0', EXTENDED_DEBUG_BUF_SIZE); panic_info = (struct macos_panic_header *)new_debug_buf; debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data)); @@ -846,8 +852,6 @@ debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned } #if CONFIG_KDP_INTERACTIVE_DEBUGGING - PE_i_can_has_debugger(NULL); - /* * If reboot on panic is enabled and the caller of panic indicated that we should skip * local coredumps, don't try to write these and instead go straight to reboot. This @@ -890,6 +894,10 @@ debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned } } + if (debug_boot_arg & DB_REBOOT_ALWAYS) { + kdp_machine_reboot_type(kPEPanicRestartCPU); + } + /* If KDP is configured, try to trap to the debugger */ if (current_debugger != NO_CUR_DB) { kdp_raise_exception(exception, code, subcode, state); diff --git a/osfmk/kern/debug.h b/osfmk/kern/debug.h index ecf996701..6289eeecd 100644 --- a/osfmk/kern/debug.h +++ b/osfmk/kern/debug.h @@ -276,6 +276,10 @@ struct embedded_panic_header { uint32_t eph_stackshot_len; /* length of the panic stackshot (0 if not valid ) */ uint32_t eph_other_log_offset; /* Offset of the other log (any logging subsequent to the stackshot) from the beginning of the header */ uint32_t eph_other_log_len; /* length of the other log */ + uint64_t eph_x86_power_state:8, + eph_x86_efi_boot_state:8, + eph_x86_system_state:8, + eph_x86_unused_bits:40; } __attribute__((packed)); #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_COMPLETE 0x01 @@ -289,7 +293,7 @@ struct embedded_panic_header { #define EMBEDDED_PANIC_HEADER_FLAG_COPROC_INITIATED_PANIC 0x100 #define EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED 0x200 -#define EMBEDDED_PANIC_HEADER_CURRENT_VERSION 1 +#define EMBEDDED_PANIC_HEADER_CURRENT_VERSION 2 #define EMBEDDED_PANIC_MAGIC 0x46554E4B /* FUNK */ struct macos_panic_header { @@ -336,6 +340,12 @@ __END_DECLS #ifdef KERNEL_PRIVATE #if DEBUG +#ifndef DKPR +#define DKPR 1 +#endif +#endif + +#if DKPR /* * For the DEBUG kernel, support the following: * sysctl -w debug.kprint_syscall= @@ -403,6 +413,12 @@ enum { #define DB_PRT_KDEBUG 0x10000 /* kprintf KDEBUG traces */ #define DB_DISABLE_LOCAL_CORE 0x20000 /* ignore local kernel core dump support */ #define DB_DISABLE_GZIP_CORE 0x40000 /* don't gzip kernel core dumps */ +#define DB_DISABLE_CROSS_PANIC 0x80000 /* x86 only - don't trigger cross panics. Only + * necessary to enable x86 kernel debugging on + * configs with a dev-fused co-processor running + * release bridgeOS. + */ +#define DB_REBOOT_ALWAYS 0x100000 /* Don't wait for debugger connection */ /* * Values for a 64-bit mask that's passed to the debugger. diff --git a/osfmk/kern/gzalloc.c b/osfmk/kern/gzalloc.c index 915b22782..9bd11481f 100644 --- a/osfmk/kern/gzalloc.c +++ b/osfmk/kern/gzalloc.c @@ -57,7 +57,7 @@ * -gzalloc_noconsistency: disable consistency checks that flag mismatched * frees, corruptions of the header/trailer signatures etc. * -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel - * enables the guard allocator for zones sized 8K-16K (if present) by + * enables the guard allocator for zones sized 1K (if present) by * default, this option can disable that behaviour. * gzname= target a zone by name. Can be coupled with size-based * targeting. Naming conventions match those of the zlog boot-arg, i.e. diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c index 97b04c739..fa13463fe 100644 --- a/osfmk/kern/kalloc.c +++ b/osfmk/kern/kalloc.c @@ -140,48 +140,63 @@ KALLOC_ZINFO_SFREE(vm_size_t bytes) * kalloc_max_prerounded is the smallest allocation size, before * rounding, for which no zone exists. * - * Also if the allocation size is more than kalloc_kernmap_size + * Also if the allocation size is more than kalloc_kernmap_size * then allocate from kernel map rather than kalloc_map. */ #if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4 -#define K_ZONE_SIZES \ - 16, \ - 32, \ - 48, \ -/* 3 */ 64, \ - 80, \ - 96, \ -/* 6 */ 128, \ - 160, 192, \ - 256, \ -/* 9 */ 288, \ - 512, 576, \ - 1024, 1152, \ -/* C */ 1280, \ - 2048, \ - 4096 - -#define K_ZONE_NAMES \ - "kalloc.16", \ - "kalloc.32", \ - "kalloc.48", \ -/* 3 */ "kalloc.64", \ - "kalloc.80", \ - "kalloc.96", \ -/* 6 */ "kalloc.128", \ - "kalloc.160", \ - "kalloc.192", \ - "kalloc.256", \ -/* 9 */ "kalloc.288", \ - "kalloc.512", \ - "kalloc.576", \ - "kalloc.1024", \ - "kalloc.1152", \ -/* C */ "kalloc.1280", \ - "kalloc.2048", \ - "kalloc.4096" +#define K_ZONE_SIZES \ + 16, \ + 32, \ + 48, \ + 64, /* 2^6 */ \ + 80, \ + 96, \ + 128, /* 2^7 */ \ + 160, \ + 192, \ + 224, \ + 256, /* 2^8 */ \ + 288, \ + 368, \ + 400, \ + 512, /* 2^9 */\ + 576, \ + 768, \ + 1024, /* 2^10 */ \ + 1152, \ + 1280, \ + 1664, \ + 2048, /* 2^11 */ \ + 4096, /* 2^12 */ \ + 6144 + +#define K_ZONE_NAMES \ + "kalloc.16", \ + "kalloc.32", \ + "kalloc.48", \ + "kalloc.64", /* 2^6 */ \ + "kalloc.80", \ + "kalloc.96", \ + "kalloc.128", /* 2^7 */ \ + "kalloc.160", \ + "kalloc.192", \ + "kalloc.224", \ + "kalloc.256", /* 2^8 */\ + "kalloc.288", \ + "kalloc.368", \ + "kalloc.400", \ + "kalloc.512", /* 2^9 */ \ + "kalloc.576", \ + "kalloc.768", \ + "kalloc.1024", /* 2^10 */ \ + "kalloc.1152", \ + "kalloc.1280", \ + "kalloc.1664", \ + "kalloc.2048", /* 2^11 */ \ + "kalloc.4096", /* 2^12 */ \ + "kalloc.6144" #elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3 @@ -214,7 +229,7 @@ KALLOC_ZINFO_SFREE(vm_size_t bytes) "kalloc.4096", "kalloc.6144" #else -#error missing zone size parameters for kalloc +#error missing or invalid zone size parameters for kalloc #endif #define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN) diff --git a/osfmk/kern/kcdata.h b/osfmk/kern/kcdata.h index 741c7f864..e358b4ec8 100644 --- a/osfmk/kern/kcdata.h +++ b/osfmk/kern/kcdata.h @@ -469,7 +469,7 @@ struct kcdata_type_definition { #define STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS 0x91bu /* struct stackshot_fault_stats */ #define STACKSHOT_KCTYPE_KERNELCACHE_LOADINFO 0x91cu /* kernelcache UUID -- same as KCDATA_TYPE_LIBRARY_LOADINFO64 */ #define STACKSHOT_KCTYPE_THREAD_WAITINFO 0x91du /* struct stackshot_thread_waitinfo */ -#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot */ +#define STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT 0x91eu /* struct thread_group_snapshot or thread_group_snapshot_v2 */ #define STACKSHOT_KCTYPE_THREAD_GROUP 0x91fu /* uint64_t */ #define STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT 0x920u /* struct jetsam_coalition_snapshot */ #define STACKSHOT_KCTYPE_JETSAM_COALITION 0x921u /* uint64_t */ @@ -659,6 +659,17 @@ struct thread_group_snapshot { char tgs_name[16]; } __attribute__((packed)); +enum thread_group_flags { + kThreadGroupEfficient = 0x1, + kThreadGroupUIApp = 0x2 +}; + +struct thread_group_snapshot_v2 { + uint64_t tgs_id; + char tgs_name[16]; + uint64_t tgs_flags; +} __attribute__((packed)); + enum coalition_flags { kCoalitionTermRequested = 0x1, kCoalitionTerminated = 0x2, diff --git a/osfmk/kern/ledger.c b/osfmk/kern/ledger.c index 4c27ea7fb..ec35a3a52 100644 --- a/osfmk/kern/ledger.c +++ b/osfmk/kern/ledger.c @@ -178,8 +178,8 @@ static kern_return_t ledger_perform_blocking(ledger_t l); static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit); static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit); -static void ledger_entry_check_new_balance(ledger_t ledger, int entry, - struct ledger_entry *le); +static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger, + int entry, struct ledger_entry *le); #if 0 static void @@ -697,7 +697,8 @@ ledger_refill(uint64_t now, ledger_t ledger, int entry) #define TOCKSTAMP_IS_STALE(now, tock) ((((now) - (tock)) < NTOCKS) ? FALSE : TRUE) void -ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry *le) +ledger_entry_check_new_balance(thread_t thread, ledger_t ledger, + int entry, struct ledger_entry *le) { ledger_amount_t credit, debit; @@ -749,7 +750,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * if ((le->le_flags & LEDGER_ACTION_BLOCK) || (!(le->le_flags & LF_CALLED_BACK) && entry_get_callback(ledger, entry))) { - set_astledger(current_thread()); + act_set_astledger_async(thread); } } else { /* @@ -778,7 +779,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * * set the AST so it can be done before returning * to userland. */ - set_astledger(current_thread()); + act_set_astledger_async(thread); } } else { /* @@ -792,7 +793,7 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * * know the ledger balance is now back below * the warning level. */ - set_astledger(current_thread()); + act_set_astledger_async(thread); } } } @@ -812,20 +813,19 @@ ledger_entry_check_new_balance(ledger_t ledger, int entry, struct ledger_entry * } void -ledger_check_new_balance(ledger_t ledger, int entry) +ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry) { struct ledger_entry *le; assert(entry > 0 && entry <= ledger->l_size); le = &ledger->l_entries[entry]; - ledger_entry_check_new_balance(ledger, entry, le); + ledger_entry_check_new_balance(thread, ledger, entry, le); } - /* - * Add value to an entry in a ledger. + * Add value to an entry in a ledger for a specific thread. */ kern_return_t -ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount) +ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount) { ledger_amount_t old, new; struct ledger_entry *le; @@ -840,12 +840,22 @@ ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount) old = OSAddAtomic64(amount, &le->le_credit); new = old + amount; - lprintf(("%p Credit %lld->%lld\n", current_thread(), old, new)); - ledger_entry_check_new_balance(ledger, entry, le); + lprintf(("%p Credit %lld->%lld\n", thread, old, new)); + + ledger_entry_check_new_balance(thread, ledger, entry, le); return (KERN_SUCCESS); } +/* + * Add value to an entry in a ledger. + */ +kern_return_t +ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount) +{ + return ledger_credit_thread(current_thread(), ledger, entry, amount); +} + /* Add all of one ledger's values into another. * They must have been created from the same template. * This is not done atomically. Another thread (if not otherwise synchronized) @@ -1285,7 +1295,7 @@ ledger_set_action(ledger_t ledger, int entry, int action) } kern_return_t -ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount) +ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount) { struct ledger_entry *le; ledger_amount_t old, new; @@ -1308,9 +1318,15 @@ ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount) } lprintf(("%p Debit %lld->%lld\n", thread, old, new)); - ledger_entry_check_new_balance(ledger, entry, le); + ledger_entry_check_new_balance(thread, ledger, entry, le); + return (KERN_SUCCESS); +} +kern_return_t +ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount) +{ + return ledger_debit_thread(current_thread(), ledger, entry, amount); } void diff --git a/osfmk/kern/ledger.h b/osfmk/kern/ledger.h index 8e627bef4..d6b27ce11 100644 --- a/osfmk/kern/ledger.h +++ b/osfmk/kern/ledger.h @@ -129,11 +129,15 @@ extern kern_return_t ledger_set_period(ledger_t ledger, int entry, uint64_t period); extern kern_return_t ledger_disable_refill(ledger_t l, int entry); extern kern_return_t ledger_entry_setactive(ledger_t ledger, int entry); -extern void ledger_check_new_balance(ledger_t ledger, int entry); +extern void ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry); extern kern_return_t ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount); extern kern_return_t ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount); +extern kern_return_t ledger_credit_thread(thread_t thread, ledger_t ledger, + int entry, ledger_amount_t amount); +extern kern_return_t ledger_debit_thread(thread_t thread, ledger_t ledger, + int entry, ledger_amount_t amount); extern kern_return_t ledger_zero_balance(ledger_t ledger, int entry); extern kern_return_t ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit, ledger_amount_t *debit); diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c index bd590a042..5f7be7132 100644 --- a/osfmk/kern/sched_prim.c +++ b/osfmk/kern/sched_prim.c @@ -744,7 +744,6 @@ thread_unblock( ml_get_power_state(&aticontext, &pidle); if (__improbable(aticontext && !(thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT))) { - ledger_credit(thread->t_ledger, task_ledgers.interrupt_wakeups, 1); DTRACE_SCHED2(iwakeup, struct thread *, thread, struct proc *, thread->task->bsd_info); uint64_t ttd = PROCESSOR_DATA(current_processor(), timer_call_ttd); @@ -757,23 +756,30 @@ thread_unblock( thread->thread_timer_wakeups_bin_2++; } + ledger_credit_thread(thread, thread->t_ledger, + task_ledgers.interrupt_wakeups, 1); if (pidle) { - ledger_credit(thread->t_ledger, task_ledgers.platform_idle_wakeups, 1); + ledger_credit_thread(thread, thread->t_ledger, + task_ledgers.platform_idle_wakeups, 1); } } else if (thread_get_tag_internal(cthread) & THREAD_TAG_CALLOUT) { + /* TODO: what about an interrupt that does a wake taken on a callout thread? */ if (cthread->callout_woken_from_icontext) { - ledger_credit(thread->t_ledger, task_ledgers.interrupt_wakeups, 1); + ledger_credit_thread(thread, thread->t_ledger, + task_ledgers.interrupt_wakeups, 1); thread->thread_callout_interrupt_wakeups++; + if (cthread->callout_woken_from_platform_idle) { - ledger_credit(thread->t_ledger, task_ledgers.platform_idle_wakeups, 1); + ledger_credit_thread(thread, thread->t_ledger, + task_ledgers.platform_idle_wakeups, 1); thread->thread_callout_platform_idle_wakeups++; } - + cthread->callout_woke_thread = TRUE; } } - + if (thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT) { thread->callout_woken_from_icontext = aticontext; thread->callout_woken_from_platform_idle = pidle; @@ -2621,17 +2627,16 @@ thread_dispatch( * Bill CPU time to both the task and * the individual thread. */ - ledger_credit(thread->t_ledger, - task_ledgers.cpu_time, consumed); - ledger_credit(thread->t_threadledger, - thread_ledgers.cpu_time, consumed); + ledger_credit_thread(thread, thread->t_ledger, + task_ledgers.cpu_time, consumed); + ledger_credit_thread(thread, thread->t_threadledger, + thread_ledgers.cpu_time, consumed); if (thread->t_bankledger) { - ledger_credit(thread->t_bankledger, - bank_ledgers.cpu_time, - (consumed - thread->t_deduct_bank_ledger_time)); - + ledger_credit_thread(thread, thread->t_bankledger, + bank_ledgers.cpu_time, + (consumed - thread->t_deduct_bank_ledger_time)); } - thread->t_deduct_bank_ledger_time =0; + thread->t_deduct_bank_ledger_time = 0; } wake_lock(thread); diff --git a/osfmk/kern/task.c b/osfmk/kern/task.c index c2a29291e..d58d29cdb 100644 --- a/osfmk/kern/task.c +++ b/osfmk/kern/task.c @@ -2267,21 +2267,6 @@ task_terminate_internal( vm_map_disable_hole_optimization(task->map); vm_map_unlock(task->map); - vm_map_remove(task->map, - task->map->min_offset, - task->map->max_offset, - /* - * Final cleanup: - * + no unnesting - * + remove immutable mappings - */ - (VM_MAP_REMOVE_NO_UNNESTING | - VM_MAP_REMOVE_IMMUTABLE)); - - /* release our shared region */ - vm_shared_region_set(task, NULL); - - #if MACH_ASSERT /* * Identify the pmap's process, in case the pmap ledgers drift @@ -2298,6 +2283,21 @@ task_terminate_internal( pmap_set_process(task->map->pmap, pid, procname); #endif /* MACH_ASSERT */ + vm_map_remove(task->map, + task->map->min_offset, + task->map->max_offset, + /* + * Final cleanup: + * + no unnesting + * + remove immutable mappings + */ + (VM_MAP_REMOVE_NO_UNNESTING | + VM_MAP_REMOVE_IMMUTABLE)); + + /* release our shared region */ + vm_shared_region_set(task, NULL); + + lck_mtx_lock(&tasks_threads_lock); queue_remove(&tasks, task, task_t, tasks); queue_enter(&terminated_tasks, task, task_t, tasks); @@ -5071,8 +5071,13 @@ PROC_CROSSED_HIGH_WATERMARK__SEND_EXC_RESOURCE_AND_SUSPEND(int max_footprint_mb, task_resume_internal(task); } } else { - task_enqueue_exception_with_corpse(task, EXC_RESOURCE, + if (audio_active) { + printf("process %s[%d] crossed memory high watermark (%d MB); EXC_RESOURCE " + "supressed due to audio playback.\n", procname, pid, max_footprint_mb); + } else { + task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL); + } } /* @@ -5248,7 +5253,8 @@ task_set_phys_footprint_limit_internal( (ledger_amount_t)new_limit_mb << 20, PHYS_FOOTPRINT_WARNING_LEVEL); if (task == current_task()) { - ledger_check_new_balance(task->ledger, task_ledgers.phys_footprint); + ledger_check_new_balance(current_thread(), task->ledger, + task_ledgers.phys_footprint); } task_unlock(task); diff --git a/osfmk/kern/task_policy.c b/osfmk/kern/task_policy.c index b13186523..f876182a5 100644 --- a/osfmk/kern/task_policy.c +++ b/osfmk/kern/task_policy.c @@ -2533,7 +2533,7 @@ task_suspend_cpumon(task_t task) */ task->rusage_cpu_flags &= ~(TASK_RUSECPU_FLAGS_PERTHR_LIMIT | TASK_RUSECPU_FLAGS_FATAL_CPUMON); queue_iterate(&task->threads, thread, thread_t, task_threads) { - set_astledger(thread); + act_set_astledger(thread); } return KERN_SUCCESS; @@ -2573,7 +2573,7 @@ task_enable_cpumon_locked(task_t task) task->rusage_cpu_flags |= TASK_RUSECPU_FLAGS_PERTHR_LIMIT; queue_iterate(&task->threads, thread, thread_t, task_threads) { - set_astledger(thread); + act_set_astledger(thread); } return KERN_SUCCESS; diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c index 0057c988f..ce815db9e 100644 --- a/osfmk/kern/thread.c +++ b/osfmk/kern/thread.c @@ -1253,7 +1253,7 @@ thread_create_internal( * This task has a per-thread CPU limit; make sure this new thread * gets its limit set too, before it gets out of the kernel. */ - set_astledger(new_thread); + act_set_astledger(new_thread); } /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */ diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h index 02bdc0607..4601a9ebb 100644 --- a/osfmk/kern/thread.h +++ b/osfmk/kern/thread.h @@ -1077,7 +1077,8 @@ extern boolean_t thread_should_abort( extern int is_64signalregset(void); extern void act_set_kperf(thread_t); -extern void set_astledger(thread_t thread); +extern void act_set_astledger(thread_t thread); +extern void act_set_astledger_async(thread_t thread); extern void act_set_io_telemetry_ast(thread_t); extern uint32_t dtrace_get_thread_predcache(thread_t); diff --git a/osfmk/kern/thread_act.c b/osfmk/kern/thread_act.c index 6f16bfd1c..3bd8b5182 100644 --- a/osfmk/kern/thread_act.c +++ b/osfmk/kern/thread_act.c @@ -1023,6 +1023,27 @@ act_set_ast( splx(s); } +/* + * set AST on thread without causing an AST check + * and without taking the thread lock + * + * If thread is not the current thread, then it may take + * up until the next context switch or quantum expiration + * on that thread for it to notice the AST. + */ +static void +act_set_ast_async(thread_t thread, + ast_t ast) +{ + thread_ast_set(thread, ast); + + if (thread == current_thread()) { + spl_t s = splsched(); + ast_propagate(thread); + splx(s); + } +} + void act_set_astbsd( thread_t thread) @@ -1033,22 +1054,10 @@ act_set_astbsd( void act_set_astkevent(thread_t thread, uint16_t bits) { - spl_t s = splsched(); - - /* - * Do not send an IPI if the thread is running on - * another processor, wait for the next quantum - * expiration to load the AST. - */ - atomic_fetch_or(&thread->kevent_ast_bits, bits); - thread_ast_set(thread, AST_KEVENT); - - if (thread == current_thread()) { - ast_propagate(thread); - } - splx(s); + /* kevent AST shouldn't send immediate IPIs */ + act_set_ast_async(thread, AST_KEVENT); } void @@ -1073,11 +1082,25 @@ act_set_astmacf( #endif void -set_astledger(thread_t thread) +act_set_astledger(thread_t thread) { act_set_ast(thread, AST_LEDGER); } +/* + * The ledger AST may need to be set while already holding + * the thread lock. This routine skips sending the IPI, + * allowing us to avoid the lock hold. + * + * However, it means the targeted thread must context switch + * to recognize the ledger AST. + */ +void +act_set_astledger_async(thread_t thread) +{ + act_set_ast_async(thread, AST_LEDGER); +} + void act_set_io_telemetry_ast(thread_t thread) { diff --git a/osfmk/kern/zalloc.c b/osfmk/kern/zalloc.c index 699dc3c74..0a21dda6a 100644 --- a/osfmk/kern/zalloc.c +++ b/osfmk/kern/zalloc.c @@ -87,6 +87,8 @@ #include #include +#include + #include #include #include @@ -215,6 +217,8 @@ uintptr_t zp_nopoison_cookie = 0; boolean_t zone_tagging_on; #endif /* VM_MAX_TAG_ZONES */ +static struct bool_gen zone_bool_gen; + /* * initialize zone poisoning * called from zone_bootstrap before any allocations are made from zalloc @@ -342,10 +346,6 @@ vm_offset_t zone_map_max_address = 0; /* Globals for random boolean generator for elements in free list */ #define MAX_ENTROPY_PER_ZCRAM 4 -#define RANDOM_BOOL_GEN_SEED_COUNT 4 -static unsigned int bool_gen_seed[RANDOM_BOOL_GEN_SEED_COUNT]; -static unsigned int bool_gen_global = 0; -decl_simple_lock_data(, bool_gen_lock) /* VM region for all metadata structures */ vm_offset_t zone_metadata_region_min = 0; @@ -368,7 +368,7 @@ decl_simple_lock_data(, all_zones_lock) unsigned int num_zones_in_use; unsigned int num_zones; -#define MAX_ZONES 288 +#define MAX_ZONES 320 struct zone zone_array[MAX_ZONES]; /* Used to keep track of empty slots in the zone_array */ @@ -2543,61 +2543,18 @@ zcram_metadata_init(vm_offset_t newmem, vm_size_t size, struct zone_page_metadat return; } - -/* - * Boolean Random Number Generator for generating booleans to randomize - * the order of elements in newly zcram()'ed memory. The algorithm is a - * modified version of the KISS RNG proposed in the paper: - * http://stat.fsu.edu/techreports/M802.pdf - * The modifications have been documented in the technical paper - * paper from UCL: - * http://www0.cs.ucl.ac.uk/staff/d.jones/GoodPracticeRNG.pdf - */ - -static void random_bool_gen_entropy( - int *buffer, - int count) -{ - - int i, t; - simple_lock(&bool_gen_lock); - for (i = 0; i < count; i++) { - bool_gen_seed[1] ^= (bool_gen_seed[1] << 5); - bool_gen_seed[1] ^= (bool_gen_seed[1] >> 7); - bool_gen_seed[1] ^= (bool_gen_seed[1] << 22); - t = bool_gen_seed[2] + bool_gen_seed[3] + bool_gen_global; - bool_gen_seed[2] = bool_gen_seed[3]; - bool_gen_global = t < 0; - bool_gen_seed[3] = t &2147483647; - bool_gen_seed[0] += 1411392427; - buffer[i] = (bool_gen_seed[0] + bool_gen_seed[1] + bool_gen_seed[3]); - } - simple_unlock(&bool_gen_lock); -} - -static boolean_t random_bool_gen( - int *buffer, - int index, - int bufsize) -{ - int valindex, bitpos; - valindex = (index / (8 * sizeof(int))) % bufsize; - bitpos = index % (8 * sizeof(int)); - return (boolean_t)(buffer[valindex] & (1 << bitpos)); -} - -static void +static void random_free_to_zone( zone_t zone, vm_offset_t newmem, vm_offset_t first_element_offset, int element_count, - int *entropy_buffer) + unsigned int *entropy_buffer) { vm_offset_t last_element_offset; vm_offset_t element_addr; vm_size_t elem_size; - int index; + int index; assert(element_count <= ZONE_CHUNK_MAXELEMENTS); elem_size = zone->elem_size; @@ -2608,7 +2565,7 @@ random_free_to_zone( #if DEBUG || DEVELOPMENT leak_scan_debug_flag || __improbable(zone->tags) || #endif /* DEBUG || DEVELOPMENT */ - random_bool_gen(entropy_buffer, index, MAX_ENTROPY_PER_ZCRAM)) { + random_bool_gen_bits(&zone_bool_gen, entropy_buffer, MAX_ENTROPY_PER_ZCRAM, 1)) { element_addr = newmem + first_element_offset; first_element_offset += elem_size; } else { @@ -2635,7 +2592,7 @@ zcram( vm_size_t elem_size; boolean_t from_zm = FALSE; int element_count; - int entropy_buffer[MAX_ENTROPY_PER_ZCRAM]; + unsigned int entropy_buffer[MAX_ENTROPY_PER_ZCRAM] = { 0 }; /* Basic sanity checks */ assert(zone != ZONE_NULL && newmem != (vm_offset_t)0); @@ -2650,12 +2607,12 @@ zcram( from_zm = TRUE; if (!from_zm) { - /* We cannot support elements larger than page size for foreign memory because we - * put metadata on the page itself for each page of foreign memory. We need to do - * this in order to be able to reach the metadata when any element is freed + /* We cannot support elements larger than page size for foreign memory because we + * put metadata on the page itself for each page of foreign memory. We need to do + * this in order to be able to reach the metadata when any element is freed */ assert((zone->allows_foreign == TRUE) && (zone->elem_size <= (PAGE_SIZE - sizeof(struct zone_page_metadata)))); - } + } if (zalloc_debug & ZALLOC_DEBUG_ZCRAM) kprintf("zcram(%p[%s], 0x%lx%s, 0x%lx)\n", zone, zone->zone_name, @@ -2663,10 +2620,8 @@ zcram( ZONE_PAGE_COUNT_INCR(zone, (size / PAGE_SIZE)); - random_bool_gen_entropy(entropy_buffer, MAX_ENTROPY_PER_ZCRAM); - - /* - * Initialize the metadata for all pages. We dont need the zone lock + /* + * Initialize the metadata for all pages. We dont need the zone lock * here because we are not manipulating any zone related state yet. */ @@ -2773,7 +2728,6 @@ void zone_bootstrap(void) { char temp_buf[16]; - unsigned int i; if (!PE_parse_boot_argn("zalloc_debug", &zalloc_debug, sizeof(zalloc_debug))) zalloc_debug = 0; @@ -2781,11 +2735,7 @@ zone_bootstrap(void) /* Set up zone element poisoning */ zp_init(); - /* Seed the random boolean generator for elements in zone free list */ - for (i = 0; i < RANDOM_BOOL_GEN_SEED_COUNT; i++) { - bool_gen_seed[i] = (unsigned int)early_random(); - } - simple_lock_init(&bool_gen_lock, 0); + random_bool_init(&zone_bool_gen); /* should zlog log to debug zone corruption instead of leaks? */ if (PE_parse_boot_argn("-zc", temp_buf, sizeof(temp_buf))) { diff --git a/osfmk/mach/arm/vm_param.h b/osfmk/mach/arm/vm_param.h index 715ee1d31..915e237e6 100644 --- a/osfmk/mach/arm/vm_param.h +++ b/osfmk/mach/arm/vm_param.h @@ -154,7 +154,7 @@ extern unsigned PAGE_SHIFT_CONST; /* system-wide values */ #define MACH_VM_MIN_ADDRESS ((mach_vm_offset_t) 0x0ULL) -#define MACH_VM_MAX_ADDRESS ((mach_vm_offset_t) 0x0000001000000000ULL) +#define MACH_VM_MAX_ADDRESS ((mach_vm_offset_t) 0x0000000FC0000000ULL) #else #error architecture not supported diff --git a/osfmk/mach/vm_statistics.h b/osfmk/mach/vm_statistics.h index f6ffb8b0a..fa0560559 100644 --- a/osfmk/mach/vm_statistics.h +++ b/osfmk/mach/vm_statistics.h @@ -362,7 +362,8 @@ typedef struct { vmkf_keep_map_locked:1, vmkf_fourk:1, vmkf_overwrite_immutable:1, - __vmkf_unused:19; + vmkf_remap_prot_copy:1, + __vmkf_unused:18; } vm_map_kernel_flags_t; #define VM_MAP_KERNEL_FLAGS_NONE (vm_map_kernel_flags_t) { \ .vmkf_atomic_entry = 0, /* keep entry atomic (no coalescing) */ \ @@ -378,6 +379,7 @@ typedef struct { .vmkf_keep_map_locked = 0, /* keep map locked when returning from vm_map_enter() */ \ .vmkf_fourk = 0, /* use fourk pager */ \ .vmkf_overwrite_immutable = 0, /* can overwrite immutable mappings */ \ + .vmkf_remap_prot_copy = 0, /* vm_remap for VM_PROT_COPY */ \ .__vmkf_unused = 0 \ } #endif /* KERNEL_PRIVATE */ diff --git a/osfmk/prng/random.c b/osfmk/prng/random.c index cb3555d39..5dc056f8a 100644 --- a/osfmk/prng/random.c +++ b/osfmk/prng/random.c @@ -626,3 +626,79 @@ write_random(void* buffer, u_int numbytes) return 0; #endif } + + +/* + * Boolean PRNG for generating booleans to randomize order of elements + * in certain kernel data structures. The algorithm is a + * modified version of the KISS RNG proposed in the paper: + * http://stat.fsu.edu/techreports/M802.pdf + * The modifications have been documented in the technical paper + * paper from UCL: + * http://www0.cs.ucl.ac.uk/staff/d.jones/GoodPracticeRNG.pdf + */ + +/* Initialize the PRNG structures. */ +void random_bool_init(struct bool_gen *bg) +{ + /* Seed the random boolean generator */ + for (int i = 0; i < RANDOM_BOOL_GEN_SEED_COUNT; i++) { + bg->seed[i] = (unsigned int)early_random(); + } + bg->state = 0; + simple_lock_init(&bg->lock, 0); +} + +/* Generate random bits and add them to an entropy pool. */ +void random_bool_gen_entropy( + struct bool_gen *bg, + unsigned int *buffer, + int count) +{ + + simple_lock(&bg->lock); + int i, t; + for (i = 0; i < count; i++) { + bg->seed[1] ^= (bg->seed[1] << 5); + bg->seed[1] ^= (bg->seed[1] >> 7); + bg->seed[1] ^= (bg->seed[1] << 22); + t = bg->seed[2] + bg->seed[3] + bg->state; + bg->seed[2] = bg->seed[3]; + bg->state = t < 0; + bg->seed[3] = t & 2147483647; + bg->seed[0] += 1411392427; + buffer[i] = (bg->seed[0] + bg->seed[1] + bg->seed[3]); + } + simple_unlock(&bg->lock); +} + +/* Get some number of bits from the entropy pool, refilling if necessary. */ +unsigned int random_bool_gen_bits( + struct bool_gen *bg, + unsigned int *buffer, + unsigned int count, + unsigned int numbits) +{ + unsigned int index = 0; + unsigned int rbits = 0; + for (unsigned int bitct = 0; bitct < numbits; bitct++) { + /* + * Find a portion of the buffer that hasn't been emptied. + * We might have emptied our last index in the previous iteration. + */ + while (index < count && buffer[index] == 0) + index++; + + /* If we've exhausted the pool, refill it. */ + if (index == count) { + random_bool_gen_entropy(bg, buffer, count); + index = 0; + } + + /* Collect-a-bit */ + unsigned int bit = buffer[index] & 1; + buffer[index] = buffer[index] >> 1; + rbits = bit | (rbits << 1); + } + return rbits; +} diff --git a/osfmk/prng/random.h b/osfmk/prng/random.h index aa269df35..7ba5f00e1 100644 --- a/osfmk/prng/random.h +++ b/osfmk/prng/random.h @@ -115,6 +115,28 @@ MACRO_END /* /dev/random's PRNG is reseeded after generating this many bytes: */ #define RESEED_BYTES (17597) +#include +/* Definitions for boolean PRNG */ +#define RANDOM_BOOL_GEN_SEED_COUNT 4 +struct bool_gen { + unsigned int seed[RANDOM_BOOL_GEN_SEED_COUNT]; + unsigned int state; + decl_simple_lock_data(, lock) +}; + +extern void random_bool_init(struct bool_gen *bg); + +extern void random_bool_gen_entropy( + struct bool_gen *bg, + unsigned int *buffer, + int count); + +extern unsigned int random_bool_gen_bits( + struct bool_gen *bg, + unsigned int *buffer, + unsigned int count, + unsigned int numbits); + __END_DECLS #endif /* _PRNG_RANDOM_H_ */ diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h index 332c6e909..4916313ce 100644 --- a/osfmk/vm/pmap.h +++ b/osfmk/vm/pmap.h @@ -660,6 +660,9 @@ extern pmap_t kernel_pmap; /* The kernel's map */ #define VM_MEM_STACK 0x200 #if __x86_64__ +/* N.B. These use the same numerical space as the PMAP_EXPAND_OPTIONS + * definitions in i386/pmap_internal.h + */ #define PMAP_CREATE_64BIT 0x1 #define PMAP_CREATE_EPT 0x2 #define PMAP_CREATE_KNOWN_FLAGS (PMAP_CREATE_64BIT | PMAP_CREATE_EPT) @@ -710,9 +713,6 @@ extern void pmap_remove_options( /* Remove mappings. */ extern void fillPage(ppnum_t pa, unsigned int fill); -extern void pmap_map_sharedpage(task_t task, pmap_t pmap); -extern void pmap_unmap_sharedpage(pmap_t pmap); - #if defined(__LP64__) void pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr); #endif diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c index a0301085e..a14a10db9 100644 --- a/osfmk/vm/vm_kern.c +++ b/osfmk/vm/vm_kern.c @@ -905,7 +905,11 @@ kmem_realloc( oldmapsize = oldmapmax - oldmapmin; newmapsize = vm_map_round_page(newsize, VM_MAP_PAGE_MASK(map)); - + if (newmapsize < newsize) { + /* overflow */ + *newaddrp = 0; + return KERN_INVALID_ARGUMENT; + } /* * Find the VM object backing the old region. @@ -1075,6 +1079,11 @@ kmem_alloc_pageable( #endif map_size = vm_map_round_page(size, VM_MAP_PAGE_MASK(map)); + if (map_size < size) { + /* overflow */ + *addrp = 0; + return KERN_INVALID_ARGUMENT; + } kr = vm_map_enter(map, &map_addr, map_size, (vm_map_offset_t) 0, @@ -1200,6 +1209,11 @@ kmem_suballoc( map_size = vm_map_round_page(size, VM_MAP_PAGE_MASK(parent)); + if (map_size < size) { + /* overflow */ + *addr = 0; + return KERN_INVALID_ARGUMENT; + } /* * Need reference on submap object because it is internal diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c index dbdf22e41..28aa94218 100644 --- a/osfmk/vm/vm_map.c +++ b/osfmk/vm/vm_map.c @@ -276,7 +276,8 @@ static kern_return_t vm_map_remap_extract( vm_prot_t *max_protection, vm_inherit_t inheritance, boolean_t pageable, - boolean_t same_map); + boolean_t same_map, + vm_map_kernel_flags_t vmk_flags); static kern_return_t vm_map_remap_range_allocate( vm_map_t map, @@ -354,7 +355,11 @@ boolean_t _vmec_reserved = (NEW)->from_reserved_zone; \ (NEW)->permanent = FALSE; \ (NEW)->used_for_jit = FALSE; \ (NEW)->from_reserved_zone = _vmec_reserved; \ - (NEW)->iokit_acct = FALSE; \ + if ((NEW)->iokit_acct) { \ + assertf(!(NEW)->use_pmap, "old %p new %p\n", (OLD), (NEW)); \ + (NEW)->iokit_acct = FALSE; \ + (NEW)->use_pmap = TRUE; \ + } \ (NEW)->vme_resilient_codesign = FALSE; \ (NEW)->vme_resilient_media = FALSE; \ (NEW)->vme_atomic = FALSE; \ @@ -5388,11 +5393,44 @@ vm_map_protect( vm_map_entry_t entry; vm_prot_t new_max; int pmap_options = 0; + kern_return_t kr; XPR(XPR_VM_MAP, "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d", map, start, end, new_prot, set_max); + if (new_prot & VM_PROT_COPY) { + vm_map_offset_t new_start; + vm_prot_t cur_prot, max_prot; + vm_map_kernel_flags_t kflags; + + /* LP64todo - see below */ + if (start >= map->max_offset) { + return KERN_INVALID_ADDRESS; + } + + kflags = VM_MAP_KERNEL_FLAGS_NONE; + kflags.vmkf_remap_prot_copy = TRUE; + new_start = start; + kr = vm_map_remap(map, + &new_start, + end - start, + 0, /* mask */ + VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, + kflags, + 0, + map, + start, + TRUE, /* copy-on-write remapping! */ + &cur_prot, + &max_prot, + VM_INHERIT_DEFAULT); + if (kr != KERN_SUCCESS) { + return kr; + } + new_prot &= ~VM_PROT_COPY; + } + vm_map_lock(map); /* LP64todo - remove this check when vm_map_commpage64() @@ -5442,17 +5480,9 @@ vm_map_protect( } new_max = current->max_protection; - if(new_prot & VM_PROT_COPY) { - new_max |= VM_PROT_WRITE; - if ((new_prot & (new_max | VM_PROT_COPY)) != new_prot) { - vm_map_unlock(map); - return(KERN_PROTECTION_FAILURE); - } - } else { - if ((new_prot & new_max) != new_prot) { - vm_map_unlock(map); - return(KERN_PROTECTION_FAILURE); - } + if ((new_prot & new_max) != new_prot) { + vm_map_unlock(map); + return(KERN_PROTECTION_FAILURE); } #if CONFIG_EMBEDDED @@ -5521,36 +5551,13 @@ vm_map_protect( old_prot = current->protection; - if(new_prot & VM_PROT_COPY) { - /* caller is asking specifically to copy the */ - /* mapped data, this implies that max protection */ - /* will include write. Caller must be prepared */ - /* for loss of shared memory communication in the */ - /* target area after taking this step */ - - if (current->is_sub_map == FALSE && - VME_OBJECT(current) == VM_OBJECT_NULL) { - VME_OBJECT_SET(current, - vm_object_allocate( - (vm_map_size_t) - (current->vme_end - - current->vme_start))); - VME_OFFSET_SET(current, 0); - assert(current->use_pmap); - } - assert(current->wired_count == 0); - current->needs_copy = TRUE; - current->max_protection |= VM_PROT_WRITE; + if (set_max) { + current->max_protection = new_prot; + current->protection = new_prot & old_prot; + } else { + current->protection = new_prot; } - if (set_max) - current->protection = - (current->max_protection = - new_prot & ~VM_PROT_COPY) & - old_prot; - else - current->protection = new_prot & ~VM_PROT_COPY; - /* * Update physical map if necessary. * If the request is to turn off write protection, @@ -11354,7 +11361,8 @@ vm_map_copy_extract( max_prot, VM_INHERIT_SHARE, TRUE, /* pageable */ - FALSE); /* same_map */ + FALSE, /* same_map */ + VM_MAP_KERNEL_FLAGS_NONE); if (kr != KERN_SUCCESS) { vm_map_copy_discard(copy); return kr; @@ -14780,7 +14788,8 @@ vm_map_remap_extract( /* What, no behavior? */ vm_inherit_t inheritance, boolean_t pageable, - boolean_t same_map) + boolean_t same_map, + vm_map_kernel_flags_t vmk_flags) { kern_return_t result; vm_map_size_t mapped_size; @@ -14961,7 +14970,20 @@ vm_map_remap_extract( new_entry->vme_start = map_address; new_entry->vme_end = map_address + tmp_size; assert(new_entry->vme_start < new_entry->vme_end); - new_entry->inheritance = inheritance; + if (copy && vmk_flags.vmkf_remap_prot_copy) { + /* + * Remapping for vm_map_protect(VM_PROT_COPY) + * to convert a read-only mapping into a + * copy-on-write version of itself but + * with write access: + * keep the original inheritance and add + * VM_PROT_WRITE to the max protection. + */ + new_entry->inheritance = src_entry->inheritance; + new_entry->max_protection |= VM_PROT_WRITE; + } else { + new_entry->inheritance = inheritance; + } VME_OFFSET_SET(new_entry, offset); /* @@ -15236,7 +15258,8 @@ vm_map_remap( max_protection, inheritance, target_map->hdr.entries_pageable, - src_map == target_map); + src_map == target_map, + vmk_flags); if (result != KERN_SUCCESS) { return result; diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c index b24e87a9a..ada634f90 100644 --- a/osfmk/vm/vm_pageout.c +++ b/osfmk/vm/vm_pageout.c @@ -420,6 +420,7 @@ unsigned int vm_stat_discard_failure = 0; /* debugging */ unsigned int vm_stat_discard_throttle = 0; /* debugging */ unsigned int vm_pageout_reactivation_limit_exceeded = 0; /* debugging */ unsigned int vm_pageout_inactive_force_reclaim = 0; /* debugging */ +unsigned int vm_pageout_skipped_external = 0; /* debugging */ unsigned int vm_pageout_scan_reclaimed_throttled = 0; unsigned int vm_pageout_scan_active_throttled = 0; @@ -1702,10 +1703,12 @@ int last_vm_pageout_reactivation_limit_exceeded = 0; int last_vm_pageout_considered_page = 0; int last_vm_compressor_pages_grabbed = 0; int last_vm_compressor_failed = 0; +int last_vm_pageout_skipped_external = 0; + void update_vm_info(void) { - int tmp1, tmp2, tmp3; + int tmp1, tmp2, tmp3, tmp4; if (!kdebug_enable) return; @@ -1773,12 +1776,14 @@ void update_vm_info(void) tmp1 = vm_pageout_scan_inactive_throttled_internal; tmp2 = vm_pageout_freed_after_compression; tmp3 = vm_compressor_pages_grabbed; + tmp4 = vm_pageout_skipped_external; KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE, tmp1 - last_vm_pageout_scan_inactive_throttled_internal, tmp2 - last_vm_pageout_freed_after_compression, tmp3 - last_vm_compressor_pages_grabbed, - 0, 0); + tmp4 - last_vm_pageout_skipped_external, + 0); vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal); vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression); @@ -1787,6 +1792,7 @@ void update_vm_info(void) last_vm_pageout_scan_inactive_throttled_internal = tmp1; last_vm_pageout_freed_after_compression = tmp2; last_vm_compressor_pages_grabbed = tmp3; + last_vm_pageout_skipped_external = tmp4; if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 && @@ -2655,6 +2661,9 @@ consider_inactive: ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) { grab_anonymous = TRUE; anons_grabbed = 0; + + vm_pageout_skipped_external++; + goto want_anonymous; } #if CONFIG_JETSAM /* If the file-backed pool has accumulated @@ -2687,6 +2696,7 @@ consider_inactive: } #endif /* CONFIG_JETSAM */ +want_anonymous: if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) { if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) { @@ -2801,6 +2811,7 @@ consider_inactive: pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m)); m->reference = FALSE; +#if !CONFIG_EMBEDDED /* * m->object must be stable since we hold the page queues lock... * we can update the scan_collisions field sans the object lock @@ -2810,7 +2821,17 @@ consider_inactive: * is possible for the value to be a bit non-determistic, but that's ok * since it's only used as a hint */ + + /* + * This is not used on EMBEDDED because having this variable set *could* lead + * us to self-cannibalize pages from m_object to fill a UPL for a pagein. + * And, there's a high probability that the object that vm_pageout_scan + * wants and collides on is a very popular object e.g. the shared cache on EMBEDDED. + * The older pages that we cannibalize from the shared cache could be really + * important text pages e.g. the system call stubs. + */ m_object->scan_collisions = 1; +#endif /* !CONFIG_EMBEDDED */ if ( !vm_page_queue_empty(&sq->age_q) ) m_want = (vm_page_t) vm_page_queue_first(&sq->age_q); diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c index feeaddd1e..b34f1b2d5 100644 --- a/osfmk/vm/vm_resident.c +++ b/osfmk/vm/vm_resident.c @@ -604,6 +604,8 @@ vm_page_init_lck_grp(void) vm_compressor_init_locks(); } +#define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1))) + void vm_page_init_local_q() { @@ -619,11 +621,12 @@ vm_page_init_local_q() if (num_cpus >= 2) { #if KASAN /* KASAN breaks the expectation of a size-aligned object by adding a - * rezone, so explicitly align. */ + * redzone, so explicitly align. */ t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq) + VM_PACKED_POINTER_ALIGNMENT); t_local_q = (void *)(((uintptr_t)t_local_q + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT-1)); #else - t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq)); + /* round the size up to the nearest power of two */ + t_local_q = (struct vplq *)kalloc(ROUNDUP_NEXTP2(num_cpus * sizeof(struct vplq))); #endif for (i = 0; i < num_cpus; i++) { diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c index 491d27fe9..2f9bdc101 100644 --- a/osfmk/vm/vm_user.c +++ b/osfmk/vm/vm_user.c @@ -3330,6 +3330,9 @@ mach_memory_object_memory_entry_64( if (pager == MEMORY_OBJECT_NULL && internal) { object = vm_object_allocate(size); + if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) { + object->copy_strategy = MEMORY_OBJECT_COPY_DELAY; + } } else { object = memory_object_to_vm_object(pager); if (object != VM_OBJECT_NULL) { diff --git a/osfmk/x86_64/bcopy.s b/osfmk/x86_64/bcopy.s index fc0af58d6..8cd909bbb 100644 --- a/osfmk/x86_64/bcopy.s +++ b/osfmk/x86_64/bcopy.s @@ -62,7 +62,8 @@ /* rdi, rsi, rdx */ /* * Note: memcpy does not support overlapping copies - */ + */ + /* TODO: movsb */ ENTRY(memcpy) movq %rdi, %rax /* return destination */ movq %rdx,%rcx diff --git a/osfmk/x86_64/copyio.c b/osfmk/x86_64/copyio.c index c86d1136c..dd4b93670 100644 --- a/osfmk/x86_64/copyio.c +++ b/osfmk/x86_64/copyio.c @@ -166,7 +166,6 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, int debug_type = 0xeff70010; debug_type += (copy_type << 2); #endif - boolean_t nopagezero = thread->map->pmap->pagezero_accessible; if (__improbable(nbytes > copysize_limit_panic)) panic("%s(%p, %p, %lu) - transfer too large", __func__, @@ -179,6 +178,7 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, goto out; pmap = thread->map->pmap; + boolean_t nopagezero = pmap->pagezero_accessible; if (__improbable((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS))) { panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr); diff --git a/osfmk/x86_64/idt64.s b/osfmk/x86_64/idt64.s index 7c42b1dee..13f3f0218 100644 --- a/osfmk/x86_64/idt64.s +++ b/osfmk/x86_64/idt64.s @@ -66,31 +66,41 @@ * syscall - synchronous system call request * fatal - fatal traps */ - /* - * Handlers: + * Indices of handlers for each exception type. */ -#define HNDL_ALLINTRS EXT(hndl_allintrs) -#define HNDL_ALLTRAPS EXT(hndl_alltraps) -#define HNDL_SYSENTER EXT(hndl_sysenter) -#define HNDL_SYSCALL EXT(hndl_syscall) -#define HNDL_UNIX_SCALL EXT(hndl_unix_scall) -#define HNDL_MACH_SCALL EXT(hndl_mach_scall) -#define HNDL_MDEP_SCALL EXT(hndl_mdep_scall) -#define HNDL_DOUBLE_FAULT EXT(hndl_double_fault) -#define HNDL_MACHINE_CHECK EXT(hndl_machine_check) - - -#if 1 -#define PUSH_FUNCTION(func) \ - sub $8, %rsp ;\ - push %rax ;\ - leaq func(%rip), %rax ;\ - movq %rax, 8(%rsp) ;\ - pop %rax -#else -#define PUSH_FUNCTION(func) pushq func -#endif +#define HNDL_ALLINTRS 0 +#define HNDL_ALLTRAPS 1 +#define HNDL_SYSENTER 2 +#define HNDL_SYSCALL 3 +#define HNDL_UNIX_SCALL 4 +#define HNDL_MACH_SCALL 5 +#define HNDL_MDEP_SCALL 6 +#define HNDL_DOUBLE_FAULT 7 +#define HNDL_MACHINE_CHECK 8 + +/* Begin double-mapped descriptor section */ + +.section __HIB, __desc +.globl EXT(idt64_hndl_table0) +EXT(idt64_hndl_table0): + .quad EXT(ks_dispatch) + .quad EXT(ks_64bit_return) + .quad 0 /* Populated with CPU shadow displacement*/ + .quad EXT(ks_return) + +EXT(idt64_hndl_table1): + .quad EXT(hndl_allintrs) + .quad EXT(hndl_alltraps) + .quad EXT(hndl_sysenter) + .quad EXT(hndl_syscall) + .quad EXT(hndl_unix_scall) + .quad EXT(hndl_mach_scall) + .quad EXT(hndl_mdep_scall) + .quad EXT(hndl_double_fault) + .quad EXT(hndl_machine_check) +.text + /* The wrapper for all non-special traps/interrupts */ /* Everything up to PUSH_FUNCTION is just to output @@ -101,12 +111,12 @@ push %rax ;\ POSTCODE2(0x6400+n) ;\ pop %rax ;\ - PUSH_FUNCTION(f) ;\ + pushq $(f) ;\ pushq $(n) ;\ jmp L_dispatch #else #define IDT_ENTRY_WRAPPER(n, f) \ - PUSH_FUNCTION(f) ;\ + pushq $(f) ;\ pushq $(n) ;\ jmp L_dispatch #endif @@ -136,9 +146,175 @@ #define TRAP_IST2(n, f) #define USER_TRAP_SPC(n, f) +/* Begin double-mapped text section */ +.section __HIB, __text /* Generate all the stubs */ #include "idt_table.h" +Entry(idt64_page_fault) + pushq $(HNDL_ALLTRAPS) + push $(T_PAGE_FAULT) + jmp L_dispatch + +Entry(idt64_debug) + push $0 /* error code */ + pushq $(HNDL_ALLTRAPS) + pushq $(T_DEBUG) + jmp L_dispatch +/* + * Legacy interrupt gate System call handlers. + * These are entered via a syscall interrupt. The system call number in %rax + * is saved to the error code slot in the stack frame. We then branch to the + * common state saving code. + */ + +#ifndef UNIX_INT +#error NO UNIX INT!!! +#endif +Entry(idt64_unix_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_UNIX_SCALL) + pushq $(UNIX_INT) + jmp L_dispatch + +Entry(idt64_mach_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_MACH_SCALL) + pushq $(MACH_INT) + jmp L_dispatch + +Entry(idt64_mdep_scall) + pushq %rax /* save system call number */ + pushq $(HNDL_MDEP_SCALL) + pushq $(MACHDEP_INT) + jmp L_dispatch + +/* + * For GP/NP/SS faults, we use the IST1 stack. + * For faults from user-space, we have to copy the machine state to the + * PCB stack and then dispatch as normal. + * For faults in kernel-space, we need to scrub for kernel exit faults and + * treat these as user-space faults. But for all other kernel-space faults + * we continue to run on the IST1 stack and we dispatch to handle the fault + * as fatal. + */ +Entry(idt64_gen_prot) + pushq $(HNDL_ALLTRAPS) + pushq $(T_GENERAL_PROTECTION) + jmp L_dispatch + +Entry(idt64_stack_fault) + pushq $(HNDL_ALLTRAPS) + pushq $(T_STACK_FAULT) + jmp L_dispatch + +Entry(idt64_segnp) + pushq $(HNDL_ALLTRAPS) + pushq $(T_SEGMENT_NOT_PRESENT) + jmp L_dispatch + +/* + * Fatal exception handlers: + */ +Entry(idt64_db_task_dbl_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_DOUBLE_FAULT) + jmp L_dispatch + +Entry(idt64_db_task_stk_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_STACK_FAULT) + jmp L_dispatch + +Entry(idt64_mc) + push $(0) /* Error */ + pushq $(HNDL_MACHINE_CHECK) + pushq $(T_MACHINE_CHECK) + jmp L_dispatch + +/* + * NMI + * This may or may not be fatal but extreme care is required + * because it may fall when control was already in another trampoline. + * + * We get here on IST2 stack which is used for NMIs only. + */ +Entry(idt64_nmi) + push %rax /* save RAX to ISF64_ERR */ + push %rcx /* save RCX to ISF64_TRAPFN */ + push %rdx /* save RDX to ISF64_TRAPNO */ + jmp L_dispatch + +Entry(idt64_double_fault) + pushq $(HNDL_DOUBLE_FAULT) + pushq $(T_DOUBLE_FAULT) + jmp L_dispatch + +Entry(hi64_syscall) +Entry(idt64_syscall) + swapgs + /* Use RAX as a temporary by shifting its contents into R11[32:63] + * The systemcall number is defined to be a 32-bit quantity, as is + * RFLAGS. + */ + shlq $32, %rax + or %rax, %r11 +.globl EXT(dblsyscall_patch_point) +EXT(dblsyscall_patch_point): +// movabsq $0x12345678ABCDEFFFULL, %rax + /* Generate offset to the double-mapped per-CPU data shadow + * into RAX + */ + leaq EXT(idt64_hndl_table0)(%rip), %rax + mov 16(%rax), %rax + mov %rsp, %gs:CPU_UBER_TMP(%rax) /* save user stack */ + mov %gs:CPU_ESTACK(%rax), %rsp /* switch stack to per-cpu estack */ + sub $(ISF64_SIZE), %rsp + + /* + * Synthesize an ISF frame on the exception stack + */ + movl $(USER_DS), ISF64_SS(%rsp) + mov %rcx, ISF64_RIP(%rsp) /* rip */ + + mov %gs:CPU_UBER_TMP(%rax), %rcx + mov %rcx, ISF64_RSP(%rsp) /* user stack --changed */ + + mov %r11, %rax + shrq $32, %rax /* Restore RAX */ + mov %r11d, %r11d /* Clear r11[32:63] */ + + mov %r11, ISF64_RFLAGS(%rsp) /* rflags */ + movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */ + mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */ + movq $(HNDL_SYSCALL), ISF64_TRAPFN(%rsp) + movq $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */ + swapgs + jmp L_dispatch /* this can only be 64-bit */ + +Entry(hi64_sysenter) +Entry(idt64_sysenter) + /* Synthesize an interrupt stack frame onto the + * exception stack. + */ + push $(USER_DS) /* ss */ + push %rcx /* uesp */ + pushf /* flags */ + /* + * Clear, among others, the Nested Task (NT) flags bit; + * this is zeroed by INT, but not by SYSENTER. + */ + push $0 + popf + push $(SYSENTER_CS) /* cs */ +L_sysenter_continue: + push %rdx /* eip */ + push %rax /* err/eax - syscall code */ + pushq $(HNDL_SYSENTER) + pushq $(T_SYSENTER) + orl $(EFL_IF), ISF64_RFLAGS(%rsp) + jmp L_dispatch + /* * Common dispatch point. * Determine what mode has been interrupted and save state accordingly. @@ -148,13 +324,113 @@ * GSBASE from user-space: pthread area, or * from kernel-space: cpu_data */ + L_dispatch: + pushq %rax + testb $3, 8+ISF64_CS(%rsp) + jz 1f + swapgs + leaq EXT(idt64_hndl_table0)(%rip), %rax + mov 16(%rax), %rax + + mov %gs:CPU_TASK_CR3(%rax), %rax + mov %rax, %cr3 +#if DEBUG + mov %rax, %gs:CPU_ENTRY_CR3 +#endif +1: + /* The text/data relationship here must be preserved in the doublemap, and the contents must be remapped */ + leaq EXT(idt64_hndl_table0)(%rip), %rax + /* Indirect branch to non-doublemapped trampolines */ + jmp *(%rax) +/* User return: register restoration and address space switch sequence */ +Entry(ks_64bit_return) + mov R64_R14(%r15), %r14 + mov R64_R13(%r15), %r13 + mov R64_R12(%r15), %r12 + mov R64_R11(%r15), %r11 + mov R64_R10(%r15), %r10 + mov R64_R9(%r15), %r9 + mov R64_R8(%r15), %r8 + mov R64_RSI(%r15), %rsi + mov R64_RDI(%r15), %rdi + mov R64_RBP(%r15), %rbp + mov R64_RDX(%r15), %rdx + mov R64_RCX(%r15), %rcx + mov R64_RBX(%r15), %rbx + mov R64_RAX(%r15), %rax + /* Switch to per-CPU exception stack */ + mov %gs:CPU_ESTACK, %rsp + + /* Synthesize interrupt stack frame from PCB savearea to exception stack */ + push R64_SS(%r15) + push R64_RSP(%r15) + push R64_RFLAGS(%r15) + push R64_CS(%r15) + push R64_RIP(%r15) + + mov R64_R15(%r15), %r15 + cmpq $(KERNEL64_CS), 8(%rsp) + jz 1f + /* Discover user cr3/ASID */ + push %rax + mov %gs:CPU_UCR3, %rax +#if DEBUG + mov %rax, %gs:CPU_EXIT_CR3 +#endif + mov %rax, %cr3 + /* Continue execution on the shared/doublemapped trampoline */ + pop %rax + swapgs +1: + cmpl $(SYSCALL_CS), 8(%rsp) /* test for exit via SYSRET */ + je L_sysret +EXT(ret64_iret): + iretq /* return from interrupt */ +L_sysret: + /* + * Here to restore rcx/r11/rsp and perform the sysret back to user-space. + * rcx user rip + * r11 user rflags + * rsp user stack pointer + */ + pop %rcx + add $8, %rsp + pop %r11 + pop %rsp + sysretq /* return from system call */ +/* End of double-mapped TEXT */ +.text + +Entry(ks_dispatch) + popq %rax cmpl $(KERNEL64_CS), ISF64_CS(%rsp) - je L_dispatch_kernel + je EXT(ks_dispatch_kernel) - swapgs + mov %rax, %gs:CPU_UBER_TMP + mov %gs:CPU_UBER_ISF, %rax + add $(ISF64_SIZE), %rax + + xchg %rsp, %rax +/* Memory to memory moves (aint x86 wonderful): + * Transfer the exception frame from the per-CPU exception stack to the + * 'PCB' stack programmed at cswitch. + */ + push ISF64_SS(%rax) + push ISF64_RSP(%rax) + push ISF64_RFLAGS(%rax) + push ISF64_CS(%rax) + push ISF64_RIP(%rax) + push ISF64_ERR(%rax) + push ISF64_TRAPFN(%rax) + push ISF64_TRAPNO(%rax) + mov %gs:CPU_UBER_TMP, %rax + jmp EXT(ks_dispatch_user) -L_dispatch_user: +Entry (ks_return) + jmp . + +Entry(ks_dispatch_user) cmpl $(TASK_MAP_32BIT), %gs:CPU_TASK_MAP je L_dispatch_U32 /* 32-bit user task */ @@ -165,7 +441,7 @@ L_dispatch_U64: mov %gs:CPU_KERNEL_STACK, %rsp jmp L_dispatch_64bit -L_dispatch_kernel: +Entry(ks_dispatch_kernel) subq $(ISS64_OFFSET), %rsp mov %r15, R64_R15(%rsp) mov %rsp, %r15 @@ -212,14 +488,11 @@ L_64bit_entry_reject: /* * Here for a 64-bit user attempting an invalid kernel entry. */ - pushq %rax - leaq HNDL_ALLTRAPS(%rip), %rax - movq %rax, ISF64_TRAPFN+8(%rsp) - popq %rax + movq $(HNDL_ALLTRAPS), ISF64_TRAPFN(%rsp) movq $(T_INVALID_OPCODE), ISF64_TRAPNO(%rsp) jmp L_dispatch_U64 -L_32bit_entry_check: +Entry(ks_32bit_entry_check) /* * Check we're not a confused 64-bit user. */ @@ -315,19 +588,12 @@ L_common_dispatch: shr $32, %rcx testl %ecx, %ecx jz 4f - testl $(1<<16), %ecx /* Global? */ - jz 3f movl $0, %gs:CPU_TLB_INVALID mov %cr4, %rcx /* RMWW CR4, for lack of an alternative*/ and $(~CR4_PGE), %rcx mov %rcx, %cr4 or $(CR4_PGE), %rcx mov %rcx, %cr4 - jmp 4f -3: - movb $0, %gs:CPU_TLB_INVALID_LOCAL - mov %cr3, %rcx - mov %rcx, %cr3 4: mov %gs:CPU_ACTIVE_THREAD, %rcx /* Get the active thread */ testq %rcx, %rcx @@ -340,7 +606,8 @@ L_common_dispatch: 5: incl %gs:hwIntCnt(,%ebx,4) // Bump the trap/intr count /* Dispatch the designated handler */ - jmp *%rdx + leaq EXT(idt64_hndl_table1)(%rip), %rax + jmp *(%rax, %rdx, 8) /* * Control is passed here to return to user. @@ -430,6 +697,40 @@ L_32bit_return: movl R32_SS(%r15), %eax movl %eax, R64_SS(%r15) + /* Validate DS/ES/FS/GS segment selectors with the Load Access Rights instruction prior to restoration */ + /* Exempt "known good" statically configured selectors, e.g. USER_DS and 0 */ + cmpl $(USER_DS), R32_DS(%r15) + jz 22f + cmpl $0, R32_DS(%r15) + jz 22f + larw R32_DS(%r15), %ax + jz 22f + movl $(USER_DS), R32_DS(%r15) +22: + cmpl $(USER_DS), R32_ES(%r15) + jz 33f + cmpl $0, R32_ES(%r15) + jz 33f + larw R32_ES(%r15), %ax + jz 33f + movl $(USER_DS), R32_ES(%r15) +33: + cmpl $(USER_DS), R32_FS(%r15) + jz 44f + cmpl $0, R32_FS(%r15) + jz 44f + larw R32_FS(%r15), %ax + jz 44f + movl $(USER_DS), R32_FS(%r15) +44: + cmpl $(USER_CTHREAD), R32_GS(%r15) + jz 55f + cmpl $0, R32_GS(%r15) + jz 55f + larw R32_GS(%r15), %ax + jz 55f + movl $(USER_CTHREAD), R32_GS(%r15) +55: /* * Restore general 32-bit registers */ @@ -458,13 +759,13 @@ L_32bit_return: xor %r15, %r15 EXT(ret32_set_ds): - movl R32_DS(%rsp), %ds + movw R32_DS(%rsp), %ds EXT(ret32_set_es): - movl R32_ES(%rsp), %es + movw R32_ES(%rsp), %es EXT(ret32_set_fs): - movl R32_FS(%rsp), %fs + movw R32_FS(%rsp), %fs EXT(ret32_set_gs): - movl R32_GS(%rsp), %gs + movw R32_GS(%rsp), %gs /* pop compat frame + trapno, trapfn and error */ add $(ISS64_OFFSET)+8+8+8, %rsp @@ -474,7 +775,6 @@ EXT(ret32_set_gs): EXT(ret32_iret): iretq /* return from interrupt */ - L_fast_exit: pop %rdx /* user return eip */ pop %rcx /* pop and toss cs */ @@ -512,176 +812,10 @@ L_64bit_return: * Restore general 64-bit registers. * Here on fault stack and PCB address in R15. */ - mov R64_R14(%r15), %r14 - mov R64_R13(%r15), %r13 - mov R64_R12(%r15), %r12 - mov R64_R11(%r15), %r11 - mov R64_R10(%r15), %r10 - mov R64_R9(%r15), %r9 - mov R64_R8(%r15), %r8 - mov R64_RSI(%r15), %rsi - mov R64_RDI(%r15), %rdi - mov R64_RBP(%r15), %rbp - mov R64_RDX(%r15), %rdx - mov R64_RCX(%r15), %rcx - mov R64_RBX(%r15), %rbx - mov R64_RAX(%r15), %rax - - /* - * We must swap GS base if we're returning to user-space, - * or we're returning from an NMI that occurred in a trampoline - * before the user GS had been swapped. In the latter case, the NMI - * handler will have flagged the high-order 32-bits of the CS. - */ - cmpq $(KERNEL64_CS), R64_CS(%r15) - jz 1f - swapgs -1: - mov R64_R15(%r15), %rsp - xchg %r15, %rsp - add $(ISS64_OFFSET)+24, %rsp /* pop saved state */ - /* + trapno/trapfn/error */ - cmpl $(SYSCALL_CS),ISF64_CS-24(%rsp) - /* test for fast entry/exit */ - je L_sysret -.globl _dump_iretq -EXT(ret64_iret): - iretq /* return from interrupt */ - -L_sysret: - /* - * Here to load rcx/r11/rsp and perform the sysret back to user-space. - * rcx user rip - * r11 user rflags - * rsp user stack pointer - */ - mov ISF64_RIP-24(%rsp), %rcx - mov ISF64_RFLAGS-24(%rsp), %r11 - mov ISF64_RSP-24(%rsp), %rsp - sysretq /* return from systen call */ - - - -/* - * System call handlers. - * These are entered via a syscall interrupt. The system call number in %rax - * is saved to the error code slot in the stack frame. We then branch to the - * common state saving code. - */ - -#ifndef UNIX_INT -#error NO UNIX INT!!! -#endif -Entry(idt64_unix_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_UNIX_SCALL) - pushq $(UNIX_INT) - jmp L_32bit_entry_check - - -Entry(idt64_mach_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_MACH_SCALL) - pushq $(MACH_INT) - jmp L_32bit_entry_check - - -Entry(idt64_mdep_scall) - swapgs /* switch to kernel gs (cpu_data) */ - pushq %rax /* save system call number */ - PUSH_FUNCTION(HNDL_MDEP_SCALL) - pushq $(MACHDEP_INT) - jmp L_32bit_entry_check - -/* Programmed into MSR_IA32_LSTAR by mp_desc.c */ -Entry(hi64_syscall) -Entry(idt64_syscall) -L_syscall_continue: - swapgs /* Kapow! get per-cpu data area */ - mov %rsp, %gs:CPU_UBER_TMP /* save user stack */ - mov %gs:CPU_UBER_ISF, %rsp /* switch stack to pcb */ - - /* - * Save values in the ISF frame in the PCB - * to cons up the saved machine state. - */ - movl $(USER_DS), ISF64_SS(%rsp) - movl $(SYSCALL_CS), ISF64_CS(%rsp) /* cs - a pseudo-segment */ - mov %r11, ISF64_RFLAGS(%rsp) /* rflags */ - mov %rcx, ISF64_RIP(%rsp) /* rip */ - mov %gs:CPU_UBER_TMP, %rcx - mov %rcx, ISF64_RSP(%rsp) /* user stack */ - mov %rax, ISF64_ERR(%rsp) /* err/rax - syscall code */ - movq $(T_SYSCALL), ISF64_TRAPNO(%rsp) /* trapno */ - leaq HNDL_SYSCALL(%rip), %r11; - movq %r11, ISF64_TRAPFN(%rsp) - mov ISF64_RFLAGS(%rsp), %r11 /* Avoid leak, restore R11 */ - jmp L_dispatch_U64 /* this can only be 64-bit */ - -/* - * sysenter entry point - * Requires user code to set up: - * edx: user instruction pointer (return address) - * ecx: user stack pointer - * on which is pushed stub ret addr and saved ebx - * Return to user-space is made using sysexit. - * Note: sysenter/sysexit cannot be used for calls returning a value in edx, - * or requiring ecx to be preserved. - */ -Entry(hi64_sysenter) -Entry(idt64_sysenter) - movq (%rsp), %rsp - /* - * Push values on to the PCB stack - * to cons up the saved machine state. - */ - push $(USER_DS) /* ss */ - push %rcx /* uesp */ - pushf /* flags */ - /* - * Clear, among others, the Nested Task (NT) flags bit; - * this is zeroed by INT, but not by SYSENTER. - */ - push $0 - popf - push $(SYSENTER_CS) /* cs */ -L_sysenter_continue: - swapgs /* switch to kernel gs (cpu_data) */ - push %rdx /* eip */ - push %rax /* err/eax - syscall code */ - PUSH_FUNCTION(HNDL_SYSENTER) - pushq $(T_SYSENTER) - orl $(EFL_IF), ISF64_RFLAGS(%rsp) - jmp L_32bit_entry_check - - -Entry(idt64_page_fault) - PUSH_FUNCTION(HNDL_ALLTRAPS) - push $(T_PAGE_FAULT) - push %rax /* save %rax temporarily */ - testb $3, 8+ISF64_CS(%rsp) /* was trap from kernel? */ - jz L_kernel_trap /* - yes, handle with care */ - pop %rax /* restore %rax, swapgs, and continue */ - swapgs - jmp L_dispatch_user - - -/* - * Debug trap. Check for single-stepping across system call into - * kernel. If this is the case, taking the debug trap has turned - * off single-stepping - save the flags register with the trace - * bit set. - */ -Entry(idt64_debug) - push $0 /* error code */ - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_DEBUG) - - testb $3, ISF64_CS(%rsp) - jnz L_dispatch + leaq EXT(idt64_hndl_table0)(%rip), %rax + jmp *8(%rax) +Entry(ks_idt64_debug_kernel) /* * trap came from kernel mode */ @@ -690,7 +824,7 @@ Entry(idt64_debug) lea EXT(idt64_sysenter)(%rip), %rax cmp %rax, ISF64_RIP+8(%rsp) pop %rax - jne L_dispatch + jne EXT(ks_dispatch_kernel) /* * Interrupt stack frame has been pushed on the temporary stack. * We have to switch to pcb stack and patch up the saved state. @@ -705,37 +839,7 @@ Entry(idt64_debug) mov ISF64_ERR(%rcx),%rcx /* restore %rcx */ jmp L_sysenter_continue /* continue sysenter entry */ - -Entry(idt64_double_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_DOUBLE_FAULT) - jmp L_dispatch_kernel - - -/* - * For GP/NP/SS faults, we use the IST1 stack. - * For faults from user-space, we have to copy the machine state to the - * PCB stack and then dispatch as normal. - * For faults in kernel-space, we need to scrub for kernel exit faults and - * treat these as user-space faults. But for all other kernel-space faults - * we continue to run on the IST1 stack and we dispatch to handle the fault - * as fatal. - */ -Entry(idt64_gen_prot) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_GENERAL_PROTECTION) - jmp trap_check_kernel_exit /* check for kernel exit sequence */ - -Entry(idt64_stack_fault) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_STACK_FAULT) - jmp trap_check_kernel_exit /* check for kernel exit sequence */ - -Entry(idt64_segnp) - PUSH_FUNCTION(HNDL_ALLTRAPS) - pushq $(T_SEGMENT_NOT_PRESENT) - /* indicate fault type */ -trap_check_kernel_exit: +Entry(ks_trap_check_kernel_exit) testb $3,ISF64_CS(%rsp) jz L_kernel_gpf @@ -763,7 +867,7 @@ trap_check_kernel_exit: pop %rax mov %gs:CPU_UBER_TMP, %rsp /* user RCX into RSP */ xchg %rcx, %rsp /* to PCB stack with user RCX */ - jmp L_dispatch_user + jmp EXT(ks_dispatch_user) L_kernel_gpf: /* Here for GPF from kernel_space. Check for recoverable cases. */ @@ -786,10 +890,10 @@ L_kernel_gpf: leaq EXT(ret32_set_gs)(%rip), %rax cmp %rax, 8+ISF64_RIP(%rsp) je L_32bit_fault_set_seg - + jmp EXT(ks_kernel_trap) /* Fall through */ -L_kernel_trap: +Entry(ks_kernel_trap) /* * Here after taking an unexpected trap from kernel mode - perhaps * while running in the trampolines hereabouts. @@ -814,7 +918,7 @@ L_kernel_trap: cmpq $(PAGE_SIZE), %rax /* current stack in PCB? */ jb 2f /* - yes, deal with it */ pop %rax /* - no, restore %rax */ - jmp L_dispatch_kernel + jmp EXT(ks_dispatch_kernel) 2: /* * Here if %rsp is in the PCB @@ -831,7 +935,7 @@ L_kernel_trap: pushq 8+ISF64_TRAPFN(%rax) pushq 8+ISF64_TRAPNO(%rax) movq (%rax), %rax - jmp L_dispatch_kernel + jmp EXT(ks_dispatch_kernel) /* @@ -917,46 +1021,8 @@ L_32bit_fault_set_seg: /* the compatibility frame */ jmp L_dispatch_U32_after_fault -/* - * Fatal exception handlers: - */ -Entry(idt64_db_task_dbl_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_DOUBLE_FAULT) - jmp L_dispatch - -Entry(idt64_db_task_stk_fault) - PUSH_FUNCTION(HNDL_DOUBLE_FAULT) - pushq $(T_STACK_FAULT) - jmp L_dispatch - -Entry(idt64_mc) - push $(0) /* Error */ - PUSH_FUNCTION(HNDL_MACHINE_CHECK) - pushq $(T_MACHINE_CHECK) - jmp L_dispatch - -/* - * NMI - * This may or may not be fatal but extreme care is required - * because it may fall when control was already in another trampoline. - * - * We get here on IST2 stack which is used for NMIs only. - * We must be aware of the interrupted state: - * - from user-space, we - * - copy state to the PCB and continue; - * - from kernel-space, we - * - copy state to the kernel stack and continue, but - * - check what GSBASE was active, set the kernel base and - * - ensure that the active state is restored when the NMI is dismissed. - */ -Entry(idt64_nmi) - push %rax /* save RAX to ISF64_ERR */ - push %rcx /* save RCX to ISF64_TRAPFN */ - push %rdx /* save RDX to ISF64_TRAPNO */ - testb $3, ISF64_CS(%rsp) /* NMI from user-space? */ - je 1f +Entry(ks_idt64_nmi_kernel) /* From user-space: copy interrupt state to user PCB */ swapgs mov %gs:CPU_UBER_ISF, %rcx /* PCB stack addr */ @@ -1015,8 +1081,7 @@ Entry(idt64_nmi) push ISF64_CS(%rcx) push ISF64_RIP(%rcx) push $(0) /* error code 0 */ - lea HNDL_ALLINTRS(%rip), %rax - push %rax /* trapfn allintrs */ + push $(HNDL_ALLINTRS) /* trapfn allintrs */ push $(T_NMI) /* trapno T_NMI */ mov ISF64_ERR(%rcx), %rax mov ISF64_TRAPNO(%rcx), %rdx @@ -1088,7 +1153,7 @@ L_return_from_trap_with_ast: je 2f /* not in the PFZ... go service AST */ movl %eax, R64_RBX(%r15) /* let the PFZ know we've pended an AST */ jmp EXT(return_to_user) -2: +2: xorq %rbp, %rbp /* clear framepointer */ CCALL(ast_taken_user) /* handle all ASTs (enables interrupts, may return via continuation) */ @@ -1180,7 +1245,7 @@ Entry(hndl_allintrs) CCALL1(interrupt, %r15) /* call generic interrupt routine */ - .globl EXT(return_to_iret) +.globl EXT(return_to_iret) LEXT(return_to_iret) /* (label for kdb_kintr and hardclock) */ decl %gs:CPU_INTERRUPT_LEVEL @@ -1442,11 +1507,16 @@ Entry(hndl_diag_scall64) sti CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1) /* no return */ - +/* TODO assert at all 'C' entry points that we're never operating on the fault stack's alias mapping */ Entry(hndl_machine_check) + /* Adjust SP and savearea to their canonical, non-aliased addresses */ + subq EXT(dblmap_dist)(%rip), %rsp + subq EXT(dblmap_dist)(%rip), %r15 CCALL1(panic_machine_check64, %r15) hlt Entry(hndl_double_fault) + subq EXT(dblmap_dist)(%rip), %rsp + subq EXT(dblmap_dist)(%rip), %r15 CCALL1(panic_double_fault64, %r15) hlt diff --git a/osfmk/x86_64/locore.s b/osfmk/x86_64/locore.s index 0638e5162..d88f2a08a 100644 --- a/osfmk/x86_64/locore.s +++ b/osfmk/x86_64/locore.s @@ -159,6 +159,12 @@ wrmsr_fail: ret #if DEBUG +#ifndef TERI +#define TERI 1 +#endif +#endif + +#if TERI .globl EXT(thread_exception_return_internal) #else .globl EXT(thread_exception_return) @@ -169,7 +175,7 @@ LEXT(thread_bootstrap_return) call EXT(dtrace_thread_bootstrap) #endif -#if DEBUG +#if TERI LEXT(thread_exception_return_internal) #else LEXT(thread_exception_return) diff --git a/osfmk/x86_64/pmap.c b/osfmk/x86_64/pmap.c index 9b2a9675e..ce64b82db 100644 --- a/osfmk/x86_64/pmap.c +++ b/osfmk/x86_64/pmap.c @@ -239,6 +239,7 @@ pmap_t kernel_pmap; struct zone *pmap_zone; /* zone of pmap structures */ struct zone *pmap_anchor_zone; +struct zone *pmap_uanchor_zone; int pmap_debug = 0; /* flag for debugging prints */ unsigned int inuse_ptepages_count = 0; @@ -323,12 +324,8 @@ void pmap_cpu_init(void) { cpu_data_t *cdp = current_cpu_datap(); - /* - * Here early in the life of a processor (from cpu_mode_init()). - * Ensure global page feature is disabled at this point. - */ - set_cr4(get_cr4() &~ CR4_PGE); + set_cr4(get_cr4() | CR4_PGE); /* * Initialize the per-cpu, TLB-related fields. @@ -337,6 +334,7 @@ pmap_cpu_init(void) cdp->cpu_active_cr3 = kernel_pmap->pm_cr3; cdp->cpu_tlb_invalid = FALSE; cdp->cpu_task_map = TASK_MAP_64BIT; + pmap_pcid_configure(); if (cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_SMEP) { pmap_smep_enabled = TRUE; @@ -415,14 +413,13 @@ pmap_bootstrap( kernel_pmap->nx_enabled = TRUE; kernel_pmap->pm_task_map = TASK_MAP_64BIT; kernel_pmap->pm_obj = (vm_object_t) NULL; - kernel_pmap->dirbase = (pd_entry_t *)((uintptr_t)IdlePTD); - kernel_pmap->pm_pdpt = (pd_entry_t *) ((uintptr_t)IdlePDPT); kernel_pmap->pm_pml4 = IdlePML4; + kernel_pmap->pm_upml4 = IdlePML4; kernel_pmap->pm_cr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4); + kernel_pmap->pm_ucr3 = (uintptr_t)ID_MAP_VTOP(IdlePML4); kernel_pmap->pm_eptp = 0; - pmap_pcid_initialize_kernel(kernel_pmap); - + pmap_pcid_initialize_kernel(kernel_pmap); current_cpu_datap()->cpu_kernel_cr3 = (addr64_t) kernel_pmap->pm_cr3; @@ -808,6 +805,18 @@ pmap_init(void) */ zone_change(pmap_anchor_zone, Z_ALIGNMENT_REQUIRED, TRUE); +/* TODO: possible general optimisation...pre-allocate via zones commonly created + * level3/2 pagetables + */ + pmap_uanchor_zone = zinit(PAGE_SIZE, task_max, PAGE_SIZE, "pagetable user anchors"); + zone_change(pmap_uanchor_zone, Z_NOENCRYPT, TRUE); + + /* The anchor is required to be page aligned. Zone debugging adds + * padding which may violate that requirement. Tell the zone + * subsystem that alignment is required. + */ + + zone_change(pmap_uanchor_zone, Z_ALIGNMENT_REQUIRED, TRUE); s = (vm_size_t) sizeof(struct pv_hashed_entry); pv_hashed_list_zone = zinit(s, 10000*s /* Expandable zone */, @@ -1288,6 +1297,7 @@ hv_ept_pmap_create(void **ept_pmap, void **eptp) * the map will be used in software only, and * is bounded by that size. */ + pmap_t pmap_create_options( ledger_t ledger, @@ -1314,24 +1324,20 @@ pmap_create_options( /* * Return error when unrecognized flags are passed. */ - if ((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0) { + if (__improbable((flags & ~(PMAP_CREATE_KNOWN_FLAGS)) != 0)) { return(PMAP_NULL); } p = (pmap_t) zalloc(pmap_zone); if (PMAP_NULL == p) panic("pmap_create zalloc"); + /* Zero all fields */ bzero(p, sizeof(*p)); /* init counts now since we'll be bumping some */ simple_lock_init(&p->lock, 0); -#if 00 - p->stats.resident_count = 0; - p->stats.resident_max = 0; - p->stats.wired_count = 0; -#else bzero(&p->stats, sizeof (p->stats)); -#endif + p->ref_count = 1; p->nx_enabled = 1; p->pm_shared = FALSE; @@ -1347,10 +1353,13 @@ pmap_create_options( } p->pm_pml4 = zalloc(pmap_anchor_zone); + p->pm_upml4 = zalloc(pmap_uanchor_zone); //cleanup for EPT pmap_assert((((uintptr_t)p->pm_pml4) & PAGE_MASK) == 0); + pmap_assert((((uintptr_t)p->pm_upml4) & PAGE_MASK) == 0); memset((char *)p->pm_pml4, 0, PAGE_SIZE); + memset((char *)p->pm_upml4, 0, PAGE_SIZE); if (flags & PMAP_CREATE_EPT) { p->pm_eptp = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4) | pmap_eptp_flags; @@ -1358,6 +1367,7 @@ pmap_create_options( } else { p->pm_eptp = 0; p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_pml4); + p->pm_ucr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_upml4); } /* allocate the vm_objs to hold the pdpt, pde and pte pages */ @@ -1381,11 +1391,13 @@ pmap_create_options( pml4[KERNEL_PML4_INDEX] = kpml4[KERNEL_PML4_INDEX]; pml4[KERNEL_KEXTS_INDEX] = kpml4[KERNEL_KEXTS_INDEX]; pml4[KERNEL_PHYSMAP_PML4_INDEX] = kpml4[KERNEL_PHYSMAP_PML4_INDEX]; - + pml4[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX]; #if KASAN pml4[KERNEL_KASAN_PML4_INDEX0] = kpml4[KERNEL_KASAN_PML4_INDEX0]; pml4[KERNEL_KASAN_PML4_INDEX1] = kpml4[KERNEL_KASAN_PML4_INDEX1]; #endif + pml4_entry_t *pml4u = pmap64_user_pml4(p, 0ULL); + pml4u[KERNEL_DBLMAP_PML4_INDEX] = kpml4[KERNEL_DBLMAP_PML4_INDEX]; } #if MACH_ASSERT @@ -1556,6 +1568,7 @@ pmap_destroy(pmap_t p) int inuse_ptepages = 0; zfree(pmap_anchor_zone, p->pm_pml4); + zfree(pmap_uanchor_zone, p->pm_upml4); inuse_ptepages += p->pm_obj_pml4->resident_page_count; vm_object_deallocate(p->pm_obj_pml4); @@ -1782,6 +1795,10 @@ pmap_expand_pml4( DBG("pmap_expand_pml4(%p,%p)\n", map, (void *)vaddr); + /* With the exception of the kext "basement", the kernel's level 4 + * pagetables must not be dynamically expanded. + */ + assert(map != kernel_pmap || (vaddr == KERNEL_BASEMENT)); /* * Allocate a VM page for the pml4 page */ @@ -1847,6 +1864,13 @@ pmap_expand_pml4( | PTE_READ(is_ept) | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER) | PTE_WRITE(is_ept)); + pml4_entry_t *upml4p; + + upml4p = pmap64_user_pml4(map, vaddr); + pmap_store_pte(upml4p, pa_to_pte(pa) + | PTE_READ(is_ept) + | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER) + | PTE_WRITE(is_ept)); PMAP_UNLOCK(map); @@ -1980,12 +2004,16 @@ pmap_expand( * which is for kexts and is in the 512GB immediately below the kernel.. * XXX - should use VM_MIN_KERNEL_AND_KEXT_ADDRESS not KERNEL_BASEMENT */ - if (map == kernel_pmap && - !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS)) - panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr); + if (__improbable(map == kernel_pmap && + !(vaddr >= KERNEL_BASEMENT && vaddr <= VM_MAX_KERNEL_ADDRESS))) { + if ((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0) { + panic("pmap_expand: bad vaddr 0x%llx for kernel pmap", vaddr); + } + } while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) { + assert((options & PMAP_EXPAND_OPTIONS_ALIASMAP) == 0); kern_return_t pepkr = pmap_expand_pdpt(map, vaddr, options); if (pepkr != KERN_SUCCESS) return pepkr; @@ -2035,7 +2063,7 @@ pmap_expand( VM_PAGE_FREE(m); - OSAddAtomic(-1, &inuse_ptepages_count); + OSAddAtomic(-1, &inuse_ptepages_count);//todo replace all with inlines PMAP_ZINFO_PFREE(map, PAGE_SIZE); return KERN_SUCCESS; } @@ -2089,6 +2117,14 @@ pmap_pre_expand(pmap_t pmap, vm_map_offset_t vaddr) | PTE_READ(is_ept) | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER) | PTE_WRITE(is_ept)); + + pte = pmap64_user_pml4(pmap, vaddr); + + pmap_store_pte(pte, pa_to_pte(i386_ptob(pn)) + | PTE_READ(is_ept) + | (is_ept ? INTEL_EPT_EX : INTEL_PTE_USER) + | PTE_WRITE(is_ept)); + } if(pmap64_pde(pmap, vaddr) == PD_ENTRY_NULL) { @@ -2552,7 +2588,7 @@ pmap_flush( } orig_acks = NMIPI_acks; NMIPI_panic(cpus_to_respond, TLB_FLUSH_TIMEOUT); - panic("TLB invalidation IPI timeout, unresponsive CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu", + panic("Uninterruptible processor(s): CPU bitmap: 0x%llx, NMIPI acks: 0x%lx, now: 0x%lx, deadline: %llu", cpus_to_respond, orig_acks, NMIPI_acks, deadline); } } @@ -2646,7 +2682,7 @@ pmap_flush_tlbs(pmap_t pmap, vm_map_offset_t startv, vm_map_offset_t endv, int o continue; uint64_t cpu_active_cr3 = CPU_GET_ACTIVE_CR3(cpu); uint64_t cpu_task_cr3 = CPU_GET_TASK_CR3(cpu); - +//recall that the shadowed task cr3 is pre-composed if ((pmap_cr3 == cpu_task_cr3) || (pmap_cr3 == cpu_active_cr3) || (pmap_is_shared)) { @@ -2781,16 +2817,9 @@ process_pmap_updates(void) pmap_assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0); if (pmap_pcid_ncpus) { pmap_pcid_validate_current(); - if (cpu_datap(ccpu)->cpu_tlb_invalid_global) { - cpu_datap(ccpu)->cpu_tlb_invalid = FALSE; - tlb_flush_global(); - } - else { - cpu_datap(ccpu)->cpu_tlb_invalid_local = FALSE; - flush_tlb_raw(); - } - } - else { + cpu_datap(ccpu)->cpu_tlb_invalid = FALSE; + tlb_flush_global(); + } else { current_cpu_datap()->cpu_tlb_invalid = FALSE; flush_tlb_raw(); } @@ -3209,7 +3238,16 @@ pmap_check_ledgers( } if (pmap->stats.resident_count != 0 || +#if 35156815 + /* + * "wired_count" is unfortunately a bit inaccurate, so let's + * tolerate some slight deviation to limit the amount of + * somewhat-spurious assertion failures. + */ + pmap->stats.wired_count > 10 || +#else /* 35156815 */ pmap->stats.wired_count != 0 || +#endif /* 35156815 */ pmap->stats.device != 0 || pmap->stats.internal != 0 || pmap->stats.external != 0 || @@ -3287,11 +3325,3 @@ void pmap_verify_noncacheable(uintptr_t vaddr) { return; panic("pmap_verify_noncacheable: IO read from a cacheable address? address: 0x%lx, PTE: %p, *PTE: 0x%llx", vaddr, ptep, *ptep); } - -#if KASAN -void kasan_map_low_fixed_regions(void) { - kasan_map_shadow(MASTER_GDT_ALIAS, PAGE_SIZE, false); - kasan_map_shadow(MASTER_IDT_ALIAS, PAGE_SIZE, false); - kasan_map_shadow(LOWGLOBAL_ALIAS, PAGE_SIZE, false); -} -#endif diff --git a/osfmk/x86_64/pmap_pcid.c b/osfmk/x86_64/pmap_pcid.c index 2a7280d41..3cf4a0e49 100644 --- a/osfmk/x86_64/pmap_pcid.c +++ b/osfmk/x86_64/pmap_pcid.c @@ -58,6 +58,7 @@ uint32_t pmap_pcid_ncpus; boolean_t pmap_pcid_disabled = FALSE; +pcid_cdata_t pcid_data[MAX_CPUS] __attribute__((aligned(64))); void pmap_pcid_configure(void) { int ccpu = cpu_number(); @@ -74,6 +75,7 @@ void pmap_pcid_configure(void) { kprintf("PMAP: PCID feature disabled %u\n", pmap_pcid_disabled); } /* no_shared_cr3+PCID is currently unsupported */ + //todo remove nscr3 #if DEBUG if (pmap_pcid_disabled == FALSE) no_shared_cr3 = FALSE; @@ -134,7 +136,8 @@ void pmap_pcid_configure(void) { cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel = &(kernel_pmap->pmap_pcid_coherency_vector[ccpu]); - cpu_datap(ccpu)->cpu_pcid_refcounts[0] = 1; + cpu_datap(ccpu)->cpu_pcid_data = &pcid_data[ccpu]; + cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0] = 1; } } @@ -167,13 +170,13 @@ pcid_t pmap_pcid_allocate_pcid(int ccpu) { int i; pcid_ref_t cur_min = 0xFF; uint32_t cur_min_index = ~1; - pcid_ref_t *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_refcounts[0]; + pcid_ref_t *cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[0]; pcid_ref_t old_count; - if ((i = cpu_datap(ccpu)->cpu_pcid_free_hint) != 0) { + if ((i = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint) != 0) { if (cpu_pcid_refcounts[i] == 0) { (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1); - cpu_datap(ccpu)->cpu_pcid_free_hint = 0; + cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = 0; return i; } } @@ -193,8 +196,7 @@ pcid_t pmap_pcid_allocate_pcid(int ccpu) { if (cur_refcount == 0) { (void)__sync_fetch_and_add(&cpu_pcid_refcounts[i], 1); return i; - } - else { + } else { if (cur_refcount < cur_min) { cur_min_index = i; cur_min = cur_refcount; @@ -208,7 +210,7 @@ pcid_t pmap_pcid_allocate_pcid(int ccpu) { old_count = __sync_fetch_and_add(&cpu_pcid_refcounts[cur_min_index], 1); pmap_assert(old_count < PMAP_PCID_MAX_REFCOUNT); - return cur_min_index; + return (cur_min_index); } void pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) { @@ -221,15 +223,15 @@ void pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) { if (pcid == PMAP_PCID_INVALID_PCID) return; - lp = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid]; + lp = cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid]; pmap_assert(pcid > 0 && pcid < PMAP_PCID_MAX_PCID); - pmap_assert(cpu_datap(ccpu)->cpu_pcid_refcounts[pcid] >= 1); + pmap_assert(cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid] >= 1); if (lp == tpmap) - (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID); + (void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID); - if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_refcounts[pcid], 1)) == 1) { - cpu_datap(ccpu)->cpu_pcid_free_hint = pcid; + if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_refcounts[pcid], 1)) == 1) { + cpu_datap(ccpu)->cpu_pcid_data->cpu_pcid_free_hint = pcid; } pmap_assert(prior_count <= PMAP_PCID_MAX_REFCOUNT); } @@ -253,6 +255,7 @@ pcid_t pcid_for_pmap_cpu_tuple(pmap_t cpmap, thread_t cthread, int ccpu) { return active_pmap->pmap_pcid_cpus[ccpu]; } +int npz = 0; #if PMAP_ASSERT #define PCID_RECORD_SIZE 128 @@ -263,6 +266,7 @@ void pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t pcid_t new_pcid = tpmap->pmap_pcid_cpus[ccpu]; pmap_t last_pmap; boolean_t pcid_conflict = FALSE, pending_flush = FALSE; + pcid_cdata_t *pcdata = cpu_datap(ccpu)->cpu_pcid_data; pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled); if (__improbable(new_pcid == PMAP_PCID_INVALID_PCID)) { @@ -277,14 +281,14 @@ void pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0); if (__probable(pending_flush == FALSE)) { - last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid]; + last_pmap = pcdata->cpu_pcid_last_pmap_dispatched[new_pcid]; pcid_conflict = ((last_pmap != NULL) && (tpmap != last_pmap)); } if (__improbable(pending_flush || pcid_conflict)) { pmap_pcid_validate_cpu(tpmap, ccpu); } /* Consider making this a unique id */ - cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap; + pcdata->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap; pmap_assert(new_pcid < PMAP_PCID_MAX_PCID); pmap_assert(((tpmap == kernel_pmap) && new_pcid == 0) || @@ -306,9 +310,24 @@ void pmap_pcid_activate(pmap_t tpmap, int ccpu, boolean_t nopagezero, boolean_t ncr3 = kernel_pmap->pm_cr3; } cpu_datap(ccpu)->cpu_kernel_pcid = kernel_pmap->pmap_pcid_cpus[ccpu]; + npz++; } - set_cr3_composed(ncr3, new_pcid, !(pending_flush || pcid_conflict)); + uint64_t preserve = !(pending_flush || pcid_conflict); + set_cr3_composed(ncr3, new_pcid, preserve); +#if DEBUG + cpu_datap(ccpu)->cpu_pcid_last_cr3 = ncr3 | new_pcid | preserve << 63; +#endif + uint64_t spcid = (new_pcid + PMAP_PCID_MAX_PCID); + if (new_pcid == 0) { + spcid = 0; + } + uint64_t scr3 = tpmap->pm_ucr3 | spcid; + + cpu_datap(ccpu)->cpu_ucr3 = scr3; + cpu_shadowp(ccpu)->cpu_ucr3 = scr3; + + cpu_shadowp(ccpu)->cpu_task_cr3 = ncr3 | new_pcid; if (!pending_flush) { /* We did not previously observe a pending invalidation for this diff --git a/pexpert/pexpert/GearImage.h b/pexpert/pexpert/GearImage.h index f074d0092..d79fa3ff4 100644 --- a/pexpert/pexpert/GearImage.h +++ b/pexpert/pexpert/GearImage.h @@ -4,7 +4,3466 @@ #define kGearFrames (6) #define kGearFPS (24) - +#if !PEXPERT_NO_3X_IMAGES +const unsigned char gGearPict3x[9*kGearFrames*kGearWidth*kGearHeight] = { + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0, + 0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x94, + 0x94,0xac,0xd6,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xa1,0x76,0x6f, + 0x6f,0x76,0x9f,0xdc,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xb4,0x7c,0x6d,0x7d, + 0x7d,0x6e,0x79,0xac,0xe8,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xff,0xfe,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd5,0x90,0x75,0x75,0x79, + 0x79,0x75,0x74,0x8e,0xd4,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf7,0xe7,0xe6,0xf6,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x82,0x76,0x7a,0x72, + 0x72,0x79,0x75,0x83,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xeb,0xca,0xae,0xac,0xc4,0xe7, + 0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x76, + 0x76,0x79,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xf2,0xed,0xed,0xf3,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa4,0x86,0x82,0x7e,0x81,0xa6, + 0xdd,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x75, + 0x75,0x78,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xf8, + 0xec,0xe4,0xe3,0xe3,0xe5,0xec,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfc,0xbc,0x78,0x70,0x79,0x76,0x71,0x77, + 0xa8,0xec,0xff,0xfc,0xfe,0xff,0xff,0xff,0xf9,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74, + 0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfb,0xed, + 0xe2,0xe0,0xe1,0xe1,0xdf,0xe1,0xf0,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xe7,0xa7,0x76,0x7a,0x7e,0x7d,0x80,0x72, + 0x81,0xc4,0xf6,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfd,0xf2,0xe4, + 0xe1,0xe3,0xe2,0xe2,0xe1,0xe0,0xeb,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe2,0xa4,0x78,0x79,0x7f,0x7b,0x7c,0x7c, + 0x79,0x91,0xd4,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xff,0xf6,0xe7,0xe2, + 0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xeb,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xf2,0xad,0x72,0x75,0x80,0x77,0x76,0x81, + 0x79,0x77,0xb4,0xf8,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xfe,0xef,0xe2,0xe1, + 0xe3,0xe2,0xe2,0xe3,0xe0,0xdf,0xec,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xc1,0x7c,0x7a,0x82,0x79,0x7a,0x7e, + 0x78,0x77,0x93,0xcb,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf3,0xe8,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa0,0x81,0x7c,0x7d,0x7c,0x78, + 0x7c,0x7e,0x76,0x9e,0xef,0xff,0xfc,0xfe,0xfc,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfc,0xea,0xe1,0xe4,0xe3, + 0xe1,0xe2,0xe2,0xe2,0xe2,0xe9,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd0,0x8b,0x73,0x80,0x7e,0x77, + 0x7e,0x7f,0x6f,0x88,0xd0,0xfb,0xfe,0xff,0xfa,0xff,0xff,0xc9,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xfe,0xf5,0xe5,0xdf,0xe3,0xe3, + 0xe1,0xe3,0xe3,0xe0,0xe5,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa6,0x7c,0x7f,0x7f,0x7a, + 0x7b,0x7b,0x7a,0x7e,0x9d,0xda,0xff,0xff,0xf6,0xff,0xff,0xc9,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xf6,0xea,0xe4,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe3,0xe1,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0x99,0x79,0x7b,0x7f, + 0x79,0x79,0x81,0x76,0x77,0xba,0xff,0xff,0xf4,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74, + 0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf8,0xff,0xff,0xf0,0xe3,0xe2,0xe4,0xe1,0xe2, + 0xe3,0xe2,0xe1,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff, + 0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfa,0xb9,0x7b,0x7a,0x82, + 0x7a,0x79,0x7f,0x77,0x73,0x99,0xda,0xfd,0xfc,0xff,0xff,0xcb,0x80,0x72,0x78,0x75, + 0x75,0x78,0x71,0x7f,0xca,0xff,0xff,0xfa,0xff,0xf8,0xe9,0xe2,0xe2,0xe3,0xe2,0xe1, + 0xe3,0xe1,0xe0,0xee,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xd9,0x9a,0x7e,0x7c, + 0x7d,0x7c,0x79,0x7c,0x7d,0x7c,0xa8,0xf1,0xff,0xff,0xff,0xcc,0x80,0x75,0x7a,0x73, + 0x73,0x79,0x74,0x80,0xca,0xff,0xff,0xfc,0xfe,0xef,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe7,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xf1,0xe4, + 0xf0,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x89,0x73, + 0x80,0x80,0x77,0x7e,0x7f,0x70,0x8a,0xd4,0xf9,0xff,0xff,0xcf,0x87,0x77,0x79,0x74, + 0x74,0x78,0x77,0x8a,0xd0,0xff,0xff,0xfb,0xfa,0xe9,0xe0,0xe3,0xe2,0xe1,0xe2,0xe3, + 0xe0,0xe5,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb, + 0xf8,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xdd,0xc2,0xb2,0xa9, + 0xaf,0xc6,0xe2,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfa,0xff,0xec,0xa2,0x7c, + 0x81,0x7e,0x79,0x7d,0x7d,0x74,0x7f,0xad,0xe4,0xff,0xff,0xde,0xa0,0x74,0x6e,0x7d, + 0x7d,0x6e,0x73,0xa1,0xe2,0xff,0xff,0xfa,0xf2,0xe6,0xe1,0xe3,0xe3,0xe1,0xe3,0xe3, + 0xe0,0xe9,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8, + 0xe7,0xe9,0xed,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xe7,0xa4,0x7b,0x7d,0x80, + 0x79,0x82,0xa7,0xd7,0xf3,0xfc,0xfe,0xff,0xfe,0xff,0xfd,0xfe,0xff,0xfa,0xcf,0x96, + 0x7a,0x7c,0x7e,0x79,0x79,0x7c,0x7b,0x8f,0xd3,0xff,0xff,0xf4,0xcd,0x8c,0x6c,0x76, + 0x76,0x6c,0x89,0xca,0xf7,0xff,0xff,0xf8,0xeb,0xe4,0xe2,0xe3,0xe2,0xe2,0xe2,0xe1, + 0xe6,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf5,0xe8,0xdd,0xd9, + 0xda,0xda,0xda,0xe5,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8d,0x7b,0x83,0x84, + 0x7d,0x7f,0x86,0x90,0xaa,0xd6,0xf7,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfa,0xb5, + 0x78,0x7c,0x83,0x77,0x77,0x7e,0x79,0x87,0xce,0xff,0xff,0xfe,0xf5,0xc6,0x93,0x7a, + 0x7a,0x92,0xc3,0xf3,0xff,0xff,0xff,0xf6,0xe7,0xe3,0xe3,0xe2,0xe2,0xe3,0xe1,0xe0, + 0xed,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xfd,0xf5,0xe9,0xe0,0xdd,0xdb,0xda, + 0xdb,0xdc,0xd9,0xde,0xee,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe3,0xae,0x8c,0x85,0x89,0x89, + 0x89,0x8a,0x83,0x79,0x81,0x9f,0xbf,0xde,0xfa,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xd8, + 0x97,0x7a,0x7b,0x81,0x81,0x7c,0x79,0x92,0xd5,0xff,0xff,0xfb,0xff,0xf3,0xd7,0xc2, + 0xc2,0xd7,0xf1,0xfe,0xff,0xff,0xff,0xf6,0xe8,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe6, + 0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf6,0xee,0xe5,0xdc,0xda,0xdc,0xde,0xde, + 0xdd,0xdc,0xdb,0xdd,0xe6,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe6,0xaf,0x88,0x80,0x86,0x84, + 0x82,0x84,0x85,0x87,0x85,0x7e,0x81,0x9f,0xcd,0xee,0xfa,0xfe,0xff,0xff,0xfe,0xf7, + 0xcd,0x91,0x76,0x7c,0x79,0x73,0x86,0xb7,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfa,0xef,0xe4,0xe0,0xe2,0xe2,0xe0,0xe5,0xf3, + 0xfd,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf1,0xe6,0xdc,0xda,0xdc,0xdd,0xdc,0xdb,0xdb, + 0xdb,0xdb,0xda,0xdc,0xe7,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8b,0x7e,0x89,0x85, + 0x80,0x84,0x86,0x88,0x86,0x81,0x80,0x83,0x8b,0xa3,0xd0,0xf9,0xff,0xfc,0xfb,0xff, + 0xf5,0xc7,0x96,0x7d,0x78,0x82,0xad,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xec,0xe2,0xe0,0xe0,0xe6,0xf2,0xfc, + 0xff,0xff,0xff,0xff,0xfd,0xf3,0xe7,0xe0,0xdc,0xdb,0xdb,0xdd,0xdd,0xdc,0xdb,0xda, + 0xdc,0xdd,0xd9,0xdc,0xec,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa6,0x7f,0x7e,0x86, + 0x8a,0x88,0x83,0x82,0x84,0x87,0x8a,0x83,0x77,0x80,0x9a,0xb8,0xdb,0xf6,0xff,0xff, + 0xfe,0xf2,0xd8,0xc2,0xbf,0xca,0xe2,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8, + 0xf8,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf3,0xf1,0xf0,0xf5,0xfc,0xff, + 0xff,0xff,0xfd,0xf6,0xec,0xe5,0xde,0xda,0xdb,0xdd,0xdd,0xdb,0xda,0xda,0xdc,0xdd, + 0xdb,0xd8,0xd8,0xe4,0xf6,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd6,0xa7,0x8b,0x85, + 0x84,0x83,0x86,0x88,0x85,0x83,0x84,0x85,0x87,0x86,0x7d,0x7c,0x9a,0xce,0xf5,0xff, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf3,0xe4,0xdb,0xdc,0xde,0xdd,0xdc,0xdc,0xdb,0xdb,0xdc,0xdc,0xda,0xda, + 0xda,0xdb,0xe3,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xc9,0x9b, + 0x82,0x83,0x88,0x88,0x85,0x83,0x84,0x86,0x88,0x84,0x82,0x80,0x7e,0x94,0xc8,0xf3, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfb,0xf0,0xe3,0xdc,0xdb,0xdc,0xdd,0xdd,0xdb,0xda,0xda,0xdb,0xdc,0xdc,0xd9,0xd8, + 0xe0,0xee,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf7,0xd8, + 0xb9,0x9c,0x82,0x7c,0x86,0x8a,0x87,0x83,0x81,0x82,0x86,0x89,0x81,0x7a,0x95,0xd8, + 0xff,0xff,0xf8,0xf9,0xf9,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xff,0xff, + 0xf5,0xe3,0xdb,0xdc,0xdd,0xdc,0xdb,0xda,0xdb,0xdc,0xdc,0xda,0xd7,0xd9,0xe0,0xe9, + 0xf3,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff, + 0xf6,0xd2,0xa4,0x8e,0x8a,0x84,0x83,0x87,0x87,0x87,0x83,0x80,0x8b,0x80,0x7b,0xc1, + 0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xef,0xdb,0xdb,0xdd,0xdb,0xdb,0xdd,0xdc,0xdb,0xda,0xda,0xda,0xdb,0xe3,0xf1,0xfc, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfe,0xfb,0xef,0xd0,0xa5,0x85,0x80,0x89,0x8a,0x88,0x83,0x81,0x8c,0x81,0x79,0xc0, + 0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xed,0xd9,0xda,0xde,0xda,0xda,0xdc,0xdc,0xda,0xd8,0xda,0xe3,0xef,0xf9,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9, + 0xfb,0xff,0xff,0xf4,0xdc,0xbf,0xa0,0x87,0x7d,0x80,0x86,0x87,0x86,0x7c,0x89,0xcd, + 0xff,0xff,0xf9,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf1,0xdc,0xd8,0xdc,0xdc,0xda,0xd8,0xd7,0xd9,0xe0,0xeb,0xf4,0xfb,0xff,0xff,0xfd, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfd,0xda,0xad,0x94,0x8b,0x89,0x87,0x84,0x90,0xb6,0xe8, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf9,0xea,0xdf,0xdb,0xdb,0xda,0xda,0xdd,0xe6,0xf4,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xda,0xb9,0xa0,0x98,0xa0,0xc2,0xee,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xee,0xe2,0xdf,0xe1,0xe9,0xf3,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa, + 0xfa,0xfc,0xfd,0xfb,0xf7,0xf7,0xfc,0xff,0xfa,0xea,0xd8,0xd4,0xdd,0xee,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xfa,0xf5,0xf1,0xf3,0xf9,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xed,0xdf,0xd7,0xd6,0xd6,0xd6,0xd6,0xd6, + 0xd6,0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xd8,0xdc,0xe7,0xf5,0xfd,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,0xf3,0xf1,0xf0,0xf0,0xf0,0xf0,0xf0,0xef,0xef, + 0xef,0xef,0xef,0xef,0xef,0xf0,0xf3,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe3,0xbc,0xa6,0x9d,0x9a,0x9a,0x9a,0x9a,0x9a, + 0x9a,0x9a,0x9a,0x9a,0x9a,0x99,0x98,0x9a,0xa2,0xb3,0xd5,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xee,0xe3,0xde,0xda,0xda,0xda,0xda,0xda,0xda,0xd8,0xd8, + 0xd8,0xd8,0xd8,0xd8,0xd8,0xd9,0xdd,0xe6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfd,0xe2,0xb2,0x91,0x8e,0x90,0x8c,0x8c,0x8d,0x8d,0x8d, + 0x8d,0x8d,0x8d,0x8d,0x8c,0x8c,0x8b,0x8d,0x8f,0x8b,0x9f,0xd0,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xeb,0xda,0xd4,0xd5,0xd4,0xd3,0xd4,0xd4,0xd4,0xd4,0xd3,0xd3, + 0xd3,0xd3,0xd3,0xd3,0xd3,0xd4,0xd4,0xd5,0xe1,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbb,0x8e,0x87,0x8d,0x90,0x90,0x90,0x90,0x90,0x90, + 0x90,0x90,0x90,0x90,0x90,0x90,0x8f,0x90,0x8f,0x86,0x86,0xa5,0xdf,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf2,0xdc,0xd2,0xd2,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd5,0xd5, + 0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd3,0xd1,0xd4,0xe6,0xfb,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x86,0x94,0x91,0x8b,0x8e,0x8d,0x8d,0x8d,0x8d, + 0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8e,0x8c,0x8c,0x94,0x8d,0x90,0xce,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xec,0xd4,0xd4,0xd6,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2, + 0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd4,0xd5,0xd0,0xdd,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa7,0x88,0x95,0x91,0x8c,0x8f,0x8e,0x8e,0x8e,0x8e, + 0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8f,0x8d,0x8d,0x95,0x8f,0x93,0xcf,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xec,0xd4,0xd2,0xd4,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1, + 0xd1,0xd1,0xd1,0xd1,0xd1,0xd0,0xd2,0xd3,0xcf,0xdc,0xf9,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbd,0x90,0x89,0x8e,0x91,0x91,0x91,0x91,0x91,0x91, + 0x91,0x91,0x91,0x91,0x91,0x91,0x90,0x91,0x90,0x88,0x89,0xa9,0xe1,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf1,0xd9,0xce,0xce,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2, + 0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd0,0xce,0xd1,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfc,0xe0,0xb1,0x91,0x8f,0x91,0x8d,0x8d,0x8e,0x8e,0x8e, + 0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8d,0x8f,0x91,0x8e,0xa1,0xd1,0xf6,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfa,0xea,0xd7,0xd0,0xd1,0xd0,0xcf,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1, + 0xd1,0xd1,0xd1,0xd1,0xd1,0xd2,0xd0,0xd2,0xdf,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe3,0xbd,0xa7,0x9e,0x9b,0x9b,0x9b,0x9b,0x9b, + 0x9b,0x9b,0x9d,0x9d,0x9d,0x9d,0x9c,0x9c,0xa3,0xb4,0xd6,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfb,0xed,0xdf,0xd8,0xd4,0xd3,0xd4,0xd6,0xd6,0xd6,0xd6,0xd6, + 0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xda,0xe3,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xdd,0xd5,0xd4,0xd4,0xd4,0xd4,0xd4, + 0xd4,0xd4,0xd5,0xd5,0xd5,0xd5,0xd6,0xd7,0xda,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xff,0xff,0xfb,0xf4,0xef,0xed,0xec,0xec,0xed,0xee,0xee,0xee,0xee, + 0xee,0xee,0xee,0xee,0xee,0xee,0xf1,0xf7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa, + 0xfa,0xfb,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xe0,0xdd,0xe5,0xf3,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xf9,0xf1,0xee,0xf0,0xf7,0xfd,0xff,0xfe,0xfc,0xfc,0xfd,0xfe,0xfe,0xfd, + 0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xc4,0xaf,0xa8,0xb0,0xcd,0xf1,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xf8,0xe5,0xd6,0xd3,0xd8,0xe3,0xf0,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfe,0xe0,0xb9,0xa5,0x9d,0x9a,0x98,0x98,0xa4,0xc4,0xed, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf4,0xde,0xce,0xc8,0xca,0xcc,0xce,0xd1,0xdb,0xf0,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa, + 0xfb,0xff,0xff,0xf6,0xe3,0xca,0xae,0x97,0x8f,0x92,0x98,0x9a,0x99,0x92,0x9f,0xd7, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xe9,0xca,0xc4,0xc8,0xc9,0xc9,0xc7,0xc6,0xcb,0xd6,0xe5,0xf1,0xfb,0xff,0xff,0xfd, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfc,0xf1,0xd7,0xb1,0x96,0x92,0x9a,0x9a,0x99,0x97,0x97,0xa1,0x97,0x91,0xcb, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe2,0xc1,0xc6,0xcc,0xc7,0xc7,0xc9,0xcb,0xcc,0xc8,0xcb,0xd9,0xec,0xf8,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf7,0xd8,0xb2,0x9f,0x9a,0x95,0x94,0x99,0x9b,0x9b,0x98,0x98,0xa2,0x99,0x95,0xce, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe3,0xc2,0xc6,0xcb,0xc6,0xc7,0xc9,0xc9,0xca,0xc9,0xc9,0xcc,0xcf,0xda,0xec,0xfb, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf9,0xe1, + 0xc7,0xac,0x94,0x8e,0x98,0x9d,0x9b,0x98,0x97,0x98,0x9c,0xa0,0x9b,0x96,0xaa,0xdf, + 0xff,0xff,0xf9,0xfb,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfc,0xfc,0xfc,0xff,0xff, + 0xed,0xce,0xc2,0xc5,0xca,0xca,0xc7,0xc6,0xc7,0xca,0xcc,0xca,0xc6,0xc9,0xd5,0xe3, + 0xef,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf0,0xd2,0xab, + 0x95,0x95,0x9a,0x9b,0x99,0x98,0x99,0x9b,0x9e,0x9d,0x9c,0x9a,0x98,0xa9,0xd1,0xf4, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xf9,0xe5,0xcc,0xc2,0xc4,0xc7,0xc8,0xc9,0xc8,0xc7,0xc7,0xc9,0xca,0xca,0xc9,0xc9, + 0xd3,0xe8,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xdd,0xb6,0x9e,0x97, + 0x97,0x98,0x9a,0x9b,0x9a,0x9a,0x9a,0x9b,0x9e,0x9f,0x99,0x97,0xad,0xd7,0xf7,0xff, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfa,0xe7,0xcf,0xc2,0xc4,0xc8,0xc9,0xc8,0xc7,0xc7,0xc8,0xc9,0xc9,0xc8,0xc8, + 0xc8,0xce,0xda,0xed,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe9,0xb5,0x93,0x92,0x99, + 0x9d,0x9b,0x97,0x97,0x9a,0x9f,0xa1,0x9b,0x93,0x99,0xac,0xc4,0xe0,0xf6,0xff,0xff, + 0xfe,0xf7,0xe5,0xd6,0xd5,0xde,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc, + 0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xe8,0xe2,0xe3,0xee,0xfb,0xff, + 0xff,0xff,0xfa,0xee,0xdd,0xce,0xc4,0xc1,0xc6,0xca,0xca,0xc7,0xc6,0xc7,0xca,0xcb, + 0xc9,0xc7,0xc7,0xd8,0xf3,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xcc,0x9d,0x93,0x9d,0x9a, + 0x96,0x9a,0x9b,0x9d,0x9c,0x9a,0x9a,0x9d,0xa2,0xb4,0xd8,0xfa,0xff,0xfd,0xfc,0xff, + 0xf8,0xd8,0xb5,0xa3,0xa1,0xaa,0xc9,0xf2,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd7,0xc2,0xbd,0xbf,0xcd,0xe6,0xfb, + 0xff,0xfd,0xfe,0xff,0xfc,0xe7,0xd2,0xc8,0xc5,0xc5,0xc7,0xc8,0xca,0xc9,0xc8,0xc6, + 0xc8,0xcb,0xc6,0xcc,0xe4,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xba,0x9b,0x97,0x9b,0x99, + 0x98,0x9b,0x9d,0x9e,0x9d,0x99,0x9b,0xb2,0xd7,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf9, + 0xda,0xaf,0x9d,0xa3,0xa2,0x9f,0xae,0xd1,0xf3,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf5,0xdb,0xc2,0xb9,0xbc,0xbe,0xbc,0xca,0xe7, + 0xfb,0xfe,0xff,0xff,0xff,0xfc,0xf6,0xe7,0xd2,0xc4,0xc4,0xc8,0xca,0xc9,0xc7,0xc6, + 0xc7,0xc8,0xc6,0xc9,0xdb,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xba,0x9e,0x9a,0x9e,0x9e, + 0x9f,0xa1,0x9b,0x94,0x9b,0xb1,0xc9,0xe2,0xfb,0xff,0xfe,0xfc,0xfb,0xff,0xff,0xe4, + 0xb5,0x9f,0xa1,0xaa,0xab,0xa6,0xa5,0xb9,0xe5,0xff,0xff,0xfd,0xff,0xf9,0xeb,0xe0, + 0xe1,0xed,0xfa,0xff,0xfd,0xff,0xff,0xea,0xc8,0xba,0xbc,0xc0,0xc2,0xbe,0xbd,0xcd, + 0xed,0xff,0xff,0xfc,0xfd,0xfe,0xff,0xfd,0xee,0xdf,0xd0,0xc5,0xc3,0xc7,0xca,0xca, + 0xca,0xc9,0xc8,0xcb,0xda,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xce,0xa1,0x93,0x9b,0x9b, + 0x97,0x9a,0x9f,0xa5,0xba,0xdd,0xf8,0xff,0xff,0xfd,0xfe,0xff,0xfc,0xff,0xfc,0xca, + 0x9f,0xa2,0xa9,0xa2,0xa3,0xa8,0xa6,0xb2,0xe0,0xff,0xff,0xfe,0xfa,0xde,0xc2,0xb6, + 0xb7,0xc6,0xe2,0xfa,0xfe,0xff,0xff,0xe6,0xc1,0xba,0xbd,0xba,0xbc,0xc2,0xbe,0xbc, + 0xdb,0xfd,0xff,0xfd,0xff,0xff,0xfd,0xfe,0xff,0xfb,0xec,0xd7,0xcc,0xc8,0xc6,0xc5, + 0xc8,0xc8,0xc4,0xcc,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xba,0x98,0x96,0x98, + 0x94,0x9c,0xb7,0xdd,0xf5,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfb,0xdc,0xb4, + 0xa1,0xa4,0xa6,0xa3,0xa4,0xa7,0xa9,0xb7,0xe3,0xff,0xff,0xf8,0xe1,0xbb,0xab,0xb2, + 0xb2,0xae,0xc1,0xe4,0xf9,0xff,0xff,0xe8,0xc5,0xbb,0xbc,0xbc,0xbc,0xc0,0xbf,0xbe, + 0xcb,0xe7,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xf9,0xec,0xd6,0xc5,0xc2, + 0xc6,0xc6,0xc6,0xd9,0xf6,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe6,0xcd,0xbe,0xb8, + 0xbe,0xcf,0xe6,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf2,0xbc,0xa2, + 0xa6,0xa4,0xa2,0xa7,0xa8,0xa4,0xab,0xc9,0xed,0xff,0xff,0xed,0xc8,0xad,0xab,0xb5, + 0xb5,0xad,0xb4,0xcf,0xf0,0xff,0xff,0xf0,0xd4,0xbd,0xb8,0xbc,0xbd,0xbb,0xbe,0xc1, + 0xbe,0xd1,0xf6,0xff,0xfc,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf0,0xe3,0xd9, + 0xd6,0xda,0xe3,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf3,0xe9, + 0xf2,0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xdb,0xab,0x9c, + 0xa6,0xa7,0xa2,0xa8,0xaa,0xa2,0xb4,0xe3,0xfb,0xff,0xff,0xe2,0xb8,0xae,0xb1,0xb0, + 0xb0,0xb3,0xb6,0xc1,0xe6,0xff,0xff,0xfb,0xe8,0xc3,0xb5,0xbd,0xbe,0xbb,0xbe,0xc0, + 0xba,0xc5,0xe6,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,0xf7, + 0xf1,0xf7,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe3,0xb6,0xa4,0xa3, + 0xa4,0xa5,0xa4,0xa7,0xa9,0xaa,0xc7,0xf6,0xff,0xff,0xff,0xe1,0xb4,0xad,0xb2,0xb0, + 0xb0,0xb5,0xb4,0xba,0xe2,0xff,0xff,0xff,0xf7,0xd0,0xba,0xbc,0xbd,0xbb,0xbe,0xbf, + 0xbe,0xbf,0xce,0xed,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff, + 0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xfd,0xce,0xa1,0xa2,0xa9, + 0xa4,0xa4,0xa9,0xa4,0xa2,0xbc,0xe6,0xfe,0xfd,0xff,0xff,0xe0,0xb3,0xab,0xb1,0xb0, + 0xb0,0xb4,0xb2,0xb9,0xe2,0xff,0xff,0xfd,0xfe,0xea,0xca,0xb8,0xb9,0xbe,0xbc,0xbd, + 0xc2,0xbe,0xbf,0xde,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe, + 0xff,0xfe,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe1,0xb8,0xa1,0xa2,0xa7, + 0xa4,0xa4,0xab,0xa5,0xa6,0xd2,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb4,0xac,0xb2,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xf9,0xff,0xff,0xdb,0xb8,0xb8,0xbf,0xbb,0xbc, + 0xbf,0xbe,0xbe,0xcd,0xe9,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xbf,0xa2,0xa6,0xa6,0xa4, + 0xa6,0xa6,0xa7,0xab,0xbf,0xe7,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb4,0xac,0xb2,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfa,0xff,0xff,0xea,0xca,0xbd,0xbc,0xbc,0xbd, + 0xbc,0xbe,0xc0,0xbf,0xd3,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xad,0x9c,0xa7,0xa7,0xa3, + 0xa9,0xa9,0xa0,0xaf,0xdc,0xfb,0xff,0xff,0xfc,0xff,0xff,0xdf,0xb4,0xad,0xb2,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfa,0xe3,0xc0,0xb5,0xbd,0xbe, + 0xbb,0xbe,0xbf,0xba,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xc0,0xa8,0xa3,0xa6,0xa6,0xa3, + 0xa7,0xa9,0xa5,0xbe,0xf1,0xff,0xfd,0xff,0xfd,0xff,0xff,0xdf,0xb4,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xfe,0xff,0xf4,0xcc,0xb8,0xbd,0xbe, + 0xbc,0xbd,0xbd,0xbd,0xc1,0xd2,0xef,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd4,0xa4,0xa3,0xa9,0xa4,0xa5,0xa7, + 0xa4,0xa5,0xb8,0xdd,0xfb,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe4,0xc7,0xb9,0xbb, + 0xbe,0xbe,0xbe,0xc2,0xbd,0xbe,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf7,0xc6,0x9d,0xa0,0xa8,0xa2,0xa3,0xaa, + 0xa5,0xa5,0xcd,0xfa,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb2,0xba,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xfa,0xd7,0xb8,0xba, + 0xbe,0xba,0xbc,0xc2,0xbb,0xba,0xd8,0xfa,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xeb,0xbf,0xa1,0xa4,0xa8,0xa4,0xa6,0xa8, + 0xa7,0xb7,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0, + 0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe8,0xc5,0xba, + 0xbc,0xbd,0xbc,0xc0,0xbe,0xbd,0xd3,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xed,0xc1,0xa1,0xa4,0xa8,0xa7,0xa9,0xa1, + 0xad,0xd8,0xf9,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0, + 0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfa,0xe0,0xbe, + 0xb7,0xbf,0xbd,0xbe,0xbd,0xbb,0xd4,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xcf,0xa2,0x9d,0xa5,0xa3,0xa0,0xa3, + 0xc4,0xf2,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0, + 0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd1, + 0xb8,0xb7,0xba,0xbc,0xb7,0xbb,0xdd,0xfe,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe8,0xc1,0xad,0xab,0xa8,0xa9,0xc2, + 0xe7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0, + 0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xec, + 0xcf,0xbd,0xbe,0xc1,0xc2,0xd0,0xee,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf0,0xd9,0xc7,0xc7,0xd6,0xee, + 0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xaf,0xb3,0xb0, + 0xb0,0xb3,0xb1,0xb9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe, + 0xf1,0xdf,0xd5,0xd6,0xe3,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xed,0xed,0xf8,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb6,0xb0,0xb3,0xaf, + 0xaf,0xb3,0xb2,0xbb,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfa,0xf1,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xbf,0xaf,0xb0,0xb2, + 0xb2,0xb1,0xb2,0xc1,0xe7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb4,0xac,0xb5, + 0xb5,0xad,0xb4,0xd1,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xec,0xc7,0xaf,0xac, + 0xad,0xb1,0xc8,0xeb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe7,0xce,0xc0, + 0xc1,0xd0,0xe7,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf5, + 0xf5,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0, + 0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x94, + 0x94,0xad,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xa1,0x76,0x6f, + 0x6f,0x76,0x9f,0xdc,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xb4,0x7c,0x6d,0x7d, + 0x7d,0x6d,0x78,0xac,0xe8,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xff,0xfe,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd5,0x90,0x75,0x75,0x79, + 0x79,0x75,0x74,0x8e,0xd4,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf7,0xe7,0xe6,0xf6,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x82,0x76,0x7a,0x72, + 0x72,0x79,0x75,0x83,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xeb,0xca,0xae,0xac,0xc4,0xe7, + 0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x76, + 0x76,0x79,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xf2,0xed,0xed,0xf3,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa4,0x86,0x82,0x7e,0x81,0xa6, + 0xdd,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xf9,0xff,0xff,0xca,0x80,0x73,0x79,0x75, + 0x75,0x78,0x72,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xf8, + 0xec,0xe4,0xe3,0xe3,0xe5,0xec,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfc,0xbc,0x78,0x70,0x79,0x75,0x70,0x77, + 0xa8,0xec,0xff,0xfc,0xfe,0xff,0xff,0xff,0xf9,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74, + 0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfb,0xed, + 0xe2,0xe0,0xe1,0xe1,0xdf,0xe1,0xf0,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xe7,0xa7,0x76,0x7a,0x7e,0x7d,0x80,0x72, + 0x81,0xc4,0xf6,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfd,0xf2,0xe4, + 0xe1,0xe3,0xe2,0xe2,0xe1,0xe0,0xeb,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe2,0xa4,0x78,0x79,0x7f,0x7b,0x7c,0x7c, + 0x79,0x91,0xd4,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xff,0xf5,0xe7,0xe3, + 0xe3,0xe2,0xe2,0xe3,0xe1,0xe1,0xeb,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xf2,0xad,0x72,0x74,0x80,0x77,0x76,0x81, + 0x79,0x77,0xb4,0xf8,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xfd,0xff,0xfe,0xef,0xe2,0xe2, + 0xe4,0xe1,0xe1,0xe3,0xe0,0xdf,0xec,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xc1,0x7c,0x7a,0x82,0x79,0x7a,0x7e, + 0x78,0x77,0x93,0xcb,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf3,0xe8,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe3,0xe1,0xe1,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa0,0x81,0x7c,0x7d,0x7c,0x78, + 0x7c,0x7e,0x76,0x9e,0xef,0xff,0xfc,0xfe,0xfc,0xff,0xff,0xca,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xff,0xfc,0xea,0xe1,0xe4,0xe3, + 0xe1,0xe2,0xe2,0xe2,0xe2,0xe9,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xd0,0x89,0x72,0x80,0x7e,0x77, + 0x7f,0x7f,0x6f,0x88,0xd0,0xfb,0xfe,0xff,0xfa,0xff,0xff,0xc9,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xfa,0xff,0xff,0xfe,0xf5,0xe5,0xdf,0xe3,0xe3, + 0xe1,0xe3,0xe3,0xe0,0xe5,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x7b,0x7f,0x7d,0x79, + 0x7c,0x7b,0x7a,0x7e,0x9d,0xda,0xff,0xff,0xf6,0xff,0xff,0xc9,0x81,0x74,0x79,0x74, + 0x74,0x79,0x73,0x7f,0xc9,0xff,0xff,0xf9,0xff,0xff,0xf6,0xea,0xe4,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe3,0xe1,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0x99,0x79,0x7a,0x7e, + 0x79,0x79,0x81,0x76,0x77,0xba,0xff,0xff,0xf4,0xff,0xff,0xc9,0x81,0x75,0x7a,0x74, + 0x74,0x7a,0x74,0x80,0xc9,0xff,0xff,0xf8,0xff,0xff,0xf0,0xe2,0xe1,0xe3,0xe2,0xe2, + 0xe3,0xe2,0xe1,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff, + 0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfa,0xb9,0x7b,0x7b,0x83, + 0x79,0x79,0x7f,0x77,0x73,0x99,0xda,0xfd,0xfc,0xff,0xff,0xcb,0x81,0x72,0x78,0x75, + 0x75,0x78,0x71,0x7f,0xca,0xff,0xff,0xfa,0xff,0xf8,0xe9,0xe1,0xe1,0xe3,0xe1,0xe1, + 0xe4,0xe2,0xe0,0xee,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xd9,0x9a,0x7f,0x7d, + 0x7d,0x7c,0x79,0x7c,0x7d,0x7c,0xa8,0xf1,0xff,0xff,0xff,0xcc,0x80,0x75,0x7a,0x73, + 0x73,0x79,0x74,0x80,0xca,0xff,0xff,0xfc,0xfe,0xef,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe1,0xe7,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xf1,0xe4, + 0xf0,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x89,0x74, + 0x80,0x7e,0x76,0x7e,0x7f,0x70,0x8a,0xd4,0xf9,0xff,0xff,0xce,0x86,0x77,0x79,0x74, + 0x74,0x78,0x77,0x8a,0xd0,0xff,0xff,0xfc,0xfa,0xe8,0xe0,0xe4,0xe3,0xe1,0xe3,0xe3, + 0xdf,0xe3,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb, + 0xf8,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xdd,0xc2,0xb2,0xa9, + 0xaf,0xc6,0xe2,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xec,0xa2,0x7c, + 0x81,0x7c,0x77,0x7d,0x7d,0x74,0x7f,0xad,0xe4,0xff,0xff,0xde,0xa0,0x74,0x6e,0x7d, + 0x7d,0x6e,0x73,0xa0,0xe1,0xff,0xff,0xf9,0xf1,0xe5,0xe0,0xe3,0xe2,0xe1,0xe3,0xe3, + 0xe0,0xe8,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8, + 0xe7,0xe9,0xed,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xe7,0xa4,0x7b,0x7d,0x80, + 0x79,0x82,0xa7,0xd7,0xf3,0xfc,0xfe,0xff,0xfe,0xff,0xfd,0xfe,0xff,0xfa,0xcf,0x96, + 0x7a,0x7c,0x7e,0x79,0x79,0x7c,0x7b,0x8f,0xd3,0xff,0xff,0xf4,0xcd,0x8c,0x6c,0x76, + 0x76,0x6c,0x89,0xc9,0xf7,0xff,0xff,0xf8,0xeb,0xe4,0xe2,0xe2,0xe2,0xe2,0xe2,0xe1, + 0xe6,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf5,0xe8,0xdd,0xd9, + 0xda,0xda,0xda,0xe5,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8d,0x7b,0x83,0x84, + 0x7d,0x7f,0x87,0x90,0xaa,0xd6,0xf7,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfa,0xb5, + 0x78,0x7c,0x84,0x78,0x77,0x7e,0x79,0x87,0xce,0xff,0xff,0xfe,0xf5,0xc6,0x93,0x7a, + 0x7a,0x92,0xc3,0xf2,0xff,0xff,0xff,0xf7,0xe8,0xe3,0xe3,0xe2,0xe1,0xe4,0xe2,0xe0, + 0xed,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xfd,0xf5,0xe9,0xe0,0xdd,0xdb,0xda, + 0xdb,0xdc,0xd9,0xde,0xee,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe3,0xae,0x8c,0x86,0x89,0x89, + 0x8a,0x89,0x81,0x78,0x82,0x9f,0xbf,0xde,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xd8, + 0x96,0x79,0x7c,0x83,0x81,0x7c,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xf3,0xd7,0xc2, + 0xc2,0xd7,0xf1,0xfe,0xff,0xff,0xff,0xf7,0xe9,0xe2,0xe2,0xe3,0xe3,0xe1,0xdf,0xe5, + 0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe5,0xdc,0xd9,0xdb,0xde,0xde, + 0xdd,0xdc,0xdb,0xdd,0xe6,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xe6,0xaf,0x88,0x81,0x86,0x84, + 0x82,0x83,0x85,0x87,0x85,0x7e,0x81,0x9f,0xcd,0xee,0xfa,0xfe,0xff,0xff,0xfe,0xf7, + 0xcd,0x91,0x77,0x7d,0x7a,0x73,0x86,0xb7,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfa,0xef,0xe4,0xe0,0xe2,0xe2,0xdf,0xe4,0xf2, + 0xfd,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf1,0xe6,0xdc,0xda,0xdc,0xdd,0xdc,0xdb,0xda, + 0xda,0xdb,0xda,0xdc,0xe7,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xc2,0x8b,0x7e,0x89,0x85, + 0x80,0x84,0x86,0x88,0x86,0x81,0x7f,0x82,0x8b,0xa4,0xd0,0xf9,0xff,0xfc,0xfb,0xff, + 0xf5,0xc7,0x96,0x7d,0x78,0x82,0xad,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xec,0xe3,0xe1,0xe0,0xe5,0xf1,0xfc, + 0xff,0xff,0xfe,0xff,0xfd,0xf3,0xe7,0xe0,0xdc,0xdb,0xdb,0xdd,0xdd,0xdd,0xdc,0xd9, + 0xdb,0xdd,0xd9,0xdc,0xec,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa6,0x7f,0x7e,0x86, + 0x8a,0x88,0x83,0x81,0x83,0x87,0x8a,0x82,0x77,0x80,0x9a,0xb8,0xdb,0xf6,0xff,0xff, + 0xfe,0xf2,0xd8,0xc2,0xbf,0xca,0xe2,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8, + 0xf8,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf3,0xf0,0xf0,0xf6,0xfd,0xff, + 0xff,0xff,0xfd,0xf5,0xec,0xe5,0xde,0xda,0xdb,0xdd,0xdd,0xdb,0xda,0xda,0xdc,0xdd, + 0xdb,0xd8,0xd8,0xe4,0xf6,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd6,0xa7,0x8b,0x85, + 0x84,0x83,0x86,0x88,0x85,0x83,0x84,0x85,0x87,0x86,0x7d,0x7b,0x99,0xcf,0xf5,0xff, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf3,0xe4,0xdb,0xdc,0xde,0xdd,0xdc,0xdc,0xdb,0xdb,0xdc,0xdc,0xda,0xda, + 0xda,0xdb,0xe3,0xf2,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xc9,0x9b, + 0x82,0x83,0x88,0x88,0x85,0x83,0x84,0x86,0x87,0x84,0x81,0x7f,0x7d,0x94,0xc8,0xf3, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfb,0xf0,0xe3,0xdc,0xdb,0xdc,0xdd,0xdd,0xdb,0xda,0xda,0xdb,0xdc,0xdc,0xd9,0xd8, + 0xe0,0xee,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf7,0xd8, + 0xb9,0x9c,0x82,0x7b,0x85,0x8a,0x88,0x83,0x81,0x82,0x86,0x89,0x81,0x7a,0x96,0xd8, + 0xff,0xff,0xf8,0xf9,0xf9,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff, + 0xf5,0xe3,0xdb,0xdc,0xdd,0xdb,0xdb,0xdb,0xdb,0xdc,0xdc,0xda,0xd7,0xd9,0xe0,0xe9, + 0xf3,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff, + 0xf5,0xd2,0xa5,0x8e,0x89,0x84,0x83,0x87,0x87,0x87,0x83,0x82,0x8c,0x80,0x7b,0xc2, + 0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xef,0xdb,0xdb,0xde,0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xda,0xda,0xdb,0xe3,0xf1,0xfc, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfe,0xfb,0xef,0xd0,0xa5,0x85,0x80,0x89,0x8a,0x88,0x83,0x82,0x8d,0x80,0x78,0xbf, + 0xff,0xff,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xed,0xd9,0xda,0xdd,0xda,0xdb,0xdb,0xda,0xd9,0xd8,0xda,0xe3,0xef,0xf9,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9, + 0xfb,0xff,0xff,0xf4,0xdc,0xbf,0xa0,0x87,0x7d,0x7f,0x85,0x87,0x86,0x7c,0x88,0xcd, + 0xff,0xff,0xf9,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf0,0xdc,0xd8,0xda,0xda,0xda,0xd7,0xd5,0xd8,0xe1,0xeb,0xf4,0xfc,0xff,0xff,0xfd, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfd,0xda,0xad,0x94,0x8b,0x88,0x87,0x84,0x91,0xb7,0xe8, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf8,0xea,0xdf,0xda,0xd9,0xda,0xdb,0xde,0xe6,0xf3,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xda,0xb9,0xa0,0x98,0xa0,0xc2,0xee,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xee,0xe2,0xdf,0xe1,0xea,0xf4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa, + 0xfa,0xfc,0xfd,0xfb,0xf7,0xf7,0xfc,0xff,0xfa,0xea,0xd8,0xd4,0xdd,0xee,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xfa,0xf5,0xf1,0xf3,0xf9,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfd, + 0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xed,0xdf,0xd7,0xd6,0xd6,0xd6,0xd6,0xd6, + 0xd6,0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xd8,0xdc,0xe7,0xf5,0xfd,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfb,0xf7,0xf3,0xf1,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0,0xf0, + 0xef,0xef,0xef,0xef,0xef,0xf0,0xf3,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe3,0xbc,0xa6,0x9d,0x9a,0x9a,0x9a,0x9a,0x9a, + 0x9a,0x9a,0x9a,0x9a,0x9a,0x99,0x98,0x9a,0xa2,0xb3,0xd5,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xee,0xe3,0xde,0xda,0xda,0xda,0xda,0xda,0xda,0xda,0xd9, + 0xd8,0xd8,0xd8,0xd8,0xd8,0xd9,0xdd,0xe6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfd,0xe2,0xb2,0x91,0x8e,0x90,0x8c,0x8c,0x8d,0x8d,0x8d, + 0x8c,0x8c,0x8d,0x8d,0x8c,0x8c,0x8b,0x8d,0x8f,0x8b,0x9f,0xd0,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xeb,0xda,0xd4,0xd5,0xd4,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4, + 0xd3,0xd3,0xd3,0xd3,0xd3,0xd4,0xd4,0xd5,0xe1,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbb,0x8e,0x87,0x8d,0x90,0x90,0x90,0x90,0x90,0x90, + 0x90,0x90,0x90,0x90,0x90,0x90,0x8f,0x90,0x8f,0x86,0x86,0xa5,0xdf,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf2,0xdc,0xd2,0xd2,0xd3,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4,0xd4, + 0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd3,0xd1,0xd4,0xe6,0xfb,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf0,0xa4,0x86,0x94,0x91,0x8b,0x8e,0x8d,0x8d,0x8d,0x8d, + 0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8e,0x8c,0x8c,0x94,0x8d,0x90,0xce,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xec,0xd4,0xd4,0xd6,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2, + 0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd4,0xd5,0xd0,0xdd,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf1,0xa7,0x88,0x95,0x91,0x8c,0x8f,0x8e,0x8e,0x8e,0x8e, + 0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8f,0x8d,0x8d,0x95,0x8f,0x93,0xcf,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xec,0xd4,0xd2,0xd4,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1,0xd1, + 0xd1,0xd1,0xd1,0xd1,0xd1,0xd0,0xd2,0xd3,0xcf,0xdc,0xf9,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf6,0xbd,0x90,0x89,0x8e,0x91,0x91,0x91,0x91,0x91,0x91, + 0x91,0x91,0x91,0x91,0x91,0x91,0x90,0x91,0x90,0x88,0x89,0xa9,0xe1,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf1,0xd9,0xce,0xce,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd2, + 0xd2,0xd2,0xd2,0xd2,0xd2,0xd2,0xd0,0xce,0xd1,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfc,0xe0,0xb1,0x91,0x8f,0x91,0x8d,0x8d,0x8e,0x8e,0x8e, + 0x8e,0x8e,0x8e,0x8e,0x8e,0x8e,0x8d,0x8f,0x91,0x8e,0xa1,0xd1,0xf6,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfa,0xea,0xd7,0xd0,0xd1,0xd0,0xcf,0xd0,0xd1,0xd1,0xd1,0xd1,0xd1, + 0xd1,0xd1,0xd1,0xd1,0xd1,0xd2,0xd0,0xd2,0xdf,0xf3,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe3,0xbd,0xa7,0x9e,0x9b,0x9b,0x9b,0x9b,0x9b, + 0x9c,0x9d,0x9d,0x9d,0x9d,0x9d,0x9c,0x9c,0xa3,0xb4,0xd6,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfb,0xed,0xdf,0xd8,0xd4,0xd3,0xd4,0xd6,0xd6,0xd6,0xd6,0xd6, + 0xd6,0xd6,0xd6,0xd6,0xd6,0xd7,0xda,0xe3,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xdd,0xd5,0xd4,0xd4,0xd4,0xd4,0xd4, + 0xd5,0xd5,0xd5,0xd5,0xd5,0xd5,0xd6,0xd7,0xda,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xff,0xff,0xfb,0xf4,0xef,0xed,0xec,0xec,0xed,0xee,0xee,0xee,0xee, + 0xee,0xee,0xee,0xee,0xee,0xee,0xf1,0xf7,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfb,0xfb,0xfb,0xfb,0xfa,0xfa,0xfa, + 0xfa,0xfb,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xee,0xe1,0xdd,0xe5,0xf3,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xf9,0xf1,0xee,0xf0,0xf7,0xfd,0xff,0xfe,0xfc,0xfc,0xfd,0xfe,0xfe,0xfd, + 0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xc4,0xaf,0xa8,0xb0,0xcd,0xf1,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xf8,0xe5,0xd6,0xd3,0xd8,0xe2,0xef,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfe,0xe0,0xb9,0xa5,0x9d,0x9a,0x99,0x97,0xa3,0xc4,0xed, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf5,0xde,0xcd,0xc8,0xca,0xcc,0xcd,0xd1,0xdc,0xef,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa, + 0xfb,0xff,0xff,0xf6,0xe3,0xca,0xae,0x97,0x8f,0x92,0x98,0x9a,0x99,0x93,0x9f,0xd7, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xe9,0xca,0xc4,0xc8,0xc9,0xc9,0xc8,0xc7,0xcb,0xd6,0xe5,0xf1,0xfb,0xff,0xff,0xfd, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfc,0xf2,0xd7,0xb1,0x96,0x93,0x9a,0x9a,0x9a,0x98,0x97,0xa2,0x99,0x92,0xcb, + 0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe3,0xc2,0xc6,0xcb,0xc6,0xc8,0xcb,0xcc,0xcc,0xc8,0xcb,0xd9,0xec,0xf8,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf7,0xd9,0xb4,0xa0,0x9a,0x95,0x95,0x9a,0x9b,0x9b,0x99,0x98,0xa1,0x99,0x96,0xce, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe3,0xc2,0xc6,0xcb,0xc6,0xc7,0xc9,0xc9,0xca,0xc9,0xc9,0xcc,0xcf,0xda,0xec,0xfb, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf9,0xe1, + 0xc7,0xac,0x94,0x8f,0x98,0x9d,0x9b,0x98,0x97,0x98,0x9c,0xa1,0x9b,0x95,0xaa,0xdf, + 0xff,0xff,0xf9,0xfc,0xfc,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfc,0xfc,0xff,0xff, + 0xec,0xcd,0xc2,0xc6,0xcb,0xc9,0xc6,0xc6,0xc7,0xca,0xcc,0xca,0xc6,0xc9,0xd5,0xe3, + 0xef,0xfc,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf0,0xd2,0xab, + 0x95,0x95,0x99,0x9b,0x9a,0x98,0x99,0x9b,0x9e,0x9d,0x9c,0x9a,0x98,0xa9,0xd1,0xf4, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xf9,0xe5,0xcc,0xc3,0xc6,0xc7,0xc8,0xc9,0xc8,0xc7,0xc7,0xc9,0xca,0xca,0xc9,0xca, + 0xd4,0xe8,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xdd,0xb6,0x9e,0x97, + 0x97,0x97,0x99,0x9a,0x9a,0x9a,0x9a,0x9b,0x9e,0x9f,0x99,0x97,0xad,0xd7,0xf7,0xff, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfa,0xe7,0xcf,0xc2,0xc4,0xc8,0xc9,0xc8,0xc7,0xc7,0xc8,0xc9,0xc9,0xc8,0xca, + 0xcb,0xce,0xda,0xee,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe9,0xb5,0x93,0x92,0x99, + 0x9d,0x9b,0x98,0x98,0x9a,0x9f,0xa2,0x9b,0x93,0x99,0xac,0xc4,0xe0,0xf6,0xff,0xff, + 0xfe,0xf7,0xe5,0xd7,0xd6,0xdf,0xed,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfc, + 0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xe8,0xe2,0xe3,0xee,0xfb,0xff, + 0xff,0xff,0xfa,0xee,0xdd,0xce,0xc3,0xc2,0xc7,0xcb,0xc9,0xc7,0xc7,0xc7,0xc9,0xcb, + 0xca,0xc7,0xc8,0xda,0xf3,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xcc,0x9d,0x92,0x9d,0x9a, + 0x96,0x9a,0x9d,0x9e,0x9c,0x9a,0x9b,0x9c,0xa2,0xb4,0xd8,0xfa,0xff,0xfd,0xfc,0xff, + 0xf8,0xd8,0xb4,0xa4,0xa3,0xab,0xc9,0xf2,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd7,0xc2,0xbd,0xbf,0xcc,0xe6,0xfb, + 0xff,0xfd,0xfe,0xff,0xfc,0xe8,0xd3,0xc8,0xc6,0xc6,0xc7,0xc9,0xc9,0xc9,0xc8,0xc6, + 0xc8,0xcb,0xc7,0xcd,0xe5,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xba,0x9b,0x97,0x9b,0x99, + 0x98,0x9b,0x9d,0x9e,0x9d,0x99,0x9b,0xb2,0xd7,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf9, + 0xdb,0xb1,0x9e,0xa3,0xa3,0xa0,0xae,0xd2,0xf4,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf5,0xdb,0xc2,0xb9,0xbc,0xbe,0xbc,0xca,0xe7, + 0xfb,0xfe,0xff,0xff,0xff,0xfc,0xf6,0xe7,0xd1,0xc5,0xc6,0xc8,0xca,0xc9,0xc7,0xc6, + 0xc7,0xc8,0xc7,0xcb,0xdb,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xba,0x9e,0x9a,0x9e,0x9e, + 0x9f,0xa1,0x9b,0x94,0x9b,0xb1,0xc9,0xe2,0xfb,0xff,0xfe,0xfc,0xfb,0xff,0xff,0xe4, + 0xb6,0xa0,0xa2,0xa9,0xaa,0xa6,0xa6,0xb9,0xe5,0xff,0xff,0xfd,0xff,0xf9,0xeb,0xe0, + 0xe0,0xec,0xfa,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbd,0xc1,0xc2,0xbe,0xbe,0xcd, + 0xec,0xff,0xff,0xfc,0xfd,0xfe,0xff,0xfd,0xee,0xe0,0xd2,0xc5,0xc2,0xc7,0xca,0xca, + 0xca,0xc9,0xc8,0xcb,0xda,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf5,0xce,0xa1,0x93,0x9b,0x9b, + 0x97,0x9a,0x9f,0xa5,0xba,0xdd,0xf8,0xff,0xff,0xfd,0xfe,0xff,0xfc,0xff,0xfc,0xca, + 0x9f,0xa2,0xa9,0xa2,0xa3,0xa9,0xa8,0xb2,0xe0,0xff,0xff,0xfe,0xfa,0xde,0xc2,0xb6, + 0xb7,0xc6,0xe2,0xfa,0xfe,0xff,0xff,0xe6,0xc2,0xbb,0xbe,0xbb,0xbc,0xc3,0xbf,0xbc, + 0xdc,0xfe,0xff,0xfd,0xff,0xff,0xfd,0xfe,0xff,0xfb,0xec,0xd7,0xcc,0xc8,0xc6,0xc5, + 0xc8,0xc8,0xc4,0xcc,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xba,0x98,0x96,0x98, + 0x94,0x9c,0xb7,0xdd,0xf5,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfb,0xdd,0xb4, + 0xa1,0xa5,0xa7,0xa3,0xa4,0xa7,0xa9,0xb7,0xe3,0xff,0xff,0xf8,0xe1,0xbb,0xaa,0xb2, + 0xb2,0xaf,0xc2,0xe5,0xfa,0xff,0xff,0xe9,0xc6,0xbb,0xbc,0xbc,0xbc,0xc0,0xbf,0xbd, + 0xcd,0xe9,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xf9,0xec,0xd6,0xc5,0xc2, + 0xc6,0xc6,0xc6,0xd9,0xf6,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe6,0xcd,0xbe,0xb8, + 0xbe,0xcf,0xe6,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf2,0xbd,0xa2, + 0xa6,0xa5,0xa3,0xa7,0xa8,0xa4,0xab,0xc9,0xed,0xff,0xff,0xed,0xc8,0xad,0xab,0xb5, + 0xb5,0xae,0xb5,0xce,0xef,0xff,0xff,0xf1,0xd4,0xbd,0xb8,0xbd,0xbe,0xbb,0xbe,0xc1, + 0xbf,0xd2,0xf6,0xff,0xfc,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf0,0xe3,0xd9, + 0xd6,0xda,0xe3,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf3,0xe9, + 0xf2,0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xdb,0xab,0x9c, + 0xa6,0xa7,0xa2,0xa8,0xaa,0xa2,0xb4,0xe3,0xfb,0xff,0xff,0xe2,0xb8,0xae,0xb1,0xb0, + 0xb0,0xb4,0xb7,0xc2,0xe7,0xff,0xff,0xfb,0xe7,0xc3,0xb6,0xbe,0xbe,0xbb,0xbe,0xc0, + 0xba,0xc5,0xe6,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfd,0xff,0xff,0xf7, + 0xf1,0xf7,0xfe,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe3,0xb6,0xa4,0xa3, + 0xa4,0xa6,0xa5,0xa7,0xa9,0xaa,0xc7,0xf6,0xff,0xff,0xff,0xe1,0xb3,0xac,0xb2,0xb0, + 0xb0,0xb5,0xb5,0xbc,0xe3,0xff,0xff,0xff,0xf7,0xd2,0xbc,0xbd,0xbd,0xbb,0xbe,0xbf, + 0xbf,0xc0,0xce,0xed,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff, + 0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xfd,0xce,0xa1,0xa2,0xa9, + 0xa4,0xa5,0xa9,0xa4,0xa3,0xbd,0xe6,0xfe,0xfd,0xff,0xff,0xe1,0xb5,0xad,0xb1,0xb0, + 0xb1,0xb5,0xb3,0xbb,0xe3,0xff,0xff,0xfd,0xfe,0xeb,0xca,0xb7,0xb9,0xbe,0xbc,0xbd, + 0xc2,0xbe,0xbf,0xde,0xfd,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe, + 0xff,0xfe,0xfd,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe1,0xb8,0xa1,0xa3,0xa7, + 0xa4,0xa4,0xab,0xa6,0xa8,0xd3,0xff,0xff,0xf9,0xff,0xff,0xdf,0xb5,0xaf,0xb3,0xb0, + 0xb1,0xb5,0xb3,0xba,0xe1,0xff,0xff,0xf9,0xff,0xff,0xdb,0xb8,0xb9,0xc0,0xbc,0xbc, + 0xbf,0xbe,0xbe,0xcd,0xe9,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xbf,0xa2,0xa6,0xa6,0xa4, + 0xa6,0xa6,0xa7,0xab,0xbf,0xe7,0xff,0xff,0xf9,0xff,0xff,0xe0,0xb5,0xad,0xb2,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfa,0xff,0xff,0xea,0xca,0xbd,0xbc,0xbc,0xbd, + 0xbd,0xc0,0xc0,0xbe,0xd3,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe1,0xad,0x9c,0xa7,0xa6,0xa3, + 0xa9,0xa9,0xa1,0xb0,0xdc,0xfa,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb2,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfa,0xe3,0xc0,0xb5,0xbd,0xbe, + 0xbb,0xbf,0xc0,0xbb,0xc7,0xe9,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe9,0xc0,0xa8,0xa3,0xa6,0xa6,0xa3, + 0xa6,0xa9,0xa6,0xc0,0xf2,0xff,0xfe,0xff,0xfd,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xfe,0xff,0xf4,0xcc,0xb8,0xbd,0xbe, + 0xbc,0xbe,0xbf,0xbf,0xc2,0xd2,0xef,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd4,0xa4,0xa3,0xa9,0xa4,0xa5,0xa7, + 0xa4,0xa5,0xb8,0xdd,0xfb,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe4,0xc7,0xb9,0xbb, + 0xbe,0xbc,0xbc,0xc0,0xbd,0xc0,0xe2,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf7,0xc7,0x9d,0x9f,0xa9,0xa3,0xa2,0xa9, + 0xa6,0xa5,0xcc,0xf9,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xfa,0xd7,0xb8,0xba, + 0xbf,0xbb,0xbb,0xc0,0xbc,0xbb,0xd8,0xfa,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xeb,0xbf,0xa1,0xa4,0xa8,0xa5,0xa6,0xa8, + 0xa7,0xb7,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe8,0xc5,0xba, + 0xbd,0xbe,0xbc,0xc0,0xbe,0xbc,0xd2,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xed,0xc1,0xa1,0xa4,0xa7,0xa8,0xab,0xa3, + 0xad,0xd8,0xf9,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfa,0xe0,0xbe, + 0xb7,0xbf,0xbe,0xbf,0xbd,0xbb,0xd4,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xcf,0xa2,0x9e,0xa4,0xa3,0xa2,0xa5, + 0xc3,0xf2,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd1, + 0xb8,0xb7,0xba,0xbc,0xb7,0xbc,0xde,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe8,0xc1,0xae,0xab,0xa8,0xab,0xc3, + 0xe7,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xec, + 0xcf,0xbe,0xbe,0xc0,0xc2,0xd1,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf0,0xd9,0xc7,0xc7,0xd7,0xef, + 0xfd,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb5,0xae,0xb3,0xb0, + 0xb0,0xb4,0xb3,0xba,0xe1,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe, + 0xf1,0xdf,0xd5,0xd6,0xe3,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xed,0xed,0xf8,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe0,0xb7,0xb1,0xb3,0xaf, + 0xaf,0xb4,0xb4,0xbc,0xe3,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xf9,0xf1,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xbf,0xb0,0xb1,0xb2, + 0xb2,0xb2,0xb4,0xc2,0xe7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb4,0xac,0xb5, + 0xb5,0xad,0xb4,0xd1,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xeb,0xc7,0xb0,0xad, + 0xad,0xb1,0xc8,0xeb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe7,0xcf,0xc2, + 0xc2,0xcf,0xe8,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf5, + 0xf5,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0, + 0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xb1,0x99, + 0x99,0xb1,0xd9,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa5,0x7b,0x74, + 0x74,0x7b,0xa4,0xde,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xb8,0x82,0x73,0x82, + 0x81,0x74,0x7f,0xaf,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd7,0x97,0x7d,0x7b,0x7d, + 0x7d,0x7b,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xe9,0xe8,0xf6,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xce,0x8b,0x7e,0x7f,0x77, + 0x77,0x7f,0x7b,0x87,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xf6,0xe5,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xed,0xce,0xb4,0xb2,0xc8,0xe9, + 0xfe,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7f,0x7b, + 0x7b,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe, + 0xe7,0xc2,0xa9,0xa9,0xc6,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe1,0xab,0x90,0x8c,0x87,0x89,0xab, + 0xdf,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7e,0x7a, + 0x7a,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xdc, + 0xa2,0x7c,0x79,0x7d,0x81,0xa1,0xdd,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc0,0x81,0x7a,0x84,0x80,0x7a,0x7f, + 0xad,0xee,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7c,0x80,0x79, + 0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xff,0xfd,0xfc,0xff,0xec,0xa6, + 0x72,0x6b,0x70,0x75,0x6a,0x71,0xb9,0xfd,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xea,0xac,0x7f,0x84,0x88,0x86,0x88,0x7a, + 0x89,0xc8,0xf7,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xf6,0xc2,0x7d, + 0x6d,0x7b,0x78,0x7a,0x75,0x6f,0xa3,0xe8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe3,0xa9,0x81,0x83,0x88,0x84,0x85,0x85, + 0x81,0x98,0xd7,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xff,0xd3,0x8e,0x74, + 0x77,0x77,0x75,0x7a,0x75,0x70,0x9e,0xe1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf3,0xb3,0x7c,0x7f,0x89,0x81,0x81,0x89, + 0x81,0x7e,0xb7,0xf7,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xce,0x89,0x7b,0x80,0x7a, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xf8,0xb2,0x72,0x74, + 0x7c,0x71,0x72,0x7b,0x6e,0x6a,0xa7,0xf0,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xc5,0x84,0x82,0x8a,0x83,0x85,0x87, + 0x81,0x7f,0x99,0xce,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xce,0x89,0x7c,0x81,0x7b, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xff,0xff,0xf9,0xca,0x90,0x72,0x73, + 0x79,0x75,0x74,0x7c,0x71,0x72,0xbb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa6,0x89,0x84,0x85,0x86,0x82, + 0x85,0x86,0x7e,0xa4,0xf0,0xff,0xfc,0xff,0xfc,0xff,0xff,0xcd,0x88,0x7c,0x81,0x7b, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfc,0xff,0xfc,0xff,0xed,0x9a,0x71,0x78,0x77, + 0x73,0x77,0x76,0x75,0x78,0x98,0xd9,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd3,0x92,0x7d,0x89,0x87,0x81, + 0x88,0x88,0x79,0x8f,0xd2,0xfb,0xfe,0xff,0xfb,0xff,0xff,0xcd,0x88,0x7b,0x80,0x7a, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xfe,0xfb,0xce,0x83,0x6a,0x7a,0x7a, + 0x72,0x79,0x79,0x6a,0x81,0xcb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfc,0xff,0xf1,0xab,0x84,0x88,0x87,0x83, + 0x86,0x86,0x83,0x85,0xa3,0xdd,0xff,0xff,0xf7,0xff,0xff,0xcd,0x89,0x7c,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf6,0xff,0xff,0xda,0x9b,0x7a,0x75,0x76,0x77, + 0x74,0x78,0x78,0x72,0x9c,0xed,0xff,0xfb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xfb,0xd6,0xa1,0x82,0x83,0x88, + 0x83,0x83,0x89,0x7f,0x80,0xbf,0xff,0xff,0xf5,0xff,0xff,0xcd,0x89,0x7d,0x80,0x79, + 0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf4,0xff,0xff,0xb9,0x74,0x72,0x7c,0x74,0x74, + 0x79,0x74,0x71,0x92,0xcf,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff, + 0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfb,0xbe,0x84,0x83,0x8b, + 0x83,0x83,0x89,0x81,0x7b,0xa0,0xdd,0xfe,0xfc,0xff,0xff,0xce,0x89,0x7c,0x7f,0x7a, + 0x7a,0x7d,0x77,0x84,0xcc,0xff,0xff,0xfb,0xfe,0xda,0x97,0x6d,0x71,0x7a,0x74,0x73, + 0x7d,0x72,0x71,0xb3,0xfa,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xdc,0xa1,0x87,0x85, + 0x85,0x85,0x84,0x86,0x85,0x82,0xad,0xf1,0xff,0xff,0xff,0xcf,0x89,0x7d,0x80,0x79, + 0x78,0x7e,0x7a,0x86,0xce,0xff,0xff,0xff,0xf0,0xa7,0x78,0x78,0x77,0x74,0x76,0x76, + 0x75,0x75,0x90,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf2,0xe7, + 0xf1,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xcf,0x90,0x7b, + 0x88,0x88,0x82,0x87,0x86,0x78,0x91,0xd6,0xf9,0xff,0xff,0xd2,0x8f,0x80,0x80,0x7a, + 0x79,0x7d,0x7d,0x8f,0xd2,0xff,0xff,0xf9,0xd2,0x87,0x6c,0x7a,0x79,0x72,0x7a,0x7a, + 0x69,0x7e,0xc6,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc, + 0xf9,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfb,0xe0,0xc7,0xb9,0xb1, + 0xb6,0xcb,0xe5,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xed,0xa7,0x84, + 0x8a,0x86,0x81,0x86,0x86,0x7d,0x86,0xb2,0xe5,0xff,0xff,0xe0,0xa5,0x7d,0x77,0x84, + 0x81,0x73,0x7a,0xa5,0xe0,0xff,0xff,0xe3,0xab,0x7b,0x6f,0x78,0x79,0x72,0x76,0x79, + 0x72,0x99,0xea,0xff,0xfb,0xfd,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,0xf2,0xec, + 0xeb,0xed,0xf1,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe9,0xab,0x86,0x89,0x8a, + 0x83,0x8d,0xb0,0xda,0xf4,0xfd,0xff,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd2,0x9d, + 0x82,0x84,0x87,0x83,0x84,0x85,0x82,0x96,0xd6,0xff,0xff,0xf4,0xcf,0x92,0x74,0x7d, + 0x7c,0x72,0x8f,0xcc,0xf3,0xff,0xff,0xd2,0x8d,0x77,0x77,0x74,0x74,0x78,0x75,0x71, + 0x8e,0xca,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xf6,0xec,0xe3,0xe0, + 0xe1,0xe1,0xdf,0xe9,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc9,0x96,0x84,0x8c,0x8c, + 0x88,0x8b,0x91,0x98,0xb1,0xda,0xf8,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfb,0xbb, + 0x80,0x84,0x8b,0x82,0x82,0x87,0x81,0x8c,0xd0,0xff,0xff,0xfe,0xf5,0xc8,0x97,0x80, + 0x80,0x96,0xc6,0xf5,0xfe,0xff,0xff,0xce,0x85,0x76,0x79,0x72,0x71,0x7c,0x73,0x6f, + 0xaf,0xf9,0xff,0xfb,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf7,0xec,0xe6,0xe5,0xe3,0xe1, + 0xe1,0xe1,0xe0,0xe4,0xf2,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xb5,0x95,0x8f,0x92,0x92, + 0x92,0x92,0x8c,0x84,0x8c,0xa7,0xc4,0xe0,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xda, + 0x9e,0x82,0x84,0x8a,0x8b,0x86,0x82,0x98,0xd7,0xff,0xff,0xfc,0xff,0xf3,0xd9,0xc5, + 0xc5,0xd9,0xf2,0xff,0xfc,0xff,0xff,0xd4,0x8f,0x75,0x77,0x7d,0x7c,0x73,0x70,0x8e, + 0xd5,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf8,0xf1,0xea,0xe3,0xe0,0xe2,0xe4,0xe4, + 0xe3,0xe3,0xe3,0xe4,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x91,0x8b,0x90,0x8e, + 0x8b,0x8c,0x8e,0x91,0x8f,0x89,0x8b,0xa8,0xd3,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf8, + 0xd0,0x98,0x7e,0x84,0x83,0x7d,0x8e,0xbb,0xec,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xea,0xb4,0x82,0x6d,0x73,0x73,0x6c,0x88,0xc8, + 0xf6,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf5,0xea,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1, + 0xe2,0xe2,0xe1,0xe3,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc7,0x93,0x88,0x93,0x8f, + 0x8b,0x8e,0x8f,0x91,0x8f,0x8b,0x8a,0x8d,0x95,0xac,0xd5,0xf9,0xff,0xfd,0xfc,0xff, + 0xf5,0xcb,0x9d,0x85,0x81,0x8b,0xb2,0xe9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe8,0xa9,0x7b,0x6f,0x72,0x8d,0xc3,0xf3, + 0xff,0xfd,0xfe,0xff,0xfc,0xf5,0xeb,0xe5,0xe3,0xe2,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2, + 0xe2,0xe3,0xe0,0xe3,0xf0,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe5,0xad,0x88,0x88,0x90, + 0x94,0x91,0x8d,0x8c,0x8c,0x90,0x93,0x8c,0x82,0x8b,0xa4,0xbf,0xdf,0xf7,0xff,0xff, + 0xfe,0xf2,0xda,0xc6,0xc3,0xce,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8, + 0xf8,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xc7,0xba,0xbc,0xd4,0xf1,0xfe, + 0xff,0xff,0xfe,0xf8,0xf0,0xe9,0xe2,0xe0,0xe2,0xe4,0xe4,0xe1,0xe1,0xe1,0xe3,0xe3, + 0xe3,0xe1,0xe0,0xe9,0xf8,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xda,0xaf,0x95,0x8e, + 0x8d,0x8d,0x90,0x91,0x90,0x8d,0x8d,0x8d,0x8f,0x90,0x89,0x87,0xa3,0xd3,0xf6,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xfd,0xf6,0xeb,0xe4,0xe3,0xe4,0xe4,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe1,0xe1, + 0xe2,0xe3,0xe9,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xef,0xce,0xa3, + 0x8c,0x8d,0x91,0x92,0x8f,0x8d,0x8d,0x8e,0x90,0x8e,0x8d,0x8a,0x88,0x9e,0xcc,0xf3, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xf4,0xe7,0xe2,0xe2,0xe3,0xe3,0xe3,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1, + 0xe6,0xf1,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf8,0xdb, + 0xbf,0xa5,0x8c,0x85,0x8f,0x94,0x92,0x8d,0x8b,0x8b,0x8f,0x93,0x8b,0x85,0x9f,0xdc, + 0xff,0xff,0xf8,0xfa,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf9,0xf8,0xf8,0xfa,0xff,0xff, + 0xf7,0xea,0xe2,0xe1,0xe2,0xe3,0xe2,0xe1,0xe2,0xe3,0xe4,0xe2,0xdf,0xe0,0xe7,0xee, + 0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf6,0xd5,0xad,0x97,0x92,0x8d,0x8c,0x90,0x91,0x91,0x8c,0x8b,0x95,0x8b,0x87,0xc8, + 0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf3,0xe4,0xe3,0xe4,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,0xe3,0xe3,0xe8,0xf3,0xfc, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfe,0xfb,0xf0,0xd4,0xac,0x8f,0x8a,0x91,0x92,0x91,0x8d,0x8c,0x96,0x8a,0x82,0xc4, + 0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf1,0xe1,0xe2,0xe4,0xe1,0xe2,0xe2,0xe3,0xe3,0xdf,0xdf,0xe8,0xf4,0xfb,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa, + 0xfb,0xff,0xff,0xf5,0xe0,0xc5,0xa6,0x90,0x87,0x89,0x8e,0x90,0x8f,0x86,0x92,0xd1, + 0xff,0xff,0xfa,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff, + 0xf3,0xe3,0xe0,0xe3,0xe3,0xe2,0xe1,0xe0,0xe1,0xe6,0xed,0xf6,0xfd,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfd,0xdd,0xb4,0x9e,0x95,0x91,0x90,0x8e,0x9a,0xbd,0xea, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xee,0xe4,0xe2,0xe2,0xe2,0xe3,0xe5,0xea,0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf3,0xdd,0xc1,0xaa,0xa1,0xa8,0xc8,0xf0,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf0,0xe7,0xe5,0xe7,0xee,0xf6,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfa, + 0xfb,0xfc,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xde,0xd8,0xe0,0xf0,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xfa,0xf6,0xf4,0xf5,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xef,0xe2,0xdb,0xda,0xda,0xda,0xda,0xda, + 0xda,0xda,0xda,0xda,0xd9,0xd9,0xda,0xdb,0xdf,0xe9,0xf5,0xfd,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfc,0xf8,0xf4,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3, + 0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf6,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc2,0xaf,0xa8,0xa3,0xa3,0xa3,0xa3,0xa3, + 0xa3,0xa3,0xa3,0xa3,0xa3,0xa2,0xa1,0xa3,0xaa,0xb9,0xd7,0xf8,0xff,0xfe,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xf3,0xe9,0xe4,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1, + 0xe1,0xe1,0xe1,0xe1,0xe1,0xe2,0xe5,0xeb,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfd,0xe4,0xb8,0x9a,0x98,0x9a,0x96,0x96,0x97,0x97,0x97, + 0x97,0x97,0x97,0x97,0x97,0x97,0x96,0x98,0x9a,0x96,0xa8,0xd5,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xf0,0xe3,0xde,0xdf,0xde,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe7,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x99,0x92,0x97,0x9a,0x9a,0x9b,0x9b,0x9a,0x9a, + 0x9a,0x9b,0x9b,0x9a,0x9a,0x9b,0x9a,0x9a,0x98,0x92,0x92,0xad,0xe2,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe4,0xdb,0xdb,0xde,0xde,0xdd,0xdd,0xdd,0xdd,0xdd,0xdc,0xdc, + 0xdc,0xdd,0xdd,0xdd,0xdd,0xdc,0xda,0xd9,0xdc,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xac,0x92,0xa0,0x9c,0x96,0x99,0x98,0x98,0x99,0x99, + 0x99,0x98,0x98,0x99,0x99,0x98,0x98,0x96,0x96,0x9e,0x99,0x9a,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xf0,0xde,0xdc,0xde,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xdc,0xdd,0xda,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xae,0x94,0xa1,0x9d,0x97,0x9a,0x99,0x9a,0x9b,0x9b, + 0x9a,0x99,0x99,0x9a,0x9b,0x99,0x99,0x98,0x99,0xa0,0x9a,0x9c,0xd3,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xef,0xdc,0xdb,0xdd,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb, + 0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xdc,0xdd,0xd9,0xe3,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x9c,0x97,0x9b,0x9d,0x9d,0x9d,0x9e,0x9e,0x9e, + 0x9e,0x9d,0x9d,0x9e,0x9e,0x9e,0x9c,0x9e,0x9d,0x96,0x95,0xb0,0xe3,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf4,0xe1,0xd8,0xd9,0xda,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xda,0xd9, + 0xda,0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xd9,0xdb,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfc,0xe3,0xb9,0x9e,0x9b,0x9d,0x9b,0x9b,0x9b,0x9b,0x9b, + 0x9b,0x9b,0x9b,0x9b,0x9b,0x9b,0x9a,0x9c,0x9e,0x9a,0xaa,0xd5,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xda,0xdb,0xdb,0xdb,0xdb,0xd9,0xd8, + 0xd9,0xdb,0xdb,0xdb,0xdb,0xdc,0xdb,0xdb,0xe5,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe5,0xc3,0xb0,0xa9,0xa6,0xa6,0xa6,0xa6,0xa6, + 0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa8,0xae,0xbc,0xd8,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xf1,0xe5,0xdf,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde, + 0xde,0xde,0xde,0xde,0xde,0xde,0xe0,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xd9,0xd9,0xd9, + 0xd9,0xd9,0xd8,0xd8,0xd8,0xd8,0xd9,0xdb,0xde,0xe8,0xf5,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xff,0xff,0xfb,0xf6,0xf2,0xf2,0xf2,0xf1,0xf1,0xf1,0xf1,0xf1,0xf1, + 0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,0xf3,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfb,0xf8,0xf8,0xfc,0xff,0xfb,0xf0,0xe3,0xe0,0xe7,0xf3,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xf9,0xf3,0xf2,0xf4,0xf9,0xfe,0xff,0xfd,0xfc,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe3,0xca,0xb7,0xb0,0xb7,0xd3,0xf3,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xf8,0xea,0xde,0xdc,0xe0,0xe8,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfc,0xff,0xfe,0xe4,0xc0,0xad,0xa7,0xa6,0xa4,0xa3,0xad,0xca,0xee, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf6,0xe4,0xd8,0xd4,0xd4,0xd5,0xd6,0xd9,0xe2,0xf2,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfb, + 0xfc,0xff,0xff,0xf7,0xe6,0xd0,0xb7,0xa3,0x9b,0x9e,0xa4,0xa6,0xa5,0x9e,0xa9,0xdb, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xec,0xd2,0xcf,0xd3,0xd4,0xd3,0xd1,0xd0,0xd4,0xdd,0xe9,0xf4,0xfc,0xff,0xff,0xfe, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfc,0xf4,0xdd,0xba,0xa2,0x9f,0xa6,0xa6,0xa5,0xa3,0xa2,0xab,0xa4,0xa0,0xd2, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe7,0xcc,0xcf,0xd5,0xd1,0xd3,0xd5,0xd5,0xd5,0xd1,0xd3,0xe0,0xef,0xf9,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf8,0xdc,0xba,0xaa,0xa6,0xa2,0xa1,0xa5,0xa7,0xa7,0xa4,0xa3,0xab,0xa4,0xa1,0xd4, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe8,0xce,0xd0,0xd4,0xd0,0xd2,0xd4,0xd4,0xd4,0xd2,0xd2,0xd5,0xd8,0xe0,0xf0,0xfc, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xe4, + 0xcd,0xb6,0x9f,0x9b,0xa5,0xa9,0xa7,0xa3,0xa2,0xa3,0xa7,0xaa,0xa6,0xa1,0xb4,0xe3, + 0xff,0xff,0xfa,0xfc,0xfb,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfd,0xfd,0xff,0xff, + 0xf0,0xd8,0xce,0xd0,0xd3,0xd2,0xd0,0xd0,0xd3,0xd6,0xd6,0xd4,0xd1,0xd4,0xdd,0xe7, + 0xf2,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf2,0xd6,0xb4, + 0xa1,0xa2,0xa6,0xa6,0xa5,0xa4,0xa4,0xa6,0xa8,0xa7,0xa6,0xa7,0xa5,0xb3,0xd6,0xf6, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xea,0xd7,0xcf,0xd1,0xd1,0xd1,0xd2,0xd3,0xd3,0xd3,0xd4,0xd5,0xd5,0xd3,0xd2, + 0xdc,0xed,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xbe,0xaa,0xa3, + 0xa2,0xa2,0xa5,0xa7,0xa6,0xa4,0xa4,0xa6,0xa9,0xaa,0xa5,0xa4,0xb8,0xdb,0xf7,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xec,0xd9,0xcf,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd5,0xd4,0xd3,0xd3, + 0xd3,0xd6,0xe0,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xeb,0xbd,0x9f,0x9f,0xa6, + 0xa8,0xa6,0xa3,0xa3,0xa3,0xa7,0xaa,0xa6,0xa0,0xa6,0xb8,0xcb,0xe3,0xf7,0xff,0xff, + 0xfe,0xf8,0xe9,0xdc,0xdb,0xe2,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc, + 0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xec,0xe7,0xe7,0xf1,0xfb,0xff, + 0xff,0xff,0xfb,0xf1,0xe4,0xd9,0xd0,0xcd,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd6, + 0xd4,0xd1,0xd1,0xe0,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd2,0xa9,0x9f,0xa8,0xa5, + 0xa2,0xa5,0xa6,0xa9,0xa7,0xa4,0xa5,0xa9,0xae,0xbd,0xdd,0xfa,0xff,0xfd,0xfd,0xff, + 0xf9,0xde,0xbe,0xaf,0xad,0xb5,0xd0,0xf3,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf7,0xdf,0xcc,0xc8,0xcb,0xd6,0xe9,0xfb, + 0xff,0xfd,0xfe,0xff,0xfc,0xec,0xdb,0xd4,0xd1,0xd0,0xd0,0xd1,0xd1,0xd1,0xd2,0xd1, + 0xd3,0xd5,0xd2,0xd6,0xe9,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc3,0xa7,0xa2,0xa6,0xa5, + 0xa3,0xa3,0xa6,0xaa,0xa9,0xa5,0xa6,0xbb,0xdd,0xf3,0xfa,0xfe,0xff,0xff,0xfe,0xf9, + 0xdf,0xba,0xa9,0xad,0xae,0xac,0xb8,0xd6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf6,0xe1,0xce,0xc5,0xc7,0xca,0xc8,0xd3,0xeb, + 0xfc,0xfe,0xff,0xff,0xff,0xfd,0xf8,0xec,0xdb,0xd0,0xcf,0xd2,0xd2,0xd1,0xd0,0xd1, + 0xd3,0xd4,0xd2,0xd5,0xe2,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc2,0xa9,0xa5,0xa8,0xa8, + 0xa8,0xa9,0xa6,0xa1,0xa7,0xba,0xce,0xe5,0xfb,0xff,0xfe,0xfc,0xfc,0xff,0xff,0xe8, + 0xbf,0xaa,0xac,0xb4,0xb5,0xb1,0xb1,0xc1,0xe8,0xff,0xff,0xfd,0xff,0xfa,0xee,0xe4, + 0xe4,0xef,0xfb,0xff,0xfd,0xff,0xff,0xed,0xd0,0xc7,0xc8,0xcb,0xcb,0xc9,0xc9,0xd6, + 0xf0,0xff,0xff,0xfd,0xfd,0xff,0xff,0xfd,0xf1,0xe5,0xda,0xd0,0xce,0xd1,0xd3,0xd3, + 0xd3,0xd3,0xd1,0xd3,0xe1,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd3,0xab,0x9f,0xa6,0xa5, + 0xa1,0xa4,0xaa,0xb0,0xc2,0xe0,0xf8,0xff,0xfe,0xfd,0xff,0xff,0xfc,0xff,0xfc,0xd2, + 0xab,0xac,0xb3,0xaf,0xaf,0xb3,0xb3,0xbc,0xe4,0xff,0xff,0xfe,0xfa,0xe3,0xcb,0xc0, + 0xc1,0xce,0xe6,0xfa,0xfe,0xff,0xff,0xea,0xcb,0xc6,0xc9,0xc6,0xc6,0xcc,0xcb,0xca, + 0xe1,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xfc,0xf0,0xdf,0xd5,0xd3,0xd1,0xcf, + 0xd1,0xd1,0xce,0xd4,0xe9,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf0,0xc2,0xa2,0xa1,0xa4, + 0xa1,0xa6,0xbe,0xe0,0xf6,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfc,0xe1,0xbd, + 0xab,0xae,0xb2,0xaf,0xb0,0xb4,0xb6,0xc1,0xe7,0xff,0xff,0xfa,0xe7,0xc6,0xb7,0xbd, + 0xbd,0xba,0xca,0xe9,0xfb,0xff,0xff,0xec,0xce,0xc5,0xc7,0xc8,0xc7,0xc9,0xca,0xca, + 0xd5,0xeb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfb,0xef,0xdd,0xd1,0xce, + 0xcf,0xce,0xcf,0xe0,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe9,0xd2,0xc4,0xc1, + 0xc5,0xd3,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xc6,0xad, + 0xb0,0xb1,0xb0,0xb2,0xb2,0xaf,0xb8,0xd1,0xef,0xff,0xff,0xef,0xd0,0xba,0xb7,0xc0, + 0xc0,0xb9,0xc0,0xd7,0xf2,0xff,0xff,0xf3,0xda,0xc7,0xc4,0xc9,0xc9,0xc6,0xc9,0xcb, + 0xca,0xdb,0xf9,0xff,0xfd,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf3,0xe9,0xe1, + 0xde,0xe0,0xe8,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf4,0xea, + 0xf4,0xff,0xff,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe0,0xb5,0xa7, + 0xb0,0xb2,0xaf,0xb2,0xb3,0xad,0xbe,0xe7,0xfb,0xff,0xff,0xe6,0xc2,0xbb,0xbd,0xbb, + 0xbc,0xc0,0xc1,0xcb,0xea,0xff,0xff,0xfc,0xeb,0xcc,0xc1,0xc9,0xc9,0xc6,0xc9,0xca, + 0xc6,0xd0,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf9, + 0xf4,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xbf,0xae,0xae, + 0xb0,0xb1,0xb0,0xb2,0xb4,0xb5,0xcf,0xf7,0xff,0xff,0xff,0xe4,0xbe,0xba,0xbe,0xbb, + 0xbe,0xc2,0xbf,0xc4,0xe7,0xff,0xff,0xff,0xf9,0xd8,0xc5,0xc7,0xc8,0xc7,0xc8,0xc9, + 0xc9,0xca,0xd6,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff, + 0xfe,0xfc,0xfd,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xfd,0xd4,0xad,0xac,0xb4, + 0xb1,0xb0,0xb3,0xb1,0xb1,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xe5,0xbf,0xb9,0xbc,0xbc, + 0xbd,0xc0,0xbd,0xc4,0xe7,0xff,0xff,0xfe,0xfe,0xed,0xd1,0xc2,0xc5,0xc9,0xc7,0xc8, + 0xcb,0xc8,0xca,0xe3,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe5,0xc0,0xab,0xae,0xb3, + 0xb0,0xb0,0xb6,0xb2,0xb3,0xd8,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfa,0xff,0xff,0xe1,0xc4,0xc5,0xca,0xc7,0xc7, + 0xc9,0xc9,0xc9,0xd6,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf5,0xc8,0xae,0xaf,0xb1,0xb0, + 0xb1,0xb2,0xb3,0xb5,0xc6,0xe9,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc, + 0xbe,0xc1,0xbe,0xc5,0xe6,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc7,0xc7,0xc8,0xc8, + 0xc7,0xc9,0xca,0xc9,0xda,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe4,0xb7,0xa8,0xb1,0xb2,0xae, + 0xb3,0xb4,0xae,0xba,0xe1,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc, + 0xbd,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xfc,0xe7,0xc9,0xc1,0xc9,0xc9, + 0xc6,0xc9,0xc9,0xc5,0xd0,0xed,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc8,0xb1,0xae,0xb2,0xb2,0xaf, + 0xb2,0xb5,0xb2,0xc8,0xf5,0xff,0xfe,0xff,0xfe,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfe,0xff,0xfe,0xff,0xf6,0xd3,0xc3,0xc9,0xc9, + 0xc7,0xc8,0xc8,0xc9,0xcc,0xda,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xd9,0xaf,0xad,0xb4,0xb0,0xb0,0xb2, + 0xb0,0xb2,0xc2,0xe1,0xfc,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe8,0xd0,0xc6,0xc7, + 0xc9,0xc7,0xc7,0xcb,0xc8,0xc9,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf9,0xcd,0xa8,0xac,0xb4,0xae,0xae,0xb4, + 0xb0,0xb1,0xd4,0xfa,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xfb,0xde,0xc4,0xc6, + 0xca,0xc6,0xc7,0xca,0xc5,0xc5,0xde,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc6,0xab,0xaf,0xb3,0xb0,0xb1,0xb2, + 0xb1,0xc1,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xeb,0xce,0xc5, + 0xc9,0xc9,0xc8,0xca,0xc8,0xc6,0xd9,0xf4,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf0,0xc9,0xac,0xb0,0xb3,0xb2,0xb4,0xae, + 0xb8,0xdd,0xfa,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfb,0xe6,0xca, + 0xc4,0xca,0xc9,0xca,0xc7,0xc5,0xdb,0xf7,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd5,0xae,0xab,0xb1,0xae,0xab,0xaf, + 0xcc,0xf4,0xff,0xfd,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf7,0xd8, + 0xc4,0xc3,0xc5,0xc7,0xc3,0xc6,0xe3,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc8,0xb7,0xb5,0xb3,0xb5,0xca, + 0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef, + 0xd7,0xc9,0xc9,0xcb,0xcc,0xd8,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xde,0xce,0xcd,0xdc,0xf1, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xf4,0xe5,0xdd,0xdd,0xe8,0xf6,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xef,0xef,0xfa,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc2,0xbc,0xbe,0xba, + 0xbb,0xc0,0xbf,0xc6,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf4,0xf4,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbc,0xbe, + 0xbe,0xbe,0xbf,0xcb,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd8,0xbd,0xb8,0xc1, + 0xc1,0xba,0xc1,0xd9,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xee,0xcf,0xbb,0xb9, + 0xb9,0xbc,0xd1,0xef,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,0xd6,0xca, + 0xca,0xd7,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf6, + 0xf6,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf5,0xf0, + 0xf0,0xf5,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xb1,0x99, + 0x99,0xb1,0xd9,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe2,0xa5,0x7b,0x74, + 0x74,0x7b,0xa4,0xde,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xb8,0x82,0x73,0x82, + 0x81,0x74,0x7f,0xaf,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd7,0x97,0x7d,0x7b,0x7d, + 0x7d,0x7b,0x7a,0x92,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf8,0xe9,0xe8,0xf6,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xce,0x8b,0x7e,0x7f,0x77, + 0x77,0x7f,0x7b,0x87,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xf6,0xe5,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xed,0xce,0xb4,0xb2,0xc8,0xe9, + 0xfe,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7f,0x7b, + 0x7b,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xfe, + 0xe7,0xc2,0xa9,0xa9,0xc6,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe1,0xab,0x90,0x8c,0x87,0x89,0xab, + 0xdf,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7a,0x7e,0x7a, + 0x7a,0x7e,0x78,0x85,0xcc,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xdc, + 0xa2,0x7c,0x79,0x7d,0x81,0xa1,0xdd,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc0,0x81,0x7a,0x84,0x80,0x7a,0x7f, + 0xad,0xee,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfa,0xff,0xff,0xcd,0x88,0x7c,0x80,0x79, + 0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xff,0xfd,0xfc,0xff,0xec,0xa6, + 0x72,0x6b,0x70,0x75,0x6a,0x71,0xb9,0xfd,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xea,0xac,0x7f,0x84,0x88,0x86,0x88,0x7a, + 0x89,0xc8,0xf7,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xf6,0xc2,0x7d, + 0x6d,0x7b,0x78,0x7a,0x75,0x6f,0xa3,0xe8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xe3,0xa9,0x81,0x83,0x88,0x84,0x85,0x85, + 0x81,0x98,0xd7,0xff,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xcd,0x88,0x7b,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xff,0xd3,0x8e,0x74, + 0x77,0x77,0x75,0x7a,0x75,0x70,0x9e,0xe1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf3,0xb3,0x7c,0x7f,0x89,0x81,0x81,0x89, + 0x81,0x7e,0xb7,0xf7,0xff,0xfa,0xff,0xff,0xf9,0xff,0xff,0xce,0x89,0x7b,0x80,0x7a, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf9,0xff,0xff,0xfa,0xff,0xf8,0xb2,0x72,0x74, + 0x7c,0x71,0x72,0x7b,0x6e,0x6a,0xa7,0xf0,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xc5,0x84,0x82,0x8a,0x83,0x85,0x87, + 0x81,0x7f,0x99,0xce,0xf9,0xff,0xff,0xff,0xfb,0xff,0xff,0xce,0x89,0x7c,0x81,0x7b, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xff,0xff,0xf9,0xca,0x90,0x72,0x73, + 0x79,0x75,0x74,0x7c,0x71,0x72,0xbb,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdf,0xa6,0x89,0x84,0x85,0x86,0x82, + 0x85,0x86,0x7e,0xa4,0xf0,0xff,0xfc,0xff,0xfc,0xff,0xff,0xcd,0x88,0x7c,0x81,0x7b, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfc,0xff,0xfc,0xff,0xed,0x9a,0x71,0x78,0x77, + 0x73,0x77,0x76,0x75,0x78,0x98,0xd9,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfb,0xd3,0x92,0x7d,0x89,0x87,0x81, + 0x88,0x88,0x79,0x8f,0xd2,0xfb,0xfe,0xff,0xfb,0xff,0xff,0xcd,0x88,0x7b,0x80,0x7a, + 0x7a,0x7e,0x79,0x85,0xcb,0xff,0xff,0xfb,0xff,0xfe,0xfb,0xce,0x83,0x6a,0x7a,0x7a, + 0x72,0x79,0x79,0x6a,0x81,0xcb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xfc,0xff,0xf1,0xab,0x84,0x88,0x87,0x83, + 0x86,0x86,0x83,0x85,0xa3,0xdd,0xff,0xff,0xf7,0xff,0xff,0xcd,0x89,0x7c,0x7f,0x79, + 0x79,0x7e,0x79,0x85,0xcb,0xff,0xff,0xf6,0xff,0xff,0xda,0x9b,0x7a,0x75,0x76,0x77, + 0x74,0x78,0x78,0x72,0x9c,0xed,0xff,0xfb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xfb,0xd6,0xa1,0x82,0x83,0x88, + 0x83,0x83,0x89,0x7f,0x80,0xbf,0xff,0xff,0xf5,0xff,0xff,0xcd,0x89,0x7d,0x80,0x79, + 0x79,0x7f,0x7a,0x85,0xcb,0xff,0xff,0xf4,0xff,0xff,0xb9,0x74,0x72,0x7c,0x74,0x74, + 0x79,0x74,0x71,0x92,0xcf,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfa,0xfd,0xff, + 0xfd,0xfb,0xfc,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xfb,0xbe,0x84,0x83,0x8b, + 0x83,0x83,0x89,0x81,0x7b,0xa0,0xdd,0xfe,0xfc,0xff,0xff,0xce,0x89,0x7c,0x7f,0x7a, + 0x7a,0x7d,0x77,0x84,0xcc,0xff,0xff,0xfb,0xfe,0xda,0x97,0x6d,0x71,0x7a,0x74,0x73, + 0x7d,0x72,0x71,0xb3,0xfa,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xdc,0xa1,0x87,0x85, + 0x85,0x85,0x84,0x86,0x85,0x82,0xad,0xf1,0xff,0xff,0xff,0xcf,0x89,0x7d,0x80,0x79, + 0x78,0x7e,0x7a,0x86,0xce,0xff,0xff,0xff,0xf0,0xa7,0x78,0x78,0x77,0x74,0x76,0x76, + 0x75,0x75,0x90,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf2,0xe7, + 0xf1,0xff,0xff,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xcf,0x90,0x7b, + 0x88,0x88,0x82,0x87,0x86,0x78,0x91,0xd6,0xf9,0xff,0xff,0xd2,0x8f,0x80,0x80,0x7a, + 0x79,0x7d,0x7d,0x8f,0xd2,0xff,0xff,0xf9,0xd2,0x87,0x6c,0x7a,0x79,0x72,0x7a,0x7a, + 0x69,0x7e,0xc6,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc, + 0xf9,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfb,0xe0,0xc7,0xb9,0xb1, + 0xb6,0xcb,0xe5,0xf8,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfb,0xff,0xed,0xa7,0x84, + 0x8a,0x86,0x81,0x86,0x86,0x7d,0x86,0xb2,0xe5,0xff,0xff,0xe0,0xa5,0x7d,0x77,0x84, + 0x81,0x73,0x7a,0xa5,0xe0,0xff,0xff,0xe3,0xab,0x7b,0x6f,0x78,0x79,0x72,0x76,0x79, + 0x72,0x99,0xea,0xff,0xfb,0xfd,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xf9,0xf2,0xec, + 0xeb,0xed,0xf1,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe9,0xab,0x86,0x89,0x8a, + 0x83,0x8d,0xb0,0xda,0xf4,0xfd,0xff,0xff,0xfe,0xff,0xfe,0xfe,0xff,0xfa,0xd2,0x9d, + 0x82,0x84,0x87,0x83,0x84,0x85,0x82,0x96,0xd6,0xff,0xff,0xf4,0xcf,0x92,0x74,0x7d, + 0x7c,0x72,0x8f,0xcc,0xf3,0xff,0xff,0xd2,0x8d,0x77,0x77,0x74,0x74,0x78,0x75,0x71, + 0x8e,0xca,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xf6,0xec,0xe3,0xe0, + 0xe1,0xe1,0xdf,0xe9,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc9,0x96,0x84,0x8c,0x8c, + 0x88,0x8b,0x91,0x98,0xb1,0xda,0xf8,0xff,0xfe,0xfc,0xfe,0xff,0xfb,0xff,0xfb,0xbb, + 0x80,0x84,0x8b,0x82,0x82,0x87,0x81,0x8c,0xd0,0xff,0xff,0xfe,0xf5,0xc8,0x97,0x80, + 0x80,0x96,0xc6,0xf5,0xfe,0xff,0xff,0xce,0x85,0x76,0x79,0x72,0x71,0x7c,0x73,0x6f, + 0xaf,0xf9,0xff,0xfb,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf7,0xec,0xe6,0xe5,0xe3,0xe1, + 0xe1,0xe1,0xe0,0xe4,0xf2,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe6,0xb5,0x95,0x8f,0x92,0x92, + 0x92,0x92,0x8c,0x84,0x8c,0xa7,0xc4,0xe0,0xfb,0xff,0xfe,0xfb,0xf9,0xff,0xff,0xda, + 0x9e,0x82,0x84,0x8a,0x8b,0x86,0x82,0x98,0xd7,0xff,0xff,0xfc,0xff,0xf3,0xd9,0xc5, + 0xc5,0xd9,0xf2,0xff,0xfc,0xff,0xff,0xd4,0x8f,0x75,0x77,0x7d,0x7c,0x73,0x70,0x8e, + 0xd5,0xff,0xff,0xfa,0xff,0xff,0xff,0xfe,0xf8,0xf1,0xea,0xe3,0xe0,0xe2,0xe4,0xe4, + 0xe3,0xe3,0xe3,0xe4,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x91,0x8b,0x90,0x8e, + 0x8b,0x8c,0x8e,0x91,0x8f,0x89,0x8b,0xa8,0xd3,0xf1,0xfb,0xfe,0xff,0xff,0xfe,0xf8, + 0xd0,0x98,0x7e,0x84,0x83,0x7d,0x8e,0xbb,0xec,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xea,0xb4,0x82,0x6d,0x73,0x73,0x6c,0x88,0xc8, + 0xf6,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf5,0xea,0xe2,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1, + 0xe2,0xe2,0xe1,0xe3,0xec,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xc7,0x93,0x88,0x93,0x8f, + 0x8b,0x8e,0x8f,0x91,0x8f,0x8b,0x8a,0x8d,0x95,0xac,0xd5,0xf9,0xff,0xfd,0xfc,0xff, + 0xf5,0xcb,0x9d,0x85,0x81,0x8b,0xb2,0xe9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xe8,0xa9,0x7b,0x6f,0x72,0x8d,0xc3,0xf3, + 0xff,0xfd,0xfe,0xff,0xfc,0xf5,0xeb,0xe5,0xe3,0xe2,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2, + 0xe2,0xe3,0xe0,0xe3,0xf0,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xe5,0xad,0x88,0x88,0x90, + 0x94,0x91,0x8d,0x8c,0x8c,0x90,0x93,0x8c,0x82,0x8b,0xa4,0xbf,0xdf,0xf7,0xff,0xff, + 0xfe,0xf2,0xda,0xc6,0xc3,0xce,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfc,0xf8, + 0xf8,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xc7,0xba,0xbc,0xd4,0xf1,0xfe, + 0xff,0xff,0xfe,0xf8,0xf0,0xe9,0xe2,0xe0,0xe2,0xe4,0xe4,0xe1,0xe1,0xe1,0xe3,0xe3, + 0xe3,0xe1,0xe0,0xe9,0xf8,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xda,0xaf,0x95,0x8e, + 0x8d,0x8d,0x90,0x91,0x90,0x8d,0x8d,0x8d,0x8f,0x90,0x89,0x87,0xa3,0xd3,0xf6,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xfd,0xf6,0xeb,0xe4,0xe3,0xe4,0xe4,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe1,0xe1, + 0xe2,0xe3,0xe9,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xef,0xce,0xa3, + 0x8c,0x8d,0x91,0x92,0x8f,0x8d,0x8d,0x8e,0x90,0x8e,0x8d,0x8a,0x88,0x9e,0xcc,0xf3, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xf4,0xe7,0xe2,0xe2,0xe3,0xe3,0xe3,0xe2,0xe1,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1, + 0xe6,0xf1,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf8,0xdb, + 0xbf,0xa5,0x8c,0x85,0x8f,0x94,0x92,0x8d,0x8b,0x8b,0x8f,0x93,0x8b,0x85,0x9f,0xdc, + 0xff,0xff,0xf8,0xfa,0xf9,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf9,0xf8,0xf8,0xfa,0xff,0xff, + 0xf7,0xea,0xe2,0xe1,0xe2,0xe3,0xe2,0xe1,0xe2,0xe3,0xe4,0xe2,0xdf,0xe0,0xe7,0xee, + 0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf6,0xd5,0xad,0x97,0x92,0x8d,0x8c,0x90,0x91,0x91,0x8c,0x8b,0x95,0x8b,0x87,0xc8, + 0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf3,0xe4,0xe3,0xe4,0xe1,0xe2,0xe3,0xe3,0xe2,0xe1,0xe1,0xe3,0xe3,0xe8,0xf3,0xfc, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfe,0xfb,0xf0,0xd4,0xac,0x8f,0x8a,0x91,0x92,0x91,0x8d,0x8c,0x96,0x8a,0x82,0xc4, + 0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf1,0xe1,0xe2,0xe4,0xe1,0xe2,0xe2,0xe3,0xe3,0xdf,0xdf,0xe8,0xf4,0xfb,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfa, + 0xfb,0xff,0xff,0xf5,0xe0,0xc5,0xa6,0x90,0x87,0x89,0x8e,0x90,0x8f,0x86,0x92,0xd1, + 0xff,0xff,0xfa,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xfe,0xfd,0xff,0xff, + 0xf3,0xe3,0xe0,0xe3,0xe3,0xe2,0xe1,0xe0,0xe1,0xe6,0xed,0xf6,0xfd,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xfd,0xdd,0xb4,0x9e,0x95,0x91,0x90,0x8e,0x9a,0xbd,0xea, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xee,0xe4,0xe2,0xe2,0xe2,0xe3,0xe5,0xea,0xf5,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf3,0xdd,0xc1,0xaa,0xa1,0xa8,0xc8,0xf0,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf0,0xe7,0xe5,0xe7,0xee,0xf6,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfa, + 0xfb,0xfc,0xfd,0xfc,0xf8,0xf7,0xfc,0xff,0xfb,0xed,0xde,0xd8,0xe0,0xf0,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xfa,0xf6,0xf4,0xf5,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xef,0xe2,0xdb,0xda,0xda,0xda,0xda,0xda, + 0xda,0xda,0xda,0xda,0xd9,0xd9,0xda,0xdb,0xdf,0xe9,0xf5,0xfd,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfc,0xf8,0xf4,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf3, + 0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf6,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc2,0xaf,0xa8,0xa3,0xa3,0xa3,0xa3,0xa3, + 0xa3,0xa3,0xa3,0xa3,0xa3,0xa2,0xa1,0xa3,0xaa,0xb9,0xd7,0xf8,0xff,0xfe,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xf3,0xe9,0xe4,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1, + 0xe1,0xe1,0xe1,0xe1,0xe1,0xe2,0xe5,0xeb,0xf7,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfd,0xe4,0xb8,0x9a,0x98,0x9a,0x96,0x96,0x97,0x97,0x97, + 0x97,0x97,0x97,0x97,0x97,0x97,0x96,0x98,0x9a,0x96,0xa8,0xd5,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xf0,0xe3,0xde,0xdf,0xde,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe7,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x99,0x92,0x97,0x9a,0x9a,0x9b,0x9b,0x9a,0x9a, + 0x9a,0x9b,0x9b,0x9a,0x9a,0x9b,0x9a,0x9a,0x98,0x92,0x92,0xad,0xe2,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe4,0xdb,0xdb,0xde,0xde,0xdd,0xdd,0xdd,0xdd,0xdd,0xdc,0xdc, + 0xdc,0xdd,0xdd,0xdd,0xdd,0xdc,0xda,0xd9,0xdc,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xac,0x92,0xa0,0x9c,0x96,0x99,0x98,0x98,0x99,0x99, + 0x99,0x98,0x98,0x99,0x99,0x98,0x98,0x96,0x96,0x9e,0x99,0x9a,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xf0,0xde,0xdc,0xde,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xdc,0xdd,0xda,0xe4,0xfb,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfd,0xfb,0xff,0xf2,0xae,0x94,0xa1,0x9d,0x97,0x9a,0x99,0x9a,0x9b,0x9b, + 0x9a,0x99,0x99,0x9a,0x9b,0x99,0x99,0x98,0x99,0xa0,0x9a,0x9c,0xd3,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfd,0xff,0xff,0xef,0xdc,0xdb,0xdd,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb,0xdb, + 0xdb,0xdb,0xdb,0xdb,0xdb,0xda,0xdc,0xdd,0xd9,0xe3,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfd,0xff,0xf7,0xc3,0x9c,0x97,0x9b,0x9d,0x9d,0x9d,0x9e,0x9e,0x9e, + 0x9e,0x9d,0x9d,0x9e,0x9e,0x9e,0x9c,0x9e,0x9d,0x96,0x95,0xb0,0xe3,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf4,0xe1,0xd8,0xd9,0xda,0xdb,0xdb,0xdc,0xdc,0xdc,0xdc,0xda,0xd9, + 0xda,0xdc,0xdc,0xdc,0xdc,0xdc,0xdb,0xd9,0xdb,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfc,0xe3,0xb9,0x9e,0x9b,0x9d,0x9b,0x9b,0x9b,0x9b,0x9b, + 0x9b,0x9b,0x9b,0x9b,0x9b,0x9b,0x9a,0x9c,0x9e,0x9a,0xaa,0xd5,0xf7,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xda,0xdb,0xdb,0xdb,0xdb,0xd9,0xd8, + 0xd9,0xdb,0xdb,0xdb,0xdb,0xdc,0xdb,0xdb,0xe5,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xe5,0xc3,0xb0,0xa9,0xa6,0xa6,0xa6,0xa6,0xa6, + 0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa6,0xa8,0xae,0xbc,0xd8,0xf7,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfc,0xf1,0xe5,0xdf,0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xde, + 0xde,0xde,0xde,0xde,0xde,0xde,0xe0,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xee,0xe1,0xda,0xd9,0xd9,0xd9,0xd9,0xd9, + 0xd9,0xd9,0xd8,0xd8,0xd8,0xd8,0xd9,0xdb,0xde,0xe8,0xf5,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xff,0xff,0xfb,0xf6,0xf2,0xf2,0xf2,0xf1,0xf1,0xf1,0xf1,0xf1,0xf1, + 0xf1,0xf1,0xf1,0xf1,0xf1,0xf1,0xf3,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfb,0xfb,0xfb,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfb,0xf8,0xf8,0xfc,0xff,0xfb,0xf0,0xe3,0xe0,0xe7,0xf3,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfe,0xf9,0xf3,0xf2,0xf4,0xf9,0xfe,0xff,0xfd,0xfc,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe3,0xca,0xb7,0xb0,0xb7,0xd3,0xf3,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xf8,0xea,0xde,0xdc,0xe0,0xe8,0xf2,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfc,0xff,0xfe,0xe4,0xc0,0xad,0xa7,0xa6,0xa4,0xa3,0xad,0xca,0xee, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf6,0xe4,0xd8,0xd4,0xd4,0xd5,0xd6,0xd9,0xe2,0xf2,0xfe,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xfe,0xfb, + 0xfc,0xff,0xff,0xf7,0xe6,0xd0,0xb7,0xa3,0x9b,0x9e,0xa4,0xa6,0xa5,0x9e,0xa9,0xdb, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xec,0xd2,0xcf,0xd3,0xd4,0xd3,0xd1,0xd0,0xd4,0xdd,0xe9,0xf4,0xfc,0xff,0xff,0xfe, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xfc,0xf4,0xdd,0xba,0xa2,0x9f,0xa6,0xa6,0xa5,0xa3,0xa2,0xab,0xa4,0xa0,0xd2, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe7,0xcc,0xcf,0xd5,0xd1,0xd3,0xd5,0xd5,0xd5,0xd1,0xd3,0xe0,0xef,0xf9,0xfd,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xf8,0xdc,0xba,0xaa,0xa6,0xa2,0xa1,0xa5,0xa7,0xa7,0xa4,0xa3,0xab,0xa4,0xa1,0xd4, + 0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff, + 0xe8,0xce,0xd0,0xd4,0xd0,0xd2,0xd4,0xd4,0xd4,0xd2,0xd2,0xd5,0xd8,0xe0,0xf0,0xfc, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xe4, + 0xcd,0xb6,0x9f,0x9b,0xa5,0xa9,0xa7,0xa3,0xa2,0xa3,0xa7,0xaa,0xa6,0xa1,0xb4,0xe3, + 0xff,0xff,0xfa,0xfc,0xfb,0xfb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc,0xfd,0xfd,0xff,0xff, + 0xf0,0xd8,0xce,0xd0,0xd3,0xd2,0xd0,0xd0,0xd3,0xd6,0xd6,0xd4,0xd1,0xd4,0xdd,0xe7, + 0xf2,0xfd,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf2,0xd6,0xb4, + 0xa1,0xa2,0xa6,0xa6,0xa5,0xa4,0xa4,0xa6,0xa8,0xa7,0xa6,0xa7,0xa5,0xb3,0xd6,0xf6, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfa,0xea,0xd7,0xcf,0xd1,0xd1,0xd1,0xd2,0xd3,0xd3,0xd3,0xd4,0xd5,0xd5,0xd3,0xd2, + 0xdc,0xed,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0,0xbe,0xaa,0xa3, + 0xa2,0xa2,0xa5,0xa7,0xa6,0xa4,0xa4,0xa6,0xa9,0xaa,0xa5,0xa4,0xb8,0xdb,0xf7,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xec,0xd9,0xcf,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd5,0xd4,0xd3,0xd3, + 0xd3,0xd6,0xe0,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xeb,0xbd,0x9f,0x9f,0xa6, + 0xa8,0xa6,0xa3,0xa3,0xa3,0xa7,0xaa,0xa6,0xa0,0xa6,0xb8,0xcb,0xe3,0xf7,0xff,0xff, + 0xfe,0xf8,0xe9,0xdc,0xdb,0xe2,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfc, + 0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xec,0xe7,0xe7,0xf1,0xfb,0xff, + 0xff,0xff,0xfb,0xf1,0xe4,0xd9,0xd0,0xcd,0xd0,0xd2,0xd2,0xd1,0xd1,0xd1,0xd4,0xd6, + 0xd4,0xd1,0xd1,0xe0,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd2,0xa9,0x9f,0xa8,0xa5, + 0xa2,0xa5,0xa6,0xa9,0xa7,0xa4,0xa5,0xa9,0xae,0xbd,0xdd,0xfa,0xff,0xfd,0xfd,0xff, + 0xf9,0xde,0xbe,0xaf,0xad,0xb5,0xd0,0xf3,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf7,0xdf,0xcc,0xc8,0xcb,0xd6,0xe9,0xfb, + 0xff,0xfd,0xfe,0xff,0xfc,0xec,0xdb,0xd4,0xd1,0xd0,0xd0,0xd1,0xd1,0xd1,0xd2,0xd1, + 0xd3,0xd5,0xd2,0xd6,0xe9,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc3,0xa7,0xa2,0xa6,0xa5, + 0xa3,0xa3,0xa6,0xaa,0xa9,0xa5,0xa6,0xbb,0xdd,0xf3,0xfa,0xfe,0xff,0xff,0xfe,0xf9, + 0xdf,0xba,0xa9,0xad,0xae,0xac,0xb8,0xd6,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf6,0xe1,0xce,0xc5,0xc7,0xca,0xc8,0xd3,0xeb, + 0xfc,0xfe,0xff,0xff,0xff,0xfd,0xf8,0xec,0xdb,0xd0,0xcf,0xd2,0xd2,0xd1,0xd0,0xd1, + 0xd3,0xd4,0xd2,0xd5,0xe2,0xf6,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc2,0xa9,0xa5,0xa8,0xa8, + 0xa8,0xa9,0xa6,0xa1,0xa7,0xba,0xce,0xe5,0xfb,0xff,0xfe,0xfc,0xfc,0xff,0xff,0xe8, + 0xbf,0xaa,0xac,0xb4,0xb5,0xb1,0xb1,0xc1,0xe8,0xff,0xff,0xfd,0xff,0xfa,0xee,0xe4, + 0xe4,0xef,0xfb,0xff,0xfd,0xff,0xff,0xed,0xd0,0xc7,0xc8,0xcb,0xcb,0xc9,0xc9,0xd6, + 0xf0,0xff,0xff,0xfd,0xfd,0xff,0xff,0xfd,0xf1,0xe5,0xda,0xd0,0xce,0xd1,0xd3,0xd3, + 0xd3,0xd3,0xd1,0xd3,0xe1,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf6,0xd3,0xab,0x9f,0xa6,0xa5, + 0xa1,0xa4,0xaa,0xb0,0xc2,0xe0,0xf8,0xff,0xfe,0xfd,0xff,0xff,0xfc,0xff,0xfc,0xd2, + 0xab,0xac,0xb3,0xaf,0xaf,0xb3,0xb3,0xbc,0xe4,0xff,0xff,0xfe,0xfa,0xe3,0xcb,0xc0, + 0xc1,0xce,0xe6,0xfa,0xfe,0xff,0xff,0xea,0xcb,0xc6,0xc9,0xc6,0xc6,0xcc,0xcb,0xca, + 0xe1,0xfd,0xff,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xfc,0xf0,0xdf,0xd5,0xd3,0xd1,0xcf, + 0xd1,0xd1,0xce,0xd4,0xe9,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf0,0xc2,0xa2,0xa1,0xa4, + 0xa1,0xa6,0xbe,0xe0,0xf6,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xfe,0xff,0xfc,0xe1,0xbd, + 0xab,0xae,0xb2,0xaf,0xb0,0xb4,0xb6,0xc1,0xe7,0xff,0xff,0xfa,0xe7,0xc6,0xb7,0xbd, + 0xbd,0xba,0xca,0xe9,0xfb,0xff,0xff,0xec,0xce,0xc5,0xc7,0xc8,0xc7,0xc9,0xca,0xca, + 0xd5,0xeb,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfb,0xef,0xdd,0xd1,0xce, + 0xcf,0xce,0xcf,0xe0,0xf8,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe9,0xd2,0xc4,0xc1, + 0xc5,0xd3,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfc,0xff,0xf4,0xc6,0xad, + 0xb0,0xb1,0xb0,0xb2,0xb2,0xaf,0xb8,0xd1,0xef,0xff,0xff,0xef,0xd0,0xba,0xb7,0xc0, + 0xc0,0xb9,0xc0,0xd7,0xf2,0xff,0xff,0xf3,0xda,0xc7,0xc4,0xc9,0xc9,0xc6,0xc9,0xcb, + 0xca,0xdb,0xf9,0xff,0xfd,0xfe,0xff,0xfe,0xfd,0xfe,0xff,0xff,0xfb,0xf3,0xe9,0xe1, + 0xde,0xe0,0xe8,0xf3,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf4,0xea, + 0xf4,0xff,0xff,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe0,0xb5,0xa7, + 0xb0,0xb2,0xaf,0xb2,0xb3,0xad,0xbe,0xe7,0xfb,0xff,0xff,0xe6,0xc2,0xbb,0xbd,0xbb, + 0xbc,0xc0,0xc1,0xcb,0xea,0xff,0xff,0xfc,0xeb,0xcc,0xc1,0xc9,0xc9,0xc6,0xc9,0xca, + 0xc6,0xd0,0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf9, + 0xf4,0xf9,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xbf,0xae,0xae, + 0xb0,0xb1,0xb0,0xb2,0xb4,0xb5,0xcf,0xf7,0xff,0xff,0xff,0xe4,0xbe,0xba,0xbe,0xbb, + 0xbe,0xc2,0xbf,0xc4,0xe7,0xff,0xff,0xff,0xf9,0xd8,0xc5,0xc7,0xc8,0xc7,0xc8,0xc9, + 0xc9,0xca,0xd6,0xf0,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfe,0xff, + 0xfe,0xfc,0xfd,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xfd,0xd4,0xad,0xac,0xb4, + 0xb1,0xb0,0xb3,0xb1,0xb1,0xc6,0xe9,0xfe,0xfe,0xff,0xff,0xe5,0xbf,0xb9,0xbc,0xbc, + 0xbd,0xc0,0xbd,0xc4,0xe7,0xff,0xff,0xfe,0xfe,0xed,0xd1,0xc2,0xc5,0xc9,0xc7,0xc8, + 0xcb,0xc8,0xca,0xe3,0xfd,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfe, + 0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe5,0xc0,0xab,0xae,0xb3, + 0xb0,0xb0,0xb6,0xb2,0xb3,0xd8,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfa,0xff,0xff,0xe1,0xc4,0xc5,0xca,0xc7,0xc7, + 0xc9,0xc9,0xc9,0xd6,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf5,0xc8,0xae,0xaf,0xb1,0xb0, + 0xb1,0xb2,0xb3,0xb5,0xc6,0xe9,0xff,0xff,0xfa,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc, + 0xbe,0xc1,0xbe,0xc5,0xe6,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc7,0xc7,0xc8,0xc8, + 0xc7,0xc9,0xca,0xc9,0xda,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe4,0xb7,0xa8,0xb1,0xb2,0xae, + 0xb3,0xb4,0xae,0xba,0xe1,0xfb,0xff,0xff,0xfc,0xff,0xff,0xe4,0xc0,0xba,0xbd,0xbc, + 0xbd,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xfc,0xe7,0xc9,0xc1,0xc9,0xc9, + 0xc6,0xc9,0xc9,0xc5,0xd0,0xed,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc8,0xb1,0xae,0xb2,0xb2,0xaf, + 0xb2,0xb5,0xb2,0xc8,0xf5,0xff,0xfe,0xff,0xfe,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfe,0xff,0xfe,0xff,0xf6,0xd3,0xc3,0xc9,0xc9, + 0xc7,0xc8,0xc8,0xc9,0xcc,0xda,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xd9,0xaf,0xad,0xb4,0xb0,0xb0,0xb2, + 0xb0,0xb2,0xc2,0xe1,0xfc,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xfc,0xe8,0xd0,0xc6,0xc7, + 0xc9,0xc7,0xc7,0xcb,0xc8,0xc9,0xe6,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf9,0xcd,0xa8,0xac,0xb4,0xae,0xae,0xb4, + 0xb0,0xb1,0xd4,0xfa,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xfb,0xde,0xc4,0xc6, + 0xca,0xc6,0xc7,0xca,0xc5,0xc5,0xde,0xfa,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc6,0xab,0xaf,0xb3,0xb0,0xb1,0xb2, + 0xb1,0xc1,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xeb,0xce,0xc5, + 0xc9,0xc9,0xc8,0xca,0xc8,0xc6,0xd9,0xf4,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf0,0xc9,0xac,0xb0,0xb3,0xb2,0xb4,0xae, + 0xb8,0xdd,0xfa,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xfb,0xe6,0xca, + 0xc4,0xca,0xc9,0xca,0xc7,0xc5,0xdb,0xf7,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd5,0xae,0xab,0xb1,0xae,0xab,0xaf, + 0xcc,0xf4,0xff,0xfd,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbe,0xbc, + 0xbc,0xbf,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf7,0xd8, + 0xc4,0xc3,0xc5,0xc7,0xc3,0xc6,0xe3,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xeb,0xc8,0xb7,0xb5,0xb3,0xb5,0xca, + 0xeb,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef, + 0xd7,0xc9,0xc9,0xcb,0xcc,0xd8,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf2,0xde,0xce,0xcd,0xdc,0xf1, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc0,0xba,0xbd,0xbb, + 0xbc,0xc0,0xbe,0xc5,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xf4,0xe5,0xdd,0xdd,0xe8,0xf6,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xef,0xef,0xfa,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc2,0xbc,0xbe,0xba, + 0xbb,0xc0,0xbf,0xc6,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf4,0xf4,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xea,0xc8,0xbb,0xbc,0xbe, + 0xbe,0xbe,0xbf,0xcb,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd8,0xbd,0xb8,0xc1, + 0xc1,0xba,0xc1,0xd9,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xee,0xcf,0xbb,0xb9, + 0xb9,0xbc,0xd1,0xef,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb,0xd6,0xca, + 0xca,0xd7,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xf9,0xf6, + 0xf6,0xf9,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf2, + 0xf1,0xf6,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xea,0xc4,0xa3, + 0x9e,0xb2,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xb8,0x8d,0x80, + 0x7d,0x83,0xa6,0xd9,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x8c,0x79,0x8b, + 0x8b,0x7f,0x84,0xa9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xaf,0x88,0x81,0x87, + 0x87,0x86,0x7f,0x8b,0xcf,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xe8,0xf3,0xfe, + 0xff,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa5,0x8c,0x87,0x81, + 0x81,0x89,0x81,0x82,0xc7,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfb,0xe6,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xd7,0xbb,0xb5,0xc6,0xe7, + 0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xde,0xa5,0x88,0x84,0x85, + 0x85,0x89,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff, + 0xee,0xcb,0xae,0xac,0xc6,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc0,0x9d,0x92,0x92,0x94,0xa9, + 0xd5,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xde,0xa4,0x86,0x84,0x85, + 0x84,0x88,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xed, + 0xb3,0x83,0x7f,0x87,0x83,0x94,0xd3,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd0,0x93,0x85,0x8b,0x8c,0x88,0x86, + 0xaa,0xe7,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf9,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf4,0xb8, + 0x81,0x71,0x76,0x7c,0x6e,0x6e,0xae,0xf3,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfb,0xbc,0x85,0x8c,0x95,0x8f,0x91,0x8a, + 0x92,0xbe,0xee,0xff,0xff,0xfd,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf8,0xff,0xff,0xff,0xfe,0xfd,0xff,0xd6,0x88, + 0x72,0x83,0x7e,0x7c,0x7c,0x79,0x9b,0xd5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf4,0xb8,0x87,0x8c,0x94,0x8d,0x8d,0x91, + 0x8b,0x96,0xd2,0xff,0xff,0xfa,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf8,0xff,0xff,0xfc,0xff,0xff,0xe4,0xaa,0x80, + 0x7a,0x7f,0x7b,0x7c,0x7c,0x7a,0x97,0xd0,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc2,0x89,0x89,0x93,0x8c,0x8b,0x93, + 0x8b,0x87,0xb6,0xf0,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf8,0xff,0xff,0xf9,0xff,0xff,0xc4,0x83,0x7e, + 0x82,0x77,0x78,0x7e,0x74,0x70,0xa1,0xe4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xd3,0x98,0x8e,0x92,0x8e,0x8f,0x90, + 0x8d,0x8e,0x9b,0xc5,0xf7,0xff,0xfe,0xff,0xfd,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xfe,0xe6,0xa5,0x75,0x78, + 0x81,0x79,0x78,0x82,0x77,0x74,0xb5,0xfb,0xff,0xfb,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xbf,0x97,0x8b,0x90,0x90,0x8b, + 0x8f,0x91,0x85,0xa1,0xe7,0xff,0xfd,0xfe,0xfe,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfb,0xff,0xfd,0xff,0xf4,0xb3,0x85,0x7d,0x7b, + 0x7a,0x7b,0x7a,0x7d,0x7c,0x90,0xd1,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xe3,0xa1,0x86,0x92,0x91,0x8c, + 0x91,0x90,0x84,0x92,0xc6,0xf2,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfa,0xff,0xfd,0xff,0xdd,0x92,0x76,0x82,0x7e, + 0x76,0x7d,0x7e,0x71,0x81,0xbe,0xf2,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbc,0x94,0x8e,0x8f,0x8f, + 0x8f,0x8e,0x8f,0x8e,0x9e,0xd6,0xff,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc4,0xff,0xff,0xf7,0xff,0xff,0xe6,0xb0,0x84,0x78,0x7d,0x7c, + 0x78,0x7d,0x7e,0x72,0x95,0xe6,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xe8,0xb1,0x89,0x8c,0x93, + 0x8d,0x8c,0x92,0x89,0x87,0xbc,0xf8,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf4,0xff,0xff,0xc6,0x83,0x7b,0x81,0x79,0x7a, + 0x7d,0x7a,0x79,0x8d,0xc3,0xf7,0xff,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,0xfd,0xfe, + 0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xcc,0x93,0x8d,0x93, + 0x8d,0x8e,0x90,0x8b,0x88,0x9f,0xd1,0xfa,0xff,0xff,0xff,0xdf,0xa4,0x87,0x84,0x84, + 0x84,0x89,0x7d,0x7d,0xc5,0xff,0xff,0xf8,0xff,0xea,0xa6,0x76,0x7a,0x82,0x78,0x79, + 0x81,0x77,0x75,0xaf,0xf1,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xfd,0xf9,0xfa, + 0xfe,0xfe,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x92,0x8b, + 0x90,0x90,0x8d,0x91,0x91,0x88,0xaa,0xee,0xff,0xff,0xff,0xdf,0xa4,0x8a,0x86,0x84, + 0x82,0x89,0x7f,0x7d,0xc6,0xff,0xff,0xff,0xf6,0xba,0x88,0x7e,0x7d,0x7c,0x7a,0x7a, + 0x7e,0x79,0x88,0xcd,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf6,0xee, + 0xf5,0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xde,0x9e,0x86, + 0x92,0x92,0x8c,0x91,0x91,0x85,0x96,0xcd,0xf4,0xff,0xff,0xe2,0xac,0x8d,0x85,0x83, + 0x83,0x88,0x81,0x85,0xcb,0xff,0xff,0xfe,0xe1,0x95,0x76,0x82,0x7e,0x78,0x7e,0x7d, + 0x75,0x82,0xb5,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfb,0xfc,0xff,0xff,0xf9, + 0xed,0xeb,0xf7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xea,0xd1,0xbf,0xba, + 0xc1,0xd0,0xe4,0xf6,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf4,0xb9,0x94, + 0x91,0x8e,0x8c,0x8f,0x8f,0x8c,0x8f,0xa9,0xdf,0xff,0xff,0xee,0xc0,0x89,0x7a,0x8a, + 0x8b,0x82,0x80,0x98,0xd8,0xff,0xff,0xef,0xc1,0x85,0x74,0x81,0x7f,0x76,0x7d,0x7f, + 0x72,0x90,0xdf,0xff,0xfa,0xfd,0xff,0xfe,0xfb,0xfb,0xfe,0xff,0xfc,0xeb,0xcd,0xb4, + 0xa8,0xa7,0xb3,0xd5,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf3,0xc0,0x99,0x93,0x94, + 0x90,0x95,0xb1,0xd8,0xf2,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfd,0xe2,0xae, + 0x8c,0x8e,0x92,0x8c,0x8b,0x91,0x8b,0x93,0xd2,0xff,0xff,0xfb,0xe0,0x9f,0x7e,0x86, + 0x85,0x7d,0x91,0xc1,0xec,0xff,0xff,0xde,0xa4,0x83,0x7c,0x7c,0x7a,0x7a,0x7c,0x7a, + 0x83,0xb4,0xf4,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xf7,0xde,0xae,0x7f,0x6b, + 0x70,0x70,0x6a,0x91,0xdb,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xda,0xa5,0x91,0x99,0x98, + 0x92,0x95,0x9a,0xa0,0xb4,0xd9,0xf6,0xff,0xff,0xfc,0xfe,0xff,0xfb,0xff,0xff,0xc8, + 0x91,0x90,0x95,0x8b,0x8b,0x92,0x89,0x8b,0xcc,0xff,0xff,0xfc,0xfa,0xd8,0xa9,0x8d, + 0x86,0x94,0xbf,0xee,0xfd,0xff,0xff,0xda,0x9b,0x81,0x7d,0x78,0x77,0x7f,0x78,0x71, + 0xa4,0xe8,0xff,0xfe,0xff,0xfd,0xfd,0xfc,0xfe,0xfe,0xe3,0xaf,0x8a,0x7d,0x74,0x6f, + 0x72,0x73,0x6e,0x7b,0xac,0xe7,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc8,0xa2,0x97,0x9e,0x9e, + 0x9d,0x9e,0x99,0x8f,0x95,0xad,0xc6,0xdf,0xf9,0xff,0xfe,0xfd,0xfb,0xff,0xff,0xe6, + 0xb2,0x8e,0x8b,0x94,0x94,0x93,0x8a,0x91,0xd0,0xff,0xff,0xfa,0xff,0xfa,0xe2,0xcb, + 0xc7,0xd5,0xed,0xfd,0xfe,0xff,0xff,0xe2,0xa6,0x7e,0x79,0x81,0x81,0x7d,0x75,0x82, + 0xc8,0xff,0xff,0xf7,0xfb,0xfe,0xff,0xfb,0xe2,0xc2,0x9f,0x7c,0x6c,0x71,0x7a,0x7d, + 0x7b,0x79,0x7a,0x7b,0x93,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc9,0xa1,0x95,0x9c,0x9c, + 0x98,0x97,0x99,0x9b,0x99,0x95,0x94,0xa9,0xd0,0xee,0xfa,0xfe,0xff,0xfe,0xfd,0xfe, + 0xdf,0xa6,0x8a,0x8e,0x8c,0x89,0x94,0xb4,0xe5,0xff,0xff,0xfd,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf4,0xc6,0x8c,0x73,0x79,0x79,0x73,0x86,0xb6, + 0xeb,0xff,0xff,0xfe,0xff,0xfb,0xf1,0xd6,0xa6,0x7b,0x6e,0x76,0x7a,0x77,0x74,0x73, + 0x74,0x76,0x74,0x76,0x93,0xd7,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xd9,0xa2,0x93,0xa1,0x9d, + 0x98,0x9b,0x9a,0x9a,0x99,0x97,0x95,0x97,0x9f,0xb0,0xd3,0xf7,0xff,0xfd,0xfc,0xff, + 0xfa,0xda,0xae,0x91,0x8a,0x91,0xb4,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xee,0xb7,0x88,0x79,0x78,0x88,0xb8,0xee, + 0xff,0xfc,0xfd,0xff,0xfa,0xd4,0xa4,0x85,0x7a,0x73,0x71,0x75,0x79,0x78,0x75,0x72, + 0x76,0x79,0x71,0x78,0xa7,0xe4,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xef,0xbb,0x98,0x95,0x9c, + 0xa1,0x9f,0x9a,0x98,0x98,0x9b,0x9e,0x99,0x90,0x94,0xa8,0xc0,0xdb,0xf2,0xfe,0xff, + 0xff,0xfa,0xe4,0xcc,0xc7,0xcf,0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xf9, + 0xf9,0xfb,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe8,0xcf,0xbf,0xbf,0xcf,0xe9,0xfc, + 0xff,0xff,0xfb,0xe0,0xb8,0x97,0x79,0x6a,0x71,0x7c,0x7b,0x75,0x72,0x73,0x78,0x7b, + 0x7a,0x71,0x6c,0x91,0xd4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe4,0xbc,0xa0,0x9b, + 0x9b,0x9a,0x9c,0x9e,0x9d,0x9a,0x98,0x98,0x9a,0x9c,0x96,0x91,0xa3,0xce,0xf3,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xff,0xf8,0xd6,0x9c,0x74,0x70,0x79,0x7a,0x76,0x74,0x74,0x76,0x78,0x77,0x75,0x72, + 0x72,0x7a,0x93,0xc6,0xf8,0xff,0xfd,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf3,0xd6,0xaf, + 0x99,0x99,0x9d,0x9e,0x9d,0x9a,0x99,0x99,0x9c,0x9b,0x98,0x95,0x94,0xa3,0xc6,0xee, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xf9,0xd2,0x92,0x72,0x74,0x75,0x76,0x79,0x77,0x74,0x73,0x75,0x78,0x78,0x73,0x6e, + 0x80,0xae,0xe0,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0, + 0xc7,0xb1,0x9b,0x93,0x9a,0x9f,0x9f,0x9b,0x97,0x96,0x9a,0x9d,0x9a,0x91,0x9d,0xd5, + 0xff,0xff,0xf9,0xfa,0xfa,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfa,0xf9,0xf8,0xf8,0xff,0xff, + 0xe2,0xa1,0x71,0x6e,0x7d,0x79,0x72,0x71,0x73,0x78,0x7c,0x78,0x6b,0x6d,0x87,0xa7, + 0xc8,0xe9,0xfc,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xf8,0xdd,0xba,0xa5,0x9f,0x9a,0x99,0x9d,0x9c,0x9c,0x99,0x97,0x9f,0x95,0x8f,0xc5, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff, + 0xc8,0x81,0x77,0x7b,0x74,0x74,0x77,0x79,0x78,0x73,0x71,0x76,0x7b,0x90,0xbd,0xec, + 0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff, + 0xfe,0xfc,0xf5,0xdf,0xba,0x9c,0x96,0x9d,0x9e,0x9e,0x9b,0x99,0x9f,0x97,0x91,0xc2, + 0xfb,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff, + 0xc5,0x7b,0x75,0x7c,0x73,0x75,0x78,0x7a,0x79,0x6f,0x6e,0x8a,0xbd,0xe6,0xf7,0xfd, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xfb, + 0xfc,0xff,0xff,0xf9,0xe6,0xcd,0xb4,0xa1,0x95,0x95,0x9b,0x9d,0x9e,0x94,0x98,0xcd, + 0xff,0xff,0xfb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xfa,0xff,0xff, + 0xd4,0x8c,0x6f,0x72,0x78,0x77,0x70,0x6a,0x6f,0x8b,0xaf,0xce,0xee,0xff,0xff,0xfa, + 0xf9,0xfd,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfc,0xff,0xff,0xe8,0xc3,0xaa,0xa1,0x9e,0x9b,0x9c,0xa3,0xb9,0xe5, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xed,0xb9,0x86,0x72,0x76,0x76,0x77,0x7f,0x96,0xca,0xf7,0xff,0xfd,0xfd,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe5,0xcc,0xb5,0xa8,0xad,0xc9,0xec,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xee,0xbe,0x96,0x89,0x8d,0xa4,0xc9,0xeb,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfc,0xf9,0xf8,0xfb,0xff,0xfe,0xf1,0xe1,0xda,0xdf,0xee,0xfd,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xee,0xd9,0xcc,0xd0,0xe3,0xf6,0xfe,0xfe,0xfb,0xfa,0xfc,0xff,0xff,0xfe, + 0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe9,0xe0,0xdf,0xdf,0xdf,0xdf,0xdf, + 0xdf,0xdf,0xde,0xdd,0xdd,0xdd,0xdd,0xdf,0xe3,0xe9,0xf4,0xfe,0xff,0xfd,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfb,0xfa,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf4,0xf4,0xf4,0xf4,0xf4, + 0xf4,0xf5,0xf4,0xf4,0xf5,0xf5,0xf7,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xd2,0xbe,0xb3,0xaf,0xaf,0xaf,0xaf,0xaf, + 0xaf,0xaf,0xad,0xac,0xac,0xac,0xaa,0xac,0xb4,0xbe,0xd7,0xf6,0xff,0xfe,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfa,0xf2,0xea,0xe7,0xe7,0xe7,0xe6,0xe6,0xe6,0xe6,0xe6, + 0xe6,0xe5,0xe6,0xe6,0xe5,0xe6,0xe9,0xee,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xf1,0xc9,0xa8,0xa5,0xa7,0xa3,0xa4,0xa4,0xa3,0xa3, + 0xa3,0xa3,0xa3,0xa3,0xa3,0xa3,0xa1,0xa3,0xa5,0xa4,0xaf,0xd0,0xf2,0xff,0xff,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xf8,0xea,0xe3,0xe4,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,0xe3,0xe9,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x9c,0xa2,0xa8,0xa7,0xa7,0xa7,0xa6,0xa6, + 0xa6,0xa6,0xa6,0xa7,0xa7,0xa7,0xa7,0xa7,0xa5,0xa2,0x9f,0xae,0xde,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfa,0xec,0xe1,0xde,0xe2,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe1,0xe0,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xa9,0xa6,0xa3,0xa4,0xa4,0xa4,0xa4,0xa5, + 0xa5,0xa4,0xa4,0xa4,0xa4,0xa4,0xa5,0xa3,0xa3,0xab,0xa3,0x9f,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe6,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xaa,0xa8,0xa4,0xa5,0xa5,0xa5,0xa5,0xa5, + 0xa5,0xa5,0xa5,0xa5,0xa5,0xa5,0xa6,0xa4,0xa4,0xac,0xa3,0x9f,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe5,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0xac,0x9e,0xa3,0xa8,0xa8,0xa8,0xaa,0xa9,0xa7, + 0xa7,0xa8,0xa9,0xa9,0xa9,0xa9,0xa9,0xa9,0xa7,0xa6,0xa1,0xad,0xdd,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xf9,0xeb,0xe0,0xdf,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe2,0xe2,0xe0,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc7,0xa8,0xa6,0xa8,0xa5,0xa6,0xa8,0xa7,0xa6, + 0xa7,0xa8,0xa8,0xa8,0xa8,0xa8,0xa7,0xa8,0xa9,0xa8,0xb1,0xd0,0xf1,0xff,0xff,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xf5,0xe7,0xe1,0xe2,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1, + 0xe1,0xe2,0xe2,0xe1,0xe2,0xe2,0xe2,0xe2,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xec,0xd0,0xbd,0xb3,0xb0,0xb0,0xb0,0xb0,0xb2, + 0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb6,0xbe,0xd6,0xf4,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xf6,0xec,0xe6,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe4,0xe3,0xe3,0xe4,0xe4,0xe5,0xeb,0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xe7,0xde,0xdd,0xdd,0xdd,0xdd,0xde, + 0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xdf,0xe1,0xe7,0xf2,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfd,0xf8,0xf4,0xf3,0xf3,0xf3,0xf2,0xf2,0xf3,0xf3,0xf3, + 0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf4,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfd,0xfa,0xf9,0xfc,0xff,0xfe,0xf6,0xeb,0xe5,0xe9,0xf4,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfc,0xf8,0xf5,0xf6,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xea,0xd9,0xc6,0xba,0xbf,0xd8,0xf3,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf1,0xe7,0xe4,0xe6,0xec,0xf5,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,0xcf,0xba,0xb3,0xb0,0xad,0xae,0xb7,0xca,0xec, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfb,0xee,0xe1,0xdc,0xdc,0xdd,0xde,0xe2,0xe9,0xf4,0xfd,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc, + 0xfc,0xff,0xff,0xfc,0xef,0xdb,0xc5,0xb3,0xa7,0xa8,0xae,0xb1,0xb1,0xab,0xaf,0xda, + 0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf4,0xe1,0xd9,0xda,0xdc,0xdb,0xda,0xda,0xdd,0xe4,0xed,0xf4,0xfb,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff, + 0xff,0xfe,0xf9,0xe8,0xca,0xb0,0xa9,0xae,0xb0,0xb2,0xb1,0xaf,0xb4,0xaf,0xad,0xd2, + 0xfc,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xef,0xdc,0xdb,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe3,0xf0,0xfa,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfc,0xe9,0xcc,0xb8,0xb1,0xad,0xad,0xb0,0xb2,0xb3,0xb0,0xaf,0xb5,0xb0,0xae,0xd4, + 0xfe,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf0,0xdc,0xd9,0xdc,0xdb,0xdb,0xdb,0xdc,0xdc,0xdb,0xdb,0xdd,0xdf,0xe5,0xf1,0xfb, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb, + 0xd7,0xc3,0xb0,0xa8,0xad,0xb3,0xb3,0xb0,0xaf,0xaf,0xb1,0xb4,0xb5,0xaf,0xb6,0xde, + 0xff,0xff,0xfb,0xfd,0xfc,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfd,0xfd,0xff,0xff, + 0xf7,0xe4,0xd7,0xd7,0xdc,0xdb,0xd9,0xda,0xdb,0xdc,0xdd,0xdd,0xda,0xdb,0xe2,0xea, + 0xf3,0xfc,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf7,0xe2,0xc3, + 0xad,0xab,0xb1,0xb3,0xb0,0xb0,0xb1,0xb1,0xb2,0xb2,0xb3,0xb2,0xb3,0xbb,0xd0,0xef, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xf2,0xe0,0xd7,0xd8,0xd9,0xda,0xdc,0xdb,0xdb,0xdb,0xdc,0xdd,0xdd,0xdb,0xdb, + 0xe1,0xed,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xce,0xb4,0xae, + 0xad,0xad,0xb1,0xb3,0xb2,0xb0,0xb0,0xb1,0xb2,0xb5,0xb3,0xb1,0xbb,0xd7,0xf3,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xfd,0xf2,0xe2,0xd7,0xd7,0xdb,0xdc,0xdb,0xdb,0xdb,0xdb,0xdc,0xdd,0xdb,0xda, + 0xdb,0xde,0xe4,0xf0,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf3,0xcd,0xaf,0xa9,0xae, + 0xb4,0xb3,0xb0,0xae,0xaf,0xb2,0xb5,0xb4,0xaf,0xb0,0xbd,0xce,0xe1,0xf2,0xfe,0xff, + 0xff,0xfd,0xf1,0xe2,0xe0,0xe6,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xf9,0xf1,0xec,0xec,0xf2,0xfa,0xff, + 0xff,0xff,0xfd,0xf5,0xe9,0xe1,0xda,0xd7,0xd9,0xdb,0xdb,0xda,0xda,0xda,0xdc,0xdd, + 0xdd,0xdb,0xd9,0xe3,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe1,0xb5,0xa9,0xb4,0xb3, + 0xae,0xb1,0xb2,0xb2,0xb2,0xb1,0xb1,0xb4,0xb9,0xc4,0xdb,0xf5,0xff,0xfe,0xfd,0xff, + 0xfc,0xea,0xd0,0xbe,0xb9,0xc0,0xd6,0xf4,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe8,0xd9,0xd3,0xd3,0xda,0xeb,0xfa, + 0xff,0xfe,0xfe,0xff,0xfd,0xf2,0xe5,0xde,0xda,0xd8,0xd9,0xdb,0xdc,0xdb,0xdb,0xdb, + 0xdb,0xdc,0xdb,0xde,0xea,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xd5,0xb5,0xac,0xb2,0xb1, + 0xae,0xb0,0xb2,0xb3,0xb4,0xb3,0xb2,0xbf,0xda,0xf1,0xfb,0xfe,0xff,0xfe,0xfe,0xfe, + 0xeb,0xc9,0xb9,0xbb,0xba,0xba,0xc4,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,0xed,0xd7,0xce,0xd1,0xd2,0xd2,0xd9,0xe8, + 0xf9,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf2,0xe5,0xda,0xd7,0xdb,0xdd,0xdc,0xdb,0xdb, + 0xdb,0xdb,0xdb,0xdd,0xe4,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb6,0xaf,0xb4,0xb3, + 0xb3,0xb5,0xb3,0xad,0xb0,0xc1,0xd3,0xe5,0xf8,0xff,0xff,0xfe,0xfc,0xff,0xff,0xef, + 0xcf,0xb9,0xb7,0xbd,0xbf,0xc0,0xbd,0xc4,0xe6,0xff,0xff,0xfc,0xff,0xfe,0xf4,0xea, + 0xea,0xf0,0xf9,0xff,0xff,0xff,0xff,0xf5,0xe0,0xd1,0xce,0xd3,0xd6,0xd4,0xd1,0xd8, + 0xee,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe3,0xda,0xd7,0xd9,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xe3,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xe1,0xb9,0xaa,0xb1,0xb2, + 0xae,0xb0,0xb3,0xb8,0xc6,0xdf,0xf7,0xff,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd, + 0xba,0xba,0xbe,0xba,0xbb,0xc1,0xbd,0xbf,0xe3,0xff,0xff,0xfd,0xfd,0xef,0xd9,0xcd, + 0xcd,0xd5,0xe7,0xfa,0xff,0xff,0xff,0xf1,0xda,0xd2,0xd2,0xd1,0xd2,0xd5,0xd3,0xd2, + 0xe3,0xf8,0xff,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf5,0xe6,0xdd,0xda,0xd8,0xd8, + 0xdb,0xdb,0xd8,0xdc,0xe9,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd0,0xb1,0xab,0xaf, + 0xad,0xb0,0xc3,0xe0,0xf5,0xfd,0xfe,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xed,0xcd, + 0xb8,0xba,0xbe,0xbb,0xbd,0xc2,0xbe,0xc2,0xe5,0xff,0xff,0xfd,0xf1,0xd3,0xc4,0xc8, + 0xc8,0xc7,0xd2,0xe7,0xf8,0xff,0xff,0xf2,0xdd,0xd3,0xd1,0xd1,0xd1,0xd3,0xd4,0xd3, + 0xda,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf4,0xe6,0xda,0xd7, + 0xda,0xda,0xd8,0xe2,0xf6,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,0xdc,0xcd,0xc7, + 0xcb,0xd7,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfd,0xff,0xf8,0xd3,0xbd, + 0xbc,0xbb,0xbb,0xbe,0xbf,0xbf,0xc1,0xcf,0xed,0xff,0xff,0xf8,0xe2,0xc6,0xc1,0xcb, + 0xca,0xc7,0xca,0xd5,0xef,0xff,0xff,0xf8,0xe8,0xd2,0xcd,0xd2,0xd2,0xcf,0xd2,0xd4, + 0xd3,0xde,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8, + 0xe5,0xe6,0xeb,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xec, + 0xf3,0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xea,0xc2,0xb4, + 0xbd,0xbf,0xbb,0xbe,0xc0,0xbb,0xc5,0xe4,0xf9,0xff,0xff,0xf1,0xd7,0xc7,0xc6,0xc8, + 0xc8,0xca,0xca,0xce,0xeb,0xff,0xff,0xff,0xf4,0xd7,0xcc,0xd3,0xd2,0xcf,0xd2,0xd4, + 0xd2,0xd7,0xea,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb, + 0xf6,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xd4,0xbd,0xb9, + 0xbd,0xbd,0xbd,0xc0,0xc0,0xbc,0xcf,0xf5,0xff,0xff,0xff,0xf0,0xd3,0xc7,0xc6,0xc7, + 0xc7,0xcb,0xc9,0xca,0xe8,0xff,0xff,0xff,0xfc,0xe4,0xd3,0xd1,0xd1,0xd1,0xd1,0xd2, + 0xd5,0xd3,0xd8,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff, + 0xfe,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe2,0xbf,0xba,0xbf, + 0xbc,0xbc,0xbf,0xbf,0xbe,0xc7,0xe2,0xfc,0xff,0xff,0xff,0xf0,0xd2,0xc5,0xc5,0xc7, + 0xc8,0xc9,0xc7,0xca,0xe8,0xff,0xff,0xfd,0xff,0xf6,0xdf,0xcf,0xd0,0xd3,0xd0,0xd1, + 0xd5,0xd3,0xd1,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd1,0xb6,0xb8,0xbf, + 0xbc,0xbc,0xc0,0xbd,0xbd,0xd7,0xf8,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc7, + 0xc7,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfb,0xff,0xff,0xea,0xd2,0xd0,0xd3,0xd0,0xd0, + 0xd2,0xd3,0xd3,0xda,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xc0,0xba,0xbb,0xbc, + 0xbd,0xbe,0xc0,0xbf,0xc8,0xe8,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfc,0xff,0xff,0xf5,0xe1,0xd3,0xd0,0xd1,0xd1, + 0xd1,0xd4,0xd5,0xd1,0xdc,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf0,0xc7,0xb6,0xbe,0xbe,0xbb, + 0xbe,0xc0,0xbb,0xc2,0xdd,0xf7,0xff,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xf1,0xd6,0xcd,0xd2,0xd2, + 0xd0,0xd3,0xd4,0xd0,0xd7,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd9,0xbf,0xb8,0xbd,0xbe,0xbb, + 0xbe,0xc0,0xbb,0xcb,0xf1,0xff,0xfe,0xff,0xfe,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xfe,0xff,0xfa,0xe0,0xd1,0xd2,0xd1, + 0xd0,0xd1,0xd3,0xd4,0xd5,0xdd,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc1,0xba,0xbe,0xbc,0xbc,0xbe, + 0xbf,0xbe,0xc6,0xe0,0xfb,0xff,0xfe,0xff,0xfe,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xfe,0xf2,0xdd,0xcf,0xd0, + 0xd2,0xd0,0xd0,0xd4,0xd2,0xd3,0xe8,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfe,0xda,0xb6,0xb7,0xbf,0xbc,0xbb,0xc0, + 0xbd,0xbb,0xd4,0xf6,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc7,0xc9,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xe8,0xd1,0xd1, + 0xd3,0xcf,0xcf,0xd3,0xd1,0xd0,0xe0,0xf7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf8,0xd4,0xb7,0xba,0xc0,0xbc,0xbd,0xc1, + 0xbe,0xc2,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xf4,0xde,0xd1, + 0xd0,0xd2,0xd1,0xd2,0xd3,0xd4,0xde,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd7,0xb8,0xbc,0xc0,0xbd,0xc0,0xbe, + 0xc2,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xef,0xd3, + 0xcd,0xd3,0xd2,0xd1,0xd2,0xd3,0xdf,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xbb,0xb7,0xbd,0xbd,0xbc,0xba, + 0xcc,0xef,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe3, + 0xd1,0xcd,0xcf,0xd1,0xce,0xcf,0xe5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf0,0xd3,0xc3,0xc0,0xc1,0xc2,0xcb, + 0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf7, + 0xe3,0xd2,0xd2,0xd4,0xd3,0xdb,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe4,0xd5,0xd3,0xde,0xef, + 0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc6,0xc8,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xf7,0xeb,0xe3,0xe1,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xf1,0xf0,0xf8,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xef,0xd4,0xc9,0xc8,0xc7, + 0xc6,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf6,0xf4,0xf9,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xda,0xc8,0xc6,0xca, + 0xc9,0xc9,0xc9,0xcf,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe7,0xca,0xc2,0xcb, + 0xcb,0xc6,0xc9,0xd9,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf5,0xdc,0xc8,0xc4, + 0xc4,0xc5,0xd3,0xec,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xd2, + 0xd2,0xd9,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf7, + 0xf7,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb, + 0xfb,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf8,0xf2, + 0xf1,0xf6,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xea,0xc4,0xa3, + 0x9e,0xb2,0xd7,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xed,0xb8,0x8d,0x80, + 0x7d,0x83,0xa6,0xd9,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf9,0xcc,0x8c,0x79,0x8b, + 0x8b,0x7f,0x84,0xa9,0xe2,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xff,0xff,0xfe, + 0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe5,0xaf,0x88,0x81,0x87, + 0x87,0x86,0x7f,0x8b,0xcf,0xff,0xff,0xfa,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xed,0xe8,0xf3,0xfe, + 0xff,0xfe,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa5,0x8c,0x87,0x81, + 0x81,0x89,0x81,0x82,0xc7,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xfb,0xe6,0xe5,0xf5,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf4,0xd7,0xbb,0xb5,0xc6,0xe7, + 0xfd,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff,0xfc,0xff,0xff,0xde,0xa5,0x88,0x84,0x85, + 0x85,0x89,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xff, + 0xee,0xcb,0xae,0xac,0xc6,0xe4,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xec,0xc0,0x9d,0x92,0x92,0x94,0xa9, + 0xd5,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xde,0xa4,0x86,0x84,0x85, + 0x84,0x88,0x7d,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xff,0xff,0xfe,0xff,0xed, + 0xb3,0x83,0x7f,0x87,0x83,0x94,0xd3,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xd0,0x93,0x85,0x8b,0x8c,0x88,0x86, + 0xaa,0xe7,0xff,0xfc,0xfe,0xff,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf9,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf4,0xb8, + 0x81,0x71,0x76,0x7c,0x6e,0x6e,0xae,0xf3,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfb,0xbc,0x85,0x8c,0x95,0x8f,0x91,0x8a, + 0x92,0xbe,0xee,0xff,0xff,0xfd,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x83,0x8a,0x80,0x7f,0xc5,0xff,0xff,0xf8,0xff,0xff,0xff,0xfe,0xfd,0xff,0xd6,0x88, + 0x72,0x83,0x7e,0x7c,0x7c,0x79,0x9b,0xd5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xf4,0xb8,0x87,0x8c,0x94,0x8d,0x8d,0x91, + 0x8b,0x96,0xd2,0xff,0xff,0xfa,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf8,0xff,0xff,0xfc,0xff,0xff,0xe4,0xaa,0x80, + 0x7a,0x7f,0x7b,0x7c,0x7c,0x7a,0x97,0xd0,0xfa,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xff,0xfd,0xc2,0x89,0x89,0x93,0x8c,0x8b,0x93, + 0x8b,0x87,0xb6,0xf0,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf8,0xff,0xff,0xf9,0xff,0xff,0xc4,0x83,0x7e, + 0x82,0x77,0x78,0x7e,0x74,0x70,0xa1,0xe4,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xff,0xfa,0xff,0xff,0xd3,0x98,0x8e,0x92,0x8e,0x8f,0x90, + 0x8d,0x8e,0x9b,0xc5,0xf7,0xff,0xfe,0xff,0xfd,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc6,0xff,0xff,0xf9,0xff,0xff,0xfe,0xfe,0xe6,0xa5,0x75,0x78, + 0x81,0x79,0x78,0x82,0x77,0x74,0xb5,0xfb,0xff,0xfb,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xed,0xbf,0x97,0x8b,0x90,0x90,0x8b, + 0x8f,0x91,0x85,0xa1,0xe7,0xff,0xfd,0xfe,0xfe,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfb,0xff,0xfd,0xff,0xf4,0xb3,0x85,0x7d,0x7b, + 0x7a,0x7b,0x7a,0x7d,0x7c,0x90,0xd1,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xe3,0xa1,0x86,0x92,0x91,0x8c, + 0x91,0x90,0x84,0x92,0xc6,0xf2,0xff,0xff,0xfb,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xfa,0xff,0xfd,0xff,0xdd,0x92,0x76,0x82,0x7e, + 0x76,0x7d,0x7e,0x71,0x81,0xbe,0xf2,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbc,0x94,0x8e,0x8f,0x8f, + 0x8f,0x8e,0x8f,0x8e,0x9e,0xd6,0xff,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc4,0xff,0xff,0xf7,0xff,0xff,0xe6,0xb0,0x84,0x78,0x7d,0x7c, + 0x78,0x7d,0x7e,0x72,0x95,0xe6,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xe8,0xb1,0x89,0x8c,0x93, + 0x8d,0x8c,0x92,0x89,0x87,0xbc,0xf8,0xff,0xf8,0xff,0xff,0xdd,0xa4,0x88,0x85,0x84, + 0x84,0x89,0x7e,0x7e,0xc5,0xff,0xff,0xf4,0xff,0xff,0xc6,0x83,0x7b,0x81,0x79,0x7a, + 0x7d,0x7a,0x79,0x8d,0xc3,0xf7,0xff,0xfd,0xfe,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfb,0xfd,0xfe, + 0xfc,0xfa,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff,0xfb,0xff,0xff,0xcc,0x93,0x8d,0x93, + 0x8d,0x8e,0x90,0x8b,0x88,0x9f,0xd1,0xfa,0xff,0xff,0xff,0xdf,0xa4,0x87,0x84,0x84, + 0x84,0x89,0x7d,0x7d,0xc5,0xff,0xff,0xf8,0xff,0xea,0xa6,0x76,0x7a,0x82,0x78,0x79, + 0x81,0x77,0x75,0xaf,0xf1,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xff,0xfd,0xf9,0xfa, + 0xfe,0xfe,0xfb,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xe7,0xb5,0x92,0x8b, + 0x90,0x90,0x8d,0x91,0x91,0x88,0xaa,0xee,0xff,0xff,0xff,0xdf,0xa4,0x8a,0x86,0x84, + 0x82,0x89,0x7f,0x7d,0xc6,0xff,0xff,0xff,0xf6,0xba,0x88,0x7e,0x7d,0x7c,0x7a,0x7a, + 0x7e,0x79,0x88,0xcd,0xff,0xff,0xfa,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf6,0xee, + 0xf5,0xff,0xff,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xde,0x9e,0x86, + 0x92,0x92,0x8c,0x91,0x91,0x85,0x96,0xcd,0xf4,0xff,0xff,0xe2,0xac,0x8d,0x85,0x83, + 0x83,0x88,0x81,0x85,0xcb,0xff,0xff,0xfe,0xe1,0x95,0x76,0x82,0x7e,0x78,0x7e,0x7d, + 0x75,0x82,0xb5,0xeb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfb,0xfc,0xff,0xff,0xf9, + 0xed,0xeb,0xf7,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfd,0xea,0xd1,0xbf,0xba, + 0xc1,0xd0,0xe4,0xf6,0xff,0xff,0xfc,0xfb,0xfe,0xff,0xfd,0xfc,0xff,0xf4,0xb9,0x94, + 0x91,0x8e,0x8c,0x8f,0x8f,0x8c,0x8f,0xa9,0xdf,0xff,0xff,0xee,0xc0,0x89,0x7a,0x8a, + 0x8b,0x82,0x80,0x98,0xd8,0xff,0xff,0xef,0xc1,0x85,0x74,0x81,0x7f,0x76,0x7d,0x7f, + 0x72,0x90,0xdf,0xff,0xfa,0xfd,0xff,0xfe,0xfb,0xfb,0xfe,0xff,0xfc,0xeb,0xcd,0xb4, + 0xa8,0xa7,0xb3,0xd5,0xf9,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf3,0xc0,0x99,0x93,0x94, + 0x90,0x95,0xb1,0xd8,0xf2,0xfc,0xfe,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfd,0xe2,0xae, + 0x8c,0x8e,0x92,0x8c,0x8b,0x91,0x8b,0x93,0xd2,0xff,0xff,0xfb,0xe0,0x9f,0x7e,0x86, + 0x85,0x7d,0x91,0xc1,0xec,0xff,0xff,0xde,0xa4,0x83,0x7c,0x7c,0x7a,0x7a,0x7c,0x7a, + 0x83,0xb4,0xf4,0xff,0xfc,0xfd,0xff,0xfe,0xff,0xff,0xff,0xf7,0xde,0xae,0x7f,0x6b, + 0x70,0x70,0x6a,0x91,0xdb,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xda,0xa5,0x91,0x99,0x98, + 0x92,0x95,0x9a,0xa0,0xb4,0xd9,0xf6,0xff,0xff,0xfc,0xfe,0xff,0xfb,0xff,0xff,0xc8, + 0x91,0x90,0x95,0x8b,0x8b,0x92,0x89,0x8b,0xcc,0xff,0xff,0xfc,0xfa,0xd8,0xa9,0x8d, + 0x86,0x94,0xbf,0xee,0xfd,0xff,0xff,0xda,0x9b,0x81,0x7d,0x78,0x77,0x7f,0x78,0x71, + 0xa4,0xe8,0xff,0xfe,0xff,0xfd,0xfd,0xfc,0xfe,0xfe,0xe3,0xaf,0x8a,0x7d,0x74,0x6f, + 0x72,0x73,0x6e,0x7b,0xac,0xe7,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc8,0xa2,0x97,0x9e,0x9e, + 0x9d,0x9e,0x99,0x8f,0x95,0xad,0xc6,0xdf,0xf9,0xff,0xfe,0xfd,0xfb,0xff,0xff,0xe6, + 0xb2,0x8e,0x8b,0x94,0x94,0x93,0x8a,0x91,0xd0,0xff,0xff,0xfa,0xff,0xfa,0xe2,0xcb, + 0xc7,0xd5,0xed,0xfd,0xfe,0xff,0xff,0xe2,0xa6,0x7e,0x79,0x81,0x81,0x7d,0x75,0x82, + 0xc8,0xff,0xff,0xf7,0xfb,0xfe,0xff,0xfb,0xe2,0xc2,0x9f,0x7c,0x6c,0x71,0x7a,0x7d, + 0x7b,0x79,0x7a,0x7b,0x93,0xd5,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf1,0xc9,0xa1,0x95,0x9c,0x9c, + 0x98,0x97,0x99,0x9b,0x99,0x95,0x94,0xa9,0xd0,0xee,0xfa,0xfe,0xff,0xfe,0xfd,0xfe, + 0xdf,0xa6,0x8a,0x8e,0x8c,0x89,0x94,0xb4,0xe5,0xff,0xff,0xfd,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xf4,0xc6,0x8c,0x73,0x79,0x79,0x73,0x86,0xb6, + 0xeb,0xff,0xff,0xfe,0xff,0xfb,0xf1,0xd6,0xa6,0x7b,0x6e,0x76,0x7a,0x77,0x74,0x73, + 0x74,0x76,0x74,0x76,0x93,0xd7,0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xd9,0xa2,0x93,0xa1,0x9d, + 0x98,0x9b,0x9a,0x9a,0x99,0x97,0x95,0x97,0x9f,0xb0,0xd3,0xf7,0xff,0xfd,0xfc,0xff, + 0xfa,0xda,0xae,0x91,0x8a,0x91,0xb4,0xe8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xff,0xfe,0xfd,0xff,0xee,0xb7,0x88,0x79,0x78,0x88,0xb8,0xee, + 0xff,0xfc,0xfd,0xff,0xfa,0xd4,0xa4,0x85,0x7a,0x73,0x71,0x75,0x79,0x78,0x75,0x72, + 0x76,0x79,0x71,0x78,0xa7,0xe4,0xff,0xff,0xfd,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xef,0xbb,0x98,0x95,0x9c, + 0xa1,0x9f,0x9a,0x98,0x98,0x9b,0x9e,0x99,0x90,0x94,0xa8,0xc0,0xdb,0xf2,0xfe,0xff, + 0xff,0xfa,0xe4,0xcc,0xc7,0xcf,0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xf9, + 0xf9,0xfb,0xfe,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe8,0xcf,0xbf,0xbf,0xcf,0xe9,0xfc, + 0xff,0xff,0xfb,0xe0,0xb8,0x97,0x79,0x6a,0x71,0x7c,0x7b,0x75,0x72,0x73,0x78,0x7b, + 0x7a,0x71,0x6c,0x91,0xd4,0xfb,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xe4,0xbc,0xa0,0x9b, + 0x9b,0x9a,0x9c,0x9e,0x9d,0x9a,0x98,0x98,0x9a,0x9c,0x96,0x91,0xa3,0xce,0xf3,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xff,0xf8,0xd6,0x9c,0x74,0x70,0x79,0x7a,0x76,0x74,0x74,0x76,0x78,0x77,0x75,0x72, + 0x72,0x7a,0x93,0xc6,0xf8,0xff,0xfd,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf3,0xd6,0xaf, + 0x99,0x99,0x9d,0x9e,0x9d,0x9a,0x99,0x99,0x9c,0x9b,0x98,0x95,0x94,0xa3,0xc6,0xee, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xf9,0xd2,0x92,0x72,0x74,0x75,0x76,0x79,0x77,0x74,0x73,0x75,0x78,0x78,0x73,0x6e, + 0x80,0xae,0xe0,0xfa,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfb,0xe0, + 0xc7,0xb1,0x9b,0x93,0x9a,0x9f,0x9f,0x9b,0x97,0x96,0x9a,0x9d,0x9a,0x91,0x9d,0xd5, + 0xff,0xff,0xf9,0xfa,0xfa,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfa,0xf9,0xf8,0xf8,0xff,0xff, + 0xe2,0xa1,0x71,0x6e,0x7d,0x79,0x72,0x71,0x73,0x78,0x7c,0x78,0x6b,0x6d,0x87,0xa7, + 0xc8,0xe9,0xfc,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xf8,0xdd,0xba,0xa5,0x9f,0x9a,0x99,0x9d,0x9c,0x9c,0x99,0x97,0x9f,0x95,0x8f,0xc5, + 0xff,0xff,0xfb,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff, + 0xc8,0x81,0x77,0x7b,0x74,0x74,0x77,0x79,0x78,0x73,0x71,0x76,0x7b,0x90,0xbd,0xec, + 0xff,0xff,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff, + 0xfe,0xfc,0xf5,0xdf,0xba,0x9c,0x96,0x9d,0x9e,0x9e,0x9b,0x99,0x9f,0x97,0x91,0xc2, + 0xfb,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf9,0xff,0xff, + 0xc5,0x7b,0x75,0x7c,0x73,0x75,0x78,0x7a,0x79,0x6f,0x6e,0x8a,0xbd,0xe6,0xf7,0xfd, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xfb, + 0xfc,0xff,0xff,0xf9,0xe6,0xcd,0xb4,0xa1,0x95,0x95,0x9b,0x9d,0x9e,0x94,0x98,0xcd, + 0xff,0xff,0xfb,0xff,0xff,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xff,0xfa,0xff,0xff, + 0xd4,0x8c,0x6f,0x72,0x78,0x77,0x70,0x6a,0x6f,0x8b,0xaf,0xce,0xee,0xff,0xff,0xfa, + 0xf9,0xfd,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfd,0xfc,0xff,0xff,0xe8,0xc3,0xaa,0xa1,0x9e,0x9b,0x9c,0xa3,0xb9,0xe5, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xed,0xb9,0x86,0x72,0x76,0x76,0x77,0x7f,0x96,0xca,0xf7,0xff,0xfd,0xfd,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe5,0xcc,0xb5,0xa8,0xad,0xc9,0xec,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xee,0xbe,0x96,0x89,0x8d,0xa4,0xc9,0xeb,0xfc,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfc,0xf9,0xf8,0xfb,0xff,0xfe,0xf1,0xe1,0xda,0xdf,0xee,0xfd,0xff, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xee,0xd9,0xcc,0xd0,0xe3,0xf6,0xfe,0xfe,0xfb,0xfa,0xfc,0xff,0xff,0xfe, + 0xfd,0xfd,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe9,0xe0,0xdf,0xdf,0xdf,0xdf,0xdf, + 0xdf,0xdf,0xde,0xdd,0xdd,0xdd,0xdd,0xdf,0xe3,0xe9,0xf4,0xfe,0xff,0xfd,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfb,0xfa,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf4,0xf4,0xf4,0xf4,0xf4, + 0xf4,0xf5,0xf4,0xf4,0xf5,0xf5,0xf7,0xfa,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xee,0xd2,0xbe,0xb3,0xaf,0xaf,0xaf,0xaf,0xaf, + 0xaf,0xaf,0xad,0xac,0xac,0xac,0xaa,0xac,0xb4,0xbe,0xd7,0xf6,0xff,0xfe,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfa,0xf2,0xea,0xe7,0xe7,0xe7,0xe6,0xe6,0xe6,0xe6,0xe6, + 0xe6,0xe5,0xe6,0xe6,0xe5,0xe6,0xe9,0xee,0xf7,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xf1,0xc9,0xa8,0xa5,0xa7,0xa3,0xa4,0xa4,0xa3,0xa3, + 0xa3,0xa3,0xa3,0xa3,0xa3,0xa3,0xa1,0xa3,0xa5,0xa4,0xaf,0xd0,0xf2,0xff,0xff,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xf8,0xea,0xe3,0xe4,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe2,0xe3,0xe3,0xe2,0xe2,0xe2,0xe3,0xe9,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd7,0xad,0x9c,0xa2,0xa8,0xa7,0xa7,0xa7,0xa6,0xa6, + 0xa6,0xa6,0xa6,0xa7,0xa7,0xa7,0xa7,0xa7,0xa5,0xa2,0x9f,0xae,0xde,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfa,0xec,0xe1,0xde,0xe2,0xe3,0xe2,0xe2,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe1,0xe0,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xa9,0xa6,0xa3,0xa4,0xa4,0xa4,0xa4,0xa5, + 0xa5,0xa4,0xa4,0xa4,0xa4,0xa4,0xa5,0xa3,0xa3,0xab,0xa3,0x9f,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe6,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xfe,0xff,0xfe,0xfc,0xff,0xf5,0xbf,0xa5,0xaa,0xa8,0xa4,0xa5,0xa5,0xa5,0xa5,0xa5, + 0xa5,0xa5,0xa5,0xa5,0xa5,0xa5,0xa6,0xa4,0xa4,0xac,0xa3,0x9f,0xd2,0xff,0xff,0xfa, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xff,0xff,0xf5,0xe6,0xe3,0xe3,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2,0xe2, + 0xe2,0xe2,0xe2,0xe2,0xe2,0xe1,0xe3,0xe4,0xdf,0xe5,0xfa,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd3,0xac,0x9e,0xa3,0xa8,0xa8,0xa8,0xaa,0xa9,0xa7, + 0xa7,0xa8,0xa9,0xa9,0xa9,0xa9,0xa9,0xa9,0xa7,0xa6,0xa1,0xad,0xdd,0xff,0xff,0xfc, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xf9,0xeb,0xe0,0xdf,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe3,0xe3,0xe3,0xe3,0xe2,0xe2,0xe2,0xe0,0xea,0xfc,0xff,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xc7,0xa8,0xa6,0xa8,0xa5,0xa6,0xa8,0xa7,0xa6, + 0xa7,0xa8,0xa8,0xa8,0xa8,0xa8,0xa7,0xa8,0xa9,0xa8,0xb1,0xd0,0xf1,0xff,0xff,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xf5,0xe7,0xe1,0xe2,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1,0xe1, + 0xe1,0xe2,0xe2,0xe1,0xe2,0xe2,0xe2,0xe2,0xe8,0xf4,0xfe,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xec,0xd0,0xbd,0xb3,0xb0,0xb0,0xb0,0xb0,0xb2, + 0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb6,0xbe,0xd6,0xf4,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xfe,0xf6,0xec,0xe6,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3,0xe3, + 0xe3,0xe4,0xe3,0xe3,0xe4,0xe4,0xe5,0xeb,0xf5,0xfd,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf5,0xe7,0xde,0xdd,0xdd,0xdd,0xdd,0xde, + 0xde,0xde,0xde,0xde,0xde,0xde,0xde,0xdf,0xe1,0xe7,0xf2,0xfe,0xff,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfd,0xf8,0xf4,0xf3,0xf3,0xf3,0xf2,0xf2,0xf3,0xf3,0xf3, + 0xf3,0xf3,0xf3,0xf3,0xf3,0xf3,0xf4,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xfc,0xfc,0xfb,0xfb,0xfb,0xfb, + 0xfb,0xfc,0xfd,0xfd,0xfa,0xf9,0xfc,0xff,0xfe,0xf6,0xeb,0xe5,0xe9,0xf4,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfc,0xf8,0xf5,0xf6,0xfa,0xfe,0xff,0xfe,0xfd,0xfd,0xfe,0xfe,0xfe,0xfe, + 0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xea,0xd9,0xc6,0xba,0xbf,0xd8,0xf3,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfb,0xf1,0xe7,0xe4,0xe6,0xec,0xf5,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfc,0xff,0xff,0xef,0xcf,0xba,0xb3,0xb0,0xad,0xae,0xb7,0xca,0xec, + 0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfb,0xee,0xe1,0xdc,0xdc,0xdd,0xde,0xe2,0xe9,0xf4,0xfd,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xfc, + 0xfc,0xff,0xff,0xfc,0xef,0xdb,0xc5,0xb3,0xa7,0xa8,0xae,0xb1,0xb1,0xab,0xaf,0xda, + 0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff, + 0xf4,0xe1,0xd9,0xda,0xdc,0xdb,0xda,0xda,0xdd,0xe4,0xed,0xf4,0xfb,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff, + 0xff,0xfe,0xf9,0xe8,0xca,0xb0,0xa9,0xae,0xb0,0xb2,0xb1,0xaf,0xb4,0xaf,0xad,0xd2, + 0xfc,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xef,0xdc,0xdb,0xdd,0xdc,0xdc,0xdc,0xdc,0xdc,0xdc,0xdd,0xe3,0xf0,0xfa,0xfe,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xfe,0xff, + 0xfc,0xe9,0xcc,0xb8,0xb1,0xad,0xad,0xb0,0xb2,0xb3,0xb0,0xaf,0xb5,0xb0,0xae,0xd4, + 0xfe,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff, + 0xf0,0xdc,0xd9,0xdc,0xdb,0xdb,0xdb,0xdc,0xdc,0xdb,0xdb,0xdd,0xdf,0xe5,0xf1,0xfb, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfc,0xeb, + 0xd7,0xc3,0xb0,0xa8,0xad,0xb3,0xb3,0xb0,0xaf,0xaf,0xb1,0xb4,0xb5,0xaf,0xb6,0xde, + 0xff,0xff,0xfb,0xfd,0xfc,0xfc,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xfd,0xfd,0xfd,0xff,0xff, + 0xf7,0xe4,0xd7,0xd7,0xdc,0xdb,0xd9,0xda,0xdb,0xdc,0xdd,0xdd,0xda,0xdb,0xe2,0xea, + 0xf3,0xfc,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf7,0xe2,0xc3, + 0xad,0xab,0xb1,0xb3,0xb0,0xb0,0xb1,0xb1,0xb2,0xb2,0xb3,0xb2,0xb3,0xbb,0xd0,0xef, + 0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xfe,0xf2,0xe0,0xd7,0xd8,0xd9,0xda,0xdc,0xdb,0xdb,0xdb,0xdc,0xdd,0xdd,0xdb,0xdb, + 0xe1,0xed,0xf8,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xee,0xce,0xb4,0xae, + 0xad,0xad,0xb1,0xb3,0xb2,0xb0,0xb0,0xb1,0xb2,0xb5,0xb3,0xb1,0xbb,0xd7,0xf3,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xff,0xfd,0xf2,0xe2,0xd7,0xd7,0xdb,0xdc,0xdb,0xdb,0xdb,0xdb,0xdc,0xdd,0xdb,0xda, + 0xdb,0xde,0xe4,0xf0,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf3,0xcd,0xaf,0xa9,0xae, + 0xb4,0xb3,0xb0,0xae,0xaf,0xb2,0xb5,0xb4,0xaf,0xb0,0xbd,0xce,0xe1,0xf2,0xfe,0xff, + 0xff,0xfd,0xf1,0xe2,0xe0,0xe6,0xf1,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xfe,0xf9,0xf1,0xec,0xec,0xf2,0xfa,0xff, + 0xff,0xff,0xfd,0xf5,0xe9,0xe1,0xda,0xd7,0xd9,0xdb,0xdb,0xda,0xda,0xda,0xdc,0xdd, + 0xdd,0xdb,0xd9,0xe3,0xf5,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xe1,0xb5,0xa9,0xb4,0xb3, + 0xae,0xb1,0xb2,0xb2,0xb2,0xb1,0xb1,0xb4,0xb9,0xc4,0xdb,0xf5,0xff,0xfe,0xfd,0xff, + 0xfc,0xea,0xd0,0xbe,0xb9,0xc0,0xd6,0xf4,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe8,0xd9,0xd3,0xd3,0xda,0xeb,0xfa, + 0xff,0xfe,0xfe,0xff,0xfd,0xf2,0xe5,0xde,0xda,0xd8,0xd9,0xdb,0xdc,0xdb,0xdb,0xdb, + 0xdb,0xdc,0xdb,0xde,0xea,0xf9,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xd5,0xb5,0xac,0xb2,0xb1, + 0xae,0xb0,0xb2,0xb3,0xb4,0xb3,0xb2,0xbf,0xda,0xf1,0xfb,0xfe,0xff,0xfe,0xfe,0xfe, + 0xeb,0xc9,0xb9,0xbb,0xba,0xba,0xc4,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfc,0xed,0xd7,0xce,0xd1,0xd2,0xd2,0xd9,0xe8, + 0xf9,0xff,0xff,0xff,0xff,0xfe,0xfa,0xf2,0xe5,0xda,0xd7,0xdb,0xdd,0xdc,0xdb,0xdb, + 0xdb,0xdb,0xdb,0xdd,0xe4,0xf5,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf3,0xd3,0xb6,0xaf,0xb4,0xb3, + 0xb3,0xb5,0xb3,0xad,0xb0,0xc1,0xd3,0xe5,0xf8,0xff,0xff,0xfe,0xfc,0xff,0xff,0xef, + 0xcf,0xb9,0xb7,0xbd,0xbf,0xc0,0xbd,0xc4,0xe6,0xff,0xff,0xfc,0xff,0xfe,0xf4,0xea, + 0xea,0xf0,0xf9,0xff,0xff,0xff,0xff,0xf5,0xe0,0xd1,0xce,0xd3,0xd6,0xd4,0xd1,0xd8, + 0xee,0xff,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xf6,0xed,0xe3,0xda,0xd7,0xd9,0xdc,0xdc, + 0xdc,0xdc,0xdc,0xdc,0xe3,0xf4,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xe1,0xb9,0xaa,0xb1,0xb2, + 0xae,0xb0,0xb3,0xb8,0xc6,0xdf,0xf7,0xff,0xff,0xfd,0xff,0xff,0xfc,0xff,0xff,0xdd, + 0xba,0xba,0xbe,0xba,0xbb,0xc1,0xbd,0xbf,0xe3,0xff,0xff,0xfd,0xfd,0xef,0xd9,0xcd, + 0xcd,0xd5,0xe7,0xfa,0xff,0xff,0xff,0xf1,0xda,0xd2,0xd2,0xd1,0xd2,0xd5,0xd3,0xd2, + 0xe3,0xf8,0xff,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xfe,0xf5,0xe6,0xdd,0xda,0xd8,0xd8, + 0xdb,0xdb,0xd8,0xdc,0xe9,0xf8,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xf5,0xd0,0xb1,0xab,0xaf, + 0xad,0xb0,0xc3,0xe0,0xf5,0xfd,0xfe,0xff,0xff,0xff,0xfe,0xff,0xff,0xfe,0xed,0xcd, + 0xb8,0xba,0xbe,0xbb,0xbd,0xc2,0xbe,0xc2,0xe5,0xff,0xff,0xfd,0xf1,0xd3,0xc4,0xc8, + 0xc8,0xc7,0xd2,0xe7,0xf8,0xff,0xff,0xf2,0xdd,0xd3,0xd1,0xd1,0xd1,0xd3,0xd4,0xd3, + 0xda,0xeb,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xf4,0xe6,0xda,0xd7, + 0xda,0xda,0xd8,0xe2,0xf6,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xef,0xdc,0xcd,0xc7, + 0xcb,0xd7,0xe7,0xf8,0xff,0xff,0xfd,0xfc,0xfe,0xff,0xfe,0xfd,0xff,0xf8,0xd3,0xbd, + 0xbc,0xbb,0xbb,0xbe,0xbf,0xbf,0xc1,0xcf,0xed,0xff,0xff,0xf8,0xe2,0xc6,0xc1,0xcb, + 0xca,0xc7,0xca,0xd5,0xef,0xff,0xff,0xf8,0xe8,0xd2,0xcd,0xd2,0xd2,0xcf,0xd2,0xd4, + 0xd3,0xde,0xf7,0xff,0xfd,0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfe,0xf8,0xef,0xe8, + 0xe5,0xe6,0xeb,0xf4,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf8,0xec, + 0xf3,0xff,0xff,0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xea,0xc2,0xb4, + 0xbd,0xbf,0xbb,0xbe,0xc0,0xbb,0xc5,0xe4,0xf9,0xff,0xff,0xf1,0xd7,0xc7,0xc6,0xc8, + 0xc8,0xca,0xca,0xce,0xeb,0xff,0xff,0xff,0xf4,0xd7,0xcc,0xd3,0xd2,0xcf,0xd2,0xd4, + 0xd2,0xd7,0xea,0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xfb, + 0xf6,0xf8,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf2,0xd4,0xbd,0xb9, + 0xbd,0xbd,0xbd,0xc0,0xc0,0xbc,0xcf,0xf5,0xff,0xff,0xff,0xf0,0xd3,0xc7,0xc6,0xc7, + 0xc7,0xcb,0xc9,0xca,0xe8,0xff,0xff,0xff,0xfc,0xe4,0xd3,0xd1,0xd1,0xd1,0xd1,0xd2, + 0xd5,0xd3,0xd8,0xef,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfc,0xfe,0xff, + 0xfe,0xfc,0xfc,0xfe,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe2,0xbf,0xba,0xbf, + 0xbc,0xbc,0xbf,0xbf,0xbe,0xc7,0xe2,0xfc,0xff,0xff,0xff,0xf0,0xd2,0xc5,0xc5,0xc7, + 0xc8,0xc9,0xc7,0xca,0xe8,0xff,0xff,0xfd,0xff,0xf6,0xdf,0xcf,0xd0,0xd3,0xd0,0xd1, + 0xd5,0xd3,0xd1,0xe4,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xfe, + 0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf4,0xd1,0xb6,0xb8,0xbf, + 0xbc,0xbc,0xc0,0xbd,0xbd,0xd7,0xf8,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc7, + 0xc7,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfb,0xff,0xff,0xea,0xd2,0xd0,0xd3,0xd0,0xd0, + 0xd2,0xd3,0xd3,0xda,0xec,0xfd,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xfa,0xd9,0xc0,0xba,0xbb,0xbc, + 0xbd,0xbe,0xc0,0xbf,0xc8,0xe8,0xff,0xff,0xfb,0xff,0xff,0xee,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe7,0xff,0xff,0xfc,0xff,0xff,0xf5,0xe1,0xd3,0xd0,0xd1,0xd1, + 0xd1,0xd4,0xd5,0xd1,0xdc,0xf8,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd,0xff,0xf0,0xc7,0xb6,0xbe,0xbe,0xbb, + 0xbe,0xc0,0xbb,0xc2,0xdd,0xf7,0xff,0xff,0xfd,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xf1,0xd6,0xcd,0xd2,0xd2, + 0xd0,0xd3,0xd4,0xd0,0xd7,0xed,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf5,0xd9,0xbf,0xb8,0xbd,0xbe,0xbb, + 0xbe,0xc0,0xbb,0xcb,0xf1,0xff,0xfe,0xff,0xfe,0xff,0xff,0xef,0xd3,0xc6,0xc6,0xc8, + 0xc8,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xfe,0xff,0xfa,0xe0,0xd1,0xd2,0xd1, + 0xd0,0xd1,0xd3,0xd4,0xd5,0xdd,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe5,0xc1,0xba,0xbe,0xbc,0xbc,0xbe, + 0xbf,0xbe,0xc6,0xe0,0xfb,0xff,0xfe,0xff,0xfe,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfd,0xff,0xff,0xff,0xfe,0xf2,0xdd,0xcf,0xd0, + 0xd2,0xd0,0xd0,0xd4,0xd2,0xd3,0xe8,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfe,0xda,0xb6,0xb7,0xbf,0xbc,0xbb,0xc0, + 0xbd,0xbb,0xd4,0xf6,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc7,0xc9,0xe7,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xe8,0xd1,0xd1, + 0xd3,0xcf,0xcf,0xd3,0xd1,0xd0,0xe0,0xf7,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf8,0xd4,0xb7,0xba,0xc0,0xbc,0xbd,0xc1, + 0xbe,0xc2,0xe3,0xff,0xff,0xfc,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xfe,0xff,0xff,0xf4,0xde,0xd1, + 0xd0,0xd2,0xd1,0xd2,0xd3,0xd4,0xde,0xf0,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xfc,0xd7,0xb8,0xbc,0xc0,0xbd,0xc0,0xbe, + 0xc2,0xd8,0xf3,0xff,0xff,0xfe,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xef,0xd3, + 0xcd,0xd3,0xd2,0xd1,0xd2,0xd3,0xdf,0xf2,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xff,0xff,0xe0,0xbb,0xb7,0xbd,0xbd,0xbc,0xba, + 0xcc,0xef,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xfe,0xff,0xfa,0xe3, + 0xd1,0xcd,0xcf,0xd1,0xce,0xcf,0xe5,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf0,0xd3,0xc3,0xc0,0xc1,0xc2,0xcb, + 0xe3,0xfb,0xff,0xfe,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xcb,0xc7,0xc8,0xe6,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf7, + 0xe3,0xd2,0xd2,0xd4,0xd3,0xdb,0xf1,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf6,0xe4,0xd5,0xd3,0xde,0xef, + 0xfb,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xee,0xd3,0xc8,0xc8,0xc8, + 0xc7,0xca,0xc6,0xc8,0xe7,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xf7,0xeb,0xe3,0xe1,0xe8,0xf5,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfa,0xf1,0xf0,0xf8,0xfe, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfd,0xff,0xff,0xef,0xd4,0xc9,0xc8,0xc7, + 0xc6,0xca,0xc8,0xca,0xe8,0xff,0xff,0xfc,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xfd,0xf6,0xf4,0xf9,0xfe,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xff,0xf4,0xda,0xc8,0xc6,0xca, + 0xc9,0xc9,0xc9,0xcf,0xeb,0xff,0xff,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfc,0xe7,0xca,0xc2,0xcb, + 0xcb,0xc6,0xc9,0xd9,0xf2,0xff,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfe,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xff,0xf5,0xdc,0xc8,0xc4, + 0xc4,0xc5,0xd3,0xec,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xfe,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xf4,0xe0,0xd2, + 0xd2,0xd9,0xe9,0xfc,0xff,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfb,0xf7, + 0xf7,0xfa,0xfd,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xfe,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe,0xfd, + 0xfd,0xfd,0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xfe, + 0xfe,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, +}; +#endif /* !PEXPERT_NO_3X_IMAGES */ const unsigned char gGearPict2x[4*kGearFrames*kGearWidth*kGearHeight] = { 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff, diff --git a/pexpert/pexpert/arm/board_config.h b/pexpert/pexpert/arm/board_config.h index a6d930e41..13ccdb597 100644 --- a/pexpert/pexpert/arm/board_config.h +++ b/pexpert/pexpert/arm/board_config.h @@ -12,29 +12,26 @@ #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define __ARM_L2CACHE_SIZE_LOG__ 18 - #define ARM_BOARD_CLASS_S7002 +#define PEXPERT_NO_3X_IMAGES 1 #endif /* ARM_BOARD_CONFIG_S7002 */ #ifdef ARM_BOARD_CONFIG_T8002 #define ARMA7 #include - #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define __ARM_L2CACHE_SIZE_LOG__ 19 - #define ARM_BOARD_CLASS_T8002 +#define PEXPERT_NO_3X_IMAGES 1 #endif /* ARM_BOARD_CONFIG_T8002 */ #ifdef ARM_BOARD_CONFIG_T8004 #define ARMA7 #include - #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define __ARM_L2CACHE_SIZE_LOG__ 20 - #define ARM_BOARD_CLASS_T8002 +#define PEXPERT_NO_3X_IMAGES 1 #endif /* ARM_BOARD_CONFIG_T8004 */ - #endif /* ! _PEXPERT_ARM_BOARD_CONFIG_H */ diff --git a/pexpert/pexpert/arm64/board_config.h b/pexpert/pexpert/arm64/board_config.h index ecf6fe739..0aaefb898 100644 --- a/pexpert/pexpert/arm64/board_config.h +++ b/pexpert/pexpert/arm64/board_config.h @@ -14,6 +14,7 @@ #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define ARM_BOARD_CLASS_S5L8960X #define KERNEL_INTEGRITY_WT 1 +#define PEXPERT_NO_3X_IMAGES 1 #endif /* ARM64_BOARD_CONFIG_S5L8960X */ #ifdef ARM64_BOARD_CONFIG_T7000 @@ -24,7 +25,6 @@ #define __ARM_L2CACHE_SIZE_LOG__ 20 #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define ARM_BOARD_CLASS_T7000 -#define PEXPERT_3X_IMAGES 1 #define KERNEL_INTEGRITY_WT 1 #endif /* ARM64_BOARD_CONFIG_T7000 */ @@ -36,7 +36,6 @@ #define __ARM_L2CACHE_SIZE_LOG__ 21 #define ARM_BOARD_WFE_TIMEOUT_NS 1000 #define ARM_BOARD_CLASS_T7000 -#define PEXPERT_3X_IMAGES 1 #define KERNEL_INTEGRITY_WT 1 #define CPU_COUNT 3 #endif /* ARM64_BOARD_CONFIG_T7001 */ diff --git a/pexpert/pexpert/pe_images.h b/pexpert/pexpert/pe_images.h index c7a0597b1..35f179a19 100644 --- a/pexpert/pexpert/pe_images.h +++ b/pexpert/pexpert/pe_images.h @@ -42,7 +42,11 @@ static const unsigned char * default_noroot_data; static const unsigned char * default_progress_data1x = gGearPict; static const unsigned char * default_progress_data2x = gGearPict2x; +#if !PEXPERT_NO_3X_IMAGES +static const unsigned char * default_progress_data3x = gGearPict3x; +#else static const unsigned char * default_progress_data3x = NULL; +#endif static vc_progress_element default_progress = { 0, 4|1, 1000 / kGearFPS, kGearFrames, {0, 0, 0}, diff --git a/pexpert/pexpert/pexpert.h b/pexpert/pexpert/pexpert.h index 33b07cda2..61ac16c9c 100644 --- a/pexpert/pexpert/pexpert.h +++ b/pexpert/pexpert/pexpert.h @@ -390,8 +390,21 @@ extern void PE_arm_debug_enable_trace(void); #endif #if KERNEL_PRIVATE +#if defined(__arm64__) +extern uint8_t PE_smc_stashed_x86_power_state; +extern uint8_t PE_smc_stashed_x86_efi_boot_state; +extern uint8_t PE_smc_stashed_x86_system_state; +extern uint32_t PE_pcie_stashed_link_state; +#endif + boolean_t PE_reboot_on_panic(void); void PE_sync_panic_buffers(void); + +typedef struct PE_panic_save_context { + void *psc_buffer; + uint32_t psc_offset; + uint32_t psc_length; +} PE_panic_save_context_t; #endif __END_DECLS diff --git a/san/kasan-arm64.c b/san/kasan-arm64.c index 3fbb9827b..22bb738a1 100644 --- a/san/kasan-arm64.c +++ b/san/kasan-arm64.c @@ -301,8 +301,9 @@ kasan_bootstrap(boot_args *args, vm_offset_t pgtable) kernel_vbase = args->virtBase; kernel_vtop = args->virtBase + ptop - pbase; - /* Steal ~15% of physical memory */ - tosteal = vm_map_trunc_page(args->memSize / 6, ARM_PGMASK); + tosteal = (args->memSize * STOLEN_MEM_PERCENT) / 100 + STOLEN_MEM_BYTES; + tosteal = vm_map_trunc_page(tosteal, ARM_PGMASK); + args->memSize -= tosteal; /* Initialize the page allocator */ diff --git a/san/kasan-blacklist-x86_64 b/san/kasan-blacklist-x86_64 index 189a3076d..9313e4003 100644 --- a/san/kasan-blacklist-x86_64 +++ b/san/kasan-blacklist-x86_64 @@ -68,4 +68,6 @@ src:./osfmk/i386/startup64.c src:./osfmk/i386/lapic_native.c src:./osfmk/i386/fpu.c src:./osfmk/vm/vm_compressor.c - +fun:doublemap_init +fun:getsegbynamefromheader +fun:getsectbynamefromheader diff --git a/san/kasan-x86_64.c b/san/kasan-x86_64.c index 72339d8cd..93e8e416e 100644 --- a/san/kasan-x86_64.c +++ b/san/kasan-x86_64.c @@ -274,11 +274,6 @@ kasan_arch_init(void) /* Map the physical aperture */ kasan_map_shadow_superpage_zero(physmap_base, physmap_max - physmap_base); - /* Establish shadow mappings for the x86 descriptor tables and - * "low global" page; these are specially alias-mapped at fixed VAs - * early in boot - */ - kasan_map_low_fixed_regions(); } /* @@ -311,8 +306,7 @@ kasan_reserve_memory(void *_args) total_pages += mptr_tmp->NumberOfPages; } - /* steal 25% of physical memory */ - to_steal = total_pages / 4; + to_steal = (total_pages * STOLEN_MEM_PERCENT) / 100 + (STOLEN_MEM_BYTES / I386_PGBYTES); /* Search for a range large enough to steal from */ for (i = 0, mptr_tmp = mptr; i < mcount; i++, mptr_tmp = (EfiMemoryRange *)(((vm_offset_t)mptr_tmp) + msize)) { diff --git a/san/kasan.c b/san/kasan.c index 960dc25b4..1723fcef1 100644 --- a/san/kasan.c +++ b/san/kasan.c @@ -309,17 +309,16 @@ static const char *shadow_strings[] = { [ASAN_PARTIAL5] = "PARTIAL5", [ASAN_PARTIAL6] = "PARTIAL6", [ASAN_PARTIAL7] = "PARTIAL7", - [ASAN_STACK_RZ] = "", [ASAN_STACK_LEFT_RZ] = "STACK_LEFT_RZ", [ASAN_STACK_MID_RZ] = "STACK_MID_RZ", [ASAN_STACK_RIGHT_RZ] = "STACK_RIGHT_RZ", [ASAN_STACK_FREED] = "STACK_FREED", + [ASAN_STACK_OOSCOPE] = "STACK_OOSCOPE", [ASAN_GLOBAL_RZ] = "GLOBAL_RZ", - [ASAN_HEAP_RZ] = "", [ASAN_HEAP_LEFT_RZ] = "HEAP_LEFT_RZ", [ASAN_HEAP_RIGHT_RZ] = "HEAP_RIGHT_RZ", [ASAN_HEAP_FREED] = "HEAP_FREED", - [0xff] = "", + [0xff] = NULL }; #define CRASH_CONTEXT_BEFORE 5 @@ -370,6 +369,9 @@ kasan_crash_report(uptr p, uptr width, unsigned access_type) uint8_t *shadow_ptr = SHADOW_FOR_ADDRESS(p); uint8_t shadow_type = *shadow_ptr; const char *shadow_str = shadow_strings[shadow_type]; + if (!shadow_str) { + shadow_str = ""; + } kasan_handle_test(); diff --git a/san/kasan.h b/san/kasan.h index a4d985d56..bb048d860 100644 --- a/san/kasan.h +++ b/san/kasan.h @@ -73,6 +73,7 @@ typedef uintptr_t uptr; #define ASAN_STACK_MID_RZ 0xf2 #define ASAN_STACK_RIGHT_RZ 0xf3 #define ASAN_STACK_FREED 0xf5 +#define ASAN_STACK_OOSCOPE 0xf8 #define ASAN_GLOBAL_RZ 0xf9 #define ASAN_HEAP_RZ 0xe9 #define ASAN_HEAP_LEFT_RZ 0xfa diff --git a/san/kasan_internal.h b/san/kasan_internal.h index 36ae0b234..805255178 100644 --- a/san/kasan_internal.h +++ b/san/kasan_internal.h @@ -35,6 +35,8 @@ typedef uintptr_t uptr; +#define MiB(x) ((x) * 1024UL * 1024) + /* * KASAN features and config */ @@ -46,7 +48,20 @@ typedef uintptr_t uptr; #define FAKESTACK_QUARANTINE (1 && FAKESTACK) #define QUARANTINE_ENTRIES 5000 -#define QUARANTINE_MAXSIZE (10UL * 1024 * 1024) +#define QUARANTINE_MAXSIZE MiB(10) + +/* + * The amount of physical memory stolen by KASan at boot to back the shadow memory + * and page tables. Larger memory systems need to steal proportionally less. + */ +#ifdef __arm64__ +/* Works out at about 25% of 512 MiB and 15% of 3GiB system */ +# define STOLEN_MEM_PERCENT 13UL +# define STOLEN_MEM_BYTES MiB(62) +#else +# define STOLEN_MEM_PERCENT 25UL +# define STOLEN_MEM_BYTES 0 +#endif #ifndef KASAN # error KASAN undefined @@ -138,9 +153,12 @@ struct asan_global { #if defined(__x86_64__) # define _JBLEN ((9 * 2) + 3 + 16) +#elif defined(__arm64__) +# define _JBLEN ((14 + 8 + 2) * 2) +#else +# error "Unknown arch" #endif - typedef int jmp_buf[_JBLEN]; void _longjmp(jmp_buf env, int val); int _setjmp(jmp_buf env); diff --git a/san/tools/kasan_install b/san/tools/kasan_install index 5052dc945..01e7920c3 100755 --- a/san/tools/kasan_install +++ b/san/tools/kasan_install @@ -4,16 +4,10 @@ # kasan_install: set up a system to run the KASan kernel. Run with "--uninstall" # to reverse the setup. # -# Installs a symlink to the kernel kasan in /System/Library/Kernels/kernel.kasan -# and adds kcsuffix=kasan to boot-args. +# Adds kcsuffix=kasan to boot-args. # -kernel_name=kernel.kasan -kernel_src=/AppleInternal/CoreOS/xnu_kasan/${kernel_name} -SLK=/System/Library/Kernels/ -kernel_dst=${SLK}${kernel_name} - if [[ `whoami` != root ]] ; then echo "Re-running with sudo" sudo "$0" "$@" @@ -35,38 +29,8 @@ prompt() { kasan_install() { - dosymlink=0 dobootargs=0 - if [[ ! -f $kernel_src ]] ; then - echo "No KASan kernel found at $kernel_src" - exit 1 - fi - - echo -n "Installing KASan kernel... " - - if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then - echo "already installed." - elif [[ -f $kernel_dst ]] ; then - prompt "file exists. Overwrite?" && { - echo -n "Overwriting KASan kernel... " - dosymlink=1 - } - else - dosymlink=1 - fi - - # Use a temporary directory with a symlink to kernel.kasan. We can ditto - # from there into /S/L/K, even with SIP enabled. - [[ $dosymlink -eq 1 ]] && { - tmp=$(mktemp -d) || exit $? - ln -s "$kernel_src" "$tmp" || exit $? - ditto "$tmp" "$SLK" || exit $? - rm -r "$tmp" - echo "done." - } - - echo -n "Checking KASan boot args... " bootargs=$(nvram boot-args | cut -f2) @@ -97,42 +61,11 @@ kasan_install() { echo "done." } - [[ $dosymlink -eq 1 ]] && { - echo -n "Triggering kernel cache rebuild... " - touch /System/Library/Extensions || exit $? - echo "done." - } - } kasan_uninstall() { - echo -n "Removing kasan kernel... " - - dorm=0 - - if [[ -L $kernel_dst && $kernel_dst -ef $kernel_src ]] ; then - dorm=1 - elif [[ -f $kernel_dst ]] ; then - prompt "unexpected file. Remove anyway?" && { - dorm=1 - } - else - echo "not installed." - fi - - [[ $dorm -eq 1 ]] && { - if rm "$kernel_dst" ; then - echo "done." - else - if sip_enabled ; then - echo "failed due to SIP - this is normal." - fi - fi - } - - echo -n "Removing boot args... " bootargs=$(nvram boot-args | cut -f2) diff --git a/tools/lldbmacros/kcdata.py b/tools/lldbmacros/kcdata.py index ecc899a01..8732d5e94 100755 --- a/tools/lldbmacros/kcdata.py +++ b/tools/lldbmacros/kcdata.py @@ -978,7 +978,9 @@ KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_WAITINFO')] = KCT KNOWN_TYPES_COLLECTION[GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT')] = KCTypeDescription(GetTypeForName('STACKSHOT_KCTYPE_THREAD_GROUP'), ( KCSubTypeElement.FromBasicCtype('tgs_id', KCSUBTYPE_TYPE.KC_ST_UINT64, 0), - KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1), 8, 1) + KCSubTypeElement('tgs_name', KCSUBTYPE_TYPE.KC_ST_CHAR, KCSubTypeElement.GetSizeForArray(16, 1), + 8, 1), + KCSubTypeElement.FromBasicCtype('tgs_flags', KCSUBTYPE_TYPE.KC_ST_UINT64, 24), ), 'thread_group_snapshot') @@ -1331,8 +1333,14 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete): if not ss: print "No KCDATA_BUFFER_BEGIN_STACKSHOT object found. Skipping writing report." return + timestamp = ss.get('usecs_since_epoch', int(time.time())) - timestamp = time.strftime("%Y-%m-%d %H:%M:%S %z",time.gmtime(timestamp)) + try: + timestamp = time.strftime("%Y-%m-%d %H:%M:%S %z",time.gmtime(timestamp)) + except ValueError, e: + print "couldn't convert timestamp:", str(e) + timestamp = None + os_version = ss.get('osversion', 'Unknown') timebase = ss.get('mach_timebase_info', {"denom": 1, "numer": 1}) if not dsc_uuid and 'imageSlidBaseAddress' not in ss.get('shared_cache_dyld_load_info'): @@ -1373,7 +1381,8 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete): AllImageCatalog = [] obj = {} obj["kernel"] = os_version - obj["date"] = timestamp + if timestamp is not None: + obj["date"] = timestamp obj["reason"] = "kernel panic stackshot" obj["incident"] = "ABCDEFGH-1234-56IJ-789K-0LMNOPQRSTUV" obj["crashReporterKey"] = "12ab34cd45aabbccdd6712ab34cd45aabbccdd67" @@ -1486,7 +1495,15 @@ def SaveStackshotReport(j, outfile_name, dsc_uuid, dsc_libs_arr, incomplete): fh = sys.stdout else: fh = open(outfile_name, "w") - fh.write('{"bug_type":"288", "timestamp":"'+ timestamp +'", "os_version":"'+ os_version +'"}\n') + + header = {} + header['bug_type'] = 288 + if timestamp is not None: + header['timestamp'] = timestamp + header['os_version'] = os_version + fh.write(json.dumps(header)) + fh.write("\n") + fh.write(json.dumps(obj, sort_keys=False, indent=2, separators=(',', ': '))) fh.close() diff --git a/tools/lldbmacros/mbufs.py b/tools/lldbmacros/mbufs.py index fb41b65ea..548f9870f 100755 --- a/tools/lldbmacros/mbufs.py +++ b/tools/lldbmacros/mbufs.py @@ -626,6 +626,28 @@ def GetMbufTraceLeak(trace): cnt += 1 return out_string +@lldb_command('mbuf_largefailures') +def MbufLargeFailures(cmd_args=None): + """ Print the largest allocation failures + """ + topcnt = 0 + if (int(len(cmd_args)) > 0 and int(cmd_args[0]) < 5): + maxcnt = cmd_args[0] + else: + maxcnt = 5 + while (topcnt < maxcnt): + trace = kern.globals.mtracelarge_table[topcnt] + if (trace.size == 0): + topcnt += 1 + continue + print str(trace.size) + if (trace.depth != 0): + cnt = 0 + while (cnt < trace.depth): + print str(cnt + 1) + ": " + GetPc(trace.addr[cnt]) + cnt += 1 + topcnt += 1 + # Macro: mbuf_traceleak @lldb_command('mbuf_traceleak') diff --git a/tools/tests/darwintests/proc_info.c b/tools/tests/darwintests/proc_info.c index 11b042d0d..9c2e16f05 100644 --- a/tools/tests/darwintests/proc_info.c +++ b/tools/tests/darwintests/proc_info.c @@ -145,6 +145,10 @@ next:; T_PASS("found up-pointer for %s", uptr_names[found]); } } + + uint64_t up_overflow[2] = {0}; + uptrs_count = proc_list_uptrs(getpid(), up_overflow, sizeof(uint64_t)+1); + T_ASSERT_EQ(up_overflow[1], 0 , "overflow check"); } #pragma mark dynamic kqueue info diff --git a/tools/tests/darwintests/stackshot.m b/tools/tests/darwintests/stackshot.m index eb17dc1c5..8c6219ab7 100644 --- a/tools/tests/darwintests/stackshot.m +++ b/tools/tests/darwintests/stackshot.m @@ -15,6 +15,7 @@ T_GLOBAL_META( static const char *current_process_name(void); static void parse_stackshot(bool delta, void *ssbuf, size_t sslen); +static void parse_thread_group_stackshot(void **sbuf, size_t sslen); static uint64_t stackshot_timestamp(void *ssbuf, size_t sslen); static void initialize_thread(void); @@ -231,6 +232,34 @@ T_DECL(instrs_cycles, "test a getting instructions and cycles in stackshot") }); } +static void +check_thread_groups_supported() +{ + int err; + int supported = 0; + size_t supported_size = sizeof(supported); + err = sysctlbyname("kern.thread_groups_supported", &supported, &supported_size, NULL, 0); + + if (err || !supported) + T_SKIP("thread groups not supported on this system"); +} + +T_DECL(thread_groups, "test getting thread groups in stackshot") +{ + check_thread_groups_supported(); + + struct scenario scenario = { + .flags = (STACKSHOT_SAVE_LOADINFO | STACKSHOT_THREAD_GROUP + | STACKSHOT_KCDATA_FORMAT) + }; + + T_LOG("attempting to take stackshot with thread group flag"); + take_stackshot(&scenario, ^(void *ssbuf, size_t sslen) { + parse_thread_group_stackshot(ssbuf, sslen); + }); + +} + #pragma mark performance tests #define SHOULD_REUSE_SIZE_HINT 0x01 @@ -318,6 +347,79 @@ stackshot_timestamp(void *ssbuf, size_t sslen) #define TEST_THREAD_NAME "stackshot_test_thread" +static void +parse_thread_group_stackshot(void **ssbuf, size_t sslen) +{ + bool seen_thread_group_snapshot = false; + kcdata_iter_t iter = kcdata_iter(ssbuf, sslen); + T_ASSERT_EQ(kcdata_iter_type(iter), KCDATA_BUFFER_BEGIN_STACKSHOT, + "buffer provided is a stackshot"); + + NSMutableSet *thread_groups = [[NSMutableSet alloc] init]; + + iter = kcdata_iter_next(iter); + KCDATA_ITER_FOREACH(iter) { + switch (kcdata_iter_type(iter)) { + case KCDATA_TYPE_ARRAY: { + T_QUIET; + T_ASSERT_TRUE(kcdata_iter_array_valid(iter), + "checked that array is valid"); + + if (kcdata_iter_array_elem_type(iter) != STACKSHOT_KCTYPE_THREAD_GROUP_SNAPSHOT) { + continue; + } + + seen_thread_group_snapshot = true; + + if (kcdata_iter_array_elem_size(iter) >= sizeof(struct thread_group_snapshot_v2)) { + struct thread_group_snapshot_v2 *tgs_array = kcdata_iter_payload(iter); + for (uint32_t j = 0; j < kcdata_iter_array_elem_count(iter); j++) { + struct thread_group_snapshot_v2 *tgs = tgs_array + j; + [thread_groups addObject:@(tgs->tgs_id)]; + } + + } + else { + struct thread_group_snapshot *tgs_array = kcdata_iter_payload(iter); + for (uint32_t j = 0; j < kcdata_iter_array_elem_count(iter); j++) { + struct thread_group_snapshot *tgs = tgs_array + j; + [thread_groups addObject:@(tgs->tgs_id)]; + } + } + break; + } + } + } + KCDATA_ITER_FOREACH(iter) { + NSError *error = nil; + + switch (kcdata_iter_type(iter)) { + + case KCDATA_TYPE_CONTAINER_BEGIN: { + T_QUIET; + T_ASSERT_TRUE(kcdata_iter_container_valid(iter), + "checked that container is valid"); + + NSDictionary *container = parseKCDataContainer(&iter, &error); + T_QUIET; T_ASSERT_NOTNULL(container, "parsed container from stackshot"); + T_QUIET; T_ASSERT_NULL(error, "error unset after parsing container"); + + if (kcdata_iter_container_type(iter) != STACKSHOT_KCCONTAINER_THREAD) { + break; + } + + int tg = [container[@"thread_snapshots"][@"thread_group"] intValue]; + + T_ASSERT_TRUE([thread_groups containsObject:@(tg)], "check that the thread group the thread is in exists"); + + break; + }; + + } + } + T_ASSERT_TRUE(seen_thread_group_snapshot, "check that we have seen a thread group snapshot"); +} + static void parse_stackshot(bool delta, void *ssbuf, size_t sslen) { -- 2.45.2